From 4c1aac35db4d4823a57351e5752fd65ea0dd2b0d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 09:51:20 +0100 Subject: [PATCH 001/423] Modify ModelDrivenTransforms - Add new method Jp that computes the parameters' Jacobian. - Add ne LinearOrthoMDTransform --- menpofit/transform/modeldriven.py | 173 ++++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 35 deletions(-) diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index ddf8d00..1db964b 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -1,11 +1,11 @@ import numpy as np - from menpo.base import Targetable, Vectorizable +from menpo.shape import PointCloud from menpofit.modelinstance import PDM, GlobalPDM, OrthoPDM from menpo.transform.base import Transform, VComposable, VInvertible from menpofit.differentiable import DP - +# TODO: Should MDT implement VComposable and VInvertible? class ModelDrivenTransform(Transform, Targetable, Vectorizable, VComposable, VInvertible, DP): r""" @@ -135,22 +135,17 @@ def compose_after_from_vector_inplace(self, delta): r""" Composes two ModelDrivenTransforms together based on the first order approximation proposed by Papandreou and Maragos in [1]. - Parameters ---------- delta : (N,) ndarray Vectorized :class:`ModelDrivenTransform` to be applied **before** self - Returns -------- transform : self self, updated to the result of the composition - - References ---------- - .. [1] G. Papandreou and P. Maragos, "Adaptive and Constrained Algorithms for Inverse Compositional Active Appearance Model Fitting", CVPR08 @@ -176,15 +171,6 @@ def compose_after_from_vector_inplace(self, delta): # (n_points, n_params, n_dims) dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> eilk', dW_dx, dW_dp_0) - #TODO: Can we do this without splitting across the two dimensions? - # dW_dx_x = dW_dx[:, 0, :].flatten()[..., None] - # dW_dx_y = dW_dx[:, 1, :].flatten()[..., None] - # dW_dp_0_mat = np.reshape(dW_dp_0, (n_points * self.n_dims, - # self.n_parameters)) - # dW_dx_dW_dp_0 = dW_dp_0_mat * dW_dx_x + dW_dp_0_mat * dW_dx_y - # dW_dx_dW_dp_0 = np.reshape(dW_dx_dW_dp_0, - # (n_points, self.n_parameters, self.n_dims)) - # (n_params, n_params) J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) # (n_params, n_params) @@ -203,19 +189,14 @@ def _build_pseudoinverse(self): def pseudoinverse_vector(self, vector): r""" The vectorized pseudoinverse of a provided vector instance. - Syntactic sugar for - self.from_vector(vector).pseudoinverse.as_vector() - On ModelDrivenTransform this is especially fast - we just negate the vector provided. - Parameters ---------- vector : (P,) ndarray A vectorized version of self - Returns ------- pseudoinverse_vector : (N,) ndarray @@ -262,7 +243,6 @@ def d_dp(self, points): # dW_dl: n_points x (n_dims) x n_centres x n_dims # dX_dp: (n_points x n_dims) x n_params - # The following is equivalent to # np.einsum('ild, lpd -> ipd', self.dW_dl, dX_dp) @@ -273,6 +253,47 @@ def d_dp(self, points): return dW_dp + def Jp(self): + r""" + Compute parameters' Jacobian. + + References + ---------- + + .. [1] G. Papandreou and P. Maragos, "Adaptive and Constrained + Algorithms for Inverse Compositional Active Appearance Model + Fitting", CVPR08 + """ + # the incremental warp is always evaluated at p=0, ie the mean shape + points = self.pdm.model.mean().points + + # compute: + # - dW/dp when p=0 + # - dW/dp when p!=0 + # - dW/dx when p!=0 evaluated at the source landmarks + + # dW/dp when p=0 and when p!=0 are the same and simply given by + # the Jacobian of the model + # (n_points, n_params, n_dims) + dW_dp_0 = self.pdm.d_dp(points) + # (n_points, n_params, n_dims) + dW_dp = dW_dp_0 + + # (n_points, n_dims, n_dims) + dW_dx = self.transform.d_dx(points) + + # (n_points, n_params, n_dims) + dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> eilk', dW_dx, dW_dp_0) + + # (n_params, n_params) + J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) + # (n_params, n_params) + H = np.einsum('ijk, ilk -> jl', dW_dp, dW_dp) + # (n_params, n_params) + Jp = np.linalg.solve(H, J) + + return Jp + # noinspection PyMissingConstructor class GlobalMDTransform(ModelDrivenTransform): @@ -326,18 +347,76 @@ def compose_after_from_vector_inplace(self, delta): r""" Composes two ModelDrivenTransforms together based on the first order approximation proposed by Papandreou and Maragos in [1]. - Parameters ---------- delta : (N,) ndarray Vectorized :class:`ModelDrivenTransform` to be applied **before** self - Returns -------- transform : self self, updated to the result of the composition + References + ---------- + .. [1] G. Papandreou and P. Maragos, "Adaptive and Constrained + Algorithms for Inverse Compositional Active Appearance Model + Fitting", CVPR08 + """ + # the incremental warp is always evaluated at p=0, ie the mean shape + points = self.pdm.model.mean().points + + # compute: + # - dW/dp when p=0 + # - dW/dp when p!=0 + # - dW/dx when p!=0 evaluated at the source landmarks + + # dW/dq when p=0 and when p!=0 are the same and given by the + # Jacobian of the global transform evaluated at the mean of the + # model + # (n_points, n_global_params, n_dims) + dW_dq = self.pdm._global_transform_d_dp(points) + + # dW/db when p=0, is the Jacobian of the model + # (n_points, n_weights, n_dims) + dW_db_0 = PDM.d_dp(self.pdm, points) + + # dW/dp when p=0, is simply the concatenation of the previous + # two terms + # (n_points, n_params, n_dims) + dW_dp_0 = np.hstack((dW_dq, dW_db_0)) + + # by application of the chain rule dW_db when p!=0, + # is the Jacobian of the global transform wrt the points times + # the Jacobian of the model: dX(S)/db = dX/dS * dS/db + # (n_points, n_dims, n_dims) + dW_dS = self.pdm.global_transform.d_dx(points) + # (n_points, n_weights, n_dims) + dW_db = np.einsum('ilj, idj -> idj', dW_dS, dW_db_0) + + # dW/dp is simply the concatenation of dW_dq with dW_db + # (n_points, n_params, n_dims) + dW_dp = np.hstack((dW_dq, dW_db)) + # dW/dx is the jacobian of the transform evaluated at the source + # landmarks + # (n_points, n_dims, n_dims) + dW_dx = self.transform.d_dx(points) + + # (n_points, n_params, n_dims) + dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> ilk', dW_dx, dW_dp_0) + + # (n_params, n_params) + J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) + # (n_params, n_params) + H = np.einsum('ijk, ilk -> jl', dW_dp, dW_dp) + # (n_params, n_params) + Jp = np.linalg.solve(H, J) + + self.from_vector_inplace(self.as_vector() + np.dot(Jp, delta)) + + def Jp(self): + r""" + Compute parameters Jacobian. References ---------- @@ -389,16 +468,6 @@ def compose_after_from_vector_inplace(self, delta): # (n_points, n_params, n_dims) dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> ilk', dW_dx, dW_dp_0) - #TODO: Can we do this without splitting across the two dimensions? - # dW_dx_x = dW_dx[:, 0, :].flatten()[..., None] - # dW_dx_y = dW_dx[:, 1, :].flatten()[..., None] - # dW_dp_0_mat = np.reshape(dW_dp_0, (n_points * self.n_dims, - # self.n_parameters)) - # dW_dx_dW_dp_0 = dW_dp_0_mat * dW_dx_x + dW_dp_0_mat * dW_dx_y - # # (n_points, n_params, n_dims) - # dW_dx_dW_dp_0 = np.reshape(dW_dx_dW_dp_0, - # (n_points, self.n_parameters, self.n_dims)) - # (n_params, n_params) J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) # (n_params, n_params) @@ -406,7 +475,7 @@ def compose_after_from_vector_inplace(self, delta): # (n_params, n_params) Jp = np.linalg.solve(H, J) - self.from_vector_inplace(self.as_vector() + np.dot(Jp, delta)) + return Jp # noinspection PyMissingConstructor @@ -455,3 +524,37 @@ def __init__(self, model, transform_cls, global_transform, source=None): self._cached_points = None self.transform = transform_cls(source, self.target) + +# TODO: document me! +class LinearOrthoMDTransform(OrthoPDM, Transform): + r""" + """ + def __init__(self, model, n_landmarks): + super(LinearOrthoMDTransform, self).__init__(model) + self.n_landmarks = n_landmarks + self.W = np.vstack((self.similarity_model.components, + self.model.components)) + V = self.W[:, :self.n_dims*self.n_landmarks] + self.pinv_V = np.linalg.pinv(V) + + @property + def dense_target(self): + return PointCloud(self.target.points[self.n_landmarks:]) + + @property + def sparse_target(self): + return PointCloud(self.target.points[:self.n_landmarks]) + + def set_target(self, target): + if target.n_points == self.n_landmarks: + # densify target + target = np.dot(np.dot(target.as_vector(), self.pinv_V), self.W) + target = PointCloud(np.reshape(target, (-1, self.n_dims))) + OrthoPDM.set_target(self, target) + + def _apply(self, _, **kwargs): + return self.target.points[self.n_landmarks:] + + def d_dp(self, _): + return OrthoPDM.d_dp(self, _)[self.n_landmarks:, ...] + From 2b401d4d152743616c57dbea1510c1738d97c9b2 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 10:01:06 +0100 Subject: [PATCH 002/423] Update modelinstance.py --- menpofit/modelinstance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/menpofit/modelinstance.py b/menpofit/modelinstance.py index cf6924a..169612b 100644 --- a/menpofit/modelinstance.py +++ b/menpofit/modelinstance.py @@ -1,5 +1,4 @@ import numpy as np - from menpo.base import Targetable, Vectorizable from menpo.model import MeanInstanceLinearModel from menpofit.differentiable import DP @@ -185,7 +184,7 @@ def d_dp(self, points): return d_dp.swapaxes(0, 1) -# TODO: document me +# TODO: document me! class GlobalPDM(PDM): r""" """ @@ -319,7 +318,7 @@ def _global_transform_d_dp(self, points): return self.global_transform.d_dp(points) -# TODO: document me +# TODO: document me! class OrthoPDM(GlobalPDM): r""" """ From 5531ce1134176862188100ac783ed7e9a914a28f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 16:58:48 +0100 Subject: [PATCH 003/423] Add convenient functions to builder.py - add functions: compute_features, scale_images, warp_images, extract_patches, densify_shapes, align_shapes - move funtions: build_reference_frame, build_patch_reference_frame, _build_reference_frame from aam.builder.py - delete DeformableModelBuilder --- menpofit/builder.py | 239 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 199 insertions(+), 40 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index f9a31fa..29d4ec8 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -1,13 +1,12 @@ from __future__ import division -import abc import numpy as np -from menpo.shape import mean_pointcloud +from menpo.shape import mean_pointcloud, PointCloud, TriMesh +from menpo.image import Image, MaskedImage +from menpo.feature import no_op from menpo.transform import Scale, Translation, GeneralizedProcrustesAnalysis from menpo.model.pca import PCAModel from menpo.visualize import print_dynamic, progress_bar_str -from .base import is_pyramid_on_features - def compute_reference_shape(shapes, normalization_diagonal, verbose=False): r""" @@ -47,8 +46,8 @@ def compute_reference_shape(shapes, normalization_diagonal, verbose=False): return reference_shape -def normalization_wrt_reference_shape(images, group, label, - normalization_diagonal, verbose=False): +def normalization_wrt_reference_shape(images, group, label, diagonal, + verbose=False): r""" Function that normalizes the images sizes with respect to the reference shape (mean shape) scaling. This step is essential before building a @@ -57,7 +56,7 @@ def normalization_wrt_reference_shape(images, group, label, The normalization includes: 1) Computation of the reference shape as the mean shape of the images' landmarks. - 2) Scaling of the reference shape using the normalization_diagonal. + 2) Scaling of the reference shape using the diagonal. 3) Rescaling of all the images so that their shape's scale is in correspondence with the reference shape's scale. @@ -74,10 +73,10 @@ def normalization_wrt_reference_shape(images, group, label, The label of of the landmark manager that you wish to use. If no label is passed, the convex hull of all landmarks is used. - normalization_diagonal: `int` + diagonal: `int` If int, it ensures that the mean shape is scaled so that the diagonal of the bounding box containing it matches the - normalization_diagonal value. + diagonal value. If None, the mean shape is not rescaled. Note that, because the reference frame is computed from the mean @@ -100,7 +99,7 @@ def normalization_wrt_reference_shape(images, group, label, shapes = [i.landmarks[group][label] for i in images] # compute the reference shape and fix its diagonal length - reference_shape = compute_reference_shape(shapes, normalization_diagonal, + reference_shape = compute_reference_shape(shapes, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size @@ -118,7 +117,193 @@ def normalization_wrt_reference_shape(images, group, label, return reference_shape, normalized_images -def build_shape_model(shapes, max_components): +# TODO: document me! +def compute_features(images, features, level_str='', verbose=None): + feature_images = [] + for c, i in enumerate(images): + if verbose: + print_dynamic( + '{}Computing feature space: {}'.format( + level_str, progress_bar_str((c + 1.) / len(images), + show_bar=False))) + i = features(i) + feature_images.append(i) + + return feature_images + + +# TODO: document me! +def scale_images(images, scale, level_str='', verbose=None): + if scale != 1: + scaled_images = [] + for c, i in enumerate(images): + if verbose: + print_dynamic( + '{}Scaling features: {}'.format( + level_str, progress_bar_str((c + 1.) / len(images), + show_bar=False))) + scaled_images.append(i.rescale(scale)) + return scaled_images + else: + return images + + +# TODO: document me! +def warp_images(images, shapes, reference_frame, transform, level_str='', + verbose=None): + warped_images = [] + for c, (i, s) in enumerate(zip(images, shapes)): + if verbose: + print_dynamic('{}Warping images - {}'.format( + level_str, + progress_bar_str(float(c + 1) / len(images), + show_bar=False))) + # compute transforms + t = transform(reference_frame.landmarks['source'].lms, s) + # warp images + warped_i = i.warp_to_mask(reference_frame.mask, t) + # attach reference frame landmarks to images + warped_i.landmarks['source'] = reference_frame.landmarks['source'] + warped_images.append(warped_i) + return warped_images + + +# TODO: document me! +def extract_patches(images, shapes, parts_shape, normalize_function=no_op, + level_str='', verbose=None): + parts_images = [] + for c, (i, s) in enumerate(zip(images, shapes)): + if verbose: + print_dynamic('{}Warping images - {}'.format( + level_str, + progress_bar_str(float(c + 1) / len(images), + show_bar=False))) + parts = i.extract_patches(s, patch_size=parts_shape, + as_single_array=True) + if normalize_function: + parts = normalize_function(parts) + parts_images.append(Image(parts)) + return parts_images + +def build_reference_frame(landmarks, boundary=3, group='source', + trilist=None): + r""" + Builds a reference frame from a particular set of landmarks. + + Parameters + ---------- + landmarks : :map:`PointCloud` + The landmarks that will be used to build the reference frame. + + boundary : `int`, optional + The number of pixels to be left as a safe margin on the boundaries + of the reference frame (has potential effects on the gradient + computation). + + group : `string`, optional + Group that will be assigned to the provided set of landmarks on the + reference frame. + + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. + + If ``None``, defaults to performing Delaunay triangulation on the + points. + + Returns + ------- + reference_frame : :map:`Image` + The reference frame. + """ + reference_frame = _build_reference_frame(landmarks, boundary=boundary, + group=group) + if trilist is not None: + reference_frame.landmarks[group] = TriMesh( + reference_frame.landmarks['source'].lms.points, trilist=trilist) + + # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, + # perhaps the trilist should be directly obtained from the group landmarks + reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) + + return reference_frame + + +def build_patch_reference_frame(landmarks, boundary=3, group='source', + patch_shape=(17, 17)): + r""" + Builds a reference frame from a particular set of landmarks. + + Parameters + ---------- + landmarks : :map:`PointCloud` + The landmarks that will be used to build the reference frame. + + boundary : `int`, optional + The number of pixels to be left as a safe margin on the boundaries + of the reference frame (has potential effects on the gradient + computation). + + group : `string`, optional + Group that will be assigned to the provided set of landmarks on the + reference frame. + + patch_shape : tuple of ints, optional + Tuple specifying the shape of the patches. + + Returns + ------- + patch_based_reference_frame : :map:`Image` + The patch based reference frame. + """ + boundary = np.max(patch_shape) + boundary + reference_frame = _build_reference_frame(landmarks, boundary=boundary, + group=group) + + # mask reference frame + reference_frame.build_mask_around_landmarks(patch_shape, group=group) + + return reference_frame + + +def _build_reference_frame(landmarks, boundary=3, group='source'): + # translate landmarks to the origin + minimum = landmarks.bounds(boundary=boundary)[0] + landmarks = Translation(-minimum).apply(landmarks) + + resolution = landmarks.range(boundary=boundary) + reference_frame = MaskedImage.init_blank(resolution) + reference_frame.landmarks[group] = landmarks + + return reference_frame + + +# TODO: document me! +def densify_shapes(shapes, reference_frame, transform): + # compute non-linear transforms + transforms = [transform(reference_frame.landmarks['source'].lms, s) + for s in shapes] + # build dense shapes + dense_shapes = [] + for (t, s) in zip(transforms, shapes): + warped_points = t.apply(reference_frame.mask.true_indices()) + dense_shape = PointCloud(np.vstack((s.points, warped_points))) + dense_shapes.append(dense_shape) + + return dense_shapes + + +# TODO: document me! +def align_shapes(shapes): + r""" + """ + # centralize shapes + centered_shapes = [Translation(-s.centre()).apply(s) for s in shapes] + # align centralized shape using Procrustes Analysis + gpa = GeneralizedProcrustesAnalysis(centered_shapes) + return [s.aligned_source() for s in gpa.transforms] + + +def build_shape_model(shapes, max_components=None): r""" Builds a shape model given a set of shapes. @@ -137,38 +322,12 @@ def build_shape_model(shapes, max_components): shape_model: :class:`menpo.model.pca` The PCA shape model. """ - # centralize shapes - centered_shapes = [Translation(-s.centre()).apply(s) for s in shapes] - # align centralized shape using Procrustes Analysis - gpa = GeneralizedProcrustesAnalysis(centered_shapes) - aligned_shapes = [s.aligned_source() for s in gpa.transforms] - + # compute aligned shapes + aligned_shapes = align_shapes(shapes) # build shape model shape_model = PCAModel(aligned_shapes) if max_components is not None: # trim shape model if required shape_model.trim_components(max_components) - return shape_model - - -class DeformableModelBuilder(object): - r""" - Abstract class with a set of functions useful to build a Deformable Model. - """ - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def build(self, images, group=None, label=None): - r""" - Builds a Multilevel Deformable Model. - """ - - @property - def pyramid_on_features(self): - r""" - True if feature extraction happens once and then a gaussian pyramid - is taken. False if a gaussian pyramid is taken and then features are - extracted at each level. - """ - return is_pyramid_on_features(self.features) + return shape_model \ No newline at end of file From 145dac8b241eb67defa8a36b0959ae72f867d07d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:07:16 +0100 Subject: [PATCH 004/423] Add check funtions to check.py - Add functions: check_scales, check_patch_shape - Comment functions: check_n_levels, check_downscale, check_boundary --- menpofit/checks.py | 87 +++++++++++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/menpofit/checks.py b/menpofit/checks.py index 27b4e55..5098d03 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -48,38 +48,44 @@ def check_list_callables(callables, n_callables, allow_single=True): return callables -def check_n_levels(n_levels): - r""" - Checks the number of pyramid levels - must be int > 0. - """ - if not isinstance(n_levels, int) or n_levels < 1: - raise ValueError("n_levels must be int > 0") - - -def check_downscale(downscale): - r""" - Checks the downscale factor of the pyramid that must be >= 1. - """ - if downscale < 1: - raise ValueError("downscale must be >= 1") - - -def check_normalization_diagonal(normalization_diagonal): +def check_diagonal(diagonal): r""" Checks the diagonal length used to normalize the images' size that must be >= 20. """ - if normalization_diagonal is not None and normalization_diagonal < 20: - raise ValueError("normalization_diagonal must be >= 20") - - -def check_boundary(boundary): - r""" - Checks the boundary added around the reference shape that must be - int >= 0. - """ - if not isinstance(boundary, int) or boundary < 0: - raise ValueError("boundary must be >= 0") + if diagonal is not None and diagonal < 20: + raise ValueError("diagonal must be >= 20") + + +# TODO: document me! +def check_scales(scales): + if isinstance(scales, (int, float)): + return [scales], 1 + elif len(scales) == 1 and isinstance(scales[0], (int, float)): + return list(scales), 1 + elif len(scales) > 1: + l1, n1 = check_scales(scales[0]) + l2, n2 = check_scales(scales[1:]) + return l1 + l2, n1 + n2 + else: + raise ValueError("scales must be an int/float or a list/tuple of " + "int/float") + + +# TODO: document me! +def check_patch_shape(patch_shape, n_levels): + if len(patch_shape) == 2 and isinstance(patch_shape[0], int): + return [patch_shape] * n_levels + elif len(patch_shape) == 1: + return check_patch_shape(patch_shape[0], 1) + elif len(patch_shape) == n_levels: + l1 = check_patch_shape(patch_shape[0], 1) + l2 = check_patch_shape(patch_shape[1:], n_levels-1) + return l1 + l2 + else: + raise ValueError("patch_shape must be a list/tuple of int or a " + "list/tuple of lit/tuple of int/float with the " + "same length as scales") def check_max_components(max_components, n_levels, var_name): @@ -105,3 +111,28 @@ def check_max_components(max_components, n_levels, var_name): if not isinstance(comp, float): raise ValueError(str_error) return max_components_list + + +# def check_n_levels(n_levels): +# r""" +# Checks the number of pyramid levels - must be int > 0. +# """ +# if not isinstance(n_levels, int) or n_levels < 1: +# raise ValueError("n_levels must be int > 0") +# +# +# def check_downscale(downscale): +# r""" +# Checks the downscale factor of the pyramid that must be >= 1. +# """ +# if downscale < 1: +# raise ValueError("downscale must be >= 1") +# +# +# def check_boundary(boundary): +# r""" +# Checks the boundary added around the reference shape that must be +# int >= 0. +# """ +# if not isinstance(boundary, int) or boundary < 0: +# raise ValueError("boundary must be >= 0") From b838b52b03e90d195924170a942c6548117d8d0d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:10:47 +0100 Subject: [PATCH 005/423] Update import in transforms.__init__ --- menpofit/transform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/transform/__init__.py b/menpofit/transform/__init__.py index 4c6c12b..2aee5a1 100644 --- a/menpofit/transform/__init__.py +++ b/menpofit/transform/__init__.py @@ -1,4 +1,4 @@ -from .modeldriven import ModelDrivenTransform, OrthoMDTransform +from .modeldriven import OrthoMDTransform, LinearOrthoMDTransform from .homogeneous import (DifferentiableAffine, DifferentiableSimilarity, DifferentiableAlignmentSimilarity, DifferentiableAlignmentAffine) From 968646ddfb9a1c7b5c976c7a6107cd433862b564 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:19:40 +0100 Subject: [PATCH 006/423] Add new AAM types - LinearGlobalAAM, LinearPatchAAM and PartsAAM - Modified previous GlobalAAM and PatchAAM --- menpofit/aam/base.py | 466 +++++++++++++++++++++++++------------------ 1 file changed, 270 insertions(+), 196 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index fde1033..8d66070 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,85 +1,25 @@ from __future__ import division - +import abc import numpy as np from menpo.shape import TriMesh - -from menpofit.base import DeformableModel, name_of_callable -from .builder import build_patch_reference_frame, build_reference_frame +from menpofit.transform import DifferentiableThinPlateSplines +from menpofit.base import name_of_callable +from menpofit.builder import ( + build_reference_frame, build_patch_reference_frame) -class AAM(DeformableModel): +class AAM(object): r""" - Active Appearance Model class. - - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - - n_training_images : `int` - The number of training images used to build the AAM. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. - - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. - - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. - + Abstract interface for Active Appearance Model. """ - def __init__(self, shape_models, appearance_models, n_training_images, - transform, features, reference_shape, downscale, - scaled_shape_models): - DeformableModel.__init__(self, features) - self.n_training_images = n_training_images - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.reference_shape = reference_shape - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models - @property def n_levels(self): """ - The number of multi-resolution pyramidal levels of the AAM. + The number of scale levels of the AAM. :type: `int` """ - return len(self.appearance_models) + return len(self.scales) def instance(self, shape_weights=None, appearance_weights=None, level=-1): r""" @@ -151,39 +91,13 @@ def random_instance(self, level=-1): return self._instance(level, shape_instance, appearance_instance) + @abc.abstractmethod def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() - landmarks = template.landmarks['source'].lms - - reference_frame = self._build_reference_frame( - shape_instance, landmarks) - - transform = self.transform( - reference_frame.landmarks['source'].lms, landmarks) - - return appearance_instance.warp_to_mask(reference_frame.mask, - transform, warp_landmarks=True) - - def _build_reference_frame(self, reference_shape, landmarks): - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - return build_reference_frame( - reference_shape, trilist=trilist) - - @property - def _str_title(self): - r""" - Returns a string containing name of the model. - - :type: `string` - """ - return 'Active Appearance Model' + pass def view_shape_models_widget(self, n_parameters=5, - parameters_bounds=(-3.0, 3.0), mode='multiple', - popup=False, figure_size=(10, 8)): + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): r""" Visualizes the shape models of the AAM object using the `menpo.visualize.widgets.visualize_shape_model` widget. @@ -212,86 +126,21 @@ def view_shape_models_widget(self, n_parameters=5, from menpofit.visualize import visualize_shape_model visualize_shape_model(self.shape_models, n_parameters=n_parameters, parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, popup=popup) + figure_size=figure_size, mode=mode,) + @abc.abstractmethod def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), - mode='multiple', popup=False, - figure_size=(10, 8)): - r""" - Visualizes the appearance models of the AAM object using the - `menpo.visualize.widgets.visualize_appearance_model` widget. - - Parameters - ----------- - n_parameters : `int` or `list` of `int` or ``None``, optional - The number of appearance principal components to be used for the - parameters sliders. - If `int`, then the number of sliders per level is the minimum - between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. - parameters_bounds : (`float`, `float`), optional - The minimum and maximum bounds, in std units, for the sliders. - mode : {``single``, ``multiple``}, optional - If ``'single'``, only a single slider is constructed along with a - drop down menu. - If ``'multiple'``, a slider is constructed for each parameter. - popup : `bool`, optional - If ``True``, the widget will appear as a popup window. - figure_size : (`int`, `int`), optional - The size of the plotted figures. - """ - from menpofit.visualize import visualize_appearance_model - visualize_appearance_model(self.appearance_models, - n_parameters=n_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, - popup=popup) + mode='multiple', figure_size=(10, 8)): + pass + @abc.abstractmethod def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', - popup=False, figure_size=(10, 8)): - r""" - Visualizes both the shape and appearance models of the AAM object using - the `menpo.visualize.widgets.visualize_aam` widget. - - Parameters - ----------- - n_shape_parameters : `int` or `list` of `int` or None, optional - The number of shape principal components to be used for the - parameters sliders. - If `int`, then the number of sliders per level is the minimum - between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. - n_appearance_parameters : `int` or `list` of `int` or None, optional - The number of appearance principal components to be used for the - parameters sliders. - If `int`, then the number of sliders per level is the minimum - between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. - parameters_bounds : (`float`, `float`), optional - The minimum and maximum bounds, in std units, for the sliders. - mode : {``single``, ``multiple``}, optional - If ``'single'``, only a single slider is constructed along with a - drop down menu. - If ``'multiple'``, a slider is constructed for each parameter. - popup : `bool`, optional - If ``True``, the widget will appear as a popup window. - figure_size : (`int`, `int`), optional - The size of the plotted figures. - """ - from menpofit.visualize import visualize_aam - visualize_aam(self, n_shape_parameters=n_shape_parameters, - n_appearance_parameters=n_appearance_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, popup=popup) + figure_size=(10, 8)): + pass + # TODO: fix me! def __str__(self): out = "{}\n - {} training images.\n".format(self._str_title, self.n_training_images) @@ -395,7 +244,111 @@ def __str__(self): return out -class PatchBasedAAM(AAM): +class GlobalAAM(AAM): + r""" + Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + n_training_images : `int` + The number of training images used to build the AAM. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + downscale : `float` + The downscale factor that was used to create the different pyramidal + levels. + + scaled_shape_models : `boolean`, optional + If ``True``, the reference frames are the mean shapes of each pyramid + level, so the shape models are scaled. + + If ``False``, the reference frames of all levels are the mean shape of + the highest level, so the shape models are not scaled; they have the + same size. + + Note that from our experience, if scaled_shape_models is ``False``, AAMs + tend to have slightly better performance. + + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + def _instance(self, level, shape_instance, appearance_instance): + template = self.appearance_models[level].mean() + landmarks = template.landmarks['source'].lms + + if type(landmarks) == TriMesh: + trilist = landmarks.trilist + else: + trilist = None + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) + + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + instance = appearance_instance.warp_to_mask( + reference_frame.mask, transform) + instance.landmarks = reference_frame.landmarks + + return instance + + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + from menpofit.visualize import visualize_appearance_model + visualize_appearance_model(self.appearance_models, + n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + # TODO: fix me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + from menpofit.visualize import visualize_aam + visualize_aam(self, n_shape_parameters=n_shape_parameters, + n_appearance_parameters=n_appearance_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + +class PatchAAM(AAM): r""" Patch Based Active Appearance Model class. @@ -451,31 +404,152 @@ class PatchBasedAAM(AAM): AAMs tend to have slightly better performance. """ - def __init__(self, shape_models, appearance_models, n_training_images, - patch_shape, transform, features, reference_shape, - downscale, scaled_shape_models): - super(PatchBasedAAM, self).__init__( - shape_models, appearance_models, n_training_images, transform, - features, reference_shape, downscale, scaled_shape_models) + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = DifferentiableThinPlateSplines self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features - def _build_reference_frame(self, reference_shape, landmarks): - return build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape) + def _instance(self, level, shape_instance, appearance_instance): + template = self.appearance_models[level].mean + landmarks = template.landmarks['source'].lms - @property - def _str_title(self): - r""" - Returns a string containing name of the model. + reference_frame = build_patch_reference_frame( + shape_instance, patch_shape=self.patch_shape) - :type: `string` - """ - return 'Patch-Based Active Appearance Model' + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) - def __str__(self): - out = super(PatchBasedAAM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + instance = appearance_instance.warp_to_mask(reference_frame.mask, + transform) + instance.landmarks = reference_frame.landmarks + + return instance + + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + from menpofit.visualize import visualize_appearance_model + visualize_appearance_model(self.appearance_models, + n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + # TODO: fix me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + from menpofit.visualize import visualize_aam + visualize_aam(self, n_shape_parameters=n_shape_parameters, + n_appearance_parameters=n_appearance_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + +# TODO: document me! +class LinearGlobalAAM(AAM): + r""" + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features, + n_landmarks): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks + + # TODO: implement me! + def _instance(self, level, shape_instance, appearance_instance): + raise NotImplemented + + # TODO: implement me! + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + +# TODO: document me! +class LinearPatchAAM(AAM): + r""" + """ + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, scales, scale_shapes, + scale_features, n_landmarks): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = DifferentiableThinPlateSplines + self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks + + # TODO: implement me! + def _instance(self, level, shape_instance, appearance_instance): + raise NotImplemented + + # TODO: implement me! + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + +# TODO: document me! +class PartsAAM(AAM): + r""" + """ + def __init__(self, shape_models, appearance_models, reference_shape, + parts_shape, features, normalize_parts, scales, + scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.parts_shape = parts_shape + self.features = features + self.normalize_parts = normalize_parts + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + # TODO: implement me! + def _instance(self, level, shape_instance, appearance_instance): + raise NotImplemented + + # TODO: implement me! + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented \ No newline at end of file From ee8df0d06863c60f037d4c3547c7300cfb016d82 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:20:52 +0100 Subject: [PATCH 007/423] Add builders for all AAM classes - PatchAAMs, LinearPatchAAMs and PartsAAMs can now have different patch sizes. --- menpofit/aam/builder.py | 1025 +++++++++++++++++++++++---------------- 1 file changed, 620 insertions(+), 405 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index e330a8b..886f73c 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,24 +1,146 @@ from __future__ import division -import numpy as np - -from menpo.shape import TriMesh -from menpo.image import MaskedImage -from menpo.transform import Translation -from menpo.feature import igo +import abc +from copy import deepcopy from menpo.model import PCAModel -from menpo.visualize import print_dynamic, progress_bar_str - +from menpo.shape import mean_pointcloud +from menpo.feature import no_op +from menpo.visualize import print_dynamic from menpofit import checks -from menpofit.base import create_pyramid -from menpofit.builder import (DeformableModelBuilder, build_shape_model, - normalization_wrt_reference_shape) -from menpofit.transform import (DifferentiablePiecewiseAffine, - DifferentiableThinPlateSplines) +from menpofit.builder import ( + normalization_wrt_reference_shape, compute_features, scale_images, + warp_images, extract_patches, build_shape_model, align_shapes, + build_reference_frame, build_patch_reference_frame, densify_shapes) +from menpofit.transform import ( + DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -class AAMBuilder(DeformableModelBuilder): +class AAMBuilder(object): r""" - Class that builds Multilevel Active Appearance Models. + Abstract interface for Active Appearance Model Builder. + """ + def build(self, images, group=None, label=None, verbose=False): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + + label : `string`, optional + The label of of the landmark manager that you wish to use. If no + label is passed, the convex hull of all landmarks is used. + + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest level + """ + # normalize images and compute reference shape + reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + shape_models = [] + appearance_models = [] + # for each pyramid level (high --> low) + for j, s in enumerate(self.scales): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if j == 0: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, self.features, + level_str=level_str, + verbose=verbose) + + # extract potentially rescaled shapes + level_shapes = [i.landmarks[group][label] + for i in level_images] + + # obtain shape representation + if j == 0 or self.scale_shapes: + # obtain shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # add shape model to the list + shape_models.append(shape_model) + else: + # copy precious shape model and add it to the list + shape_models.append(deepcopy(shape_model)) + + # obtain warped images + warped_images = self._warp_images(level_images, level_shapes, + shape_model.mean(), j, + level_str, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format(level_str)) + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + appearance_models.append(appearance_model) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + # reverse the list of shape and appearance models so that they are + # ordered from lower to higher resolution + shape_models.reverse() + appearance_models.reverse() + self.scales.reverse() + + aam = self._build_aam(shape_models, appearance_models, reference_shape) + + return aam + + @classmethod + def _build_shape_model(cls, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + + @abc.abstractmethod + def _build_aam(self, shape_models, appearance_models, reference_shape): + pass + + +# TODO: implement checker for conflict between features and scale_features +# TODO: document me! +class GlobalAAMBuilder(AAMBuilder): + r""" + Class that builds a Global Active Appearance Model. Parameters ---------- @@ -44,13 +166,12 @@ class AAMBuilder(DeformableModelBuilder): Triangle list that will be used to build the reference frame. If ``None``, defaults to performing Delaunay triangulation on the points. - normalization_diagonal : `int` >= ``20``, optional + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the normalization_diagonal - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. @@ -59,25 +180,11 @@ class AAMBuilder(DeformableModelBuilder): reference frame (provided that features computation does not change the image size). - n_levels : `int` > 0, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(``n_levels``) - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of - each pyramid level, so the shape models will be scaled. + scales : `int` or float` or list of those, optional - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. + scale_shapes : `boolean`, optional - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -113,11 +220,6 @@ class AAMBuilder(DeformableModelBuilder): If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- aam : :map:`AAMBuilder` @@ -126,241 +228,202 @@ class AAMBuilder(DeformableModelBuilder): Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError ``max_shape_components`` must be ``None`` or an `int` > 0 or a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``n_levels`` elements + ``len(scales)`` elements ValueError ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``n_levels`` elements - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements + ``len(scales)`` elements """ - def __init__(self, features=igo, transform=DifferentiablePiecewiseAffine, - trilist=None, normalization_diagonal=None, n_levels=3, - downscale=2, scaled_shape_models=True, - max_shape_components=None, max_appearance_components=None, - boundary=3): + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, 0.5), + scale_shapes=False, scale_features=True, + max_shape_components=None, max_appearance_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') - features = checks.check_features(features, n_levels) - # store parameters + # set parameters self.features = features self.transform = transform self.trilist = trilist - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - self.boundary = boundary - def build(self, images, group=None, label=None, verbose=False): - r""" - Builds a Multilevel Active Appearance Model from a list of - landmarked images. + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + self.reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. + def _build_aam(self, shape_models, appearance_models, reference_shape): + return GlobalAAM(shape_models, appearance_models, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features) - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. +# TODO: document me! +class PatchAAMBuilder(AAMBuilder): + r""" + Class that builds a Patch Active Appearance Model. - verbose : `boolean`, optional - Flag that controls information and progress printing. + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from lowest - to highest level - """ - # compute reference_shape and normalize images size - self.reference_shape, normalized_images = \ - normalization_wrt_reference_shape(images, group, label, - self.normalization_diagonal, - verbose=verbose) - - # create pyramid - generators = create_pyramid(normalized_images, self.n_levels, - self.downscale, self.features, - verbose=verbose) - - # build the model at each pyramid level - if verbose: - if self.n_levels > 1: - print_dynamic('- Building model for each of the {} pyramid ' - 'levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building model\n') + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - shape_models = [] - appearance_models = [] - # for each pyramid level (high --> low) - for j in range(self.n_levels): - # since models are built from highest to lowest level, the - # parameters in form of list need to use a reversed index - rj = self.n_levels - j - 1 + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - if verbose: - level_str = ' - ' - if self.n_levels > 1: - level_str = ' - Level {}: '.format(j + 1) + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - # get feature images of current level - feature_images = [] - for c, g in enumerate(generators): - if verbose: - print_dynamic( - '{}Computing feature space/rescaling - {}'.format( - level_str, - progress_bar_str((c + 1.) / len(generators), - show_bar=False))) - feature_images.append(next(g)) + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - # extract potentially rescaled shapes - shapes = [i.landmarks[group][label] for i in feature_images] + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - # define shapes that will be used for training - if j == 0: - original_shapes = shapes - train_shapes = shapes - else: - if self.scaled_shape_models: - train_shapes = shapes - else: - train_shapes = original_shapes + If ``None``, the mean shape is not rescaled. - # train shape model and find reference frame - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = build_shape_model( - train_shapes, self.max_shape_components[rj]) - reference_frame = self._build_reference_frame(shape_model.mean()) + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - # add shape model to the list - shape_models.append(shape_model) + scales : `int` or float` or list of those, optional - # compute transforms - if verbose: - print_dynamic('{}Computing transforms'.format(level_str)) + scale_shapes : `boolean`, optional + scale_features : `boolean`, optional - # Create a dummy initial transform - s_to_t_transform = self.transform( - reference_frame.landmarks['source'].lms, - reference_frame.landmarks['source'].lms) + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - # warp images to reference frame - warped_images = [] - for c, i in enumerate(feature_images): - if verbose: - print_dynamic('{}Warping images - {}'.format( - level_str, - progress_bar_str(float(c + 1) / len(feature_images), - show_bar=False))) - # Setting the target can be significantly faster for transforms - # such as CachedPiecewiseAffine - s_to_t_transform.set_target(i.landmarks[group][label]) - warped_images.append(i.warp_to_mask(reference_frame.mask, - s_to_t_transform)) - - # attach reference_frame to images' source shape - for i in warped_images: - i.landmarks['source'] = reference_frame.landmarks['source'] - - # build appearance model - if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components[rj] is not None: - appearance_model.trim_components( - self.max_appearance_components[rj]) + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. - # add appearance model to the list - appearance_models.append(appearance_model) + Per level: + If `int`, it specifies the exact number of components to be + retained. - if verbose: - print_dynamic('{}Done\n'.format(level_str)) + If `float`, it specifies the percentage of variance to be retained. - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - appearance_models.reverse() - n_training_images = len(images) + If ``None``, all the available components are kept + (100% of variance). - return self._build_aam(shape_models, appearance_models, - n_training_images) + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + Per level: + If `int`, it specifies the exact number of components to be + retained. - Returns - ------- - reference_frame : :map:`MaskedImage` - The reference frame. - """ - return build_reference_frame(mean_shape, boundary=self.boundary, - trilist=self.trilist) + If `float`, it specifies the percentage of variance to be retained. - def _build_aam(self, shape_models, appearance_models, n_training_images): - r""" - Returns an AAM object. + If ``None``, all the available components are kept + (100% of variance). - Parameters - ---------- - shape_models : :map:`PCAModel` - The trained multilevel shape models. - - appearance_models : :map:`PCAModel` - The trained multilevel appearance models. - - n_training_images : `int` - The number of training images. + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object - Returns - ------- - aam : :map:`AAM` - The trained AAM object. - """ - from .base import AAM - return AAM(shape_models, appearance_models, n_training_images, - self.transform, self.features, self.reference_shape, - self.downscale, self.scaled_shape_models) + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=True, + scale_features=True, max_shape_components=None, + max_appearance_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + self.reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[level]) + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) + def _build_aam(self, shape_models, appearance_models, reference_shape): + return PatchAAM(shape_models, appearance_models, reference_shape, + self.patch_shape, self.features, self.scales, + self.scale_shapes, self.scale_features) -class PatchBasedAAMBuilder(AAMBuilder): + +# TODO: document me! +class LinearGlobalAAMBuilder(AAMBuilder): r""" - Class that builds Multilevel Patch-Based Active Appearance Models. + Class that builds a Linear Global Active Appearance Model. Parameters ---------- @@ -378,45 +441,33 @@ class PatchBasedAAMBuilder(AAMBuilder): once and then creating a pyramid on top tends to lead to better performing AAMs. - patch_shape : tuple of `int`, optional - The appearance model of the Patch-Based AAM will be obtained by - sampling appearance patches with the specified shape around each - landmark. + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. - normalization_diagonal : `int` >= ``20``, optional + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. + + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the ``normalization_diagonal`` - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. - .. note:: - - Because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - downscale : `float` >= 1, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: + scales : `int` or float` or list of those, optional - (downscale ** k) for k in range(``n_levels``) + scale_shapes : `boolean`, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of each - pyramid level, so the shape models will be scaled. - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -446,215 +497,379 @@ class PatchBasedAAMBuilder(AAMBuilder): Per level: If `int`, it specifies the exact number of components to be retained. + If `float`, it specifies the percentage of variance to be retained. + If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- - aam : ::map:`PatchBasedAAMBuilder` - The Patch-Based AAM Builder object + aam : :map:`AAMBuilder` + The AAM Builder object Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` - ValueError - ``downscale`` must be >= ``1`` + ``diagonal`` must be >= ``20``. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``max_shape_components must be ``None`` or an `int` > ``0`` or - a ``0`` <= `float` <= ``1`` or a list of those containing ``1`` - or ``n_levels`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > 0 or a - ``0`` <= `float` <= ``1`` or a list of those containing ``1`` - or ``n_levels`` elements + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError - ``features`` must be a `string` or a `function` or a list of those - containing 1 or ``n_levels`` elements + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements ValueError - ``pyramid_on_features`` is enabled so ``features`` must be a - `string` or a `function` or a list containing one of those + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, features=igo, patch_shape=(16, 16), - normalization_diagonal=None, n_levels=3, downscale=2, - scaled_shape_models=True, max_shape_components=None, - max_appearance_components=None, boundary=3): + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None, max_appearance_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') - features = checks.check_features(features, n_levels) - - # store parameters + # set parameters self.features = features - self.patch_shape = patch_shape - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.transform = transform + self.trilist = trilist + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - self.boundary = boundary - # patch-based AAMs can only work with TPS transform - self.transform = DifferentiableThinPlateSplines + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model( + dense_shapes, max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) + + def _build_aam(self, shape_models, appearance_models, reference_shape): + return LinearGlobalAAM(shape_models, appearance_models, + reference_shape, self.transform, + self.features, self.scales, + self.scale_shapes, self.scale_features, + self.n_landmarks) + + +# TODO: document me! +class LinearPatchAAMBuilder(AAMBuilder): + r""" + Class that builds a Linear Patch Active Appearance Model. - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - Returns - ------- - reference_frame : :map:`MaskedImage` - The patch-based reference frame. - """ - return build_patch_reference_frame(mean_shape, boundary=self.boundary, - patch_shape=self.patch_shape) + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - def _mask_image(self, image): - r""" - Creates the patch-based mask of the given image. + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - Parameters - ---------- - image : :map:`MaskedImage` - The image to be masked. - """ - image.build_mask_around_landmarks(self.patch_shape, group='source') + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - def _build_aam(self, shape_models, appearance_models, n_training_images): - r""" - Returns a Patch-Based AAM object. + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - Parameters - ---------- - shape_models : :map:`PCAModel` - The trained multilevel shape models. + If ``None``, the mean shape is not rescaled. - appearance_models : :map:`PCAModel` - The trained multilevel appearance models. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - n_training_images : `int` - The number of training images. + scales : `int` or float` or list of those, optional - Returns - ------- - aam : :map:`PatchBasedAAM` - The trained Patched-Based AAM object. - """ - from .base import PatchBasedAAM - return PatchBasedAAM(shape_models, appearance_models, - n_training_images, self.patch_shape, - self.transform, self.features, - self.reference_shape, self.downscale, - self.scaled_shape_models) + scale_shapes : `boolean`, optional + scale_features : `boolean`, optional -def build_reference_frame(landmarks, boundary=3, group='source', - trilist=None): - r""" - Builds a reference frame from a particular set of landmarks. + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - Parameters - ---------- - landmarks : :map:`PointCloud` - The landmarks that will be used to build the reference frame. + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. - boundary : `int`, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). + Per level: + If `int`, it specifies the exact number of components to be + retained. - group : `string`, optional - Group that will be assigned to the provided set of landmarks on the - reference frame. + If `float`, it specifies the percentage of variance to be retained. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. + If ``None``, all the available components are kept + (100% of variance). - If ``None``, defaults to performing Delaunay triangulation on the - points. + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - Returns - ------- - reference_frame : :map:`Image` - The reference frame. - """ - reference_frame = _build_reference_frame(landmarks, boundary=boundary, - group=group) - if trilist is not None: - reference_frame.landmarks[group] = TriMesh( - reference_frame.landmarks['source'].lms.points, trilist=trilist) + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. - # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, - # perhaps the trilist should be directly obtained from the group landmarks - reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) + If `float`, it specifies the percentage of variance to be retained. - return reference_frame + If ``None``, all the available components are kept + (100% of variance). + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=False, + scale_features=True, max_shape_components=None, + max_appearance_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components -def build_patch_reference_frame(landmarks, boundary=3, group='source', - patch_shape=(16, 16)): + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[level]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes, + max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) + + def _build_aam(self, shape_models, appearance_models, reference_shape): + return LinearPatchAAM(shape_models, appearance_models, + reference_shape, self.patch_shape, + self.features, self.scales, self.scale_shapes, + self.scale_features, self.n_landmarks) + + +# TODO: document me! +# TODO: implement offsets support? +class PartsAAMBuilder(AAMBuilder): r""" - Builds a reference frame from a particular set of landmarks. + Class that builds a Parts Active Appearance Model. Parameters ---------- - landmarks : :map:`PointCloud` - The landmarks that will be used to build the reference frame. + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + normalize_parts : `callable`, optional + + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). + + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - boundary : `int`, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. - group : `string`, optional - Group that will be assigned to the provided set of landmarks on the - reference frame. + If `float`, it specifies the percentage of variance to be retained. - patch_shape : tuple of ints, optional - Tuple specifying the shape of the patches. + If ``None``, all the available components are kept + (100% of variance). + + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). Returns ------- - patch_based_reference_frame : :map:`Image` - The patch based reference frame. + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - boundary = np.max(patch_shape) + boundary - reference_frame = _build_reference_frame(landmarks, boundary=boundary, - group=group) + def __init__(self, patch_shape=(17, 17), features=no_op, + normalize_parts=no_op, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None, max_appearance_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components - # mask reference frame - reference_frame.build_mask_around_landmarks(patch_shape, group=group) + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return extract_patches(images, shapes, self.patch_shape[level], + normalize_function=self.normalize_parts, + level_str=level_str, verbose=verbose) - return reference_frame + def _build_aam(self, shape_models, appearance_models, reference_shape): + return PartsAAM(shape_models, appearance_models, reference_shape, + self.patch_shape, self.features, + self.normalize_parts, self.scales, + self.scale_shapes, self.scale_features) -def _build_reference_frame(landmarks, boundary=3, group='source'): - # translate landmarks to the origin - minimum = landmarks.bounds(boundary=boundary)[0] - landmarks = Translation(-minimum).apply(landmarks) +from .base import ( + GlobalAAM, PatchAAM, LinearGlobalAAM, LinearPatchAAM, PartsAAM) - resolution = landmarks.range(boundary=boundary) - reference_frame = MaskedImage.init_blank(resolution) - reference_frame.landmarks[group] = landmarks - return reference_frame From 926cc474c44de6f5ae95d2841c0c6e92d9c0afe2 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:59:38 +0100 Subject: [PATCH 008/423] Update documentation in aam.base.py --- menpofit/aam/base.py | 172 ++++++++++++++++++++++++++++++++----------- 1 file changed, 130 insertions(+), 42 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 8d66070..2c39d1e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -256,14 +256,15 @@ class GlobalAAM(AAM): appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - n_training_images : `int` - The number of training images used to build the AAM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional + features : `callable` or ``[callable]``, If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -277,24 +278,11 @@ class GlobalAAM(AAM): once and then creating a pyramid on top tends to lead to better performing AAMs. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. + scales : `int` or float` or list of those, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + scale_shapes : `boolean` - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. - - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + scale_features : `boolean` """ def __init__(self, shape_models, appearance_models, reference_shape, @@ -360,17 +348,14 @@ class PatchAAM(AAM): appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - n_training_images : `int` - The number of training images used to build the AAM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, optional + features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -384,24 +369,11 @@ class PatchAAM(AAM): once and then creating a pyramid on top tends to lead to better performing AAMs. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + scales : `int` or float` or list of those - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. + scale_shapes : `boolean` - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean` """ def __init__(self, shape_models, appearance_models, reference_shape, @@ -455,6 +427,44 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearGlobalAAM(AAM): r""" + Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + """ def __init__(self, shape_models, appearance_models, reference_shape, transform, features, scales, scale_shapes, scale_features, @@ -489,6 +499,45 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearPatchAAM(AAM): r""" + Linear Patch Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + n_landmarks: `int` + """ def __init__(self, shape_models, appearance_models, reference_shape, patch_shape, features, scales, scale_shapes, @@ -524,6 +573,45 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class PartsAAM(AAM): r""" + Parts Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + normalize_parts: `callable` + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + """ def __init__(self, shape_models, appearance_models, reference_shape, parts_shape, features, normalize_parts, scales, From fe082538139952468758c36bb67e39a9b1bc6aa7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 18:37:41 +0100 Subject: [PATCH 009/423] Minor documnetation changes in aam.base and aam.builder --- menpofit/aam/base.py | 8 ++++---- menpofit/aam/builder.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 2c39d1e..af9915e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -338,7 +338,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, class PatchAAM(AAM): r""" - Patch Based Active Appearance Model class. + Patch based Based Active Appearance Model class. Parameters ----------- @@ -427,7 +427,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearGlobalAAM(AAM): r""" - Active Appearance Model class. + Linear Active Appearance Model class. Parameters ----------- @@ -499,7 +499,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearPatchAAM(AAM): r""" - Linear Patch Active Appearance Model class. + Linear Patch based Active Appearance Model class. Parameters ----------- @@ -573,7 +573,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class PartsAAM(AAM): r""" - Parts Active Appearance Model class. + Parts based Active Appearance Model class. Parameters ----------- diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 886f73c..3a8543a 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -140,7 +140,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class GlobalAAMBuilder(AAMBuilder): r""" - Class that builds a Global Active Appearance Model. + Class that builds Active Appearance Models. Parameters ---------- @@ -282,7 +282,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class PatchAAMBuilder(AAMBuilder): r""" - Class that builds a Patch Active Appearance Model. + Class that builds Patch based Active Appearance Models. Parameters ---------- @@ -423,7 +423,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class LinearGlobalAAMBuilder(AAMBuilder): r""" - Class that builds a Linear Global Active Appearance Model. + Class that builds Linear Active Appearance Models. Parameters ---------- @@ -577,7 +577,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class LinearPatchAAMBuilder(AAMBuilder): r""" - Class that builds a Linear Patch Active Appearance Model. + Class that builds Linear Patch based Active Appearance Models. Parameters ---------- @@ -730,7 +730,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: implement offsets support? class PartsAAMBuilder(AAMBuilder): r""" - Class that builds a Parts Active Appearance Model. + Class that builds Parts based Active Appearance Models. Parameters ---------- From 7833be897f9669068d7b46e3505a592b561f7929 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:20:16 +0100 Subject: [PATCH 010/423] Restructure aam.base - Remove pure AAM interface --- menpofit/aam/base.py | 212 ++++++++++++++++--------------------------- 1 file changed, 79 insertions(+), 133 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index af9915e..4266ae7 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -10,8 +10,56 @@ class AAM(object): r""" - Abstract interface for Active Appearance Model. + Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean` + + scale_features : `boolean` + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + @property def n_levels(self): """ @@ -91,54 +139,44 @@ def random_instance(self, level=-1): return self._instance(level, shape_instance, appearance_instance) - @abc.abstractmethod def _instance(self, level, shape_instance, appearance_instance): - pass + template = self.appearance_models[level].mean() + landmarks = template.landmarks['source'].lms - def view_shape_models_widget(self, n_parameters=5, - parameters_bounds=(-3.0, 3.0), - mode='multiple', figure_size=(10, 8)): - r""" - Visualizes the shape models of the AAM object using the - `menpo.visualize.widgets.visualize_shape_model` widget. + if type(landmarks) == TriMesh: + trilist = landmarks.trilist + else: + trilist = None + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) - Parameters - ----------- - n_parameters : `int` or `list` of `int` or ``None``, optional - The number of shape principal components to be used for the - parameters sliders. - If `int`, then the number of sliders per level is the minimum - between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. - parameters_bounds : (`float`, `float`), optional - The minimum and maximum bounds, in std units, for the sliders. - mode : {``single``, ``multiple``}, optional - If ``'single'``, only a single slider is constructed along with a - drop down menu. - If ``'multiple'``, a slider is constructed for each parameter. - popup : `bool`, optional - If ``True``, the widget will appear as a popup window. - figure_size : (`int`, `int`), optional - The size of the plotted figures. - """ - from menpofit.visualize import visualize_shape_model - visualize_shape_model(self.shape_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode,) + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + instance = appearance_instance.warp_to_mask( + reference_frame.mask, transform) + instance.landmarks = reference_frame.landmarks + + return instance - @abc.abstractmethod def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): - pass + from menpofit.visualize import visualize_appearance_model + visualize_appearance_model(self.appearance_models, + n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) - @abc.abstractmethod + # TODO: fix me! def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): - pass + from menpofit.visualize import visualize_aam + visualize_aam(self, n_shape_parameters=n_shape_parameters, + n_appearance_parameters=n_appearance_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) # TODO: fix me! def __str__(self): @@ -244,98 +282,6 @@ def __str__(self): return out -class GlobalAAM(AAM): - r""" - Active Appearance Model class. - - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean` - - scale_features : `boolean` - - """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() - landmarks = template.landmarks['source'].lms - - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) - - transform = self.transform( - reference_frame.landmarks['source'].lms, landmarks) - - instance = appearance_instance.warp_to_mask( - reference_frame.mask, transform) - instance.landmarks = reference_frame.landmarks - - return instance - - def view_appearance_models_widget(self, n_parameters=5, - parameters_bounds=(-3.0, 3.0), - mode='multiple', figure_size=(10, 8)): - from menpofit.visualize import visualize_appearance_model - visualize_appearance_model(self.appearance_models, - n_parameters=n_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode) - - # TODO: fix me! - def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, - parameters_bounds=(-3.0, 3.0), mode='multiple', - figure_size=(10, 8)): - from menpofit.visualize import visualize_aam - visualize_aam(self, n_shape_parameters=n_shape_parameters, - n_appearance_parameters=n_appearance_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode) - - class PatchAAM(AAM): r""" Patch based Based Active Appearance Model class. @@ -425,7 +371,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! -class LinearGlobalAAM(AAM): +class LinearAAM(AAM): r""" Linear Active Appearance Model class. @@ -614,11 +560,11 @@ class PartsAAM(AAM): """ def __init__(self, shape_models, appearance_models, reference_shape, - parts_shape, features, normalize_parts, scales, + patch_shape, features, normalize_parts, scales, scale_shapes, scale_features): self.shape_models = shape_models self.appearance_models = appearance_models - self.parts_shape = parts_shape + self.patch_shape = patch_shape self.features = features self.normalize_parts = normalize_parts self.reference_shape = reference_shape From fe2e86e779a5298c465341a4a0601681fa1d9a8f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:21:41 +0100 Subject: [PATCH 011/423] Restructure aam.builder - Remove pure AAMBuilder interface --- menpofit/aam/builder.py | 258 +++++++++++++++++++--------------------- 1 file changed, 124 insertions(+), 134 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 3a8543a..6ad44f2 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -14,131 +14,9 @@ DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -class AAMBuilder(object): - r""" - Abstract interface for Active Appearance Model Builder. - """ - def build(self, images, group=None, label=None, verbose=False): - r""" - Builds an Active Appearance Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest level - """ - # normalize images and compute reference shape - reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - shape_models = [] - appearance_models = [] - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if j == 0: - # compute features at highest level - feature_images = compute_features(images, self.features, - level_str=level_str, - verbose=verbose) - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, self.features, - level_str=level_str, - verbose=verbose) - - # extract potentially rescaled shapes - level_shapes = [i.landmarks[group][label] - for i in level_images] - - # obtain shape representation - if j == 0 or self.scale_shapes: - # obtain shape model - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) - # add shape model to the list - shape_models.append(shape_model) - else: - # copy precious shape model and add it to the list - shape_models.append(deepcopy(shape_model)) - - # obtain warped images - warped_images = self._warp_images(level_images, level_shapes, - shape_model.mean(), j, - level_str, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - appearance_models.append(appearance_model) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - appearance_models.reverse() - self.scales.reverse() - - aam = self._build_aam(shape_models, appearance_models, reference_shape) - - return aam - - @classmethod - def _build_shape_model(cls, shapes, max_components, level): - return build_shape_model(shapes, max_components=max_components) - - @abc.abstractmethod - def _build_aam(self, shape_models, appearance_models, reference_shape): - pass - - # TODO: implement checker for conflict between features and scale_features # TODO: document me! -class GlobalAAMBuilder(AAMBuilder): +class AAMBuilder(object): r""" Class that builds Active Appearance Models. @@ -266,6 +144,119 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components + def build(self, images, group=None, label=None, verbose=False): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + + label : `string`, optional + The label of of the landmark manager that you wish to use. If no + label is passed, the convex hull of all landmarks is used. + + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest level + """ + # normalize images and compute reference shape + reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + shape_models = [] + appearance_models = [] + # for each pyramid level (high --> low) + for j, s in enumerate(self.scales): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if j == 0: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, self.features, + level_str=level_str, + verbose=verbose) + + # extract potentially rescaled shapes + level_shapes = [i.landmarks[group][label] + for i in level_images] + + # obtain shape representation + if j == 0 or self.scale_shapes: + # obtain shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # add shape model to the list + shape_models.append(shape_model) + else: + # copy precious shape model and add it to the list + shape_models.append(deepcopy(shape_model)) + + # obtain warped images + warped_images = self._warp_images(level_images, level_shapes, + shape_model.mean(), j, + level_str, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format(level_str)) + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + appearance_models.append(appearance_model) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + # reverse the list of shape and appearance models so that they are + # ordered from lower to higher resolution + shape_models.reverse() + appearance_models.reverse() + self.scales.reverse() + + aam = self._build_aam(shape_models, appearance_models, reference_shape) + + return aam + + @classmethod + def _build_shape_model(cls, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): self.reference_frame = build_reference_frame(reference_shape) @@ -274,9 +265,9 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): - return GlobalAAM(shape_models, appearance_models, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features) + return AAM(shape_models, appearance_models, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features) # TODO: document me! @@ -421,7 +412,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! -class LinearGlobalAAMBuilder(AAMBuilder): +class LinearAAMBuilder(AAMBuilder): r""" Class that builds Linear Active Appearance Models. @@ -567,11 +558,11 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): - return LinearGlobalAAM(shape_models, appearance_models, - reference_shape, self.transform, - self.features, self.scales, - self.scale_shapes, self.scale_features, - self.n_landmarks) + return LinearAAM(shape_models, appearance_models, + reference_shape, self.transform, + self.features, self.scales, + self.scale_shapes, self.scale_features, + self.n_landmarks) # TODO: document me! @@ -869,7 +860,6 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): self.scale_shapes, self.scale_features) -from .base import ( - GlobalAAM, PatchAAM, LinearGlobalAAM, LinearPatchAAM, PartsAAM) +from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM From d257b83bc68158e5c1b2b3dda326c716c820ddef Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:24:23 +0100 Subject: [PATCH 012/423] Restructure fitter.py - Add new MultiFitter, ModelFitter - Move noisy_align from base.py to fitter.py --- menpofit/base.py | 34 ---- menpofit/fitter.py | 395 ++++++++++++++++++++++----------------------- 2 files changed, 190 insertions(+), 239 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index 41e8fc4..23f683b 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -98,40 +98,6 @@ def pyramid_on_features(self): return is_pyramid_on_features(self.features) -# TODO: Should this be a method on Similarity? AlignableTransforms? -def noisy_align(source, target, noise_std=0.04, rotation=False): - r""" - Constructs and perturbs the optimal similarity transform between source - to the target by adding white noise to its weights. - - Parameters - ---------- - source: :class:`menpo.shape.PointCloud` - The source pointcloud instance used in the alignment - target: :class:`menpo.shape.PointCloud` - The target pointcloud instance used in the alignment - noise_std: float - The standard deviation of the white noise - - Default: 0.04 - rotation: boolean - If False the second parameter of the Similarity, - which captures captures inplane rotations, is set to 0. - - Default:False - - Returns - ------- - noisy_transform : :class: `menpo.transform.Similarity` - The noisy Similarity Transform - """ - transform = AlignmentSimilarity(source, target, rotation=rotation) - parameters = transform.as_vector() - parameter_range = np.hstack((parameters[:2], target.range())) - noise = (parameter_range * noise_std * - np.random.randn(transform.n_parameters)) - return Similarity.init_identity(source.n_dims).from_vector(parameters + noise) - def build_sampling_grid(patch_shape): r""" diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 971e2ad..70c262f 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,112 +1,21 @@ from __future__ import division -import abc -from menpo.transform import AlignmentAffine, Scale, AlignmentSimilarity import numpy as np from menpo.shape import PointCloud -from menpofit.base import is_pyramid_on_features, pyramid_of_feature_images, \ - noisy_align -from menpofit.fittingresult import MultilevelFittingResult +from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity -class Fitter(object): +# TODO: document me! +class MultiFitter(object): r""" - Abstract interface that all :map:`Fitter` objects must implement. """ - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def _set_up(self, **kwargs): - r""" - Abstract method that sets up the fitter object. - """ - pass - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - r""" - Fits the fitter to an image. - - Parameters - ----------- - image: :map:`Image` or subclass - The image to be fitted. - initial_parameters: list - The initial parameters of the model. - gt_shape: :map:`PointCloud` - The ground truth shape associated to the image. - - Returns - ------- - fitting_result: :map:`FittingResult` - The fitting result containing the result of fitting procedure. - """ - fitting_result = self._create_fitting_result( - image, initial_parameters, gt_shape=gt_shape) - return self._fit(fitting_result, **kwargs) - - @abc.abstractmethod - def _create_fitting_result(self, **kwargs): - r""" - Abstract method that defines the fitting result object associated to - the fitter object. - """ - pass - - @abc.abstractmethod - def _fit(self, **kwargs): - r""" - Abstract method implements a particular alignment algorithm. - """ - pass - - def get_parameters(self, shape): - r""" - Abstract method that gets the parameters. - """ - pass - - -class MultilevelFitter(Fitter): - r""" - Abstract interface that all :map:`MultilevelFitter` must implement. - """ - - @abc.abstractproperty - def reference_shape(self): - r""" - The reference shape of the multilevel fitter. - """ - pass - - @abc.abstractproperty - def features(self): - r""" - Returns the feature computation functions applied at each pyramidal - level. - """ - pass - - @abc.abstractproperty + @property def n_levels(self): r""" - The number of pyramidal levels. - """ - pass - - @abc.abstractproperty - def downscale(self): - r""" - The downscale factor used by the multiple fitter. - """ - pass + The number of pyramidal levels used during alignment. - @property - def pyramid_on_features(self): - r""" - Returns True if the pyramid is computed on the feature image and False - if it is computed on the original (intensities) image and features are - extracted at each level. + :type: `int` """ - return is_pyramid_on_features(self.features) + return len(self.scales) def fit(self, image, initial_shape, max_iters=50, gt_shape=None, **kwargs): @@ -155,64 +64,16 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, affine_correction = AlignmentAffine(initial_shapes[-1], initial_shape) # run multilevel fitting - fitting_results = self._fit(images, initial_shapes[0], - max_iters=max_iters, - gt_shapes=gt_shapes, **kwargs) + algorithm_results = self._fit(images, initial_shapes[0], + max_iters=max_iters, + gt_shapes=gt_shapes, **kwargs) # build multilevel fitting result - multi_fitting_result = self._create_fitting_result( - image, fitting_results, affine_correction, gt_shape=gt_shape) - - return multi_fitting_result - - def perturb_shape(self, gt_shape, noise_std=0.04, rotation=False): - r""" - Generates an initial shape by adding gaussian noise to the perfect - similarity alignment between the ground truth and reference_shape. - - Parameters - ----------- - gt_shape: :class:`menpo.shape.PointCloud` - The ground truth shape. - noise_std: float, optional - The standard deviation of the gaussian noise used to produce the - initial shape. - - Default: 0.04 - rotation: boolean, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the initial shape. - - Default: False - - Returns - ------- - initial_shape: :class:`menpo.shape.PointCloud` - The initial shape. - """ - reference_shape = self.reference_shape - return noisy_align(reference_shape, gt_shape, noise_std=noise_std, - rotation=rotation).apply(reference_shape) - - def obtain_shape_from_bb(self, bounding_box): - r""" - Generates an initial shape given a bounding box detection. + fitter_result = self._fitter_result( + image, self, algorithm_results, affine_correction, + gt_shape=gt_shape) - Parameters - ----------- - bounding_box: (2, 2) ndarray - The bounding box specified as: - - np.array([[x_min, y_min], [x_max, y_max]]) - - Returns - ------- - initial_shape: :class:`menpo.shape.PointCloud` - The initial shape. - """ - reference_shape = self.reference_shape - return align_shape_with_bb(reference_shape, - bounding_box).apply(reference_shape) + return fitter_result def _prepare_image(self, image, initial_shape, gt_shape=None): r""" @@ -246,6 +107,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None): gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ + # attach landmarks to the image image.landmarks['initial_shape'] = initial_shape if gt_shape: @@ -256,8 +118,24 @@ def _prepare_image(self, image, initial_shape, gt_shape=None): image = image.rescale_to_reference_shape(self.reference_shape, group='initial_shape') - images = list(reversed(list(pyramid_of_feature_images( - self.n_levels, self.downscale, self.features, image)))) + # obtain image representation + from copy import deepcopy + scales = deepcopy(self.scales) + scales.reverse() + images = [] + for j, s in enumerate(scales): + if j == 0: + # compute features at highest level + feature_image = self.features(image) + elif self.scale_features: + # scale features at other levels + feature_image = images[0].rescale(s) + else: + # scale image and compute features at other levels + scaled_image = image.rescale(s) + feature_image = self.features(scaled_image) + images.append(feature_image) + images.reverse() # get initial shapes per level initial_shapes = [i.landmarks['initial_shape'].lms for i in images] @@ -265,46 +143,11 @@ def _prepare_image(self, image, initial_shape, gt_shape=None): # get ground truth shapes per level if gt_shape: gt_shapes = [i.landmarks['gt_shape'].lms for i in images] - del image.landmarks['gt_shape'] else: gt_shapes = None return images, initial_shapes, gt_shapes - def _create_fitting_result(self, image, fitting_results, affine_correction, - gt_shape=None): - r""" - Creates the :class: `menpo.aam.fitting.MultipleFitting` object - associated with a particular Fitter object. - - Parameters - ----------- - image: :class:`menpo.image.masked.MaskedImage` - The original image to be fitted. - fitting_results: :class:`menpo.fit.fittingresult.FittingResultList` - A list of basic fitting objects containing the state of the - different fitting levels. - affine_correction: :class: `menpo.transforms.affine.Affine` - An affine transform that maps the result of the top resolution - fitting level to the space scale of the original image. - gt_shape: class:`menpo.shape.PointCloud`, optional - The ground truth shape associated to the image. - - Default: None - error_type: 'me_norm', 'me' or 'rmse', optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Default: 'me_norm' - - Returns - ------- - fitting: :class:`menpo.fitmultilevel.fittingresult.MultilevelFittingResult` - The fitting object that will hold the state of the fitter. - """ - return MultilevelFittingResult(image, self, fitting_results, - affine_correction, gt_shape=gt_shape) - def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, **kwargs): r""" @@ -332,14 +175,32 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, Returns ------- - fitting_results: :class:`menpo.fit.fittingresult.FittingResult` list + algorithm_results: :class:`menpo.fg2015.fittingresult.FittingResult` list The fitting object containing the state of the whole fitting procedure. """ + max_iters = self._prepare_max_iters(max_iters) shape = initial_shape gt_shape = None - n_levels = self.n_levels + algorithm_results = [] + for j, (i, alg, it, s) in enumerate(zip(images, self.algorithms, + max_iters, self.scales)): + if gt_shapes: + gt_shape = gt_shapes[j] + + algorithm_result = alg.run(i, shape, gt_shape=gt_shape, + max_iters=it, **kwargs) + algorithm_results.append(algorithm_result) + + shape = algorithm_result.final_shape + if s != self.scales[-1]: + Scale(self.scales[j+1]/s, + n_dims=shape.n_dims).apply_inplace(shape) + + return algorithm_results + def _prepare_max_iters(self, max_iters): + n_levels = self.n_levels # check max_iters parameter if type(max_iters) is int: max_iters = [np.round(max_iters/n_levels) @@ -351,22 +212,111 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, raise ValueError('max_iters can be integer, integer list ' 'containing 1 or {} elements or ' 'None'.format(self.n_levels)) + return np.require(max_iters, dtype=np.int) - # fit images - fitting_results = [] - for j, (i, f, it) in enumerate(zip(images, self._fitters, max_iters)): - if gt_shapes is not None: - gt_shape = gt_shapes[j] - parameters = f.get_parameters(shape) - fitting_result = f.fit(i, parameters, gt_shape=gt_shape, - max_iters=it, **kwargs) - fitting_results.append(fitting_result) +# TODO: document me! +class ModelFitter(MultiFitter): + r""" + """ + @property + def reference_shape(self): + r""" + The reference shape of the AAM. + + :type: :map:`PointCloud` + """ + return self._model.reference_shape + + @property + def features(self): + r""" + The feature extracted at each pyramidal level during AAM building. + Stored in ascending pyramidal order. + + :type: `list` + """ + return self._model.features + + @property + def n_levels(self): + r""" + The number of pyramidal levels used during AAM building. + + :type: `int` + """ + return self._model.n_levels + + @property + def scales(self): + return self._model.scales + + @property + def scale_features(self): + r""" + Flag that defined the nature of Gaussian pyramid used to build the + AAM. + If ``True``, the feature space is computed once at the highest scale + and the Gaussian pyramid is applied to the feature images. + If ``False``, the Gaussian pyramid is applied to the original images + and features are extracted at each level. + + :type: `boolean` + """ + return self._model.scale_features + + def _check_n_shape(self, n_shape): + if n_shape is not None: + if type(n_shape) is int or type(n_shape) is float: + for sm in self._model.shape_models: + sm.n_active_components = n_shape + elif len(n_shape) == 1 and self._model.n_levels > 1: + for sm in self._model.shape_models: + sm.n_active_components = n_shape[0] + elif len(n_shape) == self._model.n_levels: + for sm, n in zip(self._model.shape_models, n_shape): + sm.n_active_components = n + else: + raise ValueError('n_shape can be an integer or a float or None' + 'or a list containing 1 or {} of ' + 'those'.format(self._model.n_levels)) + + def perturb_shape(self, gt_shape, noise_std=0.04, rotation=False): + transform = noisy_align(AlignmentSimilarity, self.reference_shape, + gt_shape, noise_std=noise_std, + rotation=rotation) + return transform.apply(self.reference_shape) + + def obtain_shape_from_bb(self, bounding_box): + r""" + Generates an initial shape given a bounding box detection. - shape = fitting_result.final_shape - Scale(self.downscale, n_dims=shape.n_dims).apply_inplace(shape) + Parameters + ----------- + bounding_box: (2, 2) ndarray + The bounding box specified as: + + np.array([[x_min, y_min], [x_max, y_max]]) - return fitting_results + Returns + ------- + initial_shape: :class:`menpo.shape.PointCloud` + The initial shape. + """ + + reference_shape = self.reference_shape + return align_shape_with_bb(reference_shape, + bounding_box).apply(reference_shape) + + +# TODO: document me! +def noisy_align(alignment_transform_cls, source, target, noise_std=0.04, + rotation=True): + r""" + """ + noise = noise_std * np.random.randn(target.n_points, target.n_dims) + noisy_target = PointCloud(target.points + noise) + return alignment_transform_cls(source, noisy_target, rotation=rotation) def align_shape_with_bb(shape, bounding_box): @@ -390,4 +340,39 @@ def align_shape_with_bb(shape, bounding_box): """ shape_box = PointCloud(shape.bounds()) bounding_box = PointCloud(bounding_box) - return AlignmentSimilarity(shape_box, bounding_box, rotation=False) \ No newline at end of file + return AlignmentSimilarity(shape_box, bounding_box, rotation=False) + + +# TODO: implement as a method on Similarity? AlignableTransforms? +# def noisy_align(source, target, noise_std=0.04, rotation=False): +# r""" +# Constructs and perturbs the optimal similarity transform between source +# to the target by adding white noise to its weights. +# +# Parameters +# ---------- +# source: :class:`menpo.shape.PointCloud` +# The source pointcloud instance used in the alignment +# target: :class:`menpo.shape.PointCloud` +# The target pointcloud instance used in the alignment +# noise_std: float +# The standard deviation of the white noise +# +# Default: 0.04 +# rotation: boolean +# If False the second parameter of the Similarity, +# which captures captures inplane rotations, is set to 0. +# +# Default:False +# +# Returns +# ------- +# noisy_transform : :class: `menpo.transform.Similarity` +# The noisy Similarity Transform +# """ +# transform = AlignmentSimilarity(source, target, rotation=rotation) +# parameters = transform.as_vector() +# parameter_range = np.hstack((parameters[:2], target.range())) +# noise = (parameter_range * noise_std * +# np.random.randn(transform.n_parameters)) +# return Similarity.init_identity(source.n_dims).from_vector(parameters + noise) \ No newline at end of file From d766d9591f247bb1f04b0ec8304b07f7bd0f4782 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:26:39 +0100 Subject: [PATCH 013/423] Force OrthoMDTransform and OrthoMDPDM to use only similarity transforms --- menpofit/modelinstance.py | 7 +++++-- menpofit/transform/modeldriven.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/menpofit/modelinstance.py b/menpofit/modelinstance.py index 169612b..7810e72 100644 --- a/menpofit/modelinstance.py +++ b/menpofit/modelinstance.py @@ -322,7 +322,7 @@ def _global_transform_d_dp(self, points): class OrthoPDM(GlobalPDM): r""" """ - def __init__(self, model, global_transform_cls): + def __init__(self, model): # 1. Construct similarity model from the mean of the model self.similarity_model = similarity_2d_instance_model(model.mean()) # 2. Orthonormalize model and similarity model @@ -330,7 +330,9 @@ def __init__(self, model, global_transform_cls): model_cpy.orthonormalize_against_inplace(self.similarity_model) self.similarity_weights = self.similarity_model.project( model_cpy.mean()) - super(OrthoPDM, self).__init__(model_cpy, global_transform_cls) + from menpofit.transform import DifferentiableAlignmentSimilarity + super(OrthoPDM, self).__init__(model_cpy, + DifferentiableAlignmentSimilarity) @property def global_parameters(self): @@ -353,3 +355,4 @@ def _update_global_weights(self, global_weights): def _global_transform_d_dp(self, points): return self.similarity_model.components.reshape( self.n_global_parameters, -1, self.n_dims).swapaxes(0, 1) + diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index 1db964b..fc5fd8a 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -519,8 +519,8 @@ class OrthoMDTransform(GlobalMDTransform): The source landmarks of the transform. If no `source` is provided the mean of the model is used. """ - def __init__(self, model, transform_cls, global_transform, source=None): - self.pdm = OrthoPDM(model, global_transform) + def __init__(self, model, transform_cls, source=None): + self.pdm = OrthoPDM(model) self._cached_points = None self.transform = transform_cls(source, self.target) From c797e1b90b159bd4a930a07729ab08a86077dca0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:29:16 +0100 Subject: [PATCH 014/423] Add LKAAMFitter - All AAM objects are used with a single fitter (this is different to the original implementation in my repo but seems more user friendly) --- menpofit/aam/fitter.py | 460 ++++++----------------------------------- 1 file changed, 61 insertions(+), 399 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 045f388..957bead 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,243 +1,70 @@ from __future__ import division -from itertools import chain +from menpofit.fitter import ModelFitter +from menpofit.modelinstance import OrthoPDM +from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform +from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM +from .algorithm import ( + StandardAAMInterface, LinearAAMInterface, PartsAAMInterface, AIC) +from .result import AAMFitterResult -from menpofit.base import name_of_callable -from menpofit.fitter import MultilevelFitter -from menpofit.fittingresult import AMMultilevelFittingResult -from menpofit.transform import (ModelDrivenTransform, OrthoMDTransform, - DifferentiableAlignmentSimilarity) -from menpofit.lucaskanade.appearance import SIC - -class AAMFitter(MultilevelFitter): +# TODO: document me! +class LKAAMFitter(ModelFitter): r""" - Abstract Interface for defining Active Appearance Models Fitters. - - Parameters - ----------- - aam : :map:`AAM` - The Active Appearance Model to be used. """ - def __init__(self, aam): - self.aam = aam - - @property - def reference_shape(self): - r""" - The reference shape of the AAM. - - :type: :map:`PointCloud` - """ - return self.aam.reference_shape - - @property - def features(self): - r""" - The feature extracted at each pyramidal level during AAM building. - Stored in ascending pyramidal order. - - :type: `list` - """ - return self.aam.features - - @property - def n_levels(self): - r""" - The number of pyramidal levels used during AAM building. - - :type: `int` - """ - return self.aam.n_levels - - @property - def downscale(self): - r""" - The downscale used to generate the final scale factor applied at - each pyramidal level during AAM building. - The scale factor is computed as: - - ``(downscale ** k) for k in range(n_levels)`` - - :type: `float` - """ - return self.aam.downscale - - def _create_fitting_result(self, image, fitting_results, affine_correction, - gt_shape=None): - r""" - Creates a :map:`AAMMultilevelFittingResult` associated to a - particular fitting of the AAM fitter. - - Parameters - ----------- - image : :map:`Image` or subclass - The image to be fitted. - - fitting_results : `list` of :map:`FittingResult` - A list of fitting result objects containing the state of the - the fitting for each pyramidal level. - - affine_correction : :map:`Affine` - An affine transform that maps the result of the top resolution - level to the scale space of the original image. - - gt_shape : :map:`PointCloud`, optional - The ground truth shape associated to the image. - - error_type : 'me_norm', 'me' or 'rmse', optional - Specifies how the error between the fitted and ground truth - shapes must be computed. - - Returns - ------- - fitting : :map:`AAMMultilevelFittingResult` - A fitting result object that will hold the state of the AAM - fitter for a particular fitting. - """ - return AAMMultilevelFittingResult( - image, self, fitting_results, affine_correction, gt_shape=gt_shape) - - -class LucasKanadeAAMFitter(AAMFitter): - r""" - Lucas-Kanade based :map:`Fitter` for Active Appearance Models. - - Parameters - ----------- - aam : :map:`AAM` - The Active Appearance Model to be used. - algorithm : subclass of :map:`AppearanceLucasKanade`, optional - The Appearance Lucas-Kanade class to be used. - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - n_appearance : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of appearance components or amount of appearance variance - to be used per pyramidal level. - - If `None`, all available appearance components - ``(n_active_components)`` will be used. - If `int` ``> 1``, the specified number of appearance components will - be used. - If ``0. <=`` `float` ``<= 1.``, the number of appearance components - capturing the specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - """ - def __init__(self, aam, algorithm=SIC, - md_transform=OrthoMDTransform, n_shape=None, + def __init__(self, aam, algorithm_cls=AIC, n_shape=None, n_appearance=None, **kwargs): - super(LucasKanadeAAMFitter, self).__init__(aam) - self._set_up(algorithm=algorithm, md_transform=md_transform, - n_shape=n_shape, n_appearance=n_appearance, **kwargs) - - @property - def algorithm(self): - r""" - Returns a string containing the name of fitting algorithm. - - :type: `str` - """ - return 'LK-AAM-' + self._fitters[0].algorithm - - def _set_up(self, algorithm=SIC, - md_transform=OrthoMDTransform, - global_transform=DifferentiableAlignmentSimilarity, - n_shape=None, n_appearance=None, **kwargs): - r""" - Sets up the Lucas-Kanade fitter object. - - Parameters - ----------- - algorithm : subclass of :map:`AppearanceLucasKanade`, optional - The Appearance Lucas-Kanade class to be used. - - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - - n_appearance : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of appearance components or amount of appearance variance - to be used per pyramidal level. + super(LKAAMFitter, self).__init__() + self._model = aam + self._algorithms = [] + self._check_n_shape(n_shape) + self._check_n_appearance(n_appearance) + self._set_up(algorithm_cls, **kwargs) + + def _set_up(self, algorithm_cls, **kwargs): + for j, (am, sm) in enumerate(zip(self._model.appearance_models, + self._model.shape_models)): + + if type(self.aam) is AAM or type(self.aam) is PatchAAM: + # build orthonormal model driven transform + md_transform = OrthoMDTransform( + sm, self._model.transform, + source=am.mean().landmarks['source'].lms) + # set up algorithm using standard aam interface + algorithm = algorithm_cls(StandardAAMInterface, am, + md_transform, **kwargs) + + elif (type(self.aam) is LinearAAM or + type(self.aam) is LinearPatchAAM): + # build linear version of orthogonal model driven transform + md_transform = LinearOrthoMDTransform( + sm, self._model.n_landmarks) + # set up algorithm using linear aam interface + algorithm = algorithm_cls(LinearAAMInterface, am, + md_transform, **kwargs) + + elif type(self.aam) is PartsAAM: + # build orthogonal point distribution model + pdm = OrthoPDM(sm) + # set up algorithm using parts aam interface + am.patch_shape = self._model.patch_shape[j] + am.normalize_parts = self._model.normalize_parts + algorithm = algorithm_cls(PartsAAMInterface, am, pdm, **kwargs) - If `None`, all available appearance components - ``(n_active_components)`` will be used. - If `int` ``> 1``, the specified number of appearance components will - be used. - If ``0. <=`` `float` ``<= 1.``, the number of appearance components - capturing the specified variance ratio will be computed and used. + else: + raise ValueError("AAM object must be of one of the " + "following classes: {}, {}, {}, {}, " + "{}".format(AAM, PatchAAM, LinearAAM, + LinearPatchAAM, PartsAAM)) - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. + # append algorithms to list + self._algorithms.append(algorithm) - Raises - ------- - ValueError - ``n_shape`` can be an `int`, `float`, ``None`` or a `list` - containing ``1`` or ``n_levels`` of those. - ValueError - ``n_appearance`` can be an `int`, `float`, `None` or a `list` - containing ``1`` or ``n_levels`` of those. - """ - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.aam.n_levels > 1: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.aam.n_levels: - for sm, n in zip(self.aam.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None ' - 'or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + @property + def aam(self): + return self._model - # check n_appearance parameter + def _check_n_appearance(self, n_appearance): if n_appearance is not None: if type(n_appearance) is int or type(n_appearance) is float: for am in self.aam.appearance_models: @@ -253,173 +80,8 @@ def _set_up(self, algorithm=SIC, 'or None or a list containing 1 or {} of ' 'those'.format(self.aam.n_levels)) - self._fitters = [] - for j, (am, sm) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models)): - - if md_transform is not ModelDrivenTransform: - md_trans = md_transform( - sm, self.aam.transform, global_transform, - source=am.mean().landmarks['source'].lms) - else: - md_trans = md_transform( - sm, self.aam.transform, - source=am.mean().landmarks['source'].lms) - self._fitters.append( - algorithm(am, md_trans, **kwargs)) + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return AAMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) - def __str__(self): - out = "{0} Fitter\n" \ - " - Lucas-Kanade {1}\n" \ - " - Transform is {2} and residual is {3}.\n" \ - " - {4} training images.\n".format( - self.aam._str_title, self._fitters[0].algorithm, - self._fitters[0].transform.__class__.__name__, - self._fitters[0].residual.type, self.aam.n_training_images) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_levels): - n_channels.append( - self._fitters[j].appearance_model.template_instance.n_channels) - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format(name_of_callable( - self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - self.features[j].__name__)) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.aam.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.aam.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self._fitters[0].appearance_model.n_features, - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.aam.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self._fitters[i].appearance_model.n_features, - self._fitters[i].template.n_true_pixels(), - n_channels[i], self._fitters[i].template._str_shape, - n_channels[i]) - out = "{0} - {1} motion components\n - {2} active " \ - "appearance components ({3:.2f}% of original " \ - "variance)\n".format( - out, self._fitters[i].transform.n_parameters, - self._fitters[i].appearance_model.n_active_components, - self._fitters[i].appearance_model.variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} motion parameters\n" \ - " - {10} appearance components ({11:.2f}% of original " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self._fitters[0].appearance_model.n_features, - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0], self._fitters[0].transform.n_parameters, - self._fitters[0].appearance_model.n_active_components, - self._fitters[0].appearance_model.variance_ratio() * 100) - return out - - -class AAMMultilevelFittingResult(AMMultilevelFittingResult): - r""" - Class that holds the state of a :map:`AAMFitter` object before, - during and after it has fitted a particular image. - """ - @property - def appearance_reconstructions(self): - r""" - The list containing the appearance reconstruction obtained at - each fitting iteration. - - :type: `list` of :map:`Image` or subclass - """ - return list(chain( - *[f.appearance_reconstructions for f in self.fitting_results])) - - @property - def aam_reconstructions(self): - r""" - The list containing the aam reconstruction (i.e. the appearance - reconstruction warped on the shape instance reconstruction) obtained at - each fitting iteration. - - Note that this reconstruction is only tested to work for the - :map:`OrthoMDTransform` - - :type: list` of :map:`Image` or subclass - """ - aam_reconstructions = [] - for level, f in enumerate(self.fitting_results): - if f.weights: - for shape_w, aw in zip(f.parameters, f.weights): - shape_w = shape_w[4:] - sm_level = self.fitter.aam.shape_models[level] - am_level = self.fitter.aam.appearance_models[level] - swt = shape_w / sm_level.eigenvalues[:len(shape_w)] ** 0.5 - awt = aw / am_level.eigenvalues[:len(aw)] ** 0.5 - aam_reconstructions.append(self.fitter.aam.instance( - shape_weights=swt, appearance_weights=awt, level=level)) - else: - for shape_w in f.parameters: - shape_w = shape_w[4:] - sm_level = self.fitter.aam.shape_models[level] - swt = shape_w / sm_level.eigenvalues[:len(shape_w)] ** 0.5 - aam_reconstructions.append(self.fitter.aam.instance( - shape_weights=swt, appearance_weights=None, - level=level)) - return aam_reconstructions From 479479399529fcd72bc04c0889f415223f9a7813 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:32:39 +0100 Subject: [PATCH 015/423] Add results.py - Results will substitute FittingResults --- menpofit/result.py | 903 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 903 insertions(+) create mode 100644 menpofit/result.py diff --git a/menpofit/result.py b/menpofit/result.py new file mode 100644 index 0000000..536145e --- /dev/null +++ b/menpofit/result.py @@ -0,0 +1,903 @@ +from __future__ import division +import abc +import numpy as np +from menpo.transform import Scale +from menpo.image import Image + + +# TODO: document me! +class Result(object): + r""" + """ + @abc.abstractproperty + def final_shape(self): + r""" + Returns the final fitted shape. + """ + + @abc.abstractproperty + def initial_shape(self): + r""" + Returns the initial shape from which the fitting started. + """ + + @property + def gt_shape(self): + r""" + Returns the original ground truth shape associated to the image. + """ + return self._gt_shape + + @property + def fitted_image(self): + r""" + Returns a copy of the fitted image with the following landmark + groups attached to it: + - ``initial``, containing the initial fitted shape . + - ``final``, containing the final shape. + - ``ground``, containing the ground truth shape. Only returned if + the ground truth shape was provided. + + :type: :map:`Image` + """ + image = Image(self.image.pixels) + + image.landmarks['initial'] = self.initial_shape + image.landmarks['final'] = self.final_shape + if self.gt_shape is not None: + image.landmarks['ground'] = self.gt_shape + return image + + def final_error(self, error_type='me_norm'): + r""" + Returns the final fitting error. + + Parameters + ----------- + error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + + Returns + ------- + final_error : `float` + The final error at the end of the fitting procedure. + """ + if self.gt_shape is not None: + return compute_error(self.final_shape, self.gt_shape, error_type) + else: + raise ValueError('Ground truth has not been set, final error ' + 'cannot be computed') + + def initial_error(self, error_type='me_norm'): + r""" + Returns the initial fitting error. + + Parameters + ----------- + error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + + Returns + ------- + initial_error : `float` + The initial error at the start of the fitting procedure. + """ + if self.gt_shape is not None: + return compute_error(self.initial_shape, self.gt_shape, error_type) + else: + raise ValueError('Ground truth has not been set, final error ' + 'cannot be computed') + + def as_serializableresult(self): + return SerializableIterativeResult( + self.image, self.initial_shape, self.final_shape, + gt_shape=self.gt_shape) + + def __str__(self): + out = "Initial error: {0:.4f}\nFinal error: {1:.4f}".format( + self.initial_error(), self.final_error()) + return out + + +# TODO: document me! +class IterativeResult(Result): + r""" + """ + @abc.abstractproperty + def n_iters(self): + r""" + Returns the number of iterations. + """ + + @abc.abstractproperty + def shapes(self, as_points=False): + r""" + Generates a list containing the shapes obtained at each fitting + iteration. + + Parameters + ----------- + as_points : boolean, optional + Whether the results is returned as a list of :map:`PointCloud`s or + ndarrays. + + Default: `False` + + Returns + ------- + shapes : :map:`PointCloud`s or ndarray list + A list containing the shapes obtained at each fitting iteration. + """ + + @property + def iter_image(self): + r""" + Returns a copy of the fitted image with a as many landmark groups as + iteration run by fitting procedure: + - ``iter_0``, containing the initial shape. + - ``iter_1``, containing the the fitted shape at the first + iteration. + - ``...`` + - ``iter_n``, containing the final fitted shape. + + :type: :map:`Image` + """ + image = Image(self.image.pixels) + for j, s in enumerate(self.shapes()): + image.landmarks['iter_'+str(j)] = s + return image + + def errors(self, error_type='me_norm'): + r""" + Returns a list containing the error at each fitting iteration. + + Parameters + ----------- + error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + + Returns + ------- + errors : `list` of `float` + The errors at each iteration of the fitting process. + """ + if self.gt_shape is not None: + return [compute_error(t, self.gt_shape, error_type) + for t in self.shapes()] + else: + raise ValueError('Ground truth has not been set, errors cannot ' + 'be computed') + + def plot_errors(self, error_type='me_norm', figure_id=None, + new_figure=False, render_lines=True, line_colour='b', + line_style='-', line_width=2, render_markers=True, + marker_style='o', marker_size=4, marker_face_colour='b', + marker_edge_colour='k', marker_edge_width=1., + render_axes=True, axes_font_name='sans-serif', + axes_font_size=10, axes_font_style='normal', + axes_font_weight='normal', figure_size=(10, 6), + render_grid=True, grid_line_style='--', + grid_line_width=0.5): + r""" + Plot of the error evolution at each fitting iteration. + Parameters + ---------- + error_type : {``me_norm``, ``me``, ``rmse``}, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + figure_id : `object`, optional + The id of the figure to be used. + new_figure : `bool`, optional + If ``True``, a new figure is created. + render_lines : `bool`, optional + If ``True``, the line will be rendered. + line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or + ``(3, )`` `ndarray`, optional + The colour of the lines. + line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the lines. + line_width : `float`, optional + The width of the lines. + render_markers : `bool`, optional + If ``True``, the markers will be rendered. + marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, + ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, + ``1``, ``2``, ``3``, ``4``, ``8``}, optional + The style of the markers. + marker_size : `int`, optional + The size of the markers in points^2. + marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The face (filling) colour of the markers. + marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The edge colour of the markers. + marker_edge_width : `float`, optional + The width of the markers' edge. + render_axes : `bool`, optional + If ``True``, the axes will be rendered. + axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the axes. + axes_font_size : `int`, optional + The font size of the axes. + axes_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the axes. + axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, + ``book``, ``medium``, ``roman``, ``semibold``, + ``demibold``, ``demi``, ``bold``, ``heavy``, + ``extra bold``, ``black``}, optional + The font weight of the axes. + figure_size : (`float`, `float`) or `None`, optional + The size of the figure in inches. + render_grid : `bool`, optional + If ``True``, the grid will be rendered. + grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the grid lines. + grid_line_width : `float`, optional + The width of the grid lines. + Returns + ------- + viewer : :map:`GraphPlotter` + The viewer object. + """ + from menpo.visualize import GraphPlotter + errors_list = self.errors(error_type=error_type) + return GraphPlotter(figure_id=figure_id, new_figure=new_figure, + x_axis=range(len(errors_list)), + y_axis=[errors_list], + title='Fitting Errors per Iteration', + x_label='Iteration', y_label='Fitting Error', + x_axis_limits=(0, len(errors_list)-1), + y_axis_limits=None).render( + render_lines=render_lines, line_colour=line_colour, + line_style=line_style, line_width=line_width, + render_markers=render_markers, marker_style=marker_style, + marker_size=marker_size, marker_face_colour=marker_face_colour, + marker_edge_colour=marker_edge_colour, + marker_edge_width=marker_edge_width, render_legend=False, + render_axes=render_axes, axes_font_name=axes_font_name, + axes_font_size=axes_font_size, axes_font_style=axes_font_style, + axes_font_weight=axes_font_weight, render_grid=render_grid, + grid_line_style=grid_line_style, grid_line_width=grid_line_width, + figure_size=figure_size) + + def displacements(self): + r""" + A list containing the displacement between the shape of each iteration + and the shape of the previous one. + :type: `list` of ndarray + """ + return [np.linalg.norm(s1.points - s2.points, axis=1) + for s1, s2 in zip(self.shapes, self.shapes[1:])] + + def displacements_stats(self, stat_type='mean'): + r""" + A list containing the a statistical metric on the displacement between + the shape of each iteration and the shape of the previous one. + Parameters + ----------- + stat_type : `str` ``{'mean', 'median', 'min', 'max'}``, optional + Specifies a statistic metric to be extracted from the displacements. + Returns + ------- + :type: `list` of `float` + The statistical metric on the points displacements for each + iteration. + """ + if stat_type == 'mean': + return [np.mean(d) for d in self.displacements()] + elif stat_type == 'median': + return [np.median(d) for d in self.displacements()] + elif stat_type == 'max': + return [np.max(d) for d in self.displacements()] + elif stat_type == 'min': + return [np.min(d) for d in self.displacements()] + else: + raise ValueError("type must be 'mean', 'median', 'min' or 'max'") + + def plot_displacements(self, stat_type='mean', figure_id=None, + new_figure=False, render_lines=True, line_colour='b', + line_style='-', line_width=2, render_markers=True, + marker_style='o', marker_size=4, + marker_face_colour='b', marker_edge_colour='k', + marker_edge_width=1., render_axes=True, + axes_font_name='sans-serif', axes_font_size=10, + axes_font_style='normal', axes_font_weight='normal', + figure_size=(10, 6), render_grid=True, + grid_line_style='--', grid_line_width=0.5): + r""" + Plot of a statistical metric of the displacement between the shape of + each iteration and the shape of the previous one. + Parameters + ---------- + stat_type : {``mean``, ``median``, ``min``, ``max``}, optional + Specifies a statistic metric to be extracted from the displacements + (see also `displacements_stats()` method). + figure_id : `object`, optional + The id of the figure to be used. + new_figure : `bool`, optional + If ``True``, a new figure is created. + render_lines : `bool`, optional + If ``True``, the line will be rendered. + line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or + ``(3, )`` `ndarray`, optional + The colour of the lines. + line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the lines. + line_width : `float`, optional + The width of the lines. + render_markers : `bool`, optional + If ``True``, the markers will be rendered. + marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, + ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, + ``1``, ``2``, ``3``, ``4``, ``8``}, optional + The style of the markers. + marker_size : `int`, optional + The size of the markers in points^2. + marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The face (filling) colour of the markers. + marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The edge colour of the markers. + marker_edge_width : `float`, optional + The width of the markers' edge. + render_axes : `bool`, optional + If ``True``, the axes will be rendered. + axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the axes. + axes_font_size : `int`, optional + The font size of the axes. + axes_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the axes. + axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, + ``book``, ``medium``, ``roman``, ``semibold``, + ``demibold``, ``demi``, ``bold``, ``heavy``, + ``extra bold``, ``black``}, optional + The font weight of the axes. + figure_size : (`float`, `float`) or `None`, optional + The size of the figure in inches. + render_grid : `bool`, optional + If ``True``, the grid will be rendered. + grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the grid lines. + grid_line_width : `float`, optional + The width of the grid lines. + Returns + ------- + viewer : :map:`GraphPlotter` + The viewer object. + """ + from menpo.visualize import GraphPlotter + # set labels + if stat_type == 'max': + ylabel = 'Maximum Displacement' + title = 'Maximum displacement per Iteration' + elif stat_type == 'min': + ylabel = 'Minimum Displacement' + title = 'Minimum displacement per Iteration' + elif stat_type == 'mean': + ylabel = 'Mean Displacement' + title = 'Mean displacement per Iteration' + elif stat_type == 'median': + ylabel = 'Median Displacement' + title = 'Median displacement per Iteration' + else: + raise ValueError('stat_type must be one of {max, min, mean, ' + 'median}.') + # plot + displacements_list = self.displacements_stats(stat_type=stat_type) + return GraphPlotter(figure_id=figure_id, new_figure=new_figure, + x_axis=range(len(displacements_list)), + y_axis=[displacements_list], + title=title, + x_label='Iteration', y_label=ylabel, + x_axis_limits=(0, len(displacements_list)-1), + y_axis_limits=None).render( + render_lines=render_lines, line_colour=line_colour, + line_style=line_style, line_width=line_width, + render_markers=render_markers, marker_style=marker_style, + marker_size=marker_size, marker_face_colour=marker_face_colour, + marker_edge_colour=marker_edge_colour, + marker_edge_width=marker_edge_width, render_legend=False, + render_axes=render_axes, axes_font_name=axes_font_name, + axes_font_size=axes_font_size, axes_font_style=axes_font_style, + axes_font_weight=axes_font_weight, render_grid=render_grid, + grid_line_style=grid_line_style, grid_line_width=grid_line_width, + figure_size=figure_size) + + def as_serializableresult(self): + return SerializableIterativeResult( + self.image, self.shapes, self.n_iters, gt_shape=self.gt_shape) + + +# TODO: document me! +class ParametricAlgorithmResult(IterativeResult): + r""" + """ + def __init__(self, image, fitter, shape_parameters, gt_shape=None): + self.image = image + self.fitter = fitter + self.shape_parameters = shape_parameters + self._gt_shape = gt_shape + + @property + def n_iters(self): + return len(self.shapes()) - 1 + + @property + def transforms(self): + r""" + Generates a list containing the transforms obtained at each fitting + iteration. + """ + return [self.fitter.transform.from_vector(p) + for p in self.shape_parameters] + + @property + def final_transform(self): + r""" + Returns the final transform. + """ + return self.fitter.transform.from_vector(self.shape_parameters[-1]) + + @property + def initial_transform(self): + r""" + Returns the initial transform from which the fitting started. + """ + return self.fitter.transform.from_vector(self.shape_parameters[0]) + + def shapes(self, as_points=False): + if as_points: + return [self.fitter.transform.from_vector(p).target.points + for p in self.shape_parameters] + + else: + return [self.fitter.transform.from_vector(p).target + for p in self.shape_parameters] + + @property + def final_shape(self): + return self.final_transform.target + + @property + def initial_shape(self): + return self.initial_transform.target + + +# TODO: document me! +class MultiFitterResult(IterativeResult): + r""" + """ + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(MultiFitterResult, self).__init__() + self.image = image + self.fitter = fitter + self.algorithm_results = algorithm_results + self._affine_correction = affine_correction + self._gt_shape = gt_shape + + @property + def n_levels(self): + r""" + The number of levels of the fitter object. + + :type: `int` + """ + return self.fitter.n_levels + + @property + def scales(self): + return self.fitter.scales + + @property + def n_iters(self): + r""" + The total number of iterations used to fitter the image. + + :type: `int` + """ + n_iters = 0 + for f in self.algorithm_results: + n_iters += f.n_iters + return n_iters + + def shapes(self, as_points=False): + r""" + Generates a list containing the shapes obtained at each fitting + iteration. + + Parameters + ----------- + as_points : `boolean`, optional + Whether the result is returned as a `list` of :map:`PointCloud` or + a `list` of `ndarrays`. + + Returns + ------- + shapes : `list` of :map:`PointCoulds` or `list` of `ndarray` + A list containing the fitted shapes at each iteration of + the fitting procedure. + """ + return _rescale_shapes_to_reference( + self.algorithm_results, self.scales, self._affine_correction) + + @property + def final_shape(self): + r""" + The final fitted shape. + + :type: :map:`PointCloud` + """ + final_shape = self.algorithm_results[-1].final_shape + return self._affine_correction.apply(final_shape) + + @property + def initial_shape(self): + initial_shape = self.algorithm_results[0].initial_shape + Scale(self.scales[-1]/self.scales[0], + initial_shape.n_dims).apply_inplace(initial_shape) + return self._affine_correction.apply(initial_shape) + + +# TODO: document me! +class SerializableIterativeResult(IterativeResult): + r""" + """ + def __init__(self, image, shapes, n_iters, gt_shape=None): + self.image = image + self._gt_shape = gt_shape + self._shapes = shapes + self._n_iters = n_iters + + @property + def n_iters(self): + return self._n_iters + + def shapes(self, as_points=False): + if as_points: + return [s.points for s in self._shapes] + else: + return self._shapes + + @property + def initial_shape(self): + return self._shapes[0] + + @property + def final_shape(self): + return self._shapes[-1] + + +# TODO: Document me! +def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): + r""" + """ + shapes = [] + for j, (alg, s) in enumerate(zip(algorithm_results, scales)): + transform = Scale(scales[-1]/s, alg.final_shape.n_dims) + for t in alg.shapes: + t = transform.apply(t) + shapes.append(affine_correction.apply(t)) + return shapes + + +# TODO: Document me! +def compute_error(target, ground_truth, error_type='me_norm'): + r""" + """ + gt_points = ground_truth.points + target_points = target.points + + if error_type == 'me_norm': + return _compute_me_norm(target_points, gt_points) + elif error_type == 'me': + return _compute_me(target_points, gt_points) + elif error_type == 'rmse': + return _compute_rmse(target_points, gt_points) + else: + raise ValueError("Unknown error_type string selected. Valid options " + "are: me_norm, me, rmse'") + + +# TODO: Document me! +# TODO: rename to more descriptive name +def _compute_me(target, ground_truth): + r""" + """ + return np.mean(np.sqrt(np.sum((target - ground_truth) ** 2, axis=-1))) + + +# TODO: Document me! +# TODO: rename to more descriptive name +def _compute_rmse(target, ground_truth): + r""" + """ + return np.sqrt(np.mean((target.flatten() - ground_truth.flatten()) ** 2)) + + +# TODO: Document me! +# TODO: rename to more descriptive name +def _compute_me_norm(target, ground_truth): + r""" + """ + normalizer = np.mean(np.max(ground_truth, axis=0) - + np.min(ground_truth, axis=0)) + return _compute_me(target, ground_truth) / normalizer + + +# TODO: Document me! +def compute_cumulative_error(errors, x_axis): + r""" + """ + n_errors = len(errors) + return [np.count_nonzero([errors <= x]) / n_errors for x in x_axis] + + +def plot_cumulative_error_distribution(errors, error_range=None, figure_id=None, + new_figure=False, + title='Cumulative Error Distribution', + x_label='Normalized Point-to-Point Error', + y_label='Images Proportion', + legend_entries=None, render_lines=True, + line_colour=None, line_style='-', + line_width=2, render_markers=True, + marker_style='s', marker_size=10, + marker_face_colour='w', + marker_edge_colour=None, + marker_edge_width=2, render_legend=True, + legend_title=None, + legend_font_name='sans-serif', + legend_font_style='normal', + legend_font_size=10, + legend_font_weight='normal', + legend_marker_scale=1., + legend_location=2, + legend_bbox_to_anchor=(1.05, 1.), + legend_border_axes_pad=1., + legend_n_columns=1, + legend_horizontal_spacing=1., + legend_vertical_spacing=1., + legend_border=True, + legend_border_padding=0.5, + legend_shadow=False, + legend_rounded_corners=False, + render_axes=True, + axes_font_name='sans-serif', + axes_font_size=10, + axes_font_style='normal', + axes_font_weight='normal', + axes_x_limits=None, axes_y_limits=None, + figure_size=(10, 8), render_grid=True, + grid_line_style='--', + grid_line_width=0.5): + r""" + Plot the cumulative error distribution (CED) of the provided fitting errors. + + Parameters + ---------- + errors : `list` of `lists` + A `list` with `lists` of fitting errors. A separate CED curve will be + rendered for each errors `list`. + error_range : `list` of `float` with length 3, optional + Specifies the horizontal axis range, i.e. + + :: + + error_range[0] = min_error + error_range[1] = max_error + error_range[2] = error_step + + If ``None``, then ``'error_range = [0., 0.101, 0.005]'``. + figure_id : `object`, optional + The id of the figure to be used. + new_figure : `bool`, optional + If ``True``, a new figure is created. + title : `str`, optional + The figure's title. + x_label : `str`, optional + The label of the horizontal axis. + y_label : `str`, optional + The label of the vertical axis. + legend_entries : `list of `str` or ``None``, optional + If `list` of `str`, it must have the same length as `errors` `list` and + each `str` will be used to name each curve. If ``None``, the CED curves + will be named as `'Curve %d'`. + render_lines : `bool` or `list` of `bool`, optional + If ``True``, the line will be rendered. If `bool`, this value will be + used for all curves. If `list`, a value must be specified for each + fitting errors curve, thus it must have the same length as `errors`. + line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or + ``(3, )`` `ndarray` or `list` of those or ``None``, optional + The colour of the lines. If not a `list`, this value will be + used for all curves. If `list`, a value must be specified for each + fitting errors curve, thus it must have the same length as `errors`. If + ``None``, the colours will be linearly sampled from jet colormap. + line_style : {``-``, ``--``, ``-.``, ``:``} or `list` of those, optional + The style of the lines. If not a `list`, this value will be used for all + curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + line_width : `float` or `list` of `float`, optional + The width of the lines. If `float`, this value will be used for all + curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + render_markers : `bool` or `list` of `bool`, optional + If ``True``, the markers will be rendered. If `bool`, this value will be + used for all curves. If `list`, a value must be specified for each + fitting errors curve, thus it must have the same length as `errors`. + marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, + ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, + ``1``, ``2``, ``3``, ``4``, ``8``} or `list` of those, optional + The style of the markers. If not a `list`, this value will be used for + all curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + marker_size : `int` or `list` of `int`, optional + The size of the markers in points^2. If `int`, this value will be used + for all curves. If `list`, a value must be specified for each fitting + errors curve, thus it must have the same length as `errors`. + marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray` or `list` of those or ``None``, optional + The face (filling) colour of the markers. If not a `list`, this value + will be used for all curves. If `list`, a value must be specified for + each fitting errors curve, thus it must have the same length as + `errors`. If ``None``, the colours will be linearly sampled from jet + colormap. + marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray` or `list` of those or ``None``, optional + The edge colour of the markers. If not a `list`, this value will be used + for all curves. If `list`, a value must be specified for each fitting + errors curve, thus it must have the same length as `errors`. If + ``None``, the colours will be linearly sampled from jet colormap. + marker_edge_width : `float` or `list` of `float`, optional + The width of the markers' edge. If `float`, this value will be used for + all curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + render_legend : `bool`, optional + If ``True``, the legend will be rendered. + legend_title : `str`, optional + The title of the legend. + legend_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the legend. + legend_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the legend. + legend_font_size : `int`, optional + The font size of the legend. + legend_font_weight : {``ultralight``, ``light``, ``normal``, + ``regular``, ``book``, ``medium``, ``roman``, + ``semibold``, ``demibold``, ``demi``, ``bold``, + ``heavy``, ``extra bold``, ``black``}, optional + The font weight of the legend. + legend_marker_scale : `float`, optional + The relative size of the legend markers with respect to the original + legend_location : `int`, optional + The location of the legend. The predefined values are: + + =============== === + 'best' 0 + 'upper right' 1 + 'upper left' 2 + 'lower left' 3 + 'lower right' 4 + 'right' 5 + 'center left' 6 + 'center right' 7 + 'lower center' 8 + 'upper center' 9 + 'center' 10 + =============== === + + legend_bbox_to_anchor : (`float`, `float`), optional + The bbox that the legend will be anchored. + legend_border_axes_pad : `float`, optional + The pad between the axes and legend border. + legend_n_columns : `int`, optional + The number of the legend's columns. + legend_horizontal_spacing : `float`, optional + The spacing between the columns. + legend_vertical_spacing : `float`, optional + The vertical space between the legend entries. + legend_border : `bool`, optional + If ``True``, a frame will be drawn around the legend. + legend_border_padding : `float`, optional + The fractional whitespace inside the legend border. + legend_shadow : `bool`, optional + If ``True``, a shadow will be drawn behind legend. + legend_rounded_corners : `bool`, optional + If ``True``, the frame's corners will be rounded (fancybox). + render_axes : `bool`, optional + If ``True``, the axes will be rendered. + axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the axes. + axes_font_size : `int`, optional + The font size of the axes. + axes_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the axes. + axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, + ``book``, ``medium``, ``roman``, ``semibold``, + ``demibold``, ``demi``, ``bold``, ``heavy``, + ``extra bold``, ``black``}, optional + The font weight of the axes. + axes_x_limits : (`float`, `float`) or ``None``, optional + The limits of the x axis. If ``None``, it is set to + ``(0., 'errors_max')``. + axes_y_limits : (`float`, `float`) or ``None``, optional + The limits of the y axis. If ``None``, it is set to ``(0., 1.)``. + figure_size : (`float`, `float`) or ``None``, optional + The size of the figure in inches. + render_grid : `bool`, optional + If ``True``, the grid will be rendered. + grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the grid lines. + grid_line_width : `float`, optional + The width of the grid lines. + + Raises + ------ + ValueError + legend_entries list has different length than errors list + + Returns + ------- + viewer : :map:`GraphPlotter` + The viewer object. + """ + from menpo.visualize import GraphPlotter + + # make sure that errors is a list even with one list member + if not isinstance(errors[0], list): + errors = [errors] + + # create x and y axes lists + x_axis = list(np.arange(error_range[0], error_range[1], error_range[2])) + ceds = [compute_cumulative_error(e, x_axis) for e in errors] + + # parse legend_entries, axes_x_limits and axes_y_limits + if legend_entries is None: + legend_entries = ["Curve {}".format(k) for k in range(len(ceds))] + if len(legend_entries) != len(ceds): + raise ValueError('legend_entries list has different length than errors ' + 'list') + if axes_x_limits is None: + axes_x_limits = (0., x_axis[-1]) + if axes_y_limits is None: + axes_y_limits = (0., 1.) + + # render + return GraphPlotter(figure_id=figure_id, new_figure=new_figure, + x_axis=x_axis, y_axis=ceds, title=title, + legend_entries=legend_entries, x_label=x_label, + y_label=y_label, x_axis_limits=axes_x_limits, + y_axis_limits=axes_y_limits).render( + render_lines=render_lines, line_colour=line_colour, + line_style=line_style, line_width=line_width, + render_markers=render_markers, marker_style=marker_style, + marker_size=marker_size, marker_face_colour=marker_face_colour, + marker_edge_colour=marker_edge_colour, + marker_edge_width=marker_edge_width, render_legend=render_legend, + legend_title=legend_title, legend_font_name=legend_font_name, + legend_font_style=legend_font_style, legend_font_size=legend_font_size, + legend_font_weight=legend_font_weight, + legend_marker_scale=legend_marker_scale, + legend_location=legend_location, + legend_bbox_to_anchor=legend_bbox_to_anchor, + legend_border_axes_pad=legend_border_axes_pad, + legend_n_columns=legend_n_columns, + legend_horizontal_spacing=legend_horizontal_spacing, + legend_vertical_spacing=legend_vertical_spacing, + legend_border=legend_border, + legend_border_padding=legend_border_padding, + legend_shadow=legend_shadow, + legend_rounded_corners=legend_rounded_corners, render_axes=render_axes, + axes_font_name=axes_font_name, axes_font_size=axes_font_size, + axes_font_style=axes_font_style, axes_font_weight=axes_font_weight, + figure_size=figure_size, render_grid=render_grid, + grid_line_style=grid_line_style, grid_line_width=grid_line_width) From 5462eeb471401c5185bcaacf0c2635c7a484834a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:34:23 +0100 Subject: [PATCH 016/423] Add results for AAMs --- menpofit/aam/result.py | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 menpofit/aam/result.py diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py new file mode 100644 index 0000000..5824945 --- /dev/null +++ b/menpofit/aam/result.py @@ -0,0 +1,53 @@ +from __future__ import division +from menpofit.result import ( + ParametricAlgorithmResult, MultiFitterResult, SerializableIterativeResult) + + +# TODO: document me! +# TODO: handle costs +class AAMAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + def __init__(self, image, fitter, shape_parameters, + appearance_parameters=None, gt_shape=None): + super(AAMAlgorithmResult, self).__init__( + image, fitter, shape_parameters, gt_shape=gt_shape) + self.appearance_parameters = appearance_parameters + + +# TODO: document me! +class LinearAAMAlgorithmResult(AAMAlgorithmResult): + r""" + """ + def shapes(self, as_points=False): + if as_points: + return [self.fitter.transform.from_vector(p).sparse_target.points + for p in self.shape_parameters] + + else: + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] + + @property + def final_shape(self): + return self.final_transform.sparse_target + + @property + def initial_shape(self): + return self.initial_transform.sparse_target + + +# TODO: document me! +# TODO: handle costs +class AAMFitterResult(MultiFitterResult): + r""" + """ + pass + + +# TODO: document me! +# TODO: handle costs +class SerializableAAMFitterResult(SerializableIterativeResult): + r""" + """ + pass From e85921af002bbecb821c20be5be0f2a4ddc53740 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 15:04:34 +0100 Subject: [PATCH 017/423] Add small fix in aam.fitter --- menpofit/aam/fitter.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 957bead..9bee842 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -3,8 +3,7 @@ from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM -from .algorithm import ( - StandardAAMInterface, LinearAAMInterface, PartsAAMInterface, AIC) +from .algorithm import AAMInterface, LinearAAMInterface, PartsAAMInterface, AIC from .result import AAMFitterResult @@ -15,8 +14,8 @@ class LKAAMFitter(ModelFitter): def __init__(self, aam, algorithm_cls=AIC, n_shape=None, n_appearance=None, **kwargs): super(LKAAMFitter, self).__init__() + self.algorithms = [] self._model = aam - self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) self._set_up(algorithm_cls, **kwargs) @@ -31,8 +30,8 @@ def _set_up(self, algorithm_cls, **kwargs): sm, self._model.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(StandardAAMInterface, am, - md_transform, **kwargs) + algorithm = algorithm_cls(AAMInterface, am, md_transform, + **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -58,7 +57,7 @@ def _set_up(self, algorithm_cls, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) @property def aam(self): From dd96e83c53f9fee51d6486cd366791e36925f608 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 15:19:38 +0100 Subject: [PATCH 018/423] Small corrections to fitters --- menpofit/aam/fitter.py | 10 ++++++---- menpofit/fitter.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 9bee842..4a6d010 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -13,9 +13,8 @@ class LKAAMFitter(ModelFitter): """ def __init__(self, aam, algorithm_cls=AIC, n_shape=None, n_appearance=None, **kwargs): - super(LKAAMFitter, self).__init__() - self.algorithms = [] - self._model = aam + super(LKAAMFitter, self).__init__(aam) + self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) self._set_up(algorithm_cls, **kwargs) @@ -57,12 +56,15 @@ def _set_up(self, algorithm_cls, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self.algorithms.append(algorithm) + self._algorithms.append(algorithm) @property def aam(self): return self._model + def algorithms(self): + return self._algorithms + def _check_n_appearance(self, n_appearance): if n_appearance is not None: if type(n_appearance) is int or type(n_appearance) is float: diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 70c262f..db85a5f 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,4 +1,5 @@ from __future__ import division +import abc import numpy as np from menpo.shape import PointCloud from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity @@ -17,6 +18,26 @@ def n_levels(self): """ return len(self.scales) + @abc.abstractproperty + def algorithms(self): + pass + + @abc.abstractproperty + def reference_shape(self): + pass + + @abc.abstractproperty + def scales(self): + pass + + @abc.abstractproperty + def features(self): + pass + + @abc.abstractproperty + def scale_features(self): + pass + def fit(self, image, initial_shape, max_iters=50, gt_shape=None, **kwargs): r""" @@ -214,11 +235,19 @@ def _prepare_max_iters(self, max_iters): 'None'.format(self.n_levels)) return np.require(max_iters, dtype=np.int) + @abc.abstractmethod + def _fitter_result(self): + pass + + # TODO: document me! class ModelFitter(MultiFitter): r""" """ + def __init__(self, model): + self._model = model + @property def reference_shape(self): r""" @@ -303,7 +332,6 @@ def obtain_shape_from_bb(self, bounding_box): initial_shape: :class:`menpo.shape.PointCloud` The initial shape. """ - reference_shape = self.reference_shape return align_shape_with_bb(reference_shape, bounding_box).apply(reference_shape) From 0831a0b87384d2eea227d0caff0a2d1547097491 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 15:26:20 +0100 Subject: [PATCH 019/423] More corrections to fitters --- menpofit/aam/fitter.py | 1 + menpofit/fitter.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 4a6d010..594a985 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -62,6 +62,7 @@ def _set_up(self, algorithm_cls, **kwargs): def aam(self): return self._model + @property def algorithms(self): return self._algorithms diff --git a/menpofit/fitter.py b/menpofit/fitter.py index db85a5f..ebdad02 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -91,8 +91,7 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, # build multilevel fitting result fitter_result = self._fitter_result( - image, self, algorithm_results, affine_correction, - gt_shape=gt_shape) + image, algorithm_results, affine_correction, gt_shape=gt_shape) return fitter_result @@ -236,7 +235,8 @@ def _prepare_max_iters(self, max_iters): return np.require(max_iters, dtype=np.int) @abc.abstractmethod - def _fitter_result(self): + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): pass From 1e23c75d0101ba11a7dd6e278833ee3aaa38600f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:14:42 +0100 Subject: [PATCH 020/423] Add basic AAM algorithms - Add PFC, PIC, SFC, SIC, AFC, AIC, MAFC, MAIC, WFC, WIC --- menpofit/aam/algorithm.py | 886 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 886 insertions(+) create mode 100644 menpofit/aam/algorithm.py diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py new file mode 100644 index 0000000..44d4cc5 --- /dev/null +++ b/menpofit/aam/algorithm.py @@ -0,0 +1,886 @@ +from __future__ import division +import abc +import numpy as np +from menpo.image import Image +from menpo.feature import gradient as fast_gradient +from .result import AAMAlgorithmResult, LinearAAMAlgorithmResult + + +class AAMInterface(object): + + def __init__(self, aam_algorithm, sampling_step=None): + self.algorithm = aam_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling_step is None: + sampling_step = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling_step) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def shape_model(self): + return self.transform.pdm.model + + @property + def appearance_model(self): + return self.algorithm.appearance_model + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def m(self): + return self.appearance_model.n_active_components + + @property + def true_indices(self): + return self.template.mask.true_indices() + + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + + def partial_newton_hessian(self, nabla2, dw_dp): + # reshape gradient + # gradient: n_dims x n_dims x n_channels x n_pixels + nabla2 = nabla2[self.nabla2_mask].reshape( + (2,) + nabla2.shape[:2] + (-1,)) + + # compute partial hessian + # gradient: n_dims x n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # h: n_dims x n_channels x n_pixels x n_params + h1 = 0 + aux = nabla2[..., None] * dw_dp[:, None, None, ...] + for d in aux: + h1 += d + # compute partial hessian + # h: n_dims x n_channels x n_pixels x n_params + # warp_jacobian: n_dims x x n_pixels x x n_params + # h: + h2 = 0 + aux = h1[..., None] * dw_dp[..., None, :, None, :] + for d in aux: + h2 += d + + # reshape hessian + # 2: (n_channels x n_pixels) x n_params x n_params + return h2.reshape((-1, h2.shape[3] * h2.shape[4])) + + @classmethod + def solve_shape_map(cls, H, J, e, J_prior, p): + if p.shape[0] is not H.shape[0]: + # Bidirectional Compositional case + J_prior = np.hstack((J_prior, J_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + H += np.diag(J_prior) + Je = J_prior * p + J.T.dot(e) + return - np.linalg.solve(H, Je) + + @classmethod + def solve_shape_ml(cls, H, J, e): + # compute and return ML solution + return -np.linalg.solve(H, J.T.dot(e)) + + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): + if self.n is not H.shape[0] - self.m: + # Bidirectional Compositional case + Js_prior = np.hstack((Js_prior, Js_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + J_prior = np.hstack((Ja_prior, Js_prior)) + H += np.diag(J_prior) + Je = J_prior * np.hstack((c, p)) + J.T.dot(e) + dq = - np.linalg.solve(H, Je) + return dq[:self.m], dq[self.m:] + + def solve_all_ml(self, H, J, e): + # compute ML solution + dq = - np.linalg.solve(H, J.T.dot(e)) + return dq[:self.m], dq[self.m:] + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class LinearAAMInterface(AAMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return LinearAAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class PartsAAMInterface(AAMInterface): + + def __init__(self, aam_algorithm, sampling_mask=None): + self.algorithm = aam_algorithm + + if sampling_mask is None: + sampling_mask = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling_mask[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.gradient_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.gradient2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) + + @property + def shape_model(self): + return self.transform.model + + @property + def patch_shape(self): + return self.appearance_model.patch_shape + + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + def warp(self, image): + return Image(image.extract_patches( + self.transform.target, patch_size=self.patch_shape, + as_single_array=True)) + + def gradient(self, image): + pixels = image.pixels + patch_shape = self.algorithm.appearance_model.patch_shape + g = fast_gradient(pixels.reshape((-1,) + patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return g.reshape((2,) + pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.gradient_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) + + def partial_newton_hessian(self, nabla2, dw_dp): + # reshape gradient + # gradient: dims x dims x parts x off x ch x (h x w) + nabla2 = nabla2[self.gradient2_mask].reshape( + nabla2.shape[:-2] + (-1,)) + + # compute partial hessian + # gradient: dims x dims x parts x off x ch x (h x w) + # dw_dp: dims x x parts x x params + # h: dims x parts x off x ch x (h x w) x params + h1 = 0 + aux = nabla2[..., None] * dw_dp[:, None, :, None, None, None, ...] + for d in aux: + h1 += d + # compute partial hessian + # h: dims x parts x off x ch x (h x w) x params + # dw_dp: dims x parts x x params + # h: + h2 = 0 + aux = h1[..., None] * dw_dp[..., None, None, None, None, :] + for d in aux: + h2 += d + + # reshape hessian + # 2: (parts x off x ch x w x h) x params x params + return h2.reshape((-1, h2.shape[-2] * h2.shape[-1])) + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class AAMAlgorithm(object): + + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # set common state for all AAM algorithms + self.appearance_model = appearance_model + self.template = appearance_model.mean() + self.transform = transform + self.eps = eps + # set interface + self.interface = aam_interface(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self, **kwargs): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + self.m = self.appearance_model.n_active_components + + # grab appearance model components + self.A = self.appearance_model.components + # mask them + self.A_m = self.A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + + # grab appearance model mean + self.a_bar = self.appearance_model.mean() + # vectorize it and mask it + self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] + + # compute warp jacobian + self.dW_dp = self.interface.warp_jacobian() + + # compute shape model prior + s2 = (self.appearance_model.noise_variance() / + self.interface.shape_model.noise_variance()) + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + # compute appearance model prior + S = self.appearance_model.eigenvalues + self.s2_inv_S = s2 / S + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + pass + + +class ProjectOut(AAMAlgorithm): + r""" + Abstract Interface for Project-out AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(ProjectOut, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # solve for increments on the shape parameters + self.dp = self.solve(map_inference) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + @abc.abstractmethod + def solve(self, map_inference): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class PFC(ProjectOut): + r""" + Project-out Forward Compositional (PFC) Gauss-Newton algorithm + """ + def solve(self, map_inference): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # compute masked forward Jacobian + J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) + # project out appearance model from it + QJ_m = self.project_out(J_m) + # compute masked forward Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + JQJ_m, QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class PIC(ProjectOut): + r""" + Project-out Inverse Compositional (PIC) Gauss-Newton algorithm + """ + def precompute(self): + # call super method + super(PIC, self).precompute() + # compute appearance model mean gradient + nabla_a = self.interface.gradient(self.a_bar) + # compute masked inverse Jacobian + J_m = self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + # project out appearance model from it + self.QJ_m = self.project_out(J_m) + # compute masked inverse Hessian + self.JQJ_m = self.QJ_m.T.dot(J_m) + # compute masked Jacobian pseudo-inverse + self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) + + def solve(self, map_inference): + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + self.JQJ_m, self.QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return -self.pinv_QJ_m.dot(self.e_m) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Simultaneous(AAMAlgorithm): + r""" + Abstract Interface for Simultaneous AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(Simultaneous, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [self.c] + + # compute masked error + self.e_m = i_m - a_m + + # solve for increments on the appearance and shape parameters + # simultaneously + dc, self.dp = self.solve(map_inference) + + # update appearance parameters + self.c += dc + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(self.c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + def solve(self, map_inference): + # compute masked Jacobian + J_m = self.compute_jacobian() + # assemble masked simultaneous Jacobian + J_sim_m = np.hstack((-self.A_m, J_m)) + # compute masked Hessian + H_sim_m = J_sim_m.T.dot(J_sim_m) + # solve for increments on the appearance and shape parameters + # simultaneously + if map_inference: + return self.interface.solve_all_map( + H_sim_m, J_sim_m, self.e_m, self.s2_inv_S, self.c, + self.s2_inv_L, self.transform.as_vector()) + else: + return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) + + @abc.abstractmethod + def update_warp(self): + pass + + +class SFC(Simultaneous): + r""" + Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class SIC(Simultaneous): + r""" + Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Alternating(AAMAlgorithm): + r""" + Abstract Interface for Alternating AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(Alternating, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def precompute(self, **kwargs): + # call super method + super(Alternating, self).precompute() + # compute MAP appearance Hessian + self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + Jdp = 0 + else: + Jdp = J_m.dot(self.dp) + + # compute masked error + e_m = i_m - a_m + + # solve for increment on the appearance parameters + if map_inference: + Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) + dc = np.linalg.solve(self.AA_m_map, Ae_m_map) + else: + dc = self.pinv_A_m.dot(e_m + Jdp) + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m - self.A_m.T.dot(dc), self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, + e_m - self.A_m.dot(dc)) + + # update appearance parameters + c += dc + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class AFC(Alternating): + r""" + Alternating Forward Compositional (AFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class AIC(Alternating): + r""" + Alternating Inverse Compositional (AIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class ModifiedAlternating(Alternating): + r""" + Abstract Interface for Modified Alternating AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(ModifiedAlternating, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + a_m = self.a_bar_m + c_list = [] + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + c = self.pinv_A_m.dot(i_m - a_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m, self.s2_inv_L, self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class MAFC(ModifiedAlternating): + r""" + Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class MAIC(ModifiedAlternating): + r""" + Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Wiberg(AAMAlgorithm): + r""" + Abstract Interface for Wiberg AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(Wiberg, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + else: + c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - self.a_bar_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # project out appearance models + QJ_m = self.project_out(J_m) + # compute masked Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + JQJ_m, QJ_m, e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(JQJ_m, QJ_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class WFC(Wiberg): + r""" + Wiberg Forward Compositional (WFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class WIC(Wiberg): + r""" + Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + From 6f9e6cedf6353a33a28fb82ec5d7de0d1c5acc34 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:17:23 +0100 Subject: [PATCH 021/423] Update aam.__init__.py --- menpofit/aam/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 37bd629..119cff0 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,3 +1,11 @@ -from .base import AAM, PatchBasedAAM -from .builder import AAMBuilder, PatchBasedAAMBuilder -from .fitter import LucasKanadeAAMFitter +from .builder import ( + AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, + LinearPatchAAMBuilder, PartsAAMBuilder) +from .fitter import LKAAMFitter +from .algorithm import ( + PFC, PIC, + SFC, SIC, + AFC, AIC, + MAFC, MAIC, + WFC, WIC) +from .result import SerializableAAMFitterResult From e847674a051c29e4c6d99638772181c16aca5420 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:21:48 +0100 Subject: [PATCH 022/423] Slight modification to aam.algorithm --- menpofit/aam/algorithm.py | 54 --------------------------------------- 1 file changed, 54 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 44d4cc5..1e84871 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -88,33 +88,6 @@ def steepest_descent_images(self, nabla, dW_dp): # sdi: (n_channels x n_pixels) x n_params return sdi.reshape((-1, sdi.shape[2])) - def partial_newton_hessian(self, nabla2, dw_dp): - # reshape gradient - # gradient: n_dims x n_dims x n_channels x n_pixels - nabla2 = nabla2[self.nabla2_mask].reshape( - (2,) + nabla2.shape[:2] + (-1,)) - - # compute partial hessian - # gradient: n_dims x n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # h: n_dims x n_channels x n_pixels x n_params - h1 = 0 - aux = nabla2[..., None] * dw_dp[:, None, None, ...] - for d in aux: - h1 += d - # compute partial hessian - # h: n_dims x n_channels x n_pixels x n_params - # warp_jacobian: n_dims x x n_pixels x x n_params - # h: - h2 = 0 - aux = h1[..., None] * dw_dp[..., None, :, None, :] - for d in aux: - h2 += d - - # reshape hessian - # 2: (n_channels x n_pixels) x n_params x n_params - return h2.reshape((-1, h2.shape[3] * h2.shape[4])) - @classmethod def solve_shape_map(cls, H, J, e, J_prior, p): if p.shape[0] is not H.shape[0]: @@ -227,33 +200,6 @@ def steepest_descent_images(self, nabla, dw_dp): # sdi: (parts x offsets x ch x w x h) x params return sdi.reshape((-1, sdi.shape[-1])) - def partial_newton_hessian(self, nabla2, dw_dp): - # reshape gradient - # gradient: dims x dims x parts x off x ch x (h x w) - nabla2 = nabla2[self.gradient2_mask].reshape( - nabla2.shape[:-2] + (-1,)) - - # compute partial hessian - # gradient: dims x dims x parts x off x ch x (h x w) - # dw_dp: dims x x parts x x params - # h: dims x parts x off x ch x (h x w) x params - h1 = 0 - aux = nabla2[..., None] * dw_dp[:, None, :, None, None, None, ...] - for d in aux: - h1 += d - # compute partial hessian - # h: dims x parts x off x ch x (h x w) x params - # dw_dp: dims x parts x x params - # h: - h2 = 0 - aux = h1[..., None] * dw_dp[..., None, None, None, None, :] - for d in aux: - h2 += d - - # reshape hessian - # 2: (parts x off x ch x w x h) x params x params - return h2.reshape((-1, h2.shape[-2] * h2.shape[-1])) - def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( From f88a564db9fe08fe2455c2554e39facf901d1657 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:37:32 +0100 Subject: [PATCH 023/423] Unify AAM sampling --- menpofit/aam/algorithm.py | 16 ++++++++-------- menpofit/aam/fitter.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 1e84871..58792cf 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -8,7 +8,7 @@ class AAMInterface(object): - def __init__(self, aam_algorithm, sampling_step=None): + def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm n_true_pixels = self.template.n_true_pixels() @@ -16,9 +16,9 @@ def __init__(self, aam_algorithm, sampling_step=None): n_parameters = self.transform.n_parameters sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - if sampling_step is None: - sampling_step = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling_step) + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) sampling_mask[sampling_pattern] = 1 self.i_mask = np.nonzero(np.tile( @@ -143,14 +143,14 @@ def algorithm_result(self, image, shape_parameters, class PartsAAMInterface(AAMInterface): - def __init__(self, aam_algorithm, sampling_mask=None): + def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm - if sampling_mask is None: - sampling_mask = np.ones(self.patch_shape, dtype=np.bool) + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling_mask[None, None, None, ...], + image_mask = np.tile(sampling[None, None, None, ...], image_shape[:3] + (1, 1)) self.i_mask = np.nonzero(image_mask.flatten())[0] self.gradient_mask = np.nonzero(np.tile( diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 594a985..9fe1811 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -12,14 +12,14 @@ class LKAAMFitter(ModelFitter): r""" """ def __init__(self, aam, algorithm_cls=AIC, n_shape=None, - n_appearance=None, **kwargs): + n_appearance=None, sampling=None, **kwargs): super(LKAAMFitter, self).__init__(aam) self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - self._set_up(algorithm_cls, **kwargs) + self._set_up(algorithm_cls, sampling, **kwargs) - def _set_up(self, algorithm_cls, **kwargs): + def _set_up(self, algorithm_cls, sampling, **kwargs): for j, (am, sm) in enumerate(zip(self._model.appearance_models, self._model.shape_models)): @@ -30,7 +30,7 @@ def _set_up(self, algorithm_cls, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = algorithm_cls(AAMInterface, am, md_transform, - **kwargs) + sampling=sampling, **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -39,7 +39,8 @@ def _set_up(self, algorithm_cls, **kwargs): sm, self._model.n_landmarks) # set up algorithm using linear aam interface algorithm = algorithm_cls(LinearAAMInterface, am, - md_transform, **kwargs) + md_transform, sampling=sampling, + **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model @@ -47,7 +48,8 @@ def _set_up(self, algorithm_cls, **kwargs): # set up algorithm using parts aam interface am.patch_shape = self._model.patch_shape[j] am.normalize_parts = self._model.normalize_parts - algorithm = algorithm_cls(PartsAAMInterface, am, pdm, **kwargs) + algorithm = algorithm_cls(PartsAAMInterface, am, pdm, + sampling=sampling, **kwargs) else: raise ValueError("AAM object must be of one of the " From 8cfd8e15246c6d4c96a2c9d4afc4833753a0246b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:51:36 +0100 Subject: [PATCH 024/423] Small changes to fitter.py --- menpofit/aam/algorithm.py | 1 + menpofit/fitter.py | 13 ++----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 58792cf..aa6882a 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -6,6 +6,7 @@ from .result import AAMAlgorithmResult, LinearAAMAlgorithmResult +# TODO: implement more clever sampling? class AAMInterface(object): def __init__(self, aam_algorithm, sampling=None): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ebdad02..14d9e7d 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -27,11 +27,11 @@ def reference_shape(self): pass @abc.abstractproperty - def scales(self): + def features(self): pass @abc.abstractproperty - def features(self): + def scales(self): pass @abc.abstractproperty @@ -267,15 +267,6 @@ def features(self): """ return self._model.features - @property - def n_levels(self): - r""" - The number of pyramidal levels used during AAM building. - - :type: `int` - """ - return self._model.n_levels - @property def scales(self): return self._model.scales From f1d879bd948416ecd9f430a18ddecbd8cbde8a9e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:56:35 +0100 Subject: [PATCH 025/423] Small renaming - AAMAlgorithm -> LKAAMAlgorithm - AAMInterface -> LKAAMInterface --- menpofit/aam/algorithm.py | 18 ++++++++++-------- menpofit/aam/fitter.py | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index aa6882a..9a55ff1 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -7,7 +7,7 @@ # TODO: implement more clever sampling? -class AAMInterface(object): +class LKAAMInterface(object): def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm @@ -129,7 +129,7 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class LinearAAMInterface(AAMInterface): +class LinearLKAAMInterface(LKAAMInterface): @property def shape_model(self): @@ -142,7 +142,7 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class PartsAAMInterface(AAMInterface): +class PartsLKAAMInterface(LKAAMInterface): def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm @@ -208,7 +208,9 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class AAMAlgorithm(object): +# TODO: handle costs for all LKAAMAlgorithms +# TODO document me! +class LKAAMAlgorithm(object): def __init__(self, aam_interface, appearance_model, transform, eps=10**-5, **kwargs): @@ -257,7 +259,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, pass -class ProjectOut(AAMAlgorithm): +class ProjectOut(LKAAMAlgorithm): r""" Abstract Interface for Project-out AAM algorithms """ @@ -378,7 +380,7 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Simultaneous(AAMAlgorithm): +class Simultaneous(LKAAMAlgorithm): r""" Abstract Interface for Simultaneous AAM algorithms """ @@ -498,7 +500,7 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Alternating(AAMAlgorithm): +class Alternating(LKAAMAlgorithm): r""" Abstract Interface for Alternating AAM algorithms """ @@ -723,7 +725,7 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Wiberg(AAMAlgorithm): +class Wiberg(LKAAMAlgorithm): r""" Abstract Interface for Wiberg AAM algorithms """ diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 9fe1811..f693409 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -3,7 +3,8 @@ from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM -from .algorithm import AAMInterface, LinearAAMInterface, PartsAAMInterface, AIC +from .algorithm import ( + LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) from .result import AAMFitterResult @@ -29,7 +30,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): sm, self._model.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(AAMInterface, am, md_transform, + algorithm = algorithm_cls(LKAAMInterface, am, md_transform, sampling=sampling, **kwargs) elif (type(self.aam) is LinearAAM or @@ -38,7 +39,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self._model.n_landmarks) # set up algorithm using linear aam interface - algorithm = algorithm_cls(LinearAAMInterface, am, + algorithm = algorithm_cls(LinearLKAAMInterface, am, md_transform, sampling=sampling, **kwargs) @@ -48,7 +49,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): # set up algorithm using parts aam interface am.patch_shape = self._model.patch_shape[j] am.normalize_parts = self._model.normalize_parts - algorithm = algorithm_cls(PartsAAMInterface, am, pdm, + algorithm = algorithm_cls(PartsLKAAMInterface, am, pdm, sampling=sampling, **kwargs) else: From 4399787d9f7318986140891cfbff7a8cfe7e8e04 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 17:56:31 +0100 Subject: [PATCH 026/423] Modify noisy_align in fitter.py --- menpofit/aam/fitter.py | 1 - menpofit/fitter.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f693409..a2080c4 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -89,4 +89,3 @@ def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): return AAMFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 14d9e7d..db252ac 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -240,7 +240,7 @@ def _fitter_result(self, image, algorithm_results, affine_correction, pass - +# TODO: correctly implement initialization from bounding box # TODO: document me! class ModelFitter(MultiFitter): r""" @@ -329,13 +329,12 @@ def obtain_shape_from_bb(self, bounding_box): # TODO: document me! -def noisy_align(alignment_transform_cls, source, target, noise_std=0.04, - rotation=True): +def noisy_align(alignment_transform_cls, source, target, noise_std=10): r""" """ noise = noise_std * np.random.randn(target.n_points, target.n_dims) noisy_target = PointCloud(target.points + noise) - return alignment_transform_cls(source, noisy_target, rotation=rotation) + return alignment_transform_cls(source, noisy_target) def align_shape_with_bb(shape, bounding_box): From 8b6f83e91fda1969de0c8e755b6de27749d27395 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 17:59:10 +0100 Subject: [PATCH 027/423] Add LKFitter --- menpofit/lk/fitter.py | 108 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 menpofit/lk/fitter.py diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py new file mode 100644 index 0000000..5b7d69c --- /dev/null +++ b/menpofit/lk/fitter.py @@ -0,0 +1,108 @@ +from __future__ import division +from menpo.feature import no_op +from menpofit.transform import DifferentiableAlignmentAffine +from menpofit.fitter import MultiFitter, noisy_align +from menpofit.result import MultiFitterResult +from .algorithm import IC +from .residual import SSD, FourierSSD + + +# TODO: document me! +class LKFitter(MultiFitter): + r""" + """ + def __init__(self, template, group=None, label=None, features=no_op, + transform_cls=DifferentiableAlignmentAffine, diagonal=None, + scales=(1, .5), scale_features=True, algorithm_cls=IC, + residual_cls=SSD, **kwargs): + self._features = features + self.transform_cls = transform_cls + self.diagonal = diagonal + self._scales = list(scales) + self._scales.reverse() + self._scale_features = scale_features + + self.templates, self.sources = self._prepare_template( + template, group=group, label=label) + + self._reference_shape = self.sources[0] + + self._algorithms = [] + for j, (t, s) in enumerate(zip(self.templates, self.sources)): + transform = self.transform_cls(s, s) + if ('kernel_func' in kwargs and + (residual_cls is SSD or + residual_cls is FourierSSD)): + kernel_func = kwargs.pop('kernel_func') + kernel = kernel_func(t.shape) + residual = residual_cls(kernel=kernel) + else: + residual = residual_cls() + algorithm = algorithm_cls(t, transform, residual, **kwargs) + self._algorithms.append(algorithm) + + @property + def algorithms(self): + return self._algorithms + + @property + def reference_shape(self): + return self._reference_shape + + @property + def features(self): + return self._features + + @property + def scales(self): + return self._scales + + @property + def scale_features(self): + return self._scale_features + + def _prepare_template(self, template, group=None, label=None): + # copy template + template = template.copy() + + template = template.crop_to_landmarks_inplace(group=group, label=label) + template = template.as_masked() + + # rescale template to diagonal range + if self.diagonal: + template = template.rescale_landmarks_to_diagonal_range( + self.diagonal, group=group, label=label) + + # obtain image representation + from copy import deepcopy + scales = deepcopy(self.scales) + scales.reverse() + templates = [] + for j, s in enumerate(scales): + if j == 0: + # compute features at highest level + feature_template = self.features(template) + elif self.scale_features: + # scale features at other levels + feature_template = templates[0].rescale(s) + else: + # scale image and compute features at other levels + scaled_template = template.rescale(s) + feature_template = self.features(scaled_template) + templates.append(feature_template) + templates.reverse() + + # get sources per level + sources = [i.landmarks[group][label] for i in templates] + + return templates, sources + + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return MultiFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) + + def perturb_shape(self, gt_shape, noise_std=0.04): + transform = noisy_align(self.transform_cls, self.reference_shape, + gt_shape, noise_std=noise_std) + return transform.apply(self.reference_shape) From 88627b78bd34600c29cb1e2a7df8d31bea00c098 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:04:49 +0100 Subject: [PATCH 028/423] Add results for LK --- menpofit/lk/fitter.py | 12 ++++++------ menpofit/lk/result.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 menpofit/lk/result.py diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 5b7d69c..4633904 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -2,9 +2,9 @@ from menpo.feature import no_op from menpofit.transform import DifferentiableAlignmentAffine from menpofit.fitter import MultiFitter, noisy_align -from menpofit.result import MultiFitterResult from .algorithm import IC from .residual import SSD, FourierSSD +from .result import LKFitterResult # TODO: document me! @@ -97,12 +97,12 @@ def _prepare_template(self, template, group=None, label=None): return templates, sources - def _fitter_result(self, image, algorithm_results, affine_correction, - gt_shape=None): - return MultiFitterResult(image, self, algorithm_results, - affine_correction, gt_shape=gt_shape) - def perturb_shape(self, gt_shape, noise_std=0.04): transform = noisy_align(self.transform_cls, self.reference_shape, gt_shape, noise_std=noise_std) return transform.apply(self.reference_shape) + + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return LKFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) \ No newline at end of file diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py new file mode 100644 index 0000000..1eddf5d --- /dev/null +++ b/menpofit/lk/result.py @@ -0,0 +1,18 @@ +from __future__ import division +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult + + +# TODO: document me! +# TODO: handle costs! +class LKAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + pass + + +# TODO: document me! +# TODO: handle costs +class LKFitterResult(MultiFitterResult): + r""" + """ + pass From aea81de46dda74eb3bed9620af77e23bdb70fdbe Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:05:40 +0100 Subject: [PATCH 029/423] Add residuals for LK --- menpofit/lk/residual.py | 500 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 500 insertions(+) create mode 100755 menpofit/lk/residual.py diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py new file mode 100755 index 0000000..401b1e5 --- /dev/null +++ b/menpofit/lk/residual.py @@ -0,0 +1,500 @@ +import abc +import numpy as np +from numpy.fft import fftn, ifftn, fft2 +import scipy.linalg +from menpo.feature import gradient + + +class Residual(object): + """ + An abstract base class for calculating the residual between two images + within the Lucas-Kanade algorithm. The classes were designed + specifically to work within the Lucas-Kanade framework and so no + guarantee is made that calling methods on these subclasses will generate + correct results. + """ + @classmethod + def gradient(cls, image, forward=None): + r""" + Calculates the gradients of the given method. + + If `forward` is provided, then the gradients are warped + (as required in the forward additive algorithm) + + Parameters + ---------- + image : :class:`menpo.image.base.Image` + The image to calculate the gradients for + forward : (:map:`Image`, :map:`AlignableTransform>`), optional + A tuple containing the extra weights required for the function + `warp` (which should be passed as a function handle). + + Default: `None` + """ + if forward: + # Calculate the gradient over the image + # grad: (dims x ch) x H x W + grad = gradient(image) + # Warp gradient for forward additive using the given transform + # grad: (dims x ch) x h x w + template, transform = forward + grad = grad.warp_to_mask(template.mask, transform, + warp_landmarks=False) + else: + # Calculate the gradient over the image and set one pixels along + # the boundary of the image mask to zero (no reliable gradient + # can be computed there!) + # grad: (dims x ch) x h x w + grad = gradient(image) + grad.set_boundary_pixels() + return grad + + @abc.abstractmethod + def steepest_descent_images(self, image, dW_dp, **kwargs): + r""" + Calculates the standard steepest descent images. + + Within the forward additive framework this is defined as + + .. math:: + \nabla I \frac{\partial W}{\partial p} + + The input image is vectorised (`N`-pixels) so that masked images can + be handled. + + Parameters + ---------- + image : :class:`menpo.image.base.Image` + The image to calculate the steepest descent images from, could be + either the template or input image depending on which framework is + used. + dW_dp : ndarray + The Jacobian of the warp. + + Returns + ------- + VT_dW_dp : (N, n_params) ndarray + The steepest descent images + """ + pass + + @abc.abstractmethod + def hessian(self, sdi): + r""" + Calculates the Gauss-Newton approximation to the Hessian. + + This is abstracted because some residuals expect the Hessian to be + pre-processed. The Gauss-Newton approximation to the Hessian is + defined as: + + .. math:: + \mathbf{J J^T} + + Parameters + ---------- + J : (N, n_params) ndarray + The steepest descent images. + + Returns + ------- + H : (n_params, n_params) ndarray + The approximation to the Hessian + """ + pass + + @abc.abstractmethod + def steepest_descent_update(self, sdi, image, template): + r""" + Calculates the steepest descent parameter updates. + + These are defined, for the forward additive algorithm, as: + + .. math:: + \sum_x [ \nabla I \frac{\partial W}{\partial p} ]^T [ T(x) - I(W(x;p)) ] + + Parameters + ---------- + J : (N, n_params) ndarray + The steepest descent images. + image : :class:`menpo.image.base.Image` + Either the warped image or the template + (depending on the framework) + template : :class:`menpo.image.base.Image` + Either the warped image or the template + (depending on the framework) + + Returns + ------- + sd_delta_p : (n_params,) ndarray + The steepest descent parameter updates. + """ + pass + + +class SSD(Residual): + + def __init__(self, kernel=None): + self.kernel = kernel + + def steepest_descent_images(self, image, dW_dp, forward=None): + # compute gradient + # grad: dims x ch x h x w + nabla = self.gradient(image, forward=forward) + nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + + image.shape) + + # compute steepest descent images + # gradient: dims x ch x h x w + # dw_dp: dims x x h x w x params + # sdi: ch x h x w x params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + + if self.kernel is not None: + # if required, filter steepest descent images + # fft_sdi: ch x h x w x params + filtered_sdi = ifftn(self.kernel[..., None] * + fftn(sdi, axes=(-3, -2)), + axes=(-3, -2)) + # reshape steepest descent images + # sdi: (ch x h x w) x params + # filtered_sdi: (ch x h x w) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) + filtered_sdi = filtered_sdi.reshape(sdi.shape) + else: + # reshape steepest descent images + # sdi: (ch x h x w) x params + # filtered_sdi: (ch x h x w) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) + filtered_sdi = sdi + + return filtered_sdi, sdi + + def hessian(self, sdi, sdi2=None): + # compute hessian + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params + # hessian: params x x params + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) + return H + + def steepest_descent_update(self, sdi, image, template): + error_img = image.as_vector() - template.as_vector() + return sdi.T.dot(error_img) + + +class FourierSSD(Residual): + + def __init__(self, kernel=None): + self.kernel = kernel + + def steepest_descent_images(self, image, dW_dp, forward=None): + # compute gradient + # grad: dims x ch x h x w + nabla = self.gradient(image, forward=forward) + nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + + image.shape) + + # compute steepest descent images + # gradient: dims x ch x h x w + # dw_dp: dims x x h x w x params + # sdi: ch x h x w x params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + + # compute steepest descent images fft + # fft_sdi: ch x h x w x params + fft_sdi = fftn(sdi, axes=(-3, -2)) + + if self.kernel is not None: + # if required, filter steepest descent images + filtered_fft_sdi = self.kernel[..., None] * fft_sdi + # reshape steepest descent images + # fft_sdi: (ch x h x w) x params + # filtered_fft_sdi: (ch x h x w) x params + fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) + filtered_fft_sdi = filtered_fft_sdi.reshape(fft_sdi.shape) + else: + # reshape steepest descent images + # fft_sdi: (ch x h x w) x params + # filtered_fft_sdi: (ch x h x w) x params + fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) + filtered_fft_sdi = fft_sdi + + return filtered_fft_sdi, fft_sdi + + def hessian(self, sdi, sdi2=None): + if sdi2 is None: + H = sdi.conjugate().T.dot(sdi) + else: + H = sdi.conjugate().T.dot(sdi2) + return H + + def steepest_descent_update(self, sdi, image, template): + # compute error image + # error_img: ch x h x w + error_img = image.pixels - template.pixels + + # compute error image fft + # fft_error_img: ch x (h x w) + fft_error_img = fft2(error_img) + + # compute steepest descent update + # fft_sdi: params x (ch x h x w) + # fft_error_img: (ch x h x w) + # fft_sdu: params + return sdi.conjugate().T.dot(fft_error_img.ravel()) + + +class ECC(Residual): + + def _normalise_images(self, image): + # TODO: do we need to copy the image? + # TODO: is this supposed to be per channel normalization? + norm_image = image.copy() + norm_image.normalize_norm_inplace() + return norm_image + + def steepest_descent_images(self, image, dW_dp, forward=None): + # normalize image + norm_image = self._normalise_images(image) + + # compute gradient + # gradient: dims x ch x pixels + grad = self.gradient(norm_image, forward=forward) + grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) + + # compute steepest descent images + # gradient: dims x ch x pixels + # dw_dp: dims x x pixels x params + # sdi: ch x pixels x params + sdi = 0 + a = grad[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (ch x pixels) x params + return sdi.reshape((-1, sdi.shape[-1])) + + def hessian(self, sdi): + # compute hessian + # sdi.T: params x (ch x pixels) + # sdi: (ch x pixels) x params + # hessian: params x x params + H = sdi.T.dot(sdi) + self._H_inv = scipy.linalg.inv(H) + return H + + def steepest_descent_update(self, sdi, image, template): + normalised_IWxp = self._normalise_images(image).as_vector() + normalised_template = self._normalise_images(template).as_vector() + + Gt = sdi.T.dot(normalised_template) + Gw = sdi.T.dot(normalised_IWxp) + + # Calculate the numerator + IWxp_norm = scipy.linalg.norm(normalised_IWxp) + num1 = IWxp_norm ** 2 + num2 = np.dot(Gw.T, np.dot(self._H_inv, Gw)) + num = num1 - num2 + + # Calculate the denominator + den1 = np.dot(normalised_template, normalised_IWxp) + den2 = np.dot(Gt.T, np.dot(self._H_inv, Gw)) + den = den1 - den2 + + # Calculate lambda to choose the step size + # Avoid division by zero + if den > 0: + l = num / den + else: + den3 = np.dot(Gt.T, np.dot(self._H_inv, Gt)) + l1 = np.sqrt(num2 / den3) + l2 = - den / den3 + l = np.maximum(l1, l2) + + self._error_img = l * normalised_IWxp - normalised_template + + return sdi.T.dot(self._error_img) + + +class GradientImages(Residual): + + def _regularise_gradients(self, grad): + pixels = grad.pixels + ab = np.sqrt(np.sum(pixels**2, axis=0)) + m_ab = np.median(ab) + ab = ab + m_ab + grad.pixels = pixels / ab + return grad + + def steepest_descent_images(self, image, dW_dp, forward=None): + n_dims = image.n_dims + n_channels = image.n_channels + + # compute gradient + first_grad = self.gradient(image, forward=forward) + self._template_grad = self._regularise_gradients(first_grad) + + # compute gradient + # second_grad: dims x dims x ch x pixels + second_grad = self.gradient(self._template_grad) + second_grad = second_grad.masked_pixels().flatten().reshape( + (n_dims, n_dims, n_channels, -1)) + + # Fix crossed derivatives: dydx = dxdy + second_grad[1, 0, ...] = second_grad[0, 1, ...] + + # compute steepest descent images + # gradient: dims x dims x ch x (h x w) + # dw_dp: dims x x (h x w) x params + # sdi: dims x ch x (h x w) x params + sdi = 0 + a = second_grad[..., None] * dW_dp[:, None, None, ...] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (dims x ch x h x w) x params + return sdi.reshape((-1, sdi.shape[-1])) + + def hessian(self, sdi): + # compute hessian + # sdi.T: params x (dims x ch x pixels) + # sdi: (dims x ch x pixels) x params + # hessian: params x x params + return sdi.T.dot(sdi) + + def steepest_descent_update(self, sdi, image, template): + # compute image regularized gradient + IWxp_grad = self.gradient(image) + IWxp_grad = self._regularise_gradients(IWxp_grad) + + # compute vectorized error_image + # error_img: (dims x ch x pixels) + self._error_img = (IWxp_grad.as_vector() - + self._template_grad.as_vector()) + + # compute steepest descent update + # sdi.T: params x (dims x ch x pixels) + # error_img: (dims x ch x pixels) + # sdu: params + return sdi.T.dot(self._error_img) + + +class GradientCorrelation(Residual): + + def steepest_descent_images(self, image, dW_dp, forward=None): + n_dims = image.n_dims + n_channels = image.n_channels + + # compute gradient + # grad: dims x ch x pixels + grad = self.gradient(image, forward=forward) + grad2 = grad.as_vector().reshape((n_dims, n_channels) + image.shape) + + # compute IGOs (remember axis 0 is y, axis 1 is x) + # grad: dims x ch x pixels + # phi: ch x pixels + # cos_phi: ch x pixels + # sin_phi: ch x pixels + phi = np.angle(grad2[1, ...] + 1j * grad2[0, ...]) + self._cos_phi = np.cos(phi) + self._sin_phi = np.sin(phi) + + # concatenate sin and cos terms so that we can take the second + # derivatives correctly. sin(phi) = y and cos(phi) = x which is the + # correct ordering when multiplying against the warp Jacobian + # cos_phi: ch x pixels + # sin_phi: ch x pixels + # grad: (dims x ch) x pixels + grad.from_vector_inplace( + np.concatenate((self._sin_phi[None, ...], + self._cos_phi[None, ...]), axis=0).ravel()) + + # compute IGOs gradient + # second_grad: dims x dims x ch x pixels + second_grad = self.gradient(grad) + second_grad = second_grad.masked_pixels().flatten().reshape( + (n_dims, n_dims, n_channels) + image.shape) + + # Fix crossed derivatives: dydx = dxdy + second_grad[1, 0, ...] = second_grad[0, 1, ...] + + # complete full IGOs gradient computation + # second_grad: dims x dims x ch x pixels + second_grad[1, ...] = (-self._sin_phi[None, ...] * second_grad[1, ...]) + second_grad[0, ...] = (self._cos_phi[None, ...] * second_grad[0, ...]) + + # compute steepest descent images + # gradient: dims x dims x ch x pixels + # dw_dp: dims x x pixels x params + # sdi: ch x pixels x params + sdi = 0 + aux = second_grad[..., None] * dW_dp[None, :, None, ...] + for a in aux.reshape(((-1,) + aux.shape[2:])): + sdi += a + + # compute constant N + # N: 1 + self._N = grad.n_parameters / 2 + + # reshape steepest descent images + # sdi: (ch x pixels) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) + + return sdi, sdi + + def hessian(self, sdi, sdi2=None): + # compute hessian + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params + # hessian: params x x params + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) + return H + + def steepest_descent_update(self, sdi, image, template): + n_dims = image.n_dims + n_channels = image.n_channels + + # compute image gradient + IWxp_grad = self.gradient(image) + IWxp_grad = IWxp_grad.as_vector().reshape( + (n_dims, n_channels) + image.shape) + + # compute IGOs (remember axis 0 is y, axis 1 is x) + # IWxp_grad: dims x ch x pixels + # phi: ch x pixels + # IWxp_cos_phi: ch x pixels + # IWxp_sin_phi: ch x pixels + phi = np.angle(IWxp_grad[1, ...] + 1j * IWxp_grad[0, ...]) + IWxp_cos_phi = np.cos(phi) + IWxp_sin_phi = np.sin(phi) + + # compute error image + # error_img: (ch x h x w) + self._error_img = (self._cos_phi * IWxp_sin_phi - + self._sin_phi * IWxp_cos_phi).ravel() + + # compute steepest descent update + # sdi: (ch x pixels) x params + # error_img: (ch x pixels) + # sdu: params + sdu = sdi.T.dot(self._error_img) + + # compute step size + qp = np.sum(self._cos_phi * IWxp_cos_phi + + self._sin_phi * IWxp_sin_phi) + l = self._N / qp + return l * sdu From c577c001c096a93df156153ecba124a173c0ec7d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:06:05 +0100 Subject: [PATCH 030/423] Add algorithms for LK --- menpofit/lk/algorithm.py | 179 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 menpofit/lk/algorithm.py diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py new file mode 100644 index 0000000..49d3a25 --- /dev/null +++ b/menpofit/lk/algorithm.py @@ -0,0 +1,179 @@ +from scipy.linalg import norm +import abc +import numpy as np +from .result import LKAlgorithmResult + + +# TODO: implement Linear, Parts interfaces? Will they play nice with residuals? +# TODO: implement sampling? +# TODO: handle costs for all LKAlgorithms +# TODO: document me! +class LKAlgorithm(object): + r""" + """ + def __init__(self, template, transform, residual, eps=10**-10): + self.template = template + self.transform = transform + self.residual = residual + self.eps = eps + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + pass + + +class FA(LKAlgorithm): + r""" + Forward Additive Lucas-Kanade algorithm + """ + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Forward Compositional Algorithm + while k < max_iters and eps > self.eps: + # warp image + IWxp = image.warp_to_mask(self.template.mask, self.transform) + + # compute warp jacobian + dW_dp = np.rollaxis( + self.transform.d_dp(self.template.indices()), -1) + + # compute steepest descent images + filtered_J, J = self.residual.steepest_descent_images( + image, dW_dp, forward=(self.template, self.transform)) + + # compute hessian + H = self.residual.hessian(filtered_J, sdi2=J) + + # compute steepest descent parameter updates. + sd_dp = self.residual.steepest_descent_update( + filtered_J, IWxp, self.template) + + # compute gradient descent parameter updates + dp = np.real(np.linalg.solve(H, sd_dp)) + + # Update warp weights + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(norm(dp)) + + # increase iteration counter + k += 1 + + return LKAlgorithmResult(image, self, p_list, gt_shape=None) + + +class FC(LKAlgorithm): + r""" + Forward Compositional Lucas-Kanade algorithm + """ + def __init__(self, template, transform, residual, eps=10**-10): + super(FC, self).__init__(template, transform, residual, eps=eps) + self.precompute() + + def precompute(self): + # compute warp jacobian + self.dW_dp = np.rollaxis( + self.transform.d_dp(self.template.indices()), -1) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Forward Compositional Algorithm + while k < max_iters and eps > self.eps: + # warp image + IWxp = image.warp_to_mask(self.template.mask, self.transform) + + # compute steepest descent images + filtered_J, J = self.residual.steepest_descent_images( + IWxp, self.dW_dp) + + # compute hessian + H = self.residual.hessian(filtered_J, sdi2=J) + + # compute steepest descent parameter updates. + sd_dp = self.residual.steepest_descent_update( + filtered_J, IWxp, self.template) + + # compute gradient descent parameter updates + dp = np.real(np.linalg.solve(H, sd_dp)) + + # Update warp weights + self.transform.compose_after_from_vector_inplace(dp) + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(norm(dp)) + + # increase iteration counter + k += 1 + + return LKAlgorithmResult(image, self, p_list, gt_shape=None) + + +class IC(LKAlgorithm): + r""" + Inverse Compositional Lucas-Kanade algorithm + """ + def __init__(self, template, transform, residual, eps=10**-10): + super(IC, self).__init__(template, transform, residual, eps=eps) + self.precompute() + + def precompute(self): + # compute warp jacobian + dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) + dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + + dW_dp.shape[-1:]) + # compute steepest descent images + self.filtered_J, J = self.residual.steepest_descent_images( + self.template, dW_dp) + # compute hessian + self.H = self.residual.hessian(self.filtered_J, sdi2=J) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Baker-Matthews, Inverse Compositional Algorithm + while k < max_iters and eps > self.eps: + # warp image + IWxp = image.warp_to_mask(self.template.mask, self.transform) + + # compute steepest descent parameter updates. + sd_dp = self.residual.steepest_descent_update( + self.filtered_J, IWxp, self.template) + + # compute gradient descent parameter updates + dp = np.real(np.linalg.solve(self.H, sd_dp)) + + # update warp + inv_dp = self.transform.pseudoinverse_vector(dp) + self.transform.compose_after_from_vector_inplace(inv_dp) + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(norm(dp)) + + # increase iteration counter + k += 1 + + return LKAlgorithmResult(image, self, p_list, gt_shape=None) From 6454e11b086cf992a66e78df3e46527198636458 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:06:57 +0100 Subject: [PATCH 031/423] Update lk.__init__ --- menpofit/lk/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 menpofit/lk/__init__.py diff --git a/menpofit/lk/__init__.py b/menpofit/lk/__init__.py new file mode 100644 index 0000000..b01bf94 --- /dev/null +++ b/menpofit/lk/__init__.py @@ -0,0 +1,3 @@ +from .fitter import LKFitter +from .algorithm import FA, FC, IC +from .residual import SSD, FourierSSD, ECC, GradientImages, GradientCorrelation From 37b88fbf36fa34ef56c65000e53489399de9fdad Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 13:50:03 +0100 Subject: [PATCH 032/423] Add accidentaly deleted view_shape_models_widget to AAM - Update some documentation on AAM --- menpofit/aam/base.py | 83 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 4266ae7..54b962a 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -159,9 +159,62 @@ def _instance(self, level, shape_instance, appearance_instance): return instance + def view_shape_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + r""" + Visualizes the shape models of the AAM object using the + `menpo.visualize.widgets.visualize_shape_model` widget. + + Parameters + ----------- + n_parameters : `int` or `list` of `int` or ``None``, optional + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + parameters_bounds : (`float`, `float`), optional + The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + figure_size : (`int`, `int`), optional + The size of the plotted figures. + """ + from menpofit.visualize import visualize_shape_model + visualize_shape_model(self.shape_models, n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): + r""" + Visualizes the appearance models of the AAM object using the + `menpo.visualize.widgets.visualize_appearance_model` widget. + Parameters + ----------- + n_parameters : `int` or `list` of `int` or ``None``, optional + The number of appearance principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + parameters_bounds : (`float`, `float`), optional + The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + figure_size : (`int`, `int`), optional + The size of the plotted figures. + """ from menpofit.visualize import visualize_appearance_model visualize_appearance_model(self.appearance_models, n_parameters=n_parameters, @@ -172,6 +225,36 @@ def view_appearance_models_widget(self, n_parameters=5, def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): + r""" + Visualizes both the shape and appearance models of the AAM object using + the `menpo.visualize.widgets.visualize_aam` widget. + Parameters + ----------- + n_shape_parameters : `int` or `list` of `int` or None, optional + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + n_appearance_parameters : `int` or `list` of `int` or None, optional + The number of appearance principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + parameters_bounds : (`float`, `float`), optional + The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + figure_size : (`int`, `int`), optional + The size of the plotted figures. + """ from menpofit.visualize import visualize_aam visualize_aam(self, n_shape_parameters=n_shape_parameters, n_appearance_parameters=n_appearance_parameters, From 10c565d57dbe9e760bef29b418197204a1fc5b5b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 14:49:23 +0100 Subject: [PATCH 033/423] More small changes to aam.base --- menpofit/aam/base.py | 45 ++++++++++++++++++++++++++++++++--------- menpofit/aam/builder.py | 2 +- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 54b962a..b2b11c0 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.shape import TriMesh from menpofit.transform import DifferentiableThinPlateSplines @@ -8,6 +7,7 @@ build_reference_frame, build_patch_reference_frame) +# TODO: document me! class AAM(object): r""" Active Appearance Model class. @@ -69,6 +69,15 @@ def n_levels(self): """ return len(self.scales) + # TODO: Could we directly use class names instead of this? + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Active Appearance Model' + def instance(self, shape_weights=None, appearance_weights=None, level=-1): r""" Generates a novel AAM instance given a set of shape and appearance @@ -365,6 +374,7 @@ def __str__(self): return out +# TODO: document me! class PatchAAM(AAM): r""" Patch based Based Active Appearance Model class. @@ -417,6 +427,10 @@ def __init__(self, shape_models, appearance_models, reference_shape, self.scale_shapes = scale_shapes self.scale_features = scale_features + @property + def _str_title(self): + return 'Patch-Based Active Appearance Model' + def _instance(self, level, shape_instance, appearance_instance): template = self.appearance_models[level].mean landmarks = template.landmarks['source'].lms @@ -443,14 +457,13 @@ def view_appearance_models_widget(self, n_parameters=5, figure_size=figure_size, mode=mode) # TODO: fix me! - def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, - parameters_bounds=(-3.0, 3.0), mode='multiple', - figure_size=(10, 8)): - from menpofit.visualize import visualize_aam - visualize_aam(self, n_shape_parameters=n_shape_parameters, - n_appearance_parameters=n_appearance_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode) + def __str__(self): + out = super(PatchAAM, self).__str__() + out_splitted = out.splitlines() + out_splitted[0] = self._str_title + out_splitted.insert(5, " - Patch size is {}W x {}H.".format( + self.patch_shape[1], self.patch_shape[0])) + return '\n'.join(out_splitted) # TODO: document me! @@ -524,6 +537,10 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented + # TODO: implement me! + def __str__(self): + raise NotImplemented + # TODO: document me! class LinearPatchAAM(AAM): @@ -598,6 +615,10 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented + # TODO: implement me! + def __str__(self): + raise NotImplemented + # TODO: document me! class PartsAAM(AAM): @@ -669,4 +690,8 @@ def view_appearance_models_widget(self, n_parameters=5, def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): - raise NotImplemented \ No newline at end of file + raise NotImplemented + + # TODO: implement me! + def __str__(self): + raise NotImplemented diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 6ad44f2..bb98c1a 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,5 +1,4 @@ from __future__ import division -import abc from copy import deepcopy from menpo.model import PCAModel from menpo.shape import mean_pointcloud @@ -14,6 +13,7 @@ DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) +# TODO: fix features checker # TODO: implement checker for conflict between features and scale_features # TODO: document me! class AAMBuilder(object): From 24fcff9f99edcd5ec4760616ad77254f58e06e3b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 17:52:18 +0100 Subject: [PATCH 034/423] Small changes to AAM --- menpofit/aam/algorithm.py | 37 ------------------------------------- menpofit/aam/builder.py | 16 ++++++---------- menpofit/aam/fitter.py | 12 ++++++------ 3 files changed, 12 insertions(+), 53 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 9a55ff1..6583f00 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -201,12 +201,6 @@ def steepest_descent_images(self, nabla, dw_dp): # sdi: (parts x offsets x ch x w x h) x params return sdi.reshape((-1, sdi.shape[-1])) - def algorithm_result(self, image, shape_parameters, - appearance_parameters=None, gt_shape=None): - return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, - appearance_parameters=appearance_parameters, gt_shape=gt_shape) - # TODO: handle costs for all LKAAMAlgorithms # TODO document me! @@ -263,12 +257,6 @@ class ProjectOut(LKAAMAlgorithm): r""" Abstract Interface for Project-out AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(ProjectOut, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def project_out(self, J): # project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) @@ -384,12 +372,6 @@ class Simultaneous(LKAAMAlgorithm): r""" Abstract Interface for Simultaneous AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(Simultaneous, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): # initialize transform @@ -504,12 +486,6 @@ class Alternating(LKAAMAlgorithm): r""" Abstract Interface for Alternating AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(Alternating, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def precompute(self, **kwargs): # call super method super(Alternating, self).precompute() @@ -633,12 +609,6 @@ class ModifiedAlternating(Alternating): r""" Abstract Interface for Modified Alternating AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(ModifiedAlternating, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): # initialize transform @@ -729,12 +699,6 @@ class Wiberg(LKAAMAlgorithm): r""" Abstract Interface for Wiberg AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(Wiberg, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def project_out(self, J): # project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) @@ -832,4 +796,3 @@ def update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) - diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index bb98c1a..ff8a783 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -259,10 +259,9 @@ def _build_shape_model(cls, shapes, max_components, level): def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): - self.reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return AAM(shape_models, appearance_models, reference_shape, @@ -399,11 +398,10 @@ def __init__(self, patch_shape=(17, 17), features=no_op, def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): - self.reference_frame = build_patch_reference_frame( + reference_frame = build_patch_reference_frame( reference_shape, patch_shape=self.patch_shape[level]) - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return PatchAAM(shape_models, appearance_models, reference_shape, @@ -861,5 +859,3 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM - - diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a2080c4..9e3587f 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -21,13 +21,13 @@ def __init__(self, aam, algorithm_cls=AIC, n_shape=None, self._set_up(algorithm_cls, sampling, **kwargs) def _set_up(self, algorithm_cls, sampling, **kwargs): - for j, (am, sm) in enumerate(zip(self._model.appearance_models, - self._model.shape_models)): + for j, (am, sm) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( - sm, self._model.transform, + sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = algorithm_cls(LKAAMInterface, am, md_transform, @@ -37,7 +37,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self._model.n_landmarks) + sm, self.aam.n_landmarks) # set up algorithm using linear aam interface algorithm = algorithm_cls(LinearLKAAMInterface, am, md_transform, sampling=sampling, @@ -47,10 +47,10 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - am.patch_shape = self._model.patch_shape[j] - am.normalize_parts = self._model.normalize_parts algorithm = algorithm_cls(PartsLKAAMInterface, am, pdm, sampling=sampling, **kwargs) + algorithm.patch_shape = self.aam.patch_shape[j] + algorithm.normalize_parts = self.aam.normalize_parts else: raise ValueError("AAM object must be of one of the " From e680a5587406bc790e411cbe89ba4baf2ec8fe22 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 17:55:02 +0100 Subject: [PATCH 035/423] Add TODO in lk.algorithm - Should we implement Inverse Additive? --- menpofit/lk/algorithm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 49d3a25..3034c08 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -4,6 +4,7 @@ from .result import LKAlgorithmResult +# TODO: implement Inverse Additive Algorithm? # TODO: implement Linear, Parts interfaces? Will they play nice with residuals? # TODO: implement sampling? # TODO: handle costs for all LKAlgorithms From ad511aa7896cc048fbb1af94cfcc550872e6943a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 08:42:38 +0100 Subject: [PATCH 036/423] Add ATM base classes - Add ATM, PatchATM, LinearATM, LinearPatchATM and PartsATM --- menpofit/atm/base.py | 389 ++++++++++++++++++++++++++++++++----------- 1 file changed, 288 insertions(+), 101 deletions(-) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index c56a7bd..10c513a 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -1,14 +1,13 @@ from __future__ import division - import numpy as np from menpo.shape import TriMesh - -from menpofit.base import DeformableModel, name_of_callable -from menpofit.aam.builder import (build_patch_reference_frame, - build_reference_frame) +from menpofit.transform import DifferentiableThinPlateSplines +from menpofit.base import name_of_callable +from menpofit.aam.builder import ( + build_patch_reference_frame, build_reference_frame) -class ATM(DeformableModel): +class ATM(object): r""" Active Template Model class. @@ -20,14 +19,15 @@ class ATM(DeformableModel): warped_templates : :map:`MaskedImage` list A list containing the warped templates models of the ATM. - n_training_shapes: `int` - The number of training shapes used to build the ATM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional + features : `callable` or ``[callable]``, If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -41,46 +41,42 @@ class ATM(DeformableModel): once and then creating a pyramid on top tends to lead to better performing AAMs. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. + scales : `int` or float` or list of those, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + scale_shapes : `boolean` - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. - - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + scale_features : `boolean` """ - def __init__(self, shape_models, warped_templates, n_training_shapes, - transform, features, reference_shape, downscale, - scaled_shape_models): - DeformableModel.__init__(self, features) - self.n_training_shapes = n_training_shapes + def __init__(self, shape_models, warped_templates, reference_shape, + transform, features, scales, scale_shapes, scale_features): self.shape_models = shape_models self.warped_templates = warped_templates self.transform = transform + self.features = features self.reference_shape = reference_shape - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features @property def n_levels(self): """ - The number of multi-resolution pyramidal levels of the ATM. + The number of scale level of the ATM. :type: `int` """ - return len(self.warped_templates) + return len(self.scales) + + # TODO: Could we directly use class names instead of this? + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + + :type: `string` + """ + return 'Active Template Model' def instance(self, shape_weights=None, level=-1): r""" @@ -140,35 +136,22 @@ def _instance(self, level, shape_instance): template = self.warped_templates[level] landmarks = template.landmarks['source'].lms - reference_frame = self._build_reference_frame( - shape_instance, landmarks) - - transform = self.transform( - reference_frame.landmarks['source'].lms, landmarks) - - return template.warp_to_mask(reference_frame.mask, transform, - warp_landmarks=True) - - def _build_reference_frame(self, reference_shape, landmarks): if type(landmarks) == TriMesh: trilist = landmarks.trilist else: trilist = None - return build_reference_frame( - reference_shape, trilist=trilist) + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) - @property - def _str_title(self): - r""" - Returns a string containing name of the model. + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) - :type: `string` - """ - return 'Active Template Model' + return template.warp_to_mask(reference_frame.mask, transform, + warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, - parameters_bounds=(-3.0, 3.0), mode='multiple', - popup=False, figure_size=(10, 8)): + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): r""" Visualizes the shape models of the AAM object using the `menpo.visualize.widgets.visualize_shape_model` widget. @@ -189,19 +172,18 @@ def view_shape_models_widget(self, n_parameters=5, If ``'single'``, only a single slider is constructed along with a drop down menu. If ``'multiple'``, a slider is constructed for each parameter. - popup : `bool`, optional - If ``True``, the widget will appear as a popup window. figure_size : (`int`, `int`), optional The size of the plotted figures. """ from menpofit.visualize import visualize_shape_model visualize_shape_model(self.shape_models, n_parameters=n_parameters, parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, popup=popup) + figure_size=figure_size, mode=mode) + # TODO: fix me! def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', - popup=False, figure_size=(10, 8)): + figure_size=(10, 8)): r""" Visualizes the ATM object using the menpo.visualize.widgets.visualize_atm widget. @@ -221,17 +203,16 @@ def view_atm_widget(self, n_shape_parameters=5, mode : {``single``, ``multiple``}, optional If ``'single'``, only a single slider is constructed along with a drop down menu. - If ``'multiple'``, a slider is constructed for each parameter. - popup : `bool`, optional - If ``True``, the widget will appear as a popup window. + If ``'multiple'``, a slider is constructed for each pp window. figure_size : (`int`, `int`), optional The size of the plotted figures. """ from menpofit.visualize import visualize_atm visualize_atm(self, n_shape_parameters=n_shape_parameters, parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, popup=popup) + figure_size=figure_size, mode=mode) + # TODO: fix me! def __str__(self): out = "{}\n - {} training shapes.\n".format(self._str_title, self.n_training_shapes) @@ -332,7 +313,7 @@ def __str__(self): return out -class PatchBasedATM(ATM): +class PatchATM(ATM): r""" Patch Based Active Template Model class. @@ -344,14 +325,92 @@ class PatchBasedATM(ATM): warped_templates : :map:`MaskedImage` list A list containing the warped templates models of the ATM. - n_training_shapes: `int` - The number of training shapes used to build the ATM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based ATM. + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, warped_templates, reference_shape, + patch_shape, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.transform = DifferentiableThinPlateSplines + self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + @property + def _str_title(self): + return 'Patch-Based Active Template Model' + + def _instance(self, level, shape_instance): + template = self.warped_templates[level] + landmarks = template.landmarks['source'].lms + + reference_frame = build_patch_reference_frame( + shape_instance, patch_shape=self.patch_shape) + + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + return template.warp_to_mask(reference_frame.mask, transform, + warp_landmarks=True) + + # TODO: fix me! + def __str__(self): + out = super(PatchBasedATM, self).__str__() + out_splitted = out.splitlines() + out_splitted[0] = self._str_title + out_splitted.insert(5, " - Patch size is {}W x {}H.".format( + self.patch_shape[1], self.patch_shape[0])) + return '\n'.join(out_splitted) + + +# TODO: document me! +class LinearATM(ATM): + r""" + Linear Active Template Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + warped_templates : :map:`MaskedImage` list + A list containing the warped templates models of the ATM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the ATM was + The transform used to warp the images from which the AAM was constructed. features : `callable` or ``[callable]``, optional @@ -368,51 +427,179 @@ class PatchBasedATM(ATM): once and then creating a pyramid on top tends to lead to better performing AAMs. + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, warped_templates, reference_shape, + transform, features, scales, scale_shapes, scale_features, + n_landmarks): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks + + # TODO: implement me! + def _instance(self, level, shape_instance): + raise NotImplemented + + # TODO: implement me! + def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def __str__(self): + raise NotImplemented + + +# TODO: document me! +class LinearPatchATM(ATM): + r""" + Linear Patch based Active Template Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + warped_templates : :map:`MaskedImage` list + A list containing the warped templates models of the ATM. + reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. + scale_shapes : `boolean` - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean` + + n_landmarks: `int` """ - def __init__(self, shape_models, warped_templates, n_training_shapes, - patch_shape, transform, features, reference_shape, - downscale, scaled_shape_models): - super(PatchBasedATM, self).__init__( - shape_models, warped_templates, n_training_shapes, transform, - features, reference_shape, downscale, scaled_shape_models) + def __init__(self, shape_models, warped_templates, reference_shape, + patch_shape, features, scales, scale_shapes, + scale_features, n_landmarks): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.transform = DifferentiableThinPlateSplines self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks - def _build_reference_frame(self, reference_shape, landmarks): - return build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape) + # TODO: implement me! + def _instance(self, level, shape_instance): + raise NotImplemented - @property - def _str_title(self): - r""" - Returns a string containing name of the model. + # TODO: implement me! + def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented - :type: `string` - """ - return 'Patch-Based Active Template Model' + # TODO: implement me! + def __str__(self): + raise NotImplemented + + +# TODO: document me! +class PartsATM(ATM): + r""" + Parts based Active Template Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + warped_templates : :map:`MaskedImage` list + A list containing the warped templates models of the ATM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + normalize_parts: `callable` + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, warped_templates, reference_shape, + patch_shape, features, normalize_parts, scales, + scale_shapes, scale_features): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + # TODO: implement me! + def _instance(self, level, shape_instance): + raise NotImplemented + + # TODO: implement me! + def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! def __str__(self): - out = super(PatchBasedATM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + raise NotImplemented From ce38782206e8de8b2f92acea50062c8f30452139 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:22:20 +0100 Subject: [PATCH 037/423] Remove SerializableAAMFitterResult --- menpofit/aam/__init__.py | 2 +- menpofit/aam/result.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 119cff0..673cb05 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -8,4 +8,4 @@ AFC, AIC, MAFC, MAIC, WFC, WIC) -from .result import SerializableAAMFitterResult + diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 5824945..a60c59b 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -43,11 +43,3 @@ class AAMFitterResult(MultiFitterResult): r""" """ pass - - -# TODO: document me! -# TODO: handle costs -class SerializableAAMFitterResult(SerializableIterativeResult): - r""" - """ - pass From 0801b8be3d958c5f85f08d188cf2005064db6ad3 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:23:56 +0100 Subject: [PATCH 038/423] Small changes in menpofit.builder --- menpofit/builder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 29d4ec8..735bca0 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -148,6 +148,7 @@ def scale_images(images, scale, level_str='', verbose=None): return images +# TODO: Can be done more efficiently for PWA defining a dummy transform # TODO: document me! def warp_images(images, shapes, reference_frame, transform, level_str='', verbose=None): @@ -169,7 +170,7 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! -def extract_patches(images, shapes, parts_shape, normalize_function=no_op, +def extract_patches(images, shapes, patch_shape, normalize_function=no_op, level_str='', verbose=None): parts_images = [] for c, (i, s) in enumerate(zip(images, shapes)): @@ -178,7 +179,7 @@ def extract_patches(images, shapes, parts_shape, normalize_function=no_op, level_str, progress_bar_str(float(c + 1) / len(images), show_bar=False))) - parts = i.extract_patches(s, patch_size=parts_shape, + parts = i.extract_patches(s, patch_size=patch_shape, as_single_array=True) if normalize_function: parts = normalize_function(parts) From b0343accbe49b833347c06bd8b7ef7474fe8e55c Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:25:09 +0100 Subject: [PATCH 039/423] Small changes in menpofit.fitter --- menpofit/fitter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index db252ac..da400f6 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -301,12 +301,13 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - def perturb_shape(self, gt_shape, noise_std=0.04, rotation=False): + # TODO: Fix me! + def perturb_shape(self, gt_shape, noise_std=10, rotation=False): transform = noisy_align(AlignmentSimilarity, self.reference_shape, - gt_shape, noise_std=noise_std, - rotation=rotation) + gt_shape, noise_std=noise_std) return transform.apply(self.reference_shape) + # TODO: Bounding boxes should be PointGraphs def obtain_shape_from_bb(self, bounding_box): r""" Generates an initial shape given a bounding box detection. From 616e9016095269b969b8c6cd3a83890b277b5a1a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:33:07 +0100 Subject: [PATCH 040/423] Add Builders for ATMs - Add ATMBuilder, PatchATMBuilder, LinearAT= ``20``, optional + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the normalization_diagonal - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. @@ -56,25 +57,11 @@ class ATMBuilder(DeformableModelBuilder): reference frame (provided that features computation does not change the image size). - n_levels : `int` > 0, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(``n_levels``) - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of - each pyramid level, so the shape models will be scaled. + scales : `int` or float` or list of those, optional - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. + scale_shapes : `boolean`, optional - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -93,11 +80,6 @@ class ATMBuilder(DeformableModelBuilder): If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- atm : :map:`ATMBuilder` @@ -106,41 +88,40 @@ class ATMBuilder(DeformableModelBuilder): Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError ``max_shape_components`` must be ``None`` or an `int` > 0 or a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``n_levels`` elements + ``len(scales)`` elements ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, features=igo, transform=DifferentiablePiecewiseAffine, - trilist=None, normalization_diagonal=None, n_levels=3, - downscale=2, scaled_shape_models=True, - max_shape_components=None, boundary=3): + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, 0.5), + scale_shapes=False, scale_features=True, + max_shape_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) - # store parameters + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters self.features = features self.transform = transform self.trilist = trilist - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components - self.boundary = boundary def build(self, shapes, template, group=None, label=None, verbose=False): r""" @@ -175,133 +156,235 @@ def build(self, shapes, template, group=None, label=None, verbose=False): to highest level. """ # compute reference_shape - self.reference_shape = compute_reference_shape( - shapes, self.normalization_diagonal, verbose=verbose) + reference_shape = compute_reference_shape(shapes, self.diagonal, + verbose=verbose) # normalize the template size using the reference_shape scaling - if verbose: - print_dynamic('- Normalizing template size') - normalized_template = template.rescale_to_reference_shape( - self.reference_shape, group=group, label=label) + template = template.rescale_to_reference_shape( + reference_shape, group=group, label=label) - # create pyramid for template image + # build models at each scale if verbose: - print_dynamic('- Creating template pyramid') - generator = create_pyramid([normalized_template], self.n_levels, - self.downscale, self.features) - - # build the model at each pyramid level - if verbose: - if self.n_levels > 1: - print_dynamic('- Building model for each of the {} pyramid ' - 'levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building model\n') - + print_dynamic('- Building models\n') shape_models = [] warped_templates = [] # for each pyramid level (high --> low) - for j in range(self.n_levels): - # since models are built from highest to lowest level, the - # parameters in form of list need to use a reversed index - rj = self.n_levels - j - 1 - - if verbose: - level_str = ' - ' - if self.n_levels > 1: - level_str = ' - Level {}: '.format(j + 1) - - # rescale shapes if required - if j > 0 and self.scaled_shape_models: - scale_transform = Scale(scale_factor=1.0 / self.downscale, - n_dims=2) - shapes = [scale_transform.apply(s) for s in shapes] - - # train shape model and find reference frame + for j, s in enumerate(self.scales): if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = build_shape_model(shapes, - self.max_shape_components[rj]) - reference_frame = self._build_reference_frame(shape_model.mean()) - - # add shape model to the list - shape_models.append(shape_model) + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain shape representation + if j == 0 or self.scale_shapes: + if j == 0: + level_shapes = shapes + level_reference_shape= reference_shape + else: + scale_transform = Scale(scale_factor=s, n_dims=2) + level_shapes = [scale_transform.apply(s) for s in shapes] + level_reference_shape = scale_transform.apply( + reference_shape) + # obtain shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # add shape model to the list + shape_models.append(shape_model) + else: + # copy precious shape model and add it to the list + shape_models.append(deepcopy(shape_model)) - # get template's feature image of current level if verbose: - print_dynamic('{}Warping template'.format(level_str)) - feature_template = next(generator[0]) - - # compute transform - transform = self.transform(reference_frame.landmarks['source'].lms, - feature_template.landmarks[group][label]) + print_dynamic('{}Building template model'.format(level_str)) + # obtain template representation + if j == 0: + # compute features at highest level + feature_template = self.features(template) + level_template = feature_template + elif self.scale_features: + # scale features at other levels + level_template = feature_template.rescale(s) + else: + # scale template and compute features at other levels + scaled_template = template.rescale(s) + level_template = self.features(scaled_template) - # warp template to reference frame - warped_templates.append( - feature_template.warp_to_mask(reference_frame.mask, transform)) + # extract potentially rescaled template shape + level_template_shape = level_template.landmarks[group][label] - # attach reference_frame to template's source shape - warped_templates[j].landmarks['source'] = \ - reference_frame.landmarks['source'] + # obtain warped template + warped_template = self._warp_template(level_template, + level_template_shape, + level_reference_shape, j) + # add warped template to the list + warped_templates.append(warped_template) if verbose: print_dynamic('{}Done\n'.format(level_str)) - # reverse the list of shape and appearance models so that they are + # reverse the list of shape and warped templates so that they are # ordered from lower to higher resolution shape_models.reverse() warped_templates.reverse() - n_training_shapes = len(shapes) + self.scales.reverse() + + return self._build_atm(shape_models, warped_templates, reference_shape) + + @classmethod + def _build_shape_model(cls, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + + def _warp_template(self, template, template_shape, reference_shape, level): + # build reference frame + reference_frame = build_reference_frame(reference_shape) + # compute transforms + t = self.transform(reference_frame.landmarks['source'].lms, + template_shape) + # warp template + warped_template = template.warp_to_mask(reference_frame.mask, t) + # attach landmarks + warped_template.landmarks['source'] = reference_frame.landmarks[ + 'source'] + return warped_template + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return ATM(shape_models, warped_templates, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features) + + +class PatchATMBuilder(ATMBuilder): + r""" + Class that builds Multilevel Patch-Based Active Template Models. - return self._build_atm(shape_models, warped_templates, - n_training_shapes) + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - Returns - ------- - reference_frame : :map:`MaskedImage` - The reference frame. - """ - return build_reference_frame(mean_shape, boundary=self.boundary, - trilist=self.trilist) + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - def _build_atm(self, shape_models, warped_templates, n_training_shapes): - r""" - Returns an ATM object. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - Parameters - ---------- - shape_models : `list` of :map:`PCAModel` - The trained multilevel shape models. + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - warped_templates : `list` of :map:`MaskedImage` - The warped multilevel templates. + If ``None``, the mean shape is not rescaled. - n_training_shapes : `int` - The number of training shapes. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - Returns - ------- - atm : :map:`ATM` - The trained ATM object. - """ - from .base import ATM - return ATM(shape_models, warped_templates, n_training_shapes, - self.transform, self.features, self.reference_shape, - self.downscale, self.scaled_shape_models) + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + atm : :map:`ATMBuilder` + The ATM Builder object + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=True, + scale_features=True, max_shape_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components -class PatchBasedATMBuilder(ATMBuilder): + def _warp_template(self, template, template_shape, reference_shape, level): + # build reference frame + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[level]) + # compute transforms + t = self.transform(reference_frame.landmarks['source'].lms, + template_shape) + # warp template + warped_template = template.warp_to_mask(reference_frame.mask, t) + # attach landmarks + warped_template.landmarks['source'] = reference_frame.landmarks[ + 'source'] + return warped_template + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return PatchATM(shape_models, warped_templates, reference_shape, + self.patch_shape, self.features, self.scales, + self.scale_shapes, self.scale_features) + + +# TODO: document me! +class LinearATMBuilder(ATMBuilder): r""" - Class that builds Multilevel Patch-Based Active Template Models. + Class that builds Linear Active Template Models. Parameters ---------- @@ -319,45 +402,165 @@ class PatchBasedATMBuilder(ATMBuilder): once and then creating a pyramid on top tends to lead to better performing AAMs. - patch_shape : tuple of `int`, optional - The appearance model of the Patch-Based AAM will be obtained by - sampling appearance patches with the specified shape around each - landmark. + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. - normalization_diagonal : `int` >= ``20``, optional + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. + + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the ``normalization_diagonal`` - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. - .. note:: + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). + + scales : `int` or float` or list of those, optional - Because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). + scale_shapes : `boolean`, optional - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - downscale : `float` >= 1, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. - (downscale ** k) for k in range(``n_levels``) + Per level: + If `int`, it specifies the exact number of components to be + retained. - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of each - pyramid level, so the shape models will be scaled. - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + atm : :map:`ATMBuilder` + The ATM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters + self.features = features + self.transform = transform + self.trilist = trilist + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model( + dense_shapes, max_components=max_components) + return shape_model + + def _warp_template(self, template, template_shape, reference_shape, level): + # compute transforms + t = self.transform(self.reference_frame.landmarks['source'].lms, + template_shape) + # warp template + warped_template = template.warp_to_mask(self.reference_frame.mask, t) + # attach landmarks + warped_template.landmarks['source'] = self.reference_frame.landmarks[ + 'source'] + return warped_template + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return LinearATM(shape_models, warped_templates, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features, + self.n_landmarks) + + +# TODO: document me! +class LinearPatchATMBuilder(LinearATMBuilder): + r""" + Class that builds Linear Patch based Active Template Models. + + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). + + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -376,111 +579,195 @@ class PatchBasedATMBuilder(ATMBuilder): If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- - atm : ::map:`PatchBasedATMBuilder` - The Patch-Based ATM Builder object + atm : :map:`ATMBuilder` + The ATM Builder object Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. ValueError - ``max_shape_components must be ``None`` or an `int` > ``0`` or - a ``0`` <= `float` <= ``1`` or a list of those containing ``1`` - or ``n_levels`` elements + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError - ``features`` must be a `string` or a `function` or a list of those - containing 1 or ``n_levels`` elements + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements ValueError - ``pyramid_on_features`` is enabled so ``features`` must be a - `string` or a `function` or a list containing one of those + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, features=igo, patch_shape=(16, 16), - normalization_diagonal=None, n_levels=3, downscale=2, - scaled_shape_models=True, max_shape_components=None, - boundary=3): + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=False, + scale_features=True, max_shape_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) - - # store parameters - self.features = features + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters self.patch_shape = patch_shape - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components - self.boundary = boundary - # patch-based AAMs can only work with TPS transform - self.transform = ThinPlateSplines + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[level]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes, + max_components=max_components) + return shape_model + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return LinearPatchATM(shape_models, warped_templates, + reference_shape, self.patch_shape, + self.features, self.scales, self.scale_shapes, + self.scale_features, self.n_landmarks) + + +# TODO: document me! +# TODO: implement offsets support? +class PartsATMBuilder(ATMBuilder): + r""" + Class that builds Parts based Active Template Models. - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - Returns - ------- - reference_frame : :map:`MaskedImage` - The patch-based reference frame. - """ - return build_patch_reference_frame(mean_shape, boundary=self.boundary, - patch_shape=self.patch_shape) + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - def _mask_image(self, image): - r""" - Creates the patch-based mask of the given image. + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - Parameters - ---------- - image : :map:`MaskedImage` - The image to be masked. - """ - image.build_mask_around_landmarks(self.patch_shape, group='source') + normalize_parts : `callable`, optional - def _build_atm(self, shape_models, warped_templates, n_training_shapes): - r""" - Returns a Patch-Based ATM object. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - Parameters - ---------- - shape_models : :map:`PCAModel` - The trained multilevel shape models. + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - warped_templates : `list` of :map:`MaskedImage` - The warped multilevel templates. + If ``None``, the mean shape is not rescaled. - n_training_shapes : `int` - The number of training shapes. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - Returns - ------- - atm : :map:`PatchBasedATM` - The trained Patched-Based ATM object. - """ - from .base import PatchBasedATM - return PatchBasedATM(shape_models, warped_templates, n_training_shapes, - self.patch_shape, self.transform, self.features, - self.reference_shape, self.downscale, - self.scaled_shape_models) + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + atm : :map:`ATMBuilder` + The ATM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + normalize_parts=no_op, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + + def _warp_template(self, template, template_shape, reference_shape, level): + parts = template.extract_patches(template_shape, + patch_size=self.patch_shape[level], + as_single_array=True) + if self.normalize_parts: + parts = self.normalize_parts(parts) + + return Image(parts) + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return PartsATM(shape_models, warped_templates, reference_shape, + self.patch_shape, self.features, + self.normalize_parts, self.scales, + self.scale_shapes, self.scale_features) + + +from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM diff --git a/menpofit/builder.py b/menpofit/builder.py index 735bca0..571fcf4 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -181,8 +181,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, show_bar=False))) parts = i.extract_patches(s, patch_size=patch_shape, as_single_array=True) - if normalize_function: - parts = normalize_function(parts) + parts = normalize_function(parts) parts_images.append(Image(parts)) return parts_images From 22399b59ac0921c4cbf800908aff77d450f95dba Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:34:58 +0100 Subject: [PATCH 041/423] Add Results for ATMs --- menpofit/atm/result.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 menpofit/atm/result.py diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py new file mode 100644 index 0000000..2b91276 --- /dev/null +++ b/menpofit/atm/result.py @@ -0,0 +1,38 @@ +from __future__ import division +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult + + +# TODO: document me! +# TODO: handle costs +class ATMAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + +# TODO: document me! +class LinearATMAlgorithmResult(ATMAlgorithmResult): + r""" + """ + def shapes(self, as_points=False): + if as_points: + return [self.fitter.transform.from_vector(p).sparse_target.points + for p in self.shape_parameters] + + else: + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] + + @property + def final_shape(self): + return self.final_transform.sparse_target + + @property + def initial_shape(self): + return self.initial_transform.sparse_target + + +# TODO: document me! +# TODO: handle costs +class ATMFitterResult(MultiFitterResult): + r""" + """ + pass From 4adf50e422711a8936f57795f84b8d42749d5143 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:35:40 +0100 Subject: [PATCH 042/423] Add LKFitters for ATMs --- menpofit/atm/fitter.py | 400 +++++++---------------------------------- 1 file changed, 61 insertions(+), 339 deletions(-) diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index a5bfbe4..53f8820 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -1,352 +1,74 @@ from __future__ import division +from menpofit.fitter import ModelFitter +from menpofit.modelinstance import OrthoPDM +from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform +from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM +from .algorithm import ( + LKATMInterface, LKLinearATMInterface, LKPartsATMInterface, IC) +from .result import ATMFitterResult -from menpofit.fitter import MultilevelFitter -from menpofit.fittingresult import AMMultilevelFittingResult -from menpofit.transform import (ModelDrivenTransform, OrthoMDTransform, - DifferentiableAlignmentSimilarity) -from menpofit.lucaskanade.residual import SSD, GaborFourier -from menpofit.lucaskanade.image import IC -from menpofit.base import name_of_callable - -class ATMFitter(MultilevelFitter): +# TODO: document me! +class LKATMFitter(ModelFitter): r""" - Abstract Interface for defining Active Template Models Fitters. - - Parameters - ----------- - atm : :map:`ATM` - The Active Template Model to be used. """ - def __init__(self, atm): - self.atm = atm - - @property - def reference_shape(self): - r""" - The reference shape of the ATM. - - :type: :map:`PointCloud` - """ - return self.atm.reference_shape - - @property - def features(self): - r""" - The feature extracted at each pyramidal level during ATM building. - Stored in ascending pyramidal order. - - :type: `list` - """ - return self.atm.features - - @property - def n_levels(self): - r""" - The number of pyramidal levels used during ATM building. - - :type: `int` - """ - return self.atm.n_levels - - @property - def downscale(self): - r""" - The downscale used to generate the final scale factor applied at - each pyramidal level during ATM building. - The scale factor is computed as: - - ``(downscale ** k) for k in range(n_levels)`` - - :type: `float` - """ - return self.atm.downscale - - def _create_fitting_result(self, image, fitting_results, affine_correction, - gt_shape=None): - r""" - Creates a :map:`ATMMultilevelFittingResult` associated to a - particular fitting of the ATM fitter. - - Parameters - ----------- - image : :map:`Image` or subclass - The image to be fitted. - - fitting_results : `list` of :map:`FittingResult` - A list of fitting result objects containing the state of the - the fitting for each pyramidal level. - - affine_correction : :map:`Affine` - An affine transform that maps the result of the top resolution - level to the scale space of the original image. - - gt_shape : :map:`PointCloud`, optional - The ground truth shape associated to the image. - - error_type : 'me_norm', 'me' or 'rmse', optional - Specifies how the error between the fitted and ground truth - shapes must be computed. - - Returns - ------- - fitting : :map:`ATMMultilevelFittingResult` - A fitting result object that will hold the state of the ATM - fitter for a particular fitting. - """ - return ATMMultilevelFittingResult( - image, self, fitting_results, affine_correction, gt_shape=gt_shape) - - -class LucasKanadeATMFitter(ATMFitter): - r""" - Lucas-Kanade based :map:`Fitter` for Active Template Models. - - Parameters - ----------- - atm : :map:`ATM` - The Active Template Model to be used. - - algorithm : subclass of :map:`ImageLucasKanade`, optional - The Image Lucas-Kanade class to be used. - - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - """ - def __init__(self, atm, algorithm=IC, residual=SSD, - md_transform=OrthoMDTransform, n_shape=None, **kwargs): - super(LucasKanadeATMFitter, self).__init__(atm) - self._set_up(algorithm=algorithm, residual=residual, - md_transform=md_transform, n_shape=n_shape, **kwargs) - - @property - def algorithm(self): - r""" - Returns a string containing the name of fitting algorithm. - - :type: `str` - """ - return 'LK-ATM-' + self._fitters[0].algorithm - - def _set_up(self, algorithm=IC, - residual=SSD, md_transform=OrthoMDTransform, - global_transform=DifferentiableAlignmentSimilarity, - n_shape=None, **kwargs): - r""" - Sets up the Lucas-Kanade fitter object. - - Parameters - ----------- - algorithm : subclass of :map:`ImageLucasKanade`, optional - The Image Lucas-Kanade class to be used. - - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing - the specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number - of components will be used for all levels. - - Raises - ------- - ValueError - ``n_shape`` can be an `int`, `float`, ``None`` or a `list` - containing ``1`` or ``n_levels`` of those. - """ - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.atm.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.atm.n_levels > 1: - for sm in self.atm.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.atm.n_levels: - for sm, n in zip(self.atm.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None ' - 'or a list containing 1 or {} of ' - 'those'.format(self.atm.n_levels)) - - self._fitters = [] - for j, (t, sm) in enumerate(zip(self.atm.warped_templates, - self.atm.shape_models)): - - if md_transform is not ModelDrivenTransform: - md_trans = md_transform( - sm, self.atm.transform, global_transform, - source=t.landmarks['source'].lms) - else: - md_trans = md_transform( + def __init__(self, atm, algorithm_cls=IC, n_shape=None, sampling=None, + **kwargs): + super(LKATMFitter, self).__init__(atm) + self._algorithms = [] + self._check_n_shape(n_shape) + self._set_up(algorithm_cls, sampling, **kwargs) + + def _set_up(self, algorithm_cls, sampling, **kwargs): + for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, + self.atm.shape_models)): + + if type(self.atm) is ATM or type(self.atm) is PatchATM: + # build orthonormal model driven transform + md_transform = OrthoMDTransform( sm, self.atm.transform, - source=t.landmarks['source'].lms) + source=wt.landmarks['source'].lms) + # set up algorithm using standard aam interface + algorithm = algorithm_cls(LKATMInterface, wt, md_transform, + sampling=sampling, **kwargs) + + elif (type(self.atm) is LinearATM or + type(self.atm) is LinearPatchATM): + # build linear version of orthogonal model driven transform + md_transform = LinearOrthoMDTransform( + sm, self.atm.n_landmarks) + # set up algorithm using linear aam interface + algorithm = algorithm_cls(LKLinearATMInterface, wt, + md_transform, sampling=sampling, + **kwargs) + + elif type(self.atm) is PartsATM: + # build orthogonal point distribution model + pdm = OrthoPDM(sm) + # set up algorithm using parts aam interface + algorithm = algorithm_cls( + LKPartsATMInterface, wt, pdm, sampling=sampling, + patch_shape=self.atm.patch_shape[j], + normalize_parts=self.atm.normalize_parts) - if residual is not GaborFourier: - self._fitters.append( - algorithm(t, residual(), md_trans, **kwargs)) else: - self._fitters.append( - algorithm(t, residual(t.shape), md_trans, - **kwargs)) + raise ValueError("AAM object must be of one of the " + "following classes: {}, {}, {}, {}, " + "{}".format(ATM, PatchATM, LinearATM, + LinearPatchATM, PartsATM)) - def __str__(self): - out = "{0} Fitter\n" \ - " - Lucas-Kanade {1}\n" \ - " - Transform is {2} and residual is {3}.\n" \ - " - {4} training images.\n".format( - self.atm._str_title, self._fitters[0].algorithm, - self._fitters[0].transform.__class__.__name__, - self._fitters[0].residual.type, self.atm.n_training_shapes) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_levels): - n_channels.append( - self._fitters[j].template.n_channels) - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format(name_of_callable( - self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - self.features[j].__name__)) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.atm.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, - self.downscale) + # append algorithms to list + self._algorithms.append(algorithm) - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.atm.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self._fitters[0].template.n_true_pixels() * - n_channels[0], - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.atm.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self._fitters[i].template.n_true_pixels() * - n_channels[i], - self._fitters[i].template.n_true_pixels(), - n_channels[i], self._fitters[i].template._str_shape, - n_channels[i]) - out = "{0} - {1} motion components\n\n".format( - out, self._fitters[i].transform.n_parameters) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} motion parameters\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self._fitters[0].template.n_true_pixels() * n_channels[0], - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0], self._fitters[0].transform.n_parameters) - return out - - -class ATMMultilevelFittingResult(AMMultilevelFittingResult): - r""" - Class that holds the state of a :map:`ATMFitter` object before, - during and after it has fitted a particular image. - """ @property - def atm_reconstructions(self): - r""" - The list containing the atm reconstruction (i.e. the template warped on - the shape instance reconstruction) obtained at each fitting iteration. + def atm(self): + return self._model - Note that this reconstruction is only tested to work for the - :map:`OrthoMDTransform` + @property + def algorithms(self): + return self._algorithms - :type: list` of :map:`Image` or subclass - """ - atm_reconstructions = [] - for level, f in enumerate(self.fitting_results): - for shape_w in f.parameters: - shape_w = shape_w[4:] - sm_level = self.fitter.aam.shape_models[level] - swt = shape_w / sm_level.eigenvalues[:len(shape_w)] ** 0.5 - atm_reconstructions.append(self.fitter.aam.instance( - shape_weights=swt, level=level)) - return atm_reconstructions + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return ATMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) From 10682eae34c3f8adac65c82c5caac4b1d76200c6 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:36:25 +0100 Subject: [PATCH 043/423] Add algorithms for ATMs - Add menpofit.algorithm.py --- menpofit/algorithm.py | 134 ++++++++++++++++++++++++++ menpofit/atm/algorithm.py | 193 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 menpofit/algorithm.py create mode 100644 menpofit/atm/algorithm.py diff --git a/menpofit/algorithm.py b/menpofit/algorithm.py new file mode 100644 index 0000000..b651ba6 --- /dev/null +++ b/menpofit/algorithm.py @@ -0,0 +1,134 @@ +from __future__ import division +import numpy as np +from menpo.image import Image +from menpo.feature import no_op +from menpo.feature import gradient as fast_gradient + + +# TODO: implement more clever sampling? +class LKInterface(object): + + def __init__(self, lk_algorithm, sampling=None): + self.algorithm = lk_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def true_indices(self): + return self.template.mask.true_indices() + + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + + +class LKPartsInterface(LKInterface): + + def __init__(self, lk_algorithm, patch_shape=(17, 17), + normalize_parts=no_op, sampling=None): + self.algorithm = lk_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.nabla_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.nabla2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) + + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + # TODO: add parts normalization + def warp(self, image): + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts) + + def gradient(self, image): + pixels = image.pixels + g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return g.reshape((2,) + pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.nabla_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py new file mode 100644 index 0000000..dce40fb --- /dev/null +++ b/menpofit/atm/algorithm.py @@ -0,0 +1,193 @@ +from __future__ import division +import abc +import numpy as np +from menpofit.algorithm import LKInterface, LKPartsInterface +from .result import ATMAlgorithmResult, LinearATMAlgorithmResult + + +class LKATMInterface(LKInterface): + + @property + def shape_model(self): + return self.transform.pdm.model + + @classmethod + def solve_shape_map(cls, H, J, e, J_prior, p): + if p.shape[0] is not H.shape[0]: + # Bidirectional Compositional case + J_prior = np.hstack((J_prior, J_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + H += np.diag(J_prior) + Je = J_prior * p + J.T.dot(e) + return - np.linalg.solve(H, Je) + + @classmethod + def solve_shape_ml(cls, H, J, e): + # compute and return ML solution + return -np.linalg.solve(H, J.T.dot(e)) + + def algorithm_result(self, image, shape_parameters, gt_shape=None): + return ATMAlgorithmResult( + image, self.algorithm, shape_parameters, gt_shape=gt_shape) + + +class LKLinearATMInterface(LKATMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, gt_shape=None): + return LinearATMAlgorithmResult( + image, self.algorithm, shape_parameters, gt_shape=gt_shape) + + +class LKPartsATMInterface(LKPartsInterface, LKATMInterface): + + @property + def shape_model(self): + return self.transform.model + + +# TODO: handle costs for all LKAAMAlgorithms +# TODO document me! +class LKATMAlgorithm(object): + + def __init__(self, lk_atm_interface_cls, template, transform, + eps=10**-5, **kwargs): + # set common state for all ATM algorithms + self.template = template + self.transform = transform + self.eps = eps + # set interface + self.interface = lk_atm_interface_cls(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self, **kwargs): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + + # vectorize template and mask it + self.t_m = self.template.as_vector()[self.interface.i_mask] + + # compute warp jacobian + self.dW_dp = self.interface.warp_jacobian() + + # compute shape model prior + s2 = 1 / self.interface.shape_model.noise_variance() + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + pass + + +class Compositional(LKATMAlgorithm): + r""" + Abstract Interface for Compositional ATM algorithms + """ + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.t_m + + # solve for increments on the shape parameters + self.dp = self.solve(map_inference) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + @abc.abstractmethod + def solve(self, map_inference): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class FC(Compositional): + r""" + Forward Compositional (FC) Gauss-Newton algorithm + """ + def solve(self, map_inference): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # compute masked forward Jacobian + J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) + # compute masked forward Hessian + JJ_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + JJ_m, J_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return self.interface.solve_shape_ml(JJ_m, J_m, self.e_m) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class IC(Compositional): + r""" + Inverse Compositional (IC) Gauss-Newton algorithm + """ + def precompute(self): + # call super method + super(IC, self).precompute() + # compute appearance model mean gradient + nabla_t = self.interface.gradient(self.template) + # compute masked inverse Jacobian + self.J_m = self.interface.steepest_descent_images(-nabla_t, self.dW_dp) + # compute masked inverse Hessian + self.JJ_m = self.J_m.T.dot(self.J_m) + # compute masked Jacobian pseudo-inverse + self.pinv_J_m = np.linalg.solve(self.JJ_m, self.J_m.T) + + def solve(self, map_inference): + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + self.JJ_m, self.J_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return -self.pinv_J_m.dot(self.e_m) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) From ded08f603ab58157252dc923e50a81abd1f6b39f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:37:24 +0100 Subject: [PATCH 044/423] Add __init__.py for ATMs --- menpofit/atm/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index 304b4d3..12bd84b 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,3 +1,5 @@ -from .base import ATM, PatchBasedATM -from .builder import ATMBuilder, PatchBasedATMBuilder -from .fitter import LucasKanadeATMFitter +from .builder import ( + ATMBuilder, PatchATMBuilder, LinearATMBuilder, + LinearPatchATMBuilder, PartsATMBuilder) +from .fitter import LKATMFitter +from .algorithm import FC, IC \ No newline at end of file From 6e66a513521f2ed9ac92f08f9728a9c74e766c38 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 09:11:44 +0100 Subject: [PATCH 045/423] Make shapes a property in results.py --- menpofit/result.py | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 536145e..531f759 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -112,19 +112,11 @@ def n_iters(self): """ @abc.abstractproperty - def shapes(self, as_points=False): + def shapes(self): r""" Generates a list containing the shapes obtained at each fitting iteration. - Parameters - ----------- - as_points : boolean, optional - Whether the results is returned as a list of :map:`PointCloud`s or - ndarrays. - - Default: `False` - Returns ------- shapes : :map:`PointCloud`s or ndarray list @@ -145,7 +137,7 @@ def iter_image(self): :type: :map:`Image` """ image = Image(self.image.pixels) - for j, s in enumerate(self.shapes()): + for j, s in enumerate(self.shapes): image.landmarks['iter_'+str(j)] = s return image @@ -166,7 +158,7 @@ def errors(self, error_type='me_norm'): """ if self.gt_shape is not None: return [compute_error(t, self.gt_shape, error_type) - for t in self.shapes()] + for t in self.shapes] else: raise ValueError('Ground truth has not been set, errors cannot ' 'be computed') @@ -428,7 +420,7 @@ def __init__(self, image, fitter, shape_parameters, gt_shape=None): @property def n_iters(self): - return len(self.shapes()) - 1 + return len(self.shapes) - 1 @property def transforms(self): @@ -453,14 +445,10 @@ def initial_transform(self): """ return self.fitter.transform.from_vector(self.shape_parameters[0]) - def shapes(self, as_points=False): - if as_points: - return [self.fitter.transform.from_vector(p).target.points - for p in self.shape_parameters] - - else: - return [self.fitter.transform.from_vector(p).target - for p in self.shape_parameters] + @property + def shapes(self): + return [self.fitter.transform.from_vector(p).target + for p in self.shape_parameters] @property def final_shape(self): @@ -509,7 +497,8 @@ def n_iters(self): n_iters += f.n_iters return n_iters - def shapes(self, as_points=False): + @property + def shapes(self): r""" Generates a list containing the shapes obtained at each fitting iteration. @@ -561,11 +550,9 @@ def __init__(self, image, shapes, n_iters, gt_shape=None): def n_iters(self): return self._n_iters - def shapes(self, as_points=False): - if as_points: - return [s.points for s in self._shapes] - else: - return self._shapes + @property + def shapes(self): + return self._shapes @property def initial_shape(self): From 71b917eabce9b520d8f7568d1825fa451ea2b8fb Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 09:13:06 +0100 Subject: [PATCH 046/423] Make shape a property for LinearAAMAlgorithmResult --- menpofit/aam/result.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index a60c59b..afd516c 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -19,14 +19,10 @@ def __init__(self, image, fitter, shape_parameters, class LinearAAMAlgorithmResult(AAMAlgorithmResult): r""" """ + @property def shapes(self, as_points=False): - if as_points: - return [self.fitter.transform.from_vector(p).sparse_target.points - for p in self.shape_parameters] - - else: - return [self.fitter.transform.from_vector(p).sparse_target - for p in self.shape_parameters] + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] @property def final_shape(self): From 81ad50e3d08192450dd054548f0632fa74262f0f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 09:14:41 +0100 Subject: [PATCH 047/423] Add funtion rescale_images_to_reference_frame to menpofit.builder --- menpofit/builder.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 571fcf4..b4cfa57 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -46,6 +46,26 @@ def compute_reference_shape(shapes, normalization_diagonal, verbose=False): return reference_shape +# TODO: document me! +def rescale_images_to_reference_shape(images, group, label, reference_shape, + verbose=False): + r""" + """ + # normalize the scaling of all images wrt the reference_shape size + normalized_images = [] + for c, i in enumerate(images): + if verbose: + print_dynamic('- Normalizing images size: {}'.format( + progress_bar_str((c + 1.) / len(images), + show_bar=False))) + normalized_images.append(i.rescale_to_reference_shape( + reference_shape, group=group, label=label)) + + if verbose: + print_dynamic('- Normalizing images size: Done\n') + return normalized_images + + def normalization_wrt_reference_shape(images, group, label, diagonal, verbose=False): r""" @@ -103,17 +123,8 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size - normalized_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic('- Normalizing images size: {}'.format( - progress_bar_str((c + 1.) / len(images), - show_bar=False))) - normalized_images.append(i.rescale_to_reference_shape( - reference_shape, group=group, label=label)) - - if verbose: - print_dynamic('- Normalizing images size: Done\n') + normalized_images = rescale_images_to_reference_shape( + images, group, label, reference_shape, verbose=False) return reference_shape, normalized_images From 1bef1230c3492eba5626a74a089391b0374ba92a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:18:33 +0100 Subject: [PATCH 048/423] Add CR AAM Fitter - This will substitute the previous SD AAM Fitter. --- menpofit/aam/fitter.py | 216 +++++++++++++++++++++++++++++++++-------- 1 file changed, 176 insertions(+), 40 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 9e3587f..f80d677 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,26 +1,70 @@ from __future__ import division +import abc +from menpo.transform import Scale +from menpofit.builder import ( + rescale_images_to_reference_shape, compute_features, scale_images) from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM -from .algorithm import ( +from .algorithm.lk import ( LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) +from .algorithm.cr import ( + CRAAMInterface, CRLinearAAMInterface, CRPartsAAMInterface, PAJ) from .result import AAMFitterResult # TODO: document me! -class LKAAMFitter(ModelFitter): +class AAMFitter(ModelFitter): r""" """ - def __init__(self, aam, algorithm_cls=AIC, n_shape=None, - n_appearance=None, sampling=None, **kwargs): - super(LKAAMFitter, self).__init__(aam) + def __init__(self, aam, n_shape=None, n_appearance=None): + super(AAMFitter, self).__init__(aam) self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - self._set_up(algorithm_cls, sampling, **kwargs) - def _set_up(self, algorithm_cls, sampling, **kwargs): + @property + def aam(self): + return self._model + + @property + def algorithms(self): + return self._algorithms + + def _check_n_appearance(self, n_appearance): + if n_appearance is not None: + if type(n_appearance) is int or type(n_appearance) is float: + for am in self.aam.appearance_models: + am.n_active_components = n_appearance + elif len(n_appearance) == 1 and self.aam.n_levels > 1: + for am in self.aam.appearance_models: + am.n_active_components = n_appearance[0] + elif len(n_appearance) == self.aam.n_levels: + for am, n in zip(self.aam.appearance_models, n_appearance): + am.n_active_components = n + else: + raise ValueError('n_appearance can be an integer or a float ' + 'or None or a list containing 1 or {} of ' + 'those'.format(self.aam.n_levels)) + + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return AAMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) + + +# TODO: document me! +class LKAAMFitter(AAMFitter): + r""" + """ + def __init__(self, aam, n_shape=None, n_appearance=None, + lk_algorithm_cls=AIC, sampling=None, **kwargs): + super(LKAAMFitter, self).__init__( + aam, n_shape=n_shape, n_appearance=n_appearance) + self._set_up(lk_algorithm_cls, sampling, **kwargs) + + def _set_up(self, lk_algorithm_cls, sampling, **kwargs): for j, (am, sm) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models)): @@ -30,8 +74,9 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(LKAAMInterface, am, md_transform, - sampling=sampling, **kwargs) + algorithm = lk_algorithm_cls( + LKAAMInterface, am, md_transform, sampling=sampling, + **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -39,18 +84,18 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.aam.n_landmarks) # set up algorithm using linear aam interface - algorithm = algorithm_cls(LinearLKAAMInterface, am, - md_transform, sampling=sampling, - **kwargs) + algorithm = lk_algorithm_cls( + LinearLKAAMInterface, am, md_transform, sampling=sampling, + **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - algorithm = algorithm_cls(PartsLKAAMInterface, am, pdm, - sampling=sampling, **kwargs) - algorithm.patch_shape = self.aam.patch_shape[j] - algorithm.normalize_parts = self.aam.normalize_parts + algorithm = lk_algorithm_cls( + PartsLKAAMInterface, am, pdm, + sampling=sampling, patch_shape=self.aam.patch_shape[j], + normalize_parts=self.aam.normalize_parts, **kwargs) else: raise ValueError("AAM object must be of one of the " @@ -61,31 +106,122 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): # append algorithms to list self._algorithms.append(algorithm) - @property - def aam(self): - return self._model - @property - def algorithms(self): - return self._algorithms +# TODO: document me! +class CRAAMFitter(AAMFitter): + r""" + """ + def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, + n_appearance=None, sampling=None, n_perturbations=10, + max_iters=6, **kwargs): + super(CRAAMFitter, self).__init__( + aam, n_shape=n_shape, n_appearance=n_appearance) + self.n_perturbations = n_perturbations + self.max_iters = self._prepare_max_iters(max_iters) + self._set_up(cr_algorithm_cls, sampling, **kwargs) + + def _set_up(self, cr_algorithm_cls, sampling, **kwargs): + for j, (am, sm) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models)): + + if type(self.aam) is AAM or type(self.aam) is PatchAAM: + # build orthonormal model driven transform + md_transform = OrthoMDTransform( + sm, self.aam.transform, + source=am.mean().landmarks['source'].lms) + # set up algorithm using standard aam interface + algorithm = cr_algorithm_cls( + CRAAMInterface, am, md_transform, sampling=sampling, + max_iters=self.max_iters[j], **kwargs) + + elif (type(self.aam) is LinearAAM or + type(self.aam) is LinearPatchAAM): + # build linear version of orthogonal model driven transform + md_transform = LinearOrthoMDTransform( + sm, self.aam.n_landmarks) + # set up algorithm using linear aam interface + algorithm = cr_algorithm_cls( + CRLinearAAMInterface, am, md_transform, + sampling=sampling, max_iters=self.max_iters[j], **kwargs) + + elif type(self.aam) is PartsAAM: + # build orthogonal point distribution model + pdm = OrthoPDM(sm) + # set up algorithm using parts aam interface + algorithm = cr_algorithm_cls( + CRPartsAAMInterface, am, pdm, + sampling=sampling, max_iters=self.max_iters[j], + patch_shape=self.aam.patch_shape[j], + normalize_parts=self.aam.normalize_parts, **kwargs) - def _check_n_appearance(self, n_appearance): - if n_appearance is not None: - if type(n_appearance) is int or type(n_appearance) is float: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: - for am, n in zip(self.aam.appearance_models, n_appearance): - am.n_active_components = n else: - raise ValueError('n_appearance can be an integer or a float ' - 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + raise ValueError("AAM object must be of one of the " + "following classes: {}, {}, {}, {}, " + "{}".format(AAM, PatchAAM, LinearAAM, + LinearPatchAAM, PartsAAM)) + + # append algorithms to list + self._algorithms.append(algorithm) + + def train(self, images, group=None, label=None, verbose=False, **kwargs): + # normalize images with respect to reference shape of aam + images = rescale_images_to_reference_shape( + images, group, label, self.reference_shape, verbose=verbose) + + # for each pyramid level (low --> high) + for j, s in enumerate(self.scales): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if s == self.scales[-1]: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, self.features, + level_str=level_str, + verbose=verbose) + + # extract ground truth shapes for current level + level_gt_shapes = [i.landmarks[group][label] for i in level_images] + + if j == 0: + # generate perturbed shapes + current_shapes = [] + for gt_s in level_gt_shapes: + perturbed_shapes = [] + for _ in range(self.n_perturbations): + perturbed_shapes.append(self.perturb_shape(gt_s)) + current_shapes.append(perturbed_shapes) + + # train cascaded regression algorithm + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + + # scale current shapes to next level resolution + if s != self.scales[-1]: + transform = Scale(self.scales[j+1]/s, n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + - def _fitter_result(self, image, algorithm_results, affine_correction, - gt_shape=None): - return AAMFitterResult(image, self, algorithm_results, - affine_correction, gt_shape=gt_shape) From fa3fb075c00fea1cffc4c51e8c69fe18aa166954 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:21:22 +0100 Subject: [PATCH 049/423] Rename aam.algorithm.py to aam.algorithm.lk.py - Add a new subpackage algorithm to aam --- menpofit/aam/algorithm/lk.py | 796 +++++++++++++++++++++++++++++++++++ 1 file changed, 796 insertions(+) create mode 100644 menpofit/aam/algorithm/lk.py diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py new file mode 100644 index 0000000..1241d88 --- /dev/null +++ b/menpofit/aam/algorithm/lk.py @@ -0,0 +1,796 @@ +from __future__ import division +import abc +import numpy as np +from menpo.image import Image +from menpo.feature import gradient as fast_gradient, no_op +from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult + + +# TODO: needs to use interfaces in menpofit.algorithm.py +# TODO: implement more clever sampling? +class LKAAMInterface(object): + + def __init__(self, aam_algorithm, sampling=None): + self.algorithm = aam_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def shape_model(self): + return self.transform.pdm.model + + @property + def appearance_model(self): + return self.algorithm.appearance_model + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def m(self): + return self.appearance_model.n_active_components + + @property + def true_indices(self): + return self.template.mask.true_indices() + + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + + @classmethod + def solve_shape_map(cls, H, J, e, J_prior, p): + if p.shape[0] is not H.shape[0]: + # Bidirectional Compositional case + J_prior = np.hstack((J_prior, J_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + H += np.diag(J_prior) + Je = J_prior * p + J.T.dot(e) + return - np.linalg.solve(H, Je) + + @classmethod + def solve_shape_ml(cls, H, J, e): + # compute and return ML solution + return -np.linalg.solve(H, J.T.dot(e)) + + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): + if self.n is not H.shape[0] - self.m: + # Bidirectional Compositional case + Js_prior = np.hstack((Js_prior, Js_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + J_prior = np.hstack((Ja_prior, Js_prior)) + H += np.diag(J_prior) + Je = J_prior * np.hstack((c, p)) + J.T.dot(e) + dq = - np.linalg.solve(H, Je) + return dq[:self.m], dq[self.m:] + + def solve_all_ml(self, H, J, e): + # compute ML solution + dq = - np.linalg.solve(H, J.T.dot(e)) + return dq[:self.m], dq[self.m:] + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class LinearLKAAMInterface(LKAAMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return LinearAAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class PartsLKAAMInterface(LKAAMInterface): + + def __init__(self, aam_algorithm, sampling=None, patch_shape=(17, 17), + normalize_parts=no_op): + self.algorithm = aam_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.gradient_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.gradient2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) + + @property + def shape_model(self): + return self.transform.model + + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + def warp(self, image): + return Image(image.extract_patches( + self.transform.target, patch_size=self.patch_shape, + as_single_array=True)) + + def gradient(self, image): + nabla = fast_gradient(image.pixels.reshape((-1,) + self.patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return nabla.reshape((2,) + image.pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.gradient_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) + + +# TODO: handle costs for all LKAAMAlgorithms +# TODO document me! +class LKAAMAlgorithm(object): + + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # set common state for all AAM algorithms + self.appearance_model = appearance_model + self.template = appearance_model.mean() + self.transform = transform + self.eps = eps + # set interface + self.interface = aam_interface(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self, **kwargs): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + self.m = self.appearance_model.n_active_components + + # grab appearance model components + self.A = self.appearance_model.components + # mask them + self.A_m = self.A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + + # grab appearance model mean + self.a_bar = self.appearance_model.mean() + # vectorize it and mask it + self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] + + # compute warp jacobian + self.dW_dp = self.interface.warp_jacobian() + + # compute shape model prior + s2 = (self.appearance_model.noise_variance() / + self.interface.shape_model.noise_variance()) + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + # compute appearance model prior + S = self.appearance_model.eigenvalues + self.s2_inv_S = s2 / S + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + pass + + +class ProjectOut(LKAAMAlgorithm): + r""" + Abstract Interface for Project-out AAM algorithms + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # solve for increments on the shape parameters + self.dp = self.solve(map_inference) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + @abc.abstractmethod + def solve(self, map_inference): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class PFC(ProjectOut): + r""" + Project-out Forward Compositional (PFC) Gauss-Newton algorithm + """ + def solve(self, map_inference): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # compute masked forward Jacobian + J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) + # project out appearance model from it + QJ_m = self.project_out(J_m) + # compute masked forward Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + JQJ_m, QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class PIC(ProjectOut): + r""" + Project-out Inverse Compositional (PIC) Gauss-Newton algorithm + """ + def precompute(self): + # call super method + super(PIC, self).precompute() + # compute appearance model mean gradient + nabla_a = self.interface.gradient(self.a_bar) + # compute masked inverse Jacobian + J_m = self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + # project out appearance model from it + self.QJ_m = self.project_out(J_m) + # compute masked inverse Hessian + self.JQJ_m = self.QJ_m.T.dot(J_m) + # compute masked Jacobian pseudo-inverse + self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) + + def solve(self, map_inference): + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + self.JQJ_m, self.QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return -self.pinv_QJ_m.dot(self.e_m) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Simultaneous(LKAAMAlgorithm): + r""" + Abstract Interface for Simultaneous AAM algorithms + """ + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [self.c] + + # compute masked error + self.e_m = i_m - a_m + + # solve for increments on the appearance and shape parameters + # simultaneously + dc, self.dp = self.solve(map_inference) + + # update appearance parameters + self.c += dc + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(self.c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + def solve(self, map_inference): + # compute masked Jacobian + J_m = self.compute_jacobian() + # assemble masked simultaneous Jacobian + J_sim_m = np.hstack((-self.A_m, J_m)) + # compute masked Hessian + H_sim_m = J_sim_m.T.dot(J_sim_m) + # solve for increments on the appearance and shape parameters + # simultaneously + if map_inference: + return self.interface.solve_all_map( + H_sim_m, J_sim_m, self.e_m, self.s2_inv_S, self.c, + self.s2_inv_L, self.transform.as_vector()) + else: + return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) + + @abc.abstractmethod + def update_warp(self): + pass + + +class SFC(Simultaneous): + r""" + Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class SIC(Simultaneous): + r""" + Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Alternating(LKAAMAlgorithm): + r""" + Abstract Interface for Alternating AAM algorithms + """ + def precompute(self, **kwargs): + # call super method + super(Alternating, self).precompute() + # compute MAP appearance Hessian + self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + Jdp = 0 + else: + Jdp = J_m.dot(self.dp) + + # compute masked error + e_m = i_m - a_m + + # solve for increment on the appearance parameters + if map_inference: + Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) + dc = np.linalg.solve(self.AA_m_map, Ae_m_map) + else: + dc = self.pinv_A_m.dot(e_m + Jdp) + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m - self.A_m.T.dot(dc), self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, + e_m - self.A_m.dot(dc)) + + # update appearance parameters + c += dc + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class AFC(Alternating): + r""" + Alternating Forward Compositional (AFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class AIC(Alternating): + r""" + Alternating Inverse Compositional (AIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class ModifiedAlternating(Alternating): + r""" + Abstract Interface for Modified Alternating AAM algorithms + """ + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + a_m = self.a_bar_m + c_list = [] + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + c = self.pinv_A_m.dot(i_m - a_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m, self.s2_inv_L, self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class MAFC(ModifiedAlternating): + r""" + Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class MAIC(ModifiedAlternating): + r""" + Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Wiberg(LKAAMAlgorithm): + r""" + Abstract Interface for Wiberg AAM algorithms + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + else: + c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - self.a_bar_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # project out appearance models + QJ_m = self.project_out(J_m) + # compute masked Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + JQJ_m, QJ_m, e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(JQJ_m, QJ_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class WFC(Wiberg): + r""" + Wiberg Forward Compositional (WFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class WIC(Wiberg): + r""" + Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) From 879f79a1c1390e7740a455d9a1b6c99fcc4fe8bb Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:23:34 +0100 Subject: [PATCH 050/423] Add first version of CR Algorithms - Contains 2 Project-Out algorithms PJA and PSD. --- menpofit/aam/algorithm/cr.py | 408 +++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 menpofit/aam/algorithm/cr.py diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/cr.py new file mode 100644 index 0000000..b68bb1e --- /dev/null +++ b/menpofit/aam/algorithm/cr.py @@ -0,0 +1,408 @@ +from __future__ import division +import abc +import numpy as np +from menpo.image import Image +from menpo.feature import no_op +from menpo.visualize import print_dynamic, progress_bar_str +from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult + + +# TODO: implement more clever sampling? +class CRAAMInterface(object): + + def __init__(self, cr_aam_algorithm, sampling=None): + self.algorithm = cr_aam_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + + @property + def shape_model(self): + return self.transform.pdm.model + + @property + def appearance_model(self): + return self.algorithm.appearance_model + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def m(self): + return self.appearance_model.n_active_components + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class CRLinearAAMInterface(CRAAMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return LinearAAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class CRPartsAAMInterface(CRAAMInterface): + + def __init__(self, cr_aam_algorithm, sampling=None, patch_shape=(17, 17), + normalize_parts=no_op): + self.algorithm = cr_aam_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + + @property + def shape_model(self): + return self.transform.model + + def warp(self, image): + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts) + + +# TODO document me! +class CRAAMAlgorithm(object): + + def __init__(self, aam_interface, appearance_model, transform, max_iters=3, + eps=10**-5, **kwargs): + # set common state for all AAM algorithms + self.appearance_model = appearance_model + self.template = appearance_model.mean() + self.transform = transform + self.max_iters = max_iters + self.eps = eps + # set interface + self.interface = aam_interface(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + self.m = self.appearance_model.n_active_components + + # grab appearance model components + self.A = self.appearance_model.components + # mask them + self.A_m = self.A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + + # grab appearance model mean + self.a_bar = self.appearance_model.mean() + # vectorize it and mask it + self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] + + # compute shape model prior + s2 = (self.appearance_model.noise_variance() / + self.interface.shape_model.noise_variance()) + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + # compute appearance model prior + S = self.appearance_model.eigenvalues + self.s2_inv_S = s2 / S + + def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): + # check training data + self._check_training_data(images, gt_shapes, current_shapes) + + n_images = len(images) + n_samples_image = len(current_shapes[0]) + + # set number of iterations and initialize list of regressors + self.regressors = [] + + # compute current and delta parameters from current and ground truth + # shapes + delta_params, current_params, gt_params = self._generate_params( + gt_shapes, current_shapes) + # initialize iteration counter + k = 0 + + # Cascaded Regression loop + while k < self.max_iters: + # generate regression data + features = self._generate_features(images, current_params, + verbose=verbose) + + # perform regression + if verbose: + print_dynamic('- Performing regression...') + regressor = self._perform_regression(features, delta_params, + **kwargs) + # add regressor to list + self.regressors.append(regressor) + # compute regression rmse + estimated_delta_params = regressor(features) + rmse = _compute_rmse(delta_params, estimated_delta_params) + if verbose: + print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(rmse)) + + current_params += estimated_delta_params + + delta_params = gt_params - current_params + # increase iteration counter + k += 1 + + # obtain current shapes from current parameters + current_shapes = [] + for p in current_params: + current_shapes.append(self.transform.from_vector(p).target) + + # convert current shapes into a list of list and return + final_shapes = [] + for j in range(n_images): + k = j * n_samples_image + l = k + n_samples_image + final_shapes.append(current_shapes[k:l]) + return final_shapes + + @staticmethod + def _check_training_data(images, gt_shapes, current_shapes): + if len(images) != len(gt_shapes): + raise ValueError("The number of shapes must be equal to " + "the number of images.") + elif len(images) != len(current_shapes): + raise ValueError("The number of current shapes must be " + "equal or multiple to the number of images.") + + def _generate_params(self, gt_shapes, current_shapes): + # initialize current and delta parameters arrays + n_samples = len(gt_shapes) * len(current_shapes[0]) + current_params = np.empty((n_samples, self.transform.n_parameters)) + gt_params = np.empty((n_samples, self.transform.n_parameters)) + delta_params = np.empty((n_samples, self.transform.n_parameters)) + # initialize sample counter + k = 0 + # compute ground truth and current shape parameters + for gt_s, c_s in zip(gt_shapes, current_shapes): + for s in c_s: + # compute current parameters + current_params[k] = self._compute_params(s) + # compute ground truth parameters + gt_params[k] = self._compute_params(gt_s) + # compute delta parameters + delta_params[k] = gt_params[k] - current_params[k] + # increment counter + k += 1 + + return delta_params, current_params, gt_params + + def _compute_params(self, shape): + self.transform.set_target(shape) + return self.transform.as_vector() + + def _generate_features(self, images, current_params, verbose=False): + # initialize features array + n_images = len(images) + n_samples = len(current_params) + n_samples_image = int(n_samples / n_images) + features = np.zeros((n_samples,) + self.a_bar_m.shape) + + # initialize sample counter + k = 0 + for i in images: + for _ in range(n_samples_image): + if verbose: + print_dynamic('- Generating regression features - {' + '}'.format( + progress_bar_str((k + 1.) / n_samples, + show_bar=False))) + # set transform + self.transform.from_vector_inplace(current_params[k]) + # compute regression features + f = self._compute_features(i) + # add to features array + features[k] = f + # increment counter + k += 1 + + return features + + @abc.abstractmethod + def _compute_features(self, image): + pass + + @abc.abstractmethod + def _perform_regression(self, features, deltas, gamma=None): + pass + + def run(self, image, initial_shape, gt_shape=None, **kwargs): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter + k = 0 + + # Cascaded Regression loop + while k < self.max_iters: + # compute regression features + features = self._compute_features2(image) + + # solve for increments on the shape parameters + dp = self.regressors[k](features) + + # update warp + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + +# TODO: document me! +class ProjectOut(CRAAMAlgorithm): + r""" + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def _compute_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + e_m = i_m - self.a_bar_m + return self.project_out(e_m) + + def _compute_features2(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + +# TODO: document me! +class ProjectOut2(CRAAMAlgorithm): + r""" + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def _compute_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + e_m = i_m - self.a_bar_m + return self.project_out(e_m) + + def _compute_features2(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + +# TODO: document me! +class PSD(ProjectOut): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None): + return _supervised_descent(features, deltas, gamma=gamma) + + +# TODO: document me! +class PAJ(ProjectOut): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None): + return _average_jacobian(features, deltas, gamma=gamma) + + +# TODO: document me! +class _supervised_descent(object): + r""" + """ + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = features.T.dot(features) + XT = features.T.dot(deltas) + if gamma: + XX += gamma * np.eye(features.shape[1]) + # descent direction + self.R = np.linalg.solve(XX, XT) + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +class _average_jacobian(object): + r""" + """ + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = deltas.T.dot(deltas) + XT = deltas.T.dot(features) + if gamma: + XX += gamma * np.eye(deltas.shape[1]) + # average Jacobian + self.J = np.linalg.solve(XX, XT) + # average Hessian + self.H = self.J.dot(self.J.T) + # descent direction + self.R = np.linalg.solve(self.H, self.J).T + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +def _compute_rmse(x1, x2): + return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + From c7bc348b2a691b113e4bded93748f736695cfc97 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:25:06 +0100 Subject: [PATCH 051/423] Delete old aam.alagorithm.py and update aam.algorithm.__init__.py --- menpofit/aam/__init__.py | 5 +- menpofit/aam/algorithm.py | 798 -------------------------------------- 2 files changed, 3 insertions(+), 800 deletions(-) delete mode 100644 menpofit/aam/algorithm.py diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 673cb05..32a3556 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,11 +1,12 @@ from .builder import ( AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, LinearPatchAAMBuilder, PartsAAMBuilder) -from .fitter import LKAAMFitter +from .fitter import LKAAMFitter, CRAAMFitter from .algorithm import ( PFC, PIC, SFC, SIC, AFC, AIC, MAFC, MAIC, - WFC, WIC) + WFC, WIC, + PSD, PAJ) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py deleted file mode 100644 index 6583f00..0000000 --- a/menpofit/aam/algorithm.py +++ /dev/null @@ -1,798 +0,0 @@ -from __future__ import division -import abc -import numpy as np -from menpo.image import Image -from menpo.feature import gradient as fast_gradient -from .result import AAMAlgorithmResult, LinearAAMAlgorithmResult - - -# TODO: implement more clever sampling? -class LKAAMInterface(object): - - def __init__(self, aam_algorithm, sampling=None): - self.algorithm = aam_algorithm - - n_true_pixels = self.template.n_true_pixels() - n_channels = self.template.n_channels - n_parameters = self.transform.n_parameters - sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - - if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 - - self.i_mask = np.nonzero(np.tile( - sampling_mask[None, ...], (n_channels, 1)).flatten())[0] - self.dW_dp_mask = np.nonzero(np.tile( - sampling_mask[None, ..., None], (2, 1, n_parameters))) - self.nabla_mask = np.nonzero(np.tile( - sampling_mask[None, None, ...], (2, n_channels, 1))) - self.nabla2_mask = np.nonzero(np.tile( - sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) - - @property - def shape_model(self): - return self.transform.pdm.model - - @property - def appearance_model(self): - return self.algorithm.appearance_model - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - - @property - def n(self): - return self.transform.n_parameters - - @property - def m(self): - return self.appearance_model.n_active_components - - @property - def true_indices(self): - return self.template.mask.true_indices() - - def warp_jacobian(self): - dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) - return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, - dW_dp.shape[2])) - - def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) - - def gradient(self, img): - nabla = fast_gradient(img) - nabla.set_boundary_pixels() - return nabla.as_vector().reshape((2, img.n_channels, -1)) - - def steepest_descent_images(self, nabla, dW_dp): - # reshape gradient - # nabla: n_dims x n_channels x n_pixels - nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) - # compute steepest descent images - # nabla: n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # sdi: n_channels x n_pixels x n_params - sdi = 0 - a = nabla[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - # reshape steepest descent images - # sdi: (n_channels x n_pixels) x n_params - return sdi.reshape((-1, sdi.shape[2])) - - @classmethod - def solve_shape_map(cls, H, J, e, J_prior, p): - if p.shape[0] is not H.shape[0]: - # Bidirectional Compositional case - J_prior = np.hstack((J_prior, J_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - H += np.diag(J_prior) - Je = J_prior * p + J.T.dot(e) - return - np.linalg.solve(H, Je) - - @classmethod - def solve_shape_ml(cls, H, J, e): - # compute and return ML solution - return -np.linalg.solve(H, J.T.dot(e)) - - def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): - if self.n is not H.shape[0] - self.m: - # Bidirectional Compositional case - Js_prior = np.hstack((Js_prior, Js_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - J_prior = np.hstack((Ja_prior, Js_prior)) - H += np.diag(J_prior) - Je = J_prior * np.hstack((c, p)) + J.T.dot(e) - dq = - np.linalg.solve(H, Je) - return dq[:self.m], dq[self.m:] - - def solve_all_ml(self, H, J, e): - # compute ML solution - dq = - np.linalg.solve(H, J.T.dot(e)) - return dq[:self.m], dq[self.m:] - - def algorithm_result(self, image, shape_parameters, - appearance_parameters=None, gt_shape=None): - return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, - appearance_parameters=appearance_parameters, gt_shape=gt_shape) - - -class LinearLKAAMInterface(LKAAMInterface): - - @property - def shape_model(self): - return self.transform.model - - def algorithm_result(self, image, shape_parameters, - appearance_parameters=None, gt_shape=None): - return LinearAAMAlgorithmResult( - image, self.algorithm, shape_parameters, - appearance_parameters=appearance_parameters, gt_shape=gt_shape) - - -class PartsLKAAMInterface(LKAAMInterface): - - def __init__(self, aam_algorithm, sampling=None): - self.algorithm = aam_algorithm - - if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) - - image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling[None, None, None, ...], - image_shape[:3] + (1, 1)) - self.i_mask = np.nonzero(image_mask.flatten())[0] - self.gradient_mask = np.nonzero(np.tile( - image_mask[None, ...], (2, 1, 1, 1, 1, 1))) - self.gradient2_mask = np.nonzero(np.tile( - image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) - - @property - def shape_model(self): - return self.transform.model - - @property - def patch_shape(self): - return self.appearance_model.patch_shape - - def warp_jacobian(self): - return np.rollaxis(self.transform.d_dp(None), -1) - - def warp(self, image): - return Image(image.extract_patches( - self.transform.target, patch_size=self.patch_shape, - as_single_array=True)) - - def gradient(self, image): - pixels = image.pixels - patch_shape = self.algorithm.appearance_model.patch_shape - g = fast_gradient(pixels.reshape((-1,) + patch_shape)) - # remove 1st dimension gradient which corresponds to the gradient - # between parts - return g.reshape((2,) + pixels.shape) - - def steepest_descent_images(self, nabla, dw_dp): - # reshape nabla - # nabla: dims x parts x off x ch x (h x w) - nabla = nabla[self.gradient_mask].reshape( - nabla.shape[:-2] + (-1,)) - # compute steepest descent images - # nabla: dims x parts x off x ch x (h x w) - # ds_dp: dims x parts x x params - # sdi: parts x off x ch x (h x w) x params - sdi = 0 - a = nabla[..., None] * dw_dp[..., None, None, None, :] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (parts x offsets x ch x w x h) x params - return sdi.reshape((-1, sdi.shape[-1])) - - -# TODO: handle costs for all LKAAMAlgorithms -# TODO document me! -class LKAAMAlgorithm(object): - - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # set common state for all AAM algorithms - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.transform = transform - self.eps = eps - # set interface - self.interface = aam_interface(self, **kwargs) - # perform pre-computations - self.precompute() - - def precompute(self, **kwargs): - # grab number of shape and appearance parameters - self.n = self.transform.n_parameters - self.m = self.appearance_model.n_active_components - - # grab appearance model components - self.A = self.appearance_model.components - # mask them - self.A_m = self.A.T[self.interface.i_mask, :] - # compute their pseudoinverse - self.pinv_A_m = np.linalg.pinv(self.A_m) - - # grab appearance model mean - self.a_bar = self.appearance_model.mean() - # vectorize it and mask it - self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] - - # compute warp jacobian - self.dW_dp = self.interface.warp_jacobian() - - # compute shape model prior - s2 = (self.appearance_model.noise_variance() / - self.interface.shape_model.noise_variance()) - L = self.interface.shape_model.eigenvalues - self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) - # compute appearance model prior - S = self.appearance_model.eigenvalues - self.s2_inv_S = s2 / S - - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None, - map_inference=False): - pass - - -class ProjectOut(LKAAMAlgorithm): - r""" - Abstract Interface for Project-out AAM algorithms - """ - def project_out(self, J): - # project-out appearance bases from a particular vector or matrix - return J - self.A_m.dot(self.pinv_A_m.dot(J)) - - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # vectorize it and mask it - i_m = self.i.as_vector()[self.interface.i_mask] - - # compute masked error - self.e_m = i_m - self.a_bar_m - - # solve for increments on the shape parameters - self.dp = self.solve(map_inference) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, gt_shape=gt_shape) - - @abc.abstractmethod - def solve(self, map_inference): - pass - - @abc.abstractmethod - def update_warp(self): - pass - - -class PFC(ProjectOut): - r""" - Project-out Forward Compositional (PFC) Gauss-Newton algorithm - """ - def solve(self, map_inference): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # compute masked forward Jacobian - J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) - # project out appearance model from it - QJ_m = self.project_out(J_m) - # compute masked forward Hessian - JQJ_m = QJ_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - return self.interface.solve_shape_map( - JQJ_m, QJ_m, self.e_m, self.s2_inv_L, - self.transform.as_vector()) - else: - return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class PIC(ProjectOut): - r""" - Project-out Inverse Compositional (PIC) Gauss-Newton algorithm - """ - def precompute(self): - # call super method - super(PIC, self).precompute() - # compute appearance model mean gradient - nabla_a = self.interface.gradient(self.a_bar) - # compute masked inverse Jacobian - J_m = self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - # project out appearance model from it - self.QJ_m = self.project_out(J_m) - # compute masked inverse Hessian - self.JQJ_m = self.QJ_m.T.dot(J_m) - # compute masked Jacobian pseudo-inverse - self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) - - def solve(self, map_inference): - # solve for increments on the shape parameters - if map_inference: - return self.interface.solve_shape_map( - self.JQJ_m, self.QJ_m, self.e_m, self.s2_inv_L, - self.transform.as_vector()) - else: - return -self.pinv_QJ_m.dot(self.e_m) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class Simultaneous(LKAAMAlgorithm): - r""" - Abstract Interface for Simultaneous AAM algorithms - """ - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(self.c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [self.c] - - # compute masked error - self.e_m = i_m - a_m - - # solve for increments on the appearance and shape parameters - # simultaneously - dc, self.dp = self.solve(map_inference) - - # update appearance parameters - self.c += dc - self.a = self.appearance_model.instance(self.c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(self.c) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - @abc.abstractmethod - def compute_jacobian(self): - pass - - def solve(self, map_inference): - # compute masked Jacobian - J_m = self.compute_jacobian() - # assemble masked simultaneous Jacobian - J_sim_m = np.hstack((-self.A_m, J_m)) - # compute masked Hessian - H_sim_m = J_sim_m.T.dot(J_sim_m) - # solve for increments on the appearance and shape parameters - # simultaneously - if map_inference: - return self.interface.solve_all_map( - H_sim_m, J_sim_m, self.e_m, self.s2_inv_S, self.c, - self.s2_inv_L, self.transform.as_vector()) - else: - return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) - - @abc.abstractmethod - def update_warp(self): - pass - - -class SFC(Simultaneous): - r""" - Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class SIC(Simultaneous): - r""" - Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class Alternating(LKAAMAlgorithm): - r""" - Abstract Interface for Alternating AAM algorithms - """ - def precompute(self, **kwargs): - # call super method - super(Alternating, self).precompute() - # compute MAP appearance Hessian - self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) - - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - Jdp = 0 - else: - Jdp = J_m.dot(self.dp) - - # compute masked error - e_m = i_m - a_m - - # solve for increment on the appearance parameters - if map_inference: - Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) - dc = np.linalg.solve(self.AA_m_map, Ae_m_map) - else: - dc = self.pinv_A_m.dot(e_m + Jdp) - - # compute masked Jacobian - J_m = self.compute_jacobian() - # compute masked Hessian - H_m = J_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - self.dp = self.interface.solve_shape_map( - H_m, J_m, e_m - self.A_m.T.dot(dc), self.s2_inv_L, - self.transform.as_vector()) - else: - self.dp = self.interface.solve_shape_ml(H_m, J_m, - e_m - self.A_m.dot(dc)) - - # update appearance parameters - c += dc - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - @abc.abstractmethod - def compute_jacobian(self): - pass - - @abc.abstractmethod - def update_warp(self): - pass - - -class AFC(Alternating): - r""" - Alternating Forward Compositional (AFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class AIC(Alternating): - r""" - Alternating Inverse Compositional (AIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class ModifiedAlternating(Alternating): - r""" - Abstract Interface for Modified Alternating AAM algorithms - """ - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - a_m = self.a_bar_m - c_list = [] - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - c = self.pinv_A_m.dot(i_m - a_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) - - # compute masked error - e_m = i_m - a_m - - # compute masked Jacobian - J_m = self.compute_jacobian() - # compute masked Hessian - H_m = J_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - self.dp = self.interface.solve_shape_map( - H_m, J_m, e_m, self.s2_inv_L, self.transform.as_vector()) - else: - self.dp = self.interface.solve_shape_ml(H_m, J_m, e_m) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - -class MAFC(ModifiedAlternating): - r""" - Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class MAIC(ModifiedAlternating): - r""" - Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class Wiberg(LKAAMAlgorithm): - r""" - Abstract Interface for Wiberg AAM algorithms - """ - def project_out(self, J): - # project-out appearance bases from a particular vector or matrix - return J - self.A_m.dot(self.pinv_A_m.dot(J)) - - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - else: - c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) - - # compute masked error - e_m = i_m - self.a_bar_m - - # compute masked Jacobian - J_m = self.compute_jacobian() - # project out appearance models - QJ_m = self.project_out(J_m) - # compute masked Hessian - JQJ_m = QJ_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - self.dp = self.interface.solve_shape_map( - JQJ_m, QJ_m, e_m, self.s2_inv_L, - self.transform.as_vector()) - else: - self.dp = self.interface.solve_shape_ml(JQJ_m, QJ_m, e_m) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - -class WFC(Wiberg): - r""" - Wiberg Forward Compositional (WFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class WIC(Wiberg): - r""" - Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) From 4757a5cb38e8c601b65cb2647185deb490fb68fe Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:34:29 +0100 Subject: [PATCH 052/423] Update aam.alagorithm.__init__.py --- menpofit/aam/algorithm/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 menpofit/aam/algorithm/__init__.py diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py new file mode 100644 index 0000000..3416b8c --- /dev/null +++ b/menpofit/aam/algorithm/__init__.py @@ -0,0 +1,7 @@ +from .lk import ( + PFC, PIC, + SFC, SIC, + AFC, AIC, + MAFC, MAIC, + WFC, WIC) +from .cr import PSD, PAJ From 46e079d02470f7962439d4209f1843f9a198146e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 14:23:15 +0100 Subject: [PATCH 053/423] Add new initializations for ModelFitter --- menpofit/fitter.py | 107 +++++++++++---------------------------------- 1 file changed, 25 insertions(+), 82 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 13ea1da..ae871b5 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -284,6 +284,10 @@ def reference_shape(self): """ return self._model.reference_shape + @property + def reference_bounding_box(self): + return self.reference_shape.bounding_box() + @property def features(self): r""" @@ -328,97 +332,36 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - # TODO: Fix me! - def perturb_shape(self, gt_shape, noise_std=10, rotation=False): - transform = noisy_align(AlignmentSimilarity, self.reference_shape, - gt_shape, noise_std=noise_std) - return transform.apply(self.reference_shape) - # TODO: Bounding boxes should be PointGraphs - def obtain_shape_from_bb(self, bounding_box): - r""" - Generates an initial shape given a bounding box detection. - - Parameters - ----------- - bounding_box: (2, 2) ndarray - The bounding box specified as: - - np.array([[x_min, y_min], [x_max, y_max]]) + def get_initial_shape_from_bounding_box(self, bounding_box, noise_std=0.04, + rotation=False): + transform = noisy_align(AlignmentSimilarity, + self.reference_bounding_box, bounding_box, + noise_std=noise_std, rotation=rotation) + return transform.apply(self.reference_shape) - Returns - ------- - initial_shape: :class:`menpo.shape.PointCloud` - The initial shape. - """ - reference_shape = self.reference_shape - return align_shape_with_bb(reference_shape, - bounding_box).apply(reference_shape) + def get_initial_shape_from_shape(self, shape, noise_std=0.04, + rotation=False): + return self.get_initial_shape_from_bounding_box( + shape.bounding_box(), noise_std=noise_std, rotation=rotation) # TODO: document me! -def noisy_align(alignment_transform_cls, source, target, noise_std=10): +def noisy_align(alignment_transform_cls, source, target, noise_std=0.1, + **kwargs): r""" """ - noise = noise_std * np.random.randn(target.n_points, target.n_dims) + noise = noise_std * target.range() * np.random.randn(target.n_points, + target.n_dims) noisy_target = PointCloud(target.points + noise) - return alignment_transform_cls(source, noisy_target) + return alignment_transform_cls(source, noisy_target, **kwargs) -def align_shape_with_bb(shape, bounding_box): +# TODO: document me! +def align_shape_with_bounding_box(alignment_transform_cls, shape, + bounding_box, **kwargs): r""" - Returns the Similarity transform that aligns the provided shape with the - provided bounding box. - - Parameters - ---------- - shape: :class:`menpo.shape.PointCloud` - The shape to be aligned. - bounding_box: (2, 2) ndarray - The bounding box specified as: - - np.array([[x_min, y_min], [x_max, y_max]]) - - Returns - ------- - transform : :class: `menpo.transform.Similarity` - The align transform """ - shape_box = PointCloud(shape.bounds()) - bounding_box = PointCloud(bounding_box) - return AlignmentSimilarity(shape_box, bounding_box, rotation=False) - - -# TODO: implement as a method on Similarity? AlignableTransforms? -# def noisy_align(source, target, noise_std=0.04, rotation=False): -# r""" -# Constructs and perturbs the optimal similarity transform between source -# to the target by adding white noise to its weights. -# -# Parameters -# ---------- -# source: :class:`menpo.shape.PointCloud` -# The source pointcloud instance used in the alignment -# target: :class:`menpo.shape.PointCloud` -# The target pointcloud instance used in the alignment -# noise_std: float -# The standard deviation of the white noise -# -# Default: 0.04 -# rotation: boolean -# If False the second parameter of the Similarity, -# which captures captures inplane rotations, is set to 0. -# -# Default:False -# -# Returns -# ------- -# noisy_transform : :class: `menpo.transform.Similarity` -# The noisy Similarity Transform -# """ -# transform = AlignmentSimilarity(source, target, rotation=rotation) -# parameters = transform.as_vector() -# parameter_range = np.hstack((parameters[:2], target.range())) -# noise = (parameter_range * noise_std * -# np.random.randn(transform.n_parameters)) -# return Similarity.init_identity(source.n_dims).from_vector(parameters + noise) + shape_bb = shape.bounding_box() + return alignment_transform_cls(shape_bb, bounding_box, **kwargs) + From 48c64a8e4f7bd7e26a0048bdfef702ec2f69f8b5 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 14:37:58 +0100 Subject: [PATCH 054/423] Change prepare_max_iters in Fitter for a checking function --- menpofit/aam/fitter.py | 4 ++-- menpofit/checks.py | 16 ++++++++++++++++ menpofit/fitter.py | 18 ++---------------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f80d677..93fb5d5 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,11 +1,11 @@ from __future__ import division -import abc from menpo.transform import Scale from menpofit.builder import ( rescale_images_to_reference_shape, compute_features, scale_images) from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform +import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM from .algorithm.lk import ( LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) @@ -117,7 +117,7 @@ def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, super(CRAAMFitter, self).__init__( aam, n_shape=n_shape, n_appearance=n_appearance) self.n_perturbations = n_perturbations - self.max_iters = self._prepare_max_iters(max_iters) + self.max_iters = checks.check_max_iters(max_iters, self.n_levels) self._set_up(cr_algorithm_cls, sampling, **kwargs) def _set_up(self, cr_algorithm_cls, sampling, **kwargs): diff --git a/menpofit/checks.py b/menpofit/checks.py index 5098d03..e48cb42 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,3 +1,4 @@ +import numpy as np from menpofit.base import is_pyramid_on_features @@ -113,6 +114,21 @@ def check_max_components(max_components, n_levels, var_name): return max_components_list +def check_max_iters(max_iters, n_levels): + # check max_iters parameter + if type(max_iters) is int: + max_iters = [np.round(max_iters/n_levels) + for _ in range(n_levels)] + elif len(max_iters) == 1 and n_levels > 1: + max_iters = [np.round(max_iters[0]/n_levels) + for _ in range(n_levels)] + elif len(max_iters) != n_levels: + raise ValueError('max_iters can be integer, integer list ' + 'containing 1 or {} elements or ' + 'None'.format(n_levels)) + return np.require(max_iters, dtype=np.int) + + # def check_n_levels(n_levels): # r""" # Checks the number of pyramid levels - must be int > 0. diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ae871b5..6b92870 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -3,6 +3,7 @@ import numpy as np from menpo.shape import PointCloud from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity +import menpofit.checks as checks # TODO: document me! @@ -226,7 +227,7 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, The fitting object containing the state of the whole fitting procedure. """ - max_iters = self._prepare_max_iters(max_iters) + max_iters = checks.check_max_iters(max_iters, self.n_levels) shape = initial_shape gt_shape = None algorithm_results = [] @@ -246,21 +247,6 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, return algorithm_results - def _prepare_max_iters(self, max_iters): - n_levels = self.n_levels - # check max_iters parameter - if type(max_iters) is int: - max_iters = [np.round(max_iters/n_levels) - for _ in range(n_levels)] - elif len(max_iters) == 1 and n_levels > 1: - max_iters = [np.round(max_iters[0]/n_levels) - for _ in range(n_levels)] - elif len(max_iters) != n_levels: - raise ValueError('max_iters can be integer, integer list ' - 'containing 1 or {} elements or ' - 'None'.format(self.n_levels)) - return np.require(max_iters, dtype=np.int) - @abc.abstractmethod def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): From e20fe906a6a0e7f601b80eab4880745ca992d231 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 15:34:36 +0100 Subject: [PATCH 055/423] Add support for sampling being a list --- menpofit/aam/fitter.py | 29 ++++++++++++++++------------- menpofit/checks.py | 20 +++++++++++++++++++- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 93fb5d5..f40a450 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -62,11 +62,12 @@ def __init__(self, aam, n_shape=None, n_appearance=None, lk_algorithm_cls=AIC, sampling=None, **kwargs): super(LKAAMFitter, self).__init__( aam, n_shape=n_shape, n_appearance=n_appearance) + sampling = checks.check_sampling(sampling, self.n_levels) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): - for j, (am, sm) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models)): + for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models, sampling)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform @@ -75,7 +76,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = lk_algorithm_cls( - LKAAMInterface, am, md_transform, sampling=sampling, + LKAAMInterface, am, md_transform, sampling=s, **kwargs) elif (type(self.aam) is LinearAAM or @@ -85,7 +86,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): sm, self.aam.n_landmarks) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( - LinearLKAAMInterface, am, md_transform, sampling=sampling, + LinearLKAAMInterface, am, md_transform, sampling=s, **kwargs) elif type(self.aam) is PartsAAM: @@ -93,8 +94,8 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = lk_algorithm_cls( - PartsLKAAMInterface, am, pdm, - sampling=sampling, patch_shape=self.aam.patch_shape[j], + PartsLKAAMInterface, am, pdm, sampling=s, + patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) else: @@ -116,13 +117,14 @@ def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, max_iters=6, **kwargs): super(CRAAMFitter, self).__init__( aam, n_shape=n_shape, n_appearance=n_appearance) + sampling = checks.check_sampling(sampling, self.n_levels) self.n_perturbations = n_perturbations self.max_iters = checks.check_max_iters(max_iters, self.n_levels) self._set_up(cr_algorithm_cls, sampling, **kwargs) def _set_up(self, cr_algorithm_cls, sampling, **kwargs): - for j, (am, sm) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models)): + for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models, sampling)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform @@ -131,7 +133,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = cr_algorithm_cls( - CRAAMInterface, am, md_transform, sampling=sampling, + CRAAMInterface, am, md_transform, sampling=s, max_iters=self.max_iters[j], **kwargs) elif (type(self.aam) is LinearAAM or @@ -141,8 +143,8 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): sm, self.aam.n_landmarks) # set up algorithm using linear aam interface algorithm = cr_algorithm_cls( - CRLinearAAMInterface, am, md_transform, - sampling=sampling, max_iters=self.max_iters[j], **kwargs) + CRLinearAAMInterface, am, md_transform, sampling=s, + max_iters=self.max_iters[j], **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model @@ -150,7 +152,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): # set up algorithm using parts aam interface algorithm = cr_algorithm_cls( CRPartsAAMInterface, am, pdm, - sampling=sampling, max_iters=self.max_iters[j], + sampling=s, max_iters=self.max_iters[j], patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) @@ -209,7 +211,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - perturbed_shapes.append(self.perturb_shape(gt_s)) + p_s = self.get_initial_shape_from_shape(gt_s) + perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) # train cascaded regression algorithm diff --git a/menpofit/checks.py b/menpofit/checks.py index e48cb42..2ac6e66 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -114,8 +114,8 @@ def check_max_components(max_components, n_levels, var_name): return max_components_list +# TODO: document me! def check_max_iters(max_iters, n_levels): - # check max_iters parameter if type(max_iters) is int: max_iters = [np.round(max_iters/n_levels) for _ in range(n_levels)] @@ -129,6 +129,24 @@ def check_max_iters(max_iters, n_levels): return np.require(max_iters, dtype=np.int) +# TODO: document me! +def check_sampling(sampling, n_levels): + if isinstance(sampling, (list, tuple)): + if len(sampling) == 1: + sampling = sampling * n_levels + elif len(sampling) != n_levels: + raise ValueError('A sampling list can only ' + 'contain 1 element or {} ' + 'elements'.format(n_levels)) + elif isinstance(sampling, np.ndarray): + sampling = [sampling] * n_levels + else: + raise ValueError('sampling can be a ndarray, a ndarray list ' + 'containing 1 or {} elements or ' + 'None'.format(n_levels)) + return sampling + + # def check_n_levels(n_levels): # r""" # Checks the number of pyramid levels - must be int > 0. From fa04ce93a7972c866bb16a3bae640ab6cf04b63f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 17:46:28 +0100 Subject: [PATCH 056/423] Remove ProjectOut2 from aam.algorithm.cr --- menpofit/aam/algorithm/cr.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/cr.py index b68bb1e..f9c5d7c 100644 --- a/menpofit/aam/algorithm/cr.py +++ b/menpofit/aam/algorithm/cr.py @@ -322,32 +322,6 @@ def _compute_features2(self, image): return i_m - self.a_bar_m -# TODO: document me! -class ProjectOut2(CRAAMAlgorithm): - r""" - """ - def project_out(self, J): - # project-out appearance bases from a particular vector or matrix - return J - self.A_m.dot(self.pinv_A_m.dot(J)) - - def _compute_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - self.a_bar_m - return self.project_out(e_m) - - def _compute_features2(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - return i_m - self.a_bar_m - - # TODO: document me! class PSD(ProjectOut): r""" From ecc7112e279f4efa88ce19d392fc11ed7904c8fc Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 1 Jun 2015 11:52:29 +0100 Subject: [PATCH 057/423] Small changes to training regression based AAMs - Change previous perturb_shape methods in Fitters to noisy_shape_from_shape and noisy_shape_from_bounding_box --- menpofit/aam/fitter.py | 15 ++++++--------- menpofit/fitter.py | 7 +++---- menpofit/lk/fitter.py | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f40a450..de7bed1 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -170,6 +170,11 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): images = rescale_images_to_reference_shape( images, group, label, self.reference_shape, verbose=verbose) + if self.scale_features: + # compute features at highest level + feature_images = compute_features(images, self.features, + verbose=verbose) + # for each pyramid level (low --> high) for j, s in enumerate(self.scales): if verbose: @@ -180,16 +185,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): # obtain image representation if s == self.scales[-1]: - # compute features at highest level - feature_images = compute_features(images, self.features, - level_str=level_str, - verbose=verbose) level_images = feature_images elif self.scale_features: - # compute features at highest level - feature_images = compute_features(images, self.features, - level_str=level_str, - verbose=verbose) # scale features at other levels level_images = scale_images(feature_images, s, level_str=level_str, @@ -211,7 +208,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - p_s = self.get_initial_shape_from_shape(gt_s) + p_s = self.noisy_shape_from_shape(gt_s) perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 6b92870..9b1f906 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -319,15 +319,14 @@ def _check_n_shape(self, n_shape): 'those'.format(self._model.n_levels)) # TODO: Bounding boxes should be PointGraphs - def get_initial_shape_from_bounding_box(self, bounding_box, noise_std=0.04, - rotation=False): + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, + rotation=False): transform = noisy_align(AlignmentSimilarity, self.reference_bounding_box, bounding_box, noise_std=noise_std, rotation=rotation) return transform.apply(self.reference_shape) - def get_initial_shape_from_shape(self, shape, noise_std=0.04, - rotation=False): + def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): return self.get_initial_shape_from_bounding_box( shape.bounding_box(), noise_std=noise_std, rotation=rotation) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 4633904..73f6eb1 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -97,7 +97,7 @@ def _prepare_template(self, template, group=None, label=None): return templates, sources - def perturb_shape(self, gt_shape, noise_std=0.04): + def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): transform = noisy_align(self.transform_cls, self.reference_shape, gt_shape, noise_std=noise_std) return transform.apply(self.reference_shape) From 43331244fe4bc449b51a70c600e8aecbef65ab5b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 1 Jun 2015 14:32:33 +0100 Subject: [PATCH 058/423] Removed algorithm.py - This decouples the structure of ATM and AAM fitters --- menpofit/algorithm.py | 134 -------------------------------------- menpofit/atm/algorithm.py | 130 +++++++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 137 deletions(-) delete mode 100644 menpofit/algorithm.py diff --git a/menpofit/algorithm.py b/menpofit/algorithm.py deleted file mode 100644 index b651ba6..0000000 --- a/menpofit/algorithm.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import division -import numpy as np -from menpo.image import Image -from menpo.feature import no_op -from menpo.feature import gradient as fast_gradient - - -# TODO: implement more clever sampling? -class LKInterface(object): - - def __init__(self, lk_algorithm, sampling=None): - self.algorithm = lk_algorithm - - n_true_pixels = self.template.n_true_pixels() - n_channels = self.template.n_channels - n_parameters = self.transform.n_parameters - sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - - if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 - - self.i_mask = np.nonzero(np.tile( - sampling_mask[None, ...], (n_channels, 1)).flatten())[0] - self.dW_dp_mask = np.nonzero(np.tile( - sampling_mask[None, ..., None], (2, 1, n_parameters))) - self.nabla_mask = np.nonzero(np.tile( - sampling_mask[None, None, ...], (2, n_channels, 1))) - self.nabla2_mask = np.nonzero(np.tile( - sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - - @property - def n(self): - return self.transform.n_parameters - - @property - def true_indices(self): - return self.template.mask.true_indices() - - def warp_jacobian(self): - dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) - return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, - dW_dp.shape[2])) - - def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) - - def gradient(self, img): - nabla = fast_gradient(img) - nabla.set_boundary_pixels() - return nabla.as_vector().reshape((2, img.n_channels, -1)) - - def steepest_descent_images(self, nabla, dW_dp): - # reshape gradient - # nabla: n_dims x n_channels x n_pixels - nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) - # compute steepest descent images - # nabla: n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # sdi: n_channels x n_pixels x n_params - sdi = 0 - a = nabla[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - # reshape steepest descent images - # sdi: (n_channels x n_pixels) x n_params - return sdi.reshape((-1, sdi.shape[2])) - - -class LKPartsInterface(LKInterface): - - def __init__(self, lk_algorithm, patch_shape=(17, 17), - normalize_parts=no_op, sampling=None): - self.algorithm = lk_algorithm - self.patch_shape = patch_shape - self.normalize_parts = normalize_parts - - if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) - - image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling[None, None, None, ...], - image_shape[:3] + (1, 1)) - self.i_mask = np.nonzero(image_mask.flatten())[0] - self.nabla_mask = np.nonzero(np.tile( - image_mask[None, ...], (2, 1, 1, 1, 1, 1))) - self.nabla2_mask = np.nonzero(np.tile( - image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) - - def warp_jacobian(self): - return np.rollaxis(self.transform.d_dp(None), -1) - - # TODO: add parts normalization - def warp(self, image): - parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, - as_single_array=True) - parts = self.normalize_parts(parts) - return Image(parts) - - def gradient(self, image): - pixels = image.pixels - g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) - # remove 1st dimension gradient which corresponds to the gradient - # between parts - return g.reshape((2,) + pixels.shape) - - def steepest_descent_images(self, nabla, dw_dp): - # reshape nabla - # nabla: dims x parts x off x ch x (h x w) - nabla = nabla[self.nabla_mask].reshape( - nabla.shape[:-2] + (-1,)) - # compute steepest descent images - # nabla: dims x parts x off x ch x (h x w) - # ds_dp: dims x parts x x params - # sdi: parts x off x ch x (h x w) x params - sdi = 0 - a = nabla[..., None] * dw_dp[..., None, None, None, :] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (parts x offsets x ch x w x h) x params - return sdi.reshape((-1, sdi.shape[-1])) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index dce40fb..2193d70 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,16 +1,87 @@ from __future__ import division import abc import numpy as np -from menpofit.algorithm import LKInterface, LKPartsInterface +from menpo.image import Image +from menpo.feature import no_op +from menpo.feature import gradient as fast_gradient from .result import ATMAlgorithmResult, LinearATMAlgorithmResult -class LKATMInterface(LKInterface): +# TODO: implement more clever sampling? +class LKATMInterface(object): + + def __init__(self, lk_algorithm, sampling=None): + self.algorithm = lk_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def true_indices(self): + return self.template.mask.true_indices() @property def shape_model(self): return self.transform.pdm.model + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + @classmethod def solve_shape_map(cls, H, J, e, J_prior, p): if p.shape[0] is not H.shape[0]: @@ -43,12 +114,65 @@ def algorithm_result(self, image, shape_parameters, gt_shape=None): image, self.algorithm, shape_parameters, gt_shape=gt_shape) -class LKPartsATMInterface(LKPartsInterface, LKATMInterface): +class LKPartsATMInterface(LKATMInterface): + + def __init__(self, lk_algorithm, patch_shape=(17, 17), + normalize_parts=no_op, sampling=None): + self.algorithm = lk_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.nabla_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.nabla2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) @property def shape_model(self): return self.transform.model + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + def warp(self, image): + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts) + + def gradient(self, image): + pixels = image.pixels + g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return g.reshape((2,) + pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.nabla_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) + # TODO: handle costs for all LKAAMAlgorithms # TODO document me! From 932455e4b2fbd49d394e46d13a7800154a072ad3 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 1 Jun 2015 14:53:44 +0100 Subject: [PATCH 059/423] Update fitter.py and LinearATMAlgorithmResult --- menpofit/atm/result.py | 12 ++++-------- menpofit/fitter.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 2b91276..4ee4cab 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -12,14 +12,10 @@ class ATMAlgorithmResult(ParametricAlgorithmResult): class LinearATMAlgorithmResult(ATMAlgorithmResult): r""" """ - def shapes(self, as_points=False): - if as_points: - return [self.fitter.transform.from_vector(p).sparse_target.points - for p in self.shape_parameters] - - else: - return [self.fitter.transform.from_vector(p).sparse_target - for p in self.shape_parameters] + @property + def shapes(self): + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] @property def final_shape(self): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 9b1f906..34fef7a 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -327,7 +327,7 @@ def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, return transform.apply(self.reference_shape) def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): - return self.get_initial_shape_from_bounding_box( + return self.noisy_shape_from_bounding_box( shape.bounding_box(), noise_std=noise_std, rotation=rotation) From de1c15738de3bafdb6e9b8117181dd370a8214e1 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:20:14 +0100 Subject: [PATCH 060/423] Add small changes to fitter.py --- menpofit/fitter.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 34fef7a..3d8ce11 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -23,21 +23,21 @@ def n_levels(self): def algorithms(self): pass - @abc.abstractproperty - def reference_shape(self): - pass - - @abc.abstractproperty - def features(self): - pass - - @abc.abstractproperty - def scales(self): - pass - - @abc.abstractproperty - def scale_features(self): - pass + # @abc.abstractproperty + # def reference_shape(self): + # pass + # + # @abc.abstractproperty + # def features(self): + # pass + # + # @abc.abstractproperty + # def scales(self): + # pass + # + # @abc.abstractproperty + # def scale_features(self): + # pass def fit(self, image, initial_shape, max_iters=50, gt_shape=None, crop_image=0.5, **kwargs): From 3670f43512e7062574cca9bd9a38a316daf08c49 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:21:19 +0100 Subject: [PATCH 061/423] Add first version of sdm fitter --- menpofit/sdm/fitter.py | 661 ++++++++++++++++++++++++----------------- 1 file changed, 384 insertions(+), 277 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 3993c75..2230555 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,304 +1,411 @@ -import numpy as np -from menpo.image import Image - -from menpofit.base import name_of_callable -from menpofit.aam.fitter import AAMFitter -from menpofit.clm.fitter import CLMFitter -from menpofit.fitter import MultilevelFitter - - -class SDFitter(MultilevelFitter): +from __future__ import division +from functools import partial +from menpo.transform import Scale, AlignmentSimilarity +from menpo.feature import no_op +from menpofit.builder import normalization_wrt_reference_shape, scale_images +from menpofit.fitter import MultiFitter, noisy_align +from menpofit.result import MultiFitterResult +import menpofit.checks as checks +from .algorithm import SN + + +# TODO: document me! +class CRFitter(MultiFitter): r""" - Abstract Supervised Descent Fitter. """ - def _set_up(self): - r""" - Sets up the SD fitter object. - """ - - def fit(self, image, initial_shape, max_iters=None, gt_shape=None, - **kwargs): - r""" - Fits a single image. - - Parameters - ----------- - image : :map:`MaskedImage` - The image to be fitted. - initial_shape : :map:`PointCloud` - The initial shape estimate from which the fitting procedure - will start. - max_iters : int or `list`, optional - The maximum number of iterations. - - If `int`, then this will be the overall maximum number of iterations - for all the pyramidal levels. - - If `list`, then a maximum number of iterations is specified for each - pyramidal level. - - gt_shape : :map:`PointCloud` - The ground truth shape of the image. - - **kwargs : `dict` - optional arguments to be passed through. - - Returns - ------- - fitting_list : :map:`FittingResultList` - A fitting result object. - """ - if max_iters is None: - max_iters = self.n_levels - return MultilevelFitter.fit(self, image, initial_shape, - max_iters=max_iters, gt_shape=gt_shape, - **kwargs) - - -class SDMFitter(SDFitter): - r""" - Supervised Descent Method. - - Parameters - ----------- - regressors : :map:`RegressorTrainer` - The trained regressors. - - n_training_images : `int` - The number of images that were used to train the SDM fitter. It is - only used for informational reasons. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - - References - ---------- - .. [XiongD13] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 - """ - def __init__(self, regressors, n_training_images, features, - reference_shape, downscale): - self._fitters = regressors - self._features = features - self._reference_shape = reference_shape - self._downscale = downscale - self._n_training_images = n_training_images + def __init__(self, cr_algorithm_cls=SN, features=no_op, + patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), + iterations=6, n_perturbations=10, **kwargs): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + # set parameters + self._algorithms = [] + self.diagonal = diagonal + self.scales = list(scales)[::-1] + self.n_perturbations = n_perturbations + self.iterations = checks.check_iterations(iterations, n_levels) + # set up algorithms + self._set_up(cr_algorithm_cls, features, patch_shape, **kwargs) @property - def algorithm(self): - r""" - Returns a string containing the algorithm used from the SDM family. - - : str - """ - return 'SDM-' + self._fitters[0].algorithm + def algorithms(self): + return self._algorithms @property - def reference_shape(self): - r""" - The reference shape used during training. + def reference_bounding_box(self): + return self.reference_shape.bounding_box() - :type: :map:`PointCloud` - """ - return self._reference_shape - - @property - def features(self): - r""" - The feature type per pyramid level. Note that they are stored from - lowest to highest level resolution. - - :type: `list` - """ - return self._features - - @property - def n_levels(self): - r""" - The number of pyramidal levels used during training. + def _set_up(self, cr_algorithm_cls, features, patch_shape, **kwargs): + for j in range(self.n_levels): + algorithm = cr_algorithm_cls( + features=features[j], patch_shape=patch_shape[j], + iterations=self.iterations[j], **kwargs) + self._algorithms.append(algorithm) - : int - """ - return len(self._fitters) + def train(self, images, group=None, label=None, verbose=False, **kwargs): + # normalize images and compute reference shape + self.reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) - @property - def downscale(self): + # for each pyramid level (low --> high) + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # scale images and compute features at other levels + level_images = scale_images(images, self.scales[j], + level_str=level_str, verbose=verbose) + + # extract ground truth shapes for current level + level_gt_shapes = [i.landmarks[group][label] for i in level_images] + + if j == 0: + # generate perturbed shapes + current_shapes = [] + for gt_s in level_gt_shapes: + perturbed_shapes = [] + for _ in range(self.n_perturbations): + p_s = self.noisy_shape_from_shape(gt_s) + perturbed_shapes.append(p_s) + current_shapes.append(perturbed_shapes) + + # train cascaded regression algorithm + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + + # scale current shapes to next level resolution + if self.scales[j] != (1 or self.scales[-1]): + transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + + def _prepare_image(self, image, initial_shape, gt_shape=None, + crop_image=0.5): r""" - The downscale per pyramidal level used during building the AAM. - The scale factor is: (downscale ** k) for k in range(n_levels) + Prepares the image to be fitted. - :type: `float` - """ - return self._downscale + The image is first rescaled wrt the ``reference_landmarks`` and then + a gaussian pyramid is applied. Depending on the + ``pyramid_on_features`` flag, the pyramid is either applied to the + features image computed from the rescaled imaged or applied to the + rescaled image and features extracted at each pyramidal level. - def __str__(self): - out = "Supervised Descent Method\n" \ - " - Non-Parametric '{}' Regressor\n" \ - " - {} training images.\n".format( - name_of_callable(self._fitters[0].regressor), - self._n_training_images) - # small strings about number of channels, channels string and downscale - down_str = [] - for j in range(self.n_levels): - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - temp_img = Image(image_data=np.random.rand(40, 40)) - if self.pyramid_on_features: - temp = self.features(temp_img) - n_channels = [temp.n_channels] * self.n_levels - else: - n_channels = [] - for j in range(self.n_levels): - temp = self.features[j](temp_img) - n_channels.append(temp.n_channels) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - self.features[j].__name__)) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n {}{} {} per " \ - "image.\n".format( - out, self.n_levels - i, down_str[i], feat_str[i], - n_channels[i], ch_str[i]) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( - out, feat_str[0], n_channels[0], ch_str[0]) - return out + Parameters + ---------- + image : :map:`Image` or subclass + The image to be fitted. + initial_shape : :map:`PointCloud` + The initial shape from which the fitting will start. -class SDAAMFitter(AAMFitter, SDFitter): - r""" - Supervised Descent Fitter for AAMs. + gt_shape : class : :map:`PointCloud`, optional + The original ground truth shape associated to the image. - Parameters - ----------- - aam : :map:`AAM` - The Active Appearance Model to be used. + crop_image: `None` or float`, optional + If `float`, it specifies the proportion of the border wrt the + initial shape to which the image will be internally cropped around + the initial shape range. + If `None`, no cropping is performed. - regressors : :map:``RegressorTrainer` - The trained regressors. + This will limit the fitting algorithm search region but is + likely to speed up its running time, specially when the + modeled object occupies a small portion of the image. - n_training_images : `int` - The number of training images used to train the SDM fitter. - """ - def __init__(self, aam, regressors, n_training_images): - super(SDAAMFitter, self).__init__(aam) - self._fitters = regressors - self._n_training_images = n_training_images + Returns + ------- + images : `list` of :map:`Image` or subclass + The list of images that will be fitted by the fitters. - @property - def algorithm(self): - r""" - Returns a string containing the algorithm used from the SDM family. + initial_shapes : `list` of :map:`PointCloud` + The initial shape for each one of the previous images. - :type: `string` + gt_shapes : `list` of :map:`PointCloud` + The ground truth shape for each one of the previous images. """ - return 'SD-AAM-' + self._fitters[0].algorithm + # attach landmarks to the image + image.landmarks['initial_shape'] = initial_shape + if gt_shape: + image.landmarks['gt_shape'] = gt_shape + + # if specified, crop the image + if crop_image: + image = image.copy() + image.crop_to_landmarks_proportion_inplace(crop_image, + group='initial_shape') + + # rescale image wrt the scale factor between reference_shape and + # initial_shape + image = image.rescale_to_reference_shape(self.reference_shape, + group='initial_shape') + + # obtain image representation + images = [] + for s in self.scales: + if s != 1: + # scale image + scaled_image = image.rescale(s) + else: + scaled_image = image + images.append(scaled_image) - def __str__(self): - return "{}Supervised Descent Method for AAMs:\n" \ - " - Parametric '{}' Regressor\n" \ - " - {} training images.\n".format( - self.aam.__str__(), name_of_callable(self._fitters[0].regressor), - self._n_training_images) + # get initial shapes per level + initial_shapes = [i.landmarks['initial_shape'].lms for i in images] + # get ground truth shapes per level + if gt_shape: + gt_shapes = [i.landmarks['gt_shape'].lms for i in images] + else: + gt_shapes = None -class SDCLMFitter(CLMFitter, SDFitter): - r""" - Supervised Descent Fitter for CLMs. - - Parameters - ----------- - clm : :map:`CLM` - The Constrained Local Model to be used. - - regressors : :map:`RegressorTrainer` - The trained regressors. - - n_training_images : `int` - The number of training images used to train the SDM fitter. - - References - ---------- - .. [Asthana13] Robust Discriminative Response Map Fitting with Constrained - Local Models - A. Asthana, S. Zafeiriou, S. Cheng, M. Pantic. - IEEE Conference onComputer Vision and Pattern Recognition. - Portland, Oregon, USA, June 2013. - """ - def __init__(self, clm, regressors, n_training_images): - super(SDCLMFitter, self).__init__(clm) - self._fitters = regressors - self._n_training_images = n_training_images + return images, initial_shapes, gt_shapes - @property - def algorithm(self): - r""" - Returns a string containing the algorithm used from the SDM family. + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return MultiFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) - :type: `string` - """ - return 'SD-CLM-' + self._fitters[0].algorithm + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, + rotation=False): + transform = noisy_align(AlignmentSimilarity, + self.reference_bounding_box, bounding_box, + noise_std=noise_std, rotation=rotation) + return transform.apply(self.reference_shape) + + def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): + return self.noisy_shape_from_bounding_box( + shape.bounding_box(), noise_std=noise_std, rotation=rotation) + # TODO: fix me! def __str__(self): - return "{}Supervised Descent Method for CLMs:\n" \ - " - Parametric '{}' Regressor\n" \ - " - {} training images.\n".format( - self.clm.__str__(), name_of_callable(self._fitters[0].regressor), - self._n_training_images) + pass + # out = "Supervised Descent Method\n" \ + # " - Non-Parametric '{}' Regressor\n" \ + # " - {} training images.\n".format( + # name_of_callable(self._fitters[0].regressor), + # self._n_training_images) + # # small strings about number of channels, channels string and downscale + # down_str = [] + # for j in range(self.n_levels): + # if j == self.n_levels - 1: + # down_str.append('(no downscale)') + # else: + # down_str.append('(downscale by {})'.format( + # self.downscale**(self.n_levels - j - 1))) + # temp_img = Image(image_data=np.random.rand(40, 40)) + # if self.pyramid_on_features: + # temp = self.features(temp_img) + # n_channels = [temp.n_channels] * self.n_levels + # else: + # n_channels = [] + # for j in range(self.n_levels): + # temp = self.features[j](temp_img) + # n_channels.append(temp.n_channels) + # # string about features and channels + # if self.pyramid_on_features: + # feat_str = "- Feature is {} with ".format( + # name_of_callable(self.features)) + # if n_channels[0] == 1: + # ch_str = ["channel"] + # else: + # ch_str = ["channels"] + # else: + # feat_str = [] + # ch_str = [] + # for j in range(self.n_levels): + # if isinstance(self.features[j], str): + # feat_str.append("- Feature is {} with ".format( + # self.features[j])) + # elif self.features[j] is None: + # feat_str.append("- No features extracted. ") + # else: + # feat_str.append("- Feature is {} with ".format( + # self.features[j].__name__)) + # if n_channels[j] == 1: + # ch_str.append("channel") + # else: + # ch_str.append("channels") + # if self.n_levels > 1: + # out = "{} - Gaussian pyramid with {} levels and downscale " \ + # "factor of {}.\n".format(out, self.n_levels, + # self.downscale) + # if self.pyramid_on_features: + # out = "{} - Pyramid was applied on feature space.\n " \ + # "{}{} {} per image.\n".format(out, feat_str, + # n_channels[0], ch_str[0]) + # else: + # out = "{} - Features were extracted at each pyramid " \ + # "level.\n".format(out) + # for i in range(self.n_levels - 1, -1, -1): + # out = "{} - Level {} {}: \n {}{} {} per " \ + # "image.\n".format( + # out, self.n_levels - i, down_str[i], feat_str[i], + # n_channels[i], ch_str[i]) + # else: + # if self.pyramid_on_features: + # feat_str = [feat_str] + # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( + # out, feat_str[0], n_channels[0], ch_str[0]) + # return out + + +# TODO: document me! +SDMFitter = partial(CRFitter, cr_algorithm_cls=SN) + + +# class CRFitter(MultiFitter): +# r""" +# """ +# def __init__(self, cr_algorithm_cls=SN, features=no_op, diagonal=None, +# scales=(1, 0.5), sampling=None, n_perturbations=10, +# iterations=6, **kwargs): +# # check parameters +# checks.check_diagonal(diagonal) +# scales, n_levels = checks.check_scales(scales) +# features = checks.check_features(features, n_levels) +# sampling = checks.check_sampling(sampling, n_levels) +# # set parameters +# self._algorithms = [] +# self.diagonal = diagonal +# self.scales = list(scales) +# self.n_perturbations = n_perturbations +# self.iterations = checks.check_iterations(iterations, n_levels) +# # set up algorithms +# self._set_up(cr_algorithm_cls, features, sampling, **kwargs) +# +# @property +# def algorithms(self): +# return self._algorithms +# +# def _set_up(self, cr_algorithm_cls, features, sampling, **kwargs): +# for j, s in range(self.n_levels): +# algorithm = cr_algorithm_cls( +# features=features[j], sampling=sampling[j], +# max_iters=self.iterations[j], **kwargs) +# self._algorithms.append(algorithm) +# +# def train(self, images, group=None, label=None, verbose=False, **kwargs): +# # normalize images and compute reference shape +# reference_shape, images = normalization_wrt_reference_shape( +# images, group, label, self.diagonal, verbose=verbose) +# +# # for each pyramid level (low --> high) +# for j in range(self.n_levels): +# if verbose: +# if len(self.scales) > 1: +# level_str = ' - Level {}: '.format(j) +# else: +# level_str = ' - ' +# +# # scale images and compute features at other levels +# level_images = scale_images(images, self.scales[j], +# level_str=level_str, verbose=verbose) +# +# # extract ground truth shapes for current level +# level_gt_shapes = [i.landmarks[group][label] for i in level_images] +# +# if j == 0: +# # generate perturbed shapes +# current_shapes = [] +# for gt_s in level_gt_shapes: +# perturbed_shapes = [] +# for _ in range(self.n_perturbations): +# p_s = self.noisy_shape_from_shape(gt_s) +# perturbed_shapes.append(p_s) +# current_shapes.append(perturbed_shapes) +# +# # train cascaded regression algorithm +# current_shapes = self.algorithms[j].train( +# level_images, level_gt_shapes, current_shapes, +# verbose=verbose, **kwargs) +# +# # scale current shapes to next level resolution +# if self.scales[j] != self.scales[-1]: +# transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) +# for image_shapes in current_shapes: +# for shape in image_shapes: +# transform.apply_inplace(shape) +# +# def _fitter_result(self, image, algorithm_results, affine_correction, +# gt_shape=None): +# return MultiFitterResult(image, algorithm_results, affine_correction, +# gt_shape=gt_shape) +# +# # TODO: fix me! +# def __str__(self): +# pass +# # out = "Supervised Descent Method\n" \ +# # " - Non-Parametric '{}' Regressor\n" \ +# # " - {} training images.\n".format( +# # name_of_callable(self._fitters[0].regressor), +# # self._n_training_images) +# # # small strings about number of channels, channels string and downscale +# # down_str = [] +# # for j in range(self.n_levels): +# # if j == self.n_levels - 1: +# # down_str.append('(no downscale)') +# # else: +# # down_str.append('(downscale by {})'.format( +# # self.downscale**(self.n_levels - j - 1))) +# # temp_img = Image(image_data=np.random.rand(40, 40)) +# # if self.pyramid_on_features: +# # temp = self.features(temp_img) +# # n_channels = [temp.n_channels] * self.n_levels +# # else: +# # n_channels = [] +# # for j in range(self.n_levels): +# # temp = self.features[j](temp_img) +# # n_channels.append(temp.n_channels) +# # # string about features and channels +# # if self.pyramid_on_features: +# # feat_str = "- Feature is {} with ".format( +# # name_of_callable(self.features)) +# # if n_channels[0] == 1: +# # ch_str = ["channel"] +# # else: +# # ch_str = ["channels"] +# # else: +# # feat_str = [] +# # ch_str = [] +# # for j in range(self.n_levels): +# # if isinstance(self.features[j], str): +# # feat_str.append("- Feature is {} with ".format( +# # self.features[j])) +# # elif self.features[j] is None: +# # feat_str.append("- No features extracted. ") +# # else: +# # feat_str.append("- Feature is {} with ".format( +# # self.features[j].__name__)) +# # if n_channels[j] == 1: +# # ch_str.append("channel") +# # else: +# # ch_str.append("channels") +# # if self.n_levels > 1: +# # out = "{} - Gaussian pyramid with {} levels and downscale " \ +# # "factor of {}.\n".format(out, self.n_levels, +# # self.downscale) +# # if self.pyramid_on_features: +# # out = "{} - Pyramid was applied on feature space.\n " \ +# # "{}{} {} per image.\n".format(out, feat_str, +# # n_channels[0], ch_str[0]) +# # else: +# # out = "{} - Features were extracted at each pyramid " \ +# # "level.\n".format(out) +# # for i in range(self.n_levels - 1, -1, -1): +# # out = "{} - Level {} {}: \n {}{} {} per " \ +# # "image.\n".format( +# # out, self.n_levels - i, down_str[i], feat_str[i], +# # n_channels[i], ch_str[i]) +# # else: +# # if self.pyramid_on_features: +# # feat_str = [feat_str] +# # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( +# # out, feat_str[0], n_channels[0], ch_str[0]) +# # return out \ No newline at end of file From e15c30d710677d31ee6f3d2200699035defda450 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:22:33 +0100 Subject: [PATCH 062/423] Add first version of sdm algorithms --- menpofit/sdm/algorithm.py | 337 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 menpofit/sdm/algorithm.py diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py new file mode 100644 index 0000000..ce921c1 --- /dev/null +++ b/menpofit/sdm/algorithm.py @@ -0,0 +1,337 @@ +from __future__ import division +import numpy as np +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpofit.result import NonParametricAlgorithmResult + + +# TODO document me! +class CRAlgorithm(object): + r""" + """ + def train(self, images, gt_shapes, current_shapes, verbose=False, + **kwargs): + n_images = len(images) + n_samples_image = len(current_shapes[0]) + + self._features_patch_length = compute_features_info( + images[0], gt_shapes[0], self.features, + patch_shape=self.patch_shape)[1] + + # obtain delta_x and gt_x + delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) + + # initialize iteration counter and list of regressors + k = 0 + self.regressors = [] + + # Cascaded Regression loop + while k < self.iterations: + # generate regression data + features = obtain_patch_features( + images, current_shapes, self.patch_shape, self.features, + features_patch_length=self._features_patch_length) + + # perform regression + if verbose: + print_dynamic('- Performing regression...') + regressor = self._perform_regression(features, delta_x, **kwargs) + # add regressor to list + self.regressors.append(regressor) + + # estimate delta_points + estimated_delta_x = regressor(features) + if verbose: + error = _compute_rmse(delta_x, estimated_delta_x) + print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + + j = 0 + for shapes in current_shapes: + for s in shapes: + # update current x + current_x = s.as_vector() + estimated_delta_x[j] + # update current shape inplace + s.from_vector_inplace(current_x) + # update delta_x + delta_x[j] = gt_x[j] - current_x + # increase index + j += 1 + # increase iteration counter + k += 1 + + # rearrange current shapes into their original list of list form + return current_shapes + + def run(self, image, initial_shape, gt_shape=None, **kwargs): + # set current shape and initialize list of shapes + current_shape = initial_shape + shapes = [initial_shape] + + # Cascaded Regression loop + for r in self.regressors: + # compute regression features + features = compute_patch_features( + image, current_shape, self.patch_shape, self.features, + features_patch_length=self._features_patch_length) + + # solve for increments on the shape vector + dx = r(features) + + # update current shape + current_shape = current_shape.from_vector( + current_shape.as_vector() + dx) + shapes.append(current_shape) + + # return algorithm result + return NonParametricAlgorithmResult(image, self, shapes, + gt_shape=gt_shape) + + +# TODO: document me! +class SN(CRAlgorithm): + r""" + Supervised Newton. + + This class implements the Supervised Descent Method technique, proposed + by Xiong and De la Torre in [XiongD13]. + + References + ---------- + .. [XiongD13] Supervised Descent Method and its Applications to + Face Alignment + Xuehan Xiong and Fernando De la Torre Fernando + IEEE International Conference on Computer Vision and Pattern Recognition + May, 2013 + """ + def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + eps=10 ** -5): + self.patch_shape = patch_shape + self.features = features + self.patch_shape = patch_shape + self.iterations = iterations + self.eps = eps + # wire regression callable + self._perform_regression = _supervised_newton + + +# TODO: document me! +class SGN(CRAlgorithm): + r""" + Supervised Gauss-Newton + + This class implements a variation of the Supervised Descent Method + [XiongD13] by some of the ideas incorporating ideas... + + References + ---------- + .. [XiongD13] Supervised Descent Method and its Applications to + Face Alignment + Xuehan Xiong and Fernando De la Torre Fernando + IEEE International Conference on Computer Vision and Pattern Recognition + May, 2013 + .. [Tzimiropoulos15] Supervised Descent Method and its Applications to + Face Alignment + Xuehan Xiong and Fernando De la Torre Fernando + IEEE International Conference on Computer Vision and Pattern Recognition + May, 2013 + """ + def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + eps=10 ** -5): + self.patch_shape = patch_shape + self.features = features + self.patch_shape = patch_shape + self.iterations = iterations + self.eps = eps + # wire regression callable + self._perform_regression = _supervised_gauss_newton + + +# TODO: document me! +class _supervised_newton(object): + r""" + """ + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = features.T.dot(features) + XT = features.T.dot(deltas) + if gamma: + XX += gamma * np.eye(features.shape[1]) + # descent direction + self.R = np.linalg.solve(XX, XT) + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +class _supervised_gauss_newton(object): + r""" + """ + + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = deltas.T.dot(deltas) + XT = deltas.T.dot(features) + if gamma: + XX += gamma * np.eye(deltas.shape[1]) + # average Jacobian + self.J = np.linalg.solve(XX, XT) + # average Hessian + self.H = self.J.dot(self.J.T) + # descent direction + self.R = np.linalg.solve(self.H, self.J).T + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +def _compute_rmse(x1, x2): + return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + + +# TODO: docment me! +def compute_patch_features(image, shape, patch_shape, features_callable, + features_patch_length=None): + """r + """ + patches = image.extract_patches(shape, patch_size=patch_shape, + as_single_array=True) + + if features_patch_length: + patch_features = np.empty((shape.n_points, features_patch_length)) + for j, p in enumerate(patches): + patch_features[j] = features_callable(p[0]).ravel() + else: + patch_features = [] + for j, p in enumerate(patches): + patch_features.append(features_callable(p[0]).ravel()) + patch_features = np.asarray(patch_features) + + return patch_features.ravel() + + +# TODO: docment me! +def generate_patch_features(image, shapes, patch_shape, features_callable, + features_patch_length=None): + """r + """ + if features_patch_length: + patch_features = np.empty((len(shapes), + shapes[0].n_points * features_patch_length)) + for j, s in enumerate(shapes): + patch_features[j] = compute_patch_features( + image, s, patch_shape, features_callable, + features_patch_length=features_patch_length) + else: + patch_features = [] + for j, s in enumerate(shapes): + patch_features.append(compute_patch_features( + image, s, patch_shape, features_callable, + features_patch_length=features_patch_length)) + patch_features = np.asarray(patch_features) + + return patch_features.ravel() + + +# TODO: docment me! +def obtain_patch_features(images, shapes, patch_shape, features_callable, + features_patch_length=None): + """r + """ + n_images = len(images) + n_shapes = len(shapes[0]) + n_points = shapes[0][0].n_points + + if features_patch_length: + + patch_features = np.empty((n_images, (n_shapes * n_points * + features_patch_length))) + for j, i in enumerate(images): + patch_features[j] = generate_patch_features( + i, shapes[j], patch_shape, features_callable, + features_patch_length=features_patch_length) + else: + patch_features = [] + for j, i in images: + patch_features.append(generate_patch_features( + i, shapes[j], patch_shape, features_callable, + features_patch_length=features_patch_length)) + patch_features = np.asarray(patch_features) + + return patch_features.reshape((-1, n_points * features_patch_length)) + + +def compute_delta_x(gt_shape, current_shapes): + r""" + """ + n_x = gt_shape.n_parameters + n_current_shapes = len(current_shapes) + + # initialize ground truth and delta shape vectors + gt_x = np.empty((n_current_shapes, n_x)) + delta_x = np.empty((n_current_shapes, n_x)) + + for j, s in enumerate(current_shapes): + # compute ground truth shape vector + gt_x[j] = gt_shape.as_vector() + # compute delta shape vector + delta_x[j] = gt_x[j] - s.as_vector() + + return delta_x, gt_x + + +def obtain_delta_x(gt_shapes, current_shapes): + r""" + """ + n_x = gt_shapes[0].n_parameters + n_gt_shapes = len(gt_shapes) + n_current_shapes = len(current_shapes[0]) + + # initialize current, ground truth and delta parameters + gt_x = np.empty((n_gt_shapes, n_current_shapes, n_x)) + delta_x = np.empty((n_gt_shapes, n_current_shapes, n_x)) + + # obtain ground truth points and compute delta points + for j, (gt_s, shapes) in enumerate(zip(gt_shapes, current_shapes)): + # compute ground truth par + delta_x[j], gt_x[j] = compute_delta_x(gt_s, shapes) + + return delta_x.reshape((-1, n_x)), gt_x.reshape((-1, n_x)) + + +def compute_features_info(image, shape, features_callable, + patch_shape=(17, 17)): + # TODO: include offsets support? + patches = image.extract_patches(shape, patch_size=patch_shape, + as_single_array=True) + + # TODO: include offsets support? + features_patch_shape = features_callable(patches[0, 0]).shape + features_patch_length = np.prod(features_patch_shape) + features_shape = patches.shape[:1] + features_patch_shape + features_length = np.prod(features_shape) + + return (features_patch_shape, features_patch_length, + features_shape, features_length) + +# def initialize_sampling(self, image, group=None, label=None): +# if self._sampling is None: +# sampling = np.ones(self.patch_shape, dtype=np.bool) +# else: +# sampling = self._sampling +# +# # TODO: include offsets support? +# patches = image.extract_patches_around_landmarks( +# group=group, label=label, patch_size=self.patch_shape, +# as_single_array=True) +# +# # TODO: include offsets support? +# features_patch_shape = self.features(patches[0, 0]).shape +# self._features_patch_length = np.prod(features_patch_shape) +# self._features_shape = (patches.shape[0], features_patch_shape) +# self._features_length = np.prod(self._features_shape) +# +# feature_mask = np.tile(sampling[None, None, None, ...], +# self._feature_shape[:3] + (1, 1)) +# self._feature_mask = np.nonzero(feature_mask.flatten())[0] From ec599a4edbee0329dacc9617fab26eb34233e91f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:23:45 +0100 Subject: [PATCH 063/423] Update sdm/__init__.py --- menpofit/sdm/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/__init__.py b/menpofit/sdm/__init__.py index 9661b9d..9180e28 100644 --- a/menpofit/sdm/__init__.py +++ b/menpofit/sdm/__init__.py @@ -1,2 +1,2 @@ -from .trainer import SDMTrainer, SDAAMTrainer, SDCLMTrainer -from .fitter import SDMFitter, SDAAMFitter, SDCLMFitter +from .algorithm import SN, SGN +from .fitter import CRFitter, SDMFitter From b98dfb9fbc624da9c72cd96860a9e4c1d1b70aad Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 22 Jun 2015 16:31:51 +0100 Subject: [PATCH 064/423] Slight modifications to SDM --- menpofit/result.py | 40 +++++++++++++++++++++++++++++++++------ menpofit/sdm/algorithm.py | 8 ++------ menpofit/sdm/fitter.py | 2 +- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 531f759..6fbe755 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -459,6 +459,33 @@ def initial_shape(self): return self.initial_transform.target +# TODO: document me! +class NonParametricAlgorithmResult(IterativeResult): + r""" + """ + def __init__(self, image, fitter, shapes, gt_shape=None): + self.image = image + self.fitter = fitter + self._shapes = shapes + self._gt_shape = gt_shape + + @property + def n_iters(self): + return len(self.shapes) - 1 + + @property + def shapes(self): + return self._shapes + + @property + def final_shape(self): + return self.shapes[-1] + + @property + def initial_shape(self): + return self.shapes[0] + + # TODO: document me! class MultiFitterResult(IterativeResult): r""" @@ -584,7 +611,7 @@ def compute_error(target, ground_truth, error_type='me_norm'): target_points = target.points if error_type == 'me_norm': - return _compute_me_norm(target_points, gt_points) + return _compute_norm_p2p_error(target_points, gt_points) elif error_type == 'me': return _compute_me(target_points, gt_points) elif error_type == 'rmse': @@ -611,13 +638,14 @@ def _compute_rmse(target, ground_truth): # TODO: Document me! -# TODO: rename to more descriptive name -def _compute_me_norm(target, ground_truth): +def _compute_norm_p2p_error(target, source, ground_truth=None): r""" """ + if ground_truth is None: + ground_truth = source normalizer = np.mean(np.max(ground_truth, axis=0) - np.min(ground_truth, axis=0)) - return _compute_me(target, ground_truth) / normalizer + return _compute_me(target, source) / normalizer # TODO: Document me! @@ -628,8 +656,8 @@ def compute_cumulative_error(errors, x_axis): return [np.count_nonzero([errors <= x]) / n_errors for x in x_axis] -def plot_cumulative_error_distribution(errors, error_range=None, figure_id=None, - new_figure=False, +def plot_cumulative_error_distribution(errors, error_range=None, + figure_id=None, new_figure=False, title='Cumulative Error Distribution', x_label='Normalized Point-to-Point Error', y_label='Images Proportion', diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index ce921c1..db5a195 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -11,9 +11,6 @@ class CRAlgorithm(object): """ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): - n_images = len(images) - n_samples_image = len(current_shapes[0]) - self._features_patch_length = compute_features_info( images[0], gt_shapes[0], self.features, patch_shape=self.patch_shape)[1] @@ -155,7 +152,7 @@ def __init__(self, features, deltas, gamma=None): XX = features.T.dot(features) XT = features.T.dot(deltas) if gamma: - XX += gamma * np.eye(features.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # descent direction self.R = np.linalg.solve(XX, XT) @@ -167,13 +164,12 @@ def __call__(self, features): class _supervised_gauss_newton(object): r""" """ - def __init__(self, features, deltas, gamma=None): # ridge regression XX = deltas.T.dot(deltas) XT = deltas.T.dot(features) if gamma: - XX += gamma * np.eye(deltas.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # average Jacobian self.J = np.linalg.solve(XX, XT) # average Hessian diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2230555..2c844e5 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -26,7 +26,7 @@ def __init__(self, cr_algorithm_cls=SN, features=no_op, self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations - self.iterations = checks.check_iterations(iterations, n_levels) + self.iterations = checks.check_max_iters(iterations, n_levels) # set up algorithms self._set_up(cr_algorithm_cls, features, patch_shape, **kwargs) From 9e492ed0a903a0979fb1c131da0b27f1d11b6bdb Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 22 Jun 2015 16:32:55 +0100 Subject: [PATCH 065/423] Delete sdm/trainer.py --- menpofit/sdm/trainer.py | 981 ---------------------------------------- 1 file changed, 981 deletions(-) delete mode 100644 menpofit/sdm/trainer.py diff --git a/menpofit/sdm/trainer.py b/menpofit/sdm/trainer.py deleted file mode 100644 index 83a9ec2..0000000 --- a/menpofit/sdm/trainer.py +++ /dev/null @@ -1,981 +0,0 @@ -from __future__ import division, print_function -import abc -import numpy as np -from menpo.transform import Scale -from menpo.shape import mean_pointcloud -from menpo.feature import sparse_hog, no_op -from menpofit.modelinstance import PDM, OrthoPDM -from menpo.visualize import print_dynamic, progress_bar_str - -from menpofit import checks -from menpofit.transform import (ModelDrivenTransform, OrthoMDTransform, - DifferentiableAlignmentSimilarity) -from menpofit.regression.trainer import ( - NonParametricRegressorTrainer, ParametricRegressorTrainer, - SemiParametricClassifierBasedRegressorTrainer) -from menpofit.regression.regressors import mlr -from menpofit.regression.parametricfeatures import weights -from menpofit.base import DeformableModel, create_pyramid -from .fitter import SDMFitter, SDAAMFitter, SDCLMFitter - - -def check_regression_features(regression_features, n_levels): - try: - return checks.check_list_callables(regression_features, n_levels) - except ValueError: - raise ValueError("regression_features must be a callable or a list of " - "{} callables".format(n_levels)) - - -def check_regression_type(regression_type, n_levels): - r""" - Checks the regression type (method) per level. - - It must be a callable or a list of those from the family of - functions defined in :ref:`regression_functions` - - Parameters - ---------- - regression_type : `function` or list of those - The regression type to check. - - n_levels : `int` - The number of pyramid levels. - - Returns - ------- - regression_type_list : `list` - A list of regression types that has length ``n_levels``. - """ - try: - return checks.check_list_callables(regression_type, n_levels) - except ValueError: - raise ValueError("regression_type must be a callable or a list of " - "{} callables".format(n_levels)) - - -def check_n_permutations(n_permutations): - if n_permutations < 1: - raise ValueError("n_permutations must be > 0") - - -def apply_pyramid_on_images(generators, n_levels, verbose=False): - r""" - Exhausts the pyramid generators verbosely - """ - all_images = [] - for j in range(n_levels): - - if verbose: - level_str = '- Apply pyramid: ' - if n_levels > 1: - level_str = '- Apply pyramid: [Level {} - '.format(j + 1) - - level_images = [] - for c, g in enumerate(generators): - if verbose: - print_dynamic( - '{}Computing feature space/rescaling - {}'.format( - level_str, - progress_bar_str((c + 1.) / len(generators), - show_bar=False))) - level_images.append(next(g)) - all_images.append(level_images) - if verbose: - print_dynamic('- Apply pyramid: Done\n') - return all_images - - -class SDTrainer(DeformableModel): - r""" - Mixin for Supervised Descent Trainers. - - Parameters - ---------- - regression_type : `callable`, or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - Examples of such callables can be found in :ref:`regression_callables`. - regression_features :`` None`` or `callable` or `[callable]`, optional - The features that are used during the regression. - - If `list`, a regression feature is defined per level. - - If not list or list with length ``1``, the specified regression feature - will be used for all levels. - - Depending on the :map:`SDTrainer` object, this parameter can take - different types. - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the - training shapes. - - Returns - ------- - fitter : :map:`MultilevelFitter` - The fitter object. - - Raises - ------ - ValueError - ``regression_type`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements - ValueError - n_levels must be `int` > ``0`` - ValueError - ``downscale`` must be >= ``1`` - ValueError - ``n_perturbations`` must be > 0 - ValueError - ``features`` must be a `string` or a `function` or a list of those - containing ``1`` or ``n_levels`` elements - """ - __metaclass__ = abc.ABCMeta - - def __init__(self, regression_type=mlr, regression_features=None, - features=no_op, n_levels=3, downscale=1.2, noise_std=0.04, - rotation=False, n_perturbations=10): - features = checks.check_features(features, n_levels) - DeformableModel.__init__(self, features) - - # general deformable model checks - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - - # SDM specific checks - regression_type_list = check_regression_type(regression_type, - n_levels) - regression_features = check_regression_features(regression_features, - n_levels) - check_n_permutations(n_perturbations) - - # store parameters - self.regression_type = regression_type_list - self.regression_features = regression_features - self.n_levels = n_levels - self.downscale = downscale - self.noise_std = noise_std - self.rotation = rotation - self.n_perturbations = n_perturbations - - def train(self, images, group=None, label=None, verbose=False, **kwargs): - r""" - Trains a Supervised Descent Regressor given a list of landmarked - images. - - Parameters - ---------- - images: list of :map:`MaskedImage` - The set of landmarked images from which to build the SD. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label: `string`, optional - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - verbose: `boolean`, optional - Flag that controls information and progress printing. - """ - if verbose: - print_dynamic('- Computing reference shape') - self.reference_shape = self._compute_reference_shape(images, group, - label) - # store number of training images - self.n_training_images = len(images) - - # normalize the scaling of all images wrt the reference_shape size - self._rescale_reference_shape() - normalized_images = self._normalization_wrt_reference_shape( - images, group, label, self.reference_shape, verbose=verbose) - - # create pyramid - generators = create_pyramid(normalized_images, self.n_levels, - self.downscale, self.features, - verbose=verbose) - - # get feature images of all levels - images = apply_pyramid_on_images(generators, self.n_levels, - verbose=verbose) - - # this .reverse sets the lowest resolution as the first level - images.reverse() - - # extract the ground truth shapes - gt_shapes = [[i.landmarks[group][label] for i in img] - for img in images] - - # build the regressors - if verbose: - if self.n_levels > 1: - print_dynamic('- Building regressors for each of the {} ' - 'pyramid levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building regressors\n') - - regressors = [] - # for each pyramid level (low --> high) - for j, (level_images, level_gt_shapes) in enumerate(zip(images, - gt_shapes)): - if verbose: - if self.n_levels == 1: - print_dynamic('\n') - elif self.n_levels > 1: - print_dynamic('\nLevel {}:\n'.format(j + 1)) - - # build regressor - trainer = self._set_regressor_trainer(j) - if j == 0: - regressor = trainer.train(level_images, level_gt_shapes, - verbose=verbose, **kwargs) - else: - regressor = trainer.train(level_images, level_gt_shapes, - level_shapes, verbose=verbose, - **kwargs) - - if verbose: - print_dynamic('- Perturbing shapes...') - level_shapes = trainer.perturb_shapes(gt_shapes[0]) - - regressors.append(regressor) - count = 0 - total = len(regressors) * len(images[0]) * len(level_shapes[0]) - for k, r in enumerate(regressors): - - test_images = images[k] - test_gt_shapes = gt_shapes[k] - - fitting_results = [] - for (i, gt_s, level_s) in zip(test_images, test_gt_shapes, - level_shapes): - fr_list = [] - for ls in level_s: - parameters = r.get_parameters(ls) - fr = r.fit(i, parameters) - fr.gt_shape = gt_s - fr_list.append(fr) - count += 1 - - fitting_results.append(fr_list) - if verbose: - print_dynamic('- Fitting shapes: {}'.format( - progress_bar_str((count + 1.) / total, - show_bar=False))) - - level_shapes = [[Scale(self.downscale, - n_dims=self.reference_shape.n_dims - ).apply(fr.final_shape) - for fr in fr_list] - for fr_list in fitting_results] - - if verbose: - print_dynamic('- Fitting shapes: computing mean error...') - mean_error = np.mean(np.array([fr.final_error() - for fr_list in fitting_results - for fr in fr_list])) - if verbose: - print_dynamic("- Fitting shapes: mean error " - "is {0:.6f}.\n".format(mean_error)) - - return self._build_supervised_descent_fitter(regressors) - - @classmethod - def _normalization_wrt_reference_shape(cls, images, group, label, - reference_shape, verbose=False): - r""" - Normalizes the images sizes with respect to the reference - shape (mean shape) scaling. This step is essential before building a - deformable model. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the model. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - reference_shape : :map:`PointCloud` - The reference shape that is used to resize all training images to - a consistent object size. - - verbose: bool, optional - Flag that controls information and progress printing. - - Returns - ------- - normalized_images : :map:`MaskedImage` list - A list with the normalized images. - """ - normalized_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic('- Normalizing images size: {}'.format( - progress_bar_str((c + 1.) / len(images), - show_bar=False))) - normalized_images.append(i.rescale_to_reference_shape( - reference_shape, group=group, label=label)) - - if verbose: - print_dynamic('- Normalizing images size: Done\n') - return normalized_images - - @abc.abstractmethod - def _compute_reference_shape(self, images, group, label): - r""" - Function that computes the reference shape, given a set of images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape computed based on the given images shapes. - """ - pass - - def _rescale_reference_shape(self): - r""" - Function that rescales the reference shape w.r.t. to - ``normalization_diagonal`` parameter. - """ - pass - - @abc.abstractmethod - def _set_regressor_trainer(self, **kwargs): - r""" - Function that sets the regression object to be one from - :map:`RegressorTrainer`, - """ - pass - - @abc.abstractmethod - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object. - - Parameters - ---------- - regressors : list of :map:`RegressorTrainer` - The list of regressors. - - Returns - ------- - fitter : :map:`SDMFitter` - The SDM fitter object. - """ - pass - - -class SDMTrainer(SDTrainer): - r""" - Class that trains Supervised Descent Method using Non-Parametric - Regression. - - Parameters - ---------- - regression_type : `callable` or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - The callable should be one of the methods defined in - :ref:`regression_callables` - - regression_features: ``None`` or `callable` or `[callable]`, optional - If list of length ``n_levels``, then a feature is defined per level. - - If not a list, then the specified feature will be applied to all - pyramid levels. - - Per level: - If ``None``, no features are extracted, thus specified - ``features`` is used in the regressor. - - It is recommended to set the desired features using this option, - leaving ``features`` equal to :map:`no_op`. This means that the - images will remain in the intensities space and the features will - be extracted by the regressor. - - patch_shape: tuple of `int` - The shape of the patches used by the SDM. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - initial shape. - - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the initial shape. - - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the shapes. - - normalization_diagonal : `int` >= ``20``, optional - During training, all images are rescaled to ensure that the scale of - their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the normalization_diagonal - value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - Raises - ------ - ValueError - ``regression_features`` must be ``None`` or a `string` or a `function` - or a list of those containing 1 or ``n_level`` elements - """ - def __init__(self, regression_type=mlr, regression_features=sparse_hog, - patch_shape=(16, 16), features=no_op, n_levels=3, - downscale=1.5, noise_std=0.04, - rotation=False, n_perturbations=10, - normalization_diagonal=None): - super(SDMTrainer, self).__init__( - regression_type=regression_type, - regression_features=regression_features, - features=features, n_levels=n_levels, downscale=downscale, - noise_std=noise_std, rotation=rotation, - n_perturbations=n_perturbations) - self.patch_shape = patch_shape - self.normalization_diagonal = normalization_diagonal - - def _compute_reference_shape(self, images, group, label): - r""" - Function that computes the reference shape, given a set of images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape computed based on the given images. - """ - shapes = [i.landmarks[group][label] for i in images] - return mean_pointcloud(shapes) - - def _rescale_reference_shape(self): - r""" - Function that rescales the reference shape w.r.t. to - ``normalization_diagonal`` parameter. - """ - if self.normalization_diagonal: - x, y = self.reference_shape.range() - scale = self.normalization_diagonal / np.sqrt(x**2 + y**2) - Scale(scale, self.reference_shape.n_dims).apply_inplace( - self.reference_shape) - - def _set_regressor_trainer(self, level): - r""" - Function that sets the regression class to be the - :map:`NonParametricRegressorTrainer`. - - Parameters - ---------- - level : `int` - The scale level. - - Returns - ------- - trainer : :map:`NonParametricRegressorTrainer` - The regressor object. - """ - return NonParametricRegressorTrainer( - self.reference_shape, regression_type=self.regression_type[level], - regression_features=self.regression_features[level], - patch_shape=self.patch_shape, noise_std=self.noise_std, - rotation=self.rotation, n_perturbations=self.n_perturbations) - - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object. - - Parameters - ---------- - regressors : list of :map:`RegressorTrainer` - The list of regressors. - - Returns - ------- - fitter : :map:`SDMFitter` - The SDM fitter object. - """ - return SDMFitter(regressors, self.n_training_images, self.features, - self.reference_shape, self.downscale) - - -class SDAAMTrainer(SDTrainer): - r""" - Class that trains Supervised Descent Regressor for a given Active - Appearance Model, thus uses Parametric Regression. - - Parameters - ---------- - aam : :map:`AAM` - The trained AAM object. - regression_type : `callable`, or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - Examples of such callables can be found in :ref:`regression_callables`. - regression_features: `function` or list of those, optional - If list of length ``n_levels``, then a feature is defined per level. - - If not a list or a list with length ``1``, then the specified feature - will be applied to all pyramid levels. - - The callable should be one of the methods defined in - :ref:`parametricfeatures`. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the - training shapes. - update : {'additive', 'compositional'} - Defines the way that the warp will be updated. - md_transform: :map:`ModelDrivenTransform`, optional - The model driven transform class to be used. - n_shape : `int` > ``1`` or ``0`` <= `float` <= ``1`` or ``None``, or a list of those, optional - The number of shape components to be used per fitting level. - - If list of length ``n_levels``, then a number of components is defined - per level. The first element of the list corresponds to the lowest - pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - components will be used for all levels. - - Per level: - If ``None``, all the available shape components - (``n_active_components``)will be used. - - If `int` > ``1``, a specific number of shape components is - specified. - - If ``0`` <= `float` <= ``1``, it specifies the variance percentage - that is captured by the components. - n_appearance : `int` > ``1`` or ``0`` <= `float` <= ``1`` or ``None``, or a list of those, optional - The number of appearance components to be used per fitting level. - - If list of length ``n_levels``, then a number of components is defined - per level. The first element of the list corresponds to the lowest - pyramidal level and so on. - - If not a list or a list with length 1, then the specified number of - components will be used for all levels. - - Per level: - If ``None``, all the available appearance components - (``n_active_components``) will be used. - - If `int > ``1``, a specific number of appearance components is - specified. - - If ``0`` <= `float` <= ``1``, it specifies the variance percentage - that is captured by the components. - - Raises - ------- - ValueError - n_shape can be an integer or a float or None or a list containing 1 - or ``n_levels`` of those - ValueError - n_appearance can be an integer or a float or None or a list containing - 1 or ``n_levels`` of those - ValueError - ``regression_features`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements - """ - def __init__(self, aam, regression_type=mlr, regression_features=weights, - noise_std=0.04, rotation=False, n_perturbations=10, - update='compositional', md_transform=OrthoMDTransform, - n_shape=None, n_appearance=None): - super(SDAAMTrainer, self).__init__( - regression_type=regression_type, - regression_features=regression_features, - features=aam.features, n_levels=aam.n_levels, - downscale=aam.downscale, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.aam = aam - self.update = update - self.md_transform = md_transform - # hard coded for now as this is the only supported configuration. - self.global_transform = DifferentiableAlignmentSimilarity - - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.aam.n_levels > 1: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.aam.n_levels: - for sm, n in zip(self.aam.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float, ' - 'an integer or float list containing 1 ' - 'or {} elements or else ' - 'None'.format(self.aam.n_levels)) - - # check n_appearance parameter - if n_appearance is not None: - if type(n_appearance) is int or type(n_appearance) is float: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: - for am, n in zip(self.aam.appearance_models, n_appearance): - am.n_active_components = n - else: - raise ValueError('n_appearance can be an integer or a float, ' - 'an integer or float list containing 1 ' - 'or {} elements or else ' - 'None'.format(self.aam.n_levels)) - - def _compute_reference_shape(self, images, group, label): - r""" - Function that returns the reference shape computed during AAM building. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape computed based on. - """ - return self.aam.reference_shape - - def _normalize_object_size(self, images, group, label): - r""" - Function that normalizes the images sizes with respect to the reference - shape (mean shape) scaling. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the model. - - group : `string` - The key of the landmark set that should be used. If ```None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - normalized_images : :map:`MaskedImage` list - A list with the normalized images. - """ - return [i.rescale_to_reference_shape(self.reference_shape, - group=group, label=label) - for i in images] - - def _set_regressor_trainer(self, level): - r""" - Function that sets the regression class to be the - :map:`ParametricRegressorTrainer`. - - Parameters - ---------- - level : `int` - The scale level. - - Returns - ------- - trainer: :map:`ParametricRegressorTrainer` - The regressor object. - """ - am = self.aam.appearance_models[level] - sm = self.aam.shape_models[level] - - if self.md_transform is not ModelDrivenTransform: - md_transform = self.md_transform( - sm, self.aam.transform, self.global_transform, - source=am.mean().landmarks['source'].lms) - else: - md_transform = self.md_transform( - sm, self.aam.transform, - source=am.mean().landmarks['source'].lms) - - return ParametricRegressorTrainer( - am, md_transform, self.reference_shape, - regression_type=self.regression_type[level], - regression_features=self.regression_features[level], - update=self.update, noise_std=self.noise_std, - rotation=self.rotation, n_perturbations=self.n_perturbations) - - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object for AAMs. - - Parameters - ---------- - regressors : :map:`RegressorTrainer` - The regressor to build with. - - Returns - ------- - fitter : :map:`SDAAMFitter` - The SDM fitter object. - """ - return SDAAMFitter(self.aam, regressors, self.n_training_images) - - -class SDCLMTrainer(SDTrainer): - r""" - Class that trains Supervised Descent Regressor for a given Constrained - Local Model, thus uses Semi Parametric Classifier-Based Regression. - - Parameters - ---------- - clm : :map:`CLM` - The trained CLM object. - regression_type : `callable`, or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - Examples of such callables can be found in :ref:`regression_callables`. - noise_std: float, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the - training shapes. - pdm_transform : :map:`ModelDrivenTransform`, optional - The point distribution transform class to be used. - n_shape : `int` > ``1`` or ``0`` <= `float` <= ``1`` or ``None``, or a list of those, optional - The number of shape components to be used per fitting level. - - If list of length ``n_levels``, then a number of components is defined - per level. The first element of the list corresponds to the lowest - pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - components will be used for all levels. - - Per level: - If ``None``, all the available shape components - (``n_active_components``) will be used. - - If `int` > ``1``, a specific number of shape components is - specified. - - If ``0`` <= `float` <= ``1``, it specifies the variance percentage - that is captured by the components. - - Raises - ------- - ValueError - ``n_shape`` can be an integer or a `float` or ``None`` or a list - containing ``1`` or ``n_levels`` of those. - """ - def __init__(self, clm, regression_type=mlr, noise_std=0.04, - rotation=False, n_perturbations=10, pdm_transform=OrthoPDM, - n_shape=None): - super(SDCLMTrainer, self).__init__( - regression_type=regression_type, - regression_features=[None] * clm.n_levels, - features=clm.features, n_levels=clm.n_levels, - downscale=clm.downscale, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.clm = clm - self.patch_shape = clm.patch_shape - self.pdm_transform = pdm_transform - # hard coded for now as this is the only supported configuration. - self.global_transform = DifferentiableAlignmentSimilarity - - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.clm.n_levels > 1: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.clm.n_levels: - for sm, n in zip(self.clm.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None' - 'or a list containing 1 or {} of ' - 'those'.format(self.clm.n_levels)) - - def _compute_reference_shape(self, images, group, label): - r""" - Function that returns the reference shape computed during CLM building. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape. - """ - return self.clm.reference_shape - - def _set_regressor_trainer(self, level): - r""" - Function that sets the regression class to be the - :map:`SemiParametricClassifierBasedRegressorTrainer` - - Parameters - ---------- - level : `int` - The scale level. - - Returns - ------- - trainer: :map:`SemiParametricClassifierBasedRegressorTrainer` - The regressor object. - """ - clfs = self.clm.classifiers[level] - sm = self.clm.shape_models[level] - - if self.pdm_transform is not PDM: - pdm_transform = self.pdm_transform(sm, self.global_transform) - else: - pdm_transform = self.pdm_transform(sm) - - return SemiParametricClassifierBasedRegressorTrainer( - clfs, pdm_transform, self.reference_shape, - regression_type=self.regression_type[level], - patch_shape=self.patch_shape, update='additive', - noise_std=self.noise_std, rotation=self.rotation, - n_perturbations=self.n_perturbations) - - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object for CLMs. - - Parameters - ---------- - regressors : :map:`RegressorTrainer` - Regressor to train with. - - Returns - ------- - fitter : :map:`SDCLMFitter` - The SDM fitter object. - """ - return SDCLMFitter(self.clm, regressors, self.n_training_images) From 564fa52c569debbb3e61762da322fada07e3046f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 22 Jun 2015 18:21:36 +0100 Subject: [PATCH 066/423] Changed old ...inplace calls on lk/fitter.py and fitter.py --- menpofit/fitter.py | 3 +-- menpofit/lk/fitter.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 3d8ce11..3ca7817 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -157,8 +157,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # if specified, crop the image if crop_image: - image = image.copy() - image.crop_to_landmarks_proportion_inplace(crop_image, + image = image.crop_to_landmarks_proportion(crop_image, group='initial_shape') # rescale image wrt the scale factor between reference_shape and diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 73f6eb1..68f692b 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -62,10 +62,7 @@ def scale_features(self): return self._scale_features def _prepare_template(self, template, group=None, label=None): - # copy template - template = template.copy() - - template = template.crop_to_landmarks_inplace(group=group, label=label) + template = template.crop_to_landmarks(group=group, label=label) template = template.as_masked() # rescale template to diagonal range From 9e05caa4c958bd2cd0755cef84fe60ae6e4cc054 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 23 Jun 2015 11:56:54 +0100 Subject: [PATCH 067/423] Update initialization method for ModelFitter --- menpofit/fitter.py | 106 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 3ca7817..b230e2b 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -2,7 +2,8 @@ import abc import numpy as np from menpo.shape import PointCloud -from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity +from menpo.transform import ( + Scale, Similarity, AlignmentAffine, AlignmentSimilarity) import menpofit.checks as checks @@ -317,23 +318,80 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - # TODO: Bounding boxes should be PointGraphs - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, - rotation=False): - transform = noisy_align(AlignmentSimilarity, - self.reference_bounding_box, bounding_box, - noise_std=noise_std, rotation=rotation) + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.5): + transform = noisy_params_alignment_similarity( + self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): + def noisy_shape_from_shape(self, shape, noise_std=0.5): return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std, rotation=rotation) + shape.bounding_box(), noise_std=noise_std) -# TODO: document me! -def noisy_align(alignment_transform_cls, source, target, noise_std=0.1, - **kwargs): +def noisy_params_alignment_similarity(source, target, noise_std=0.5): r""" + Constructs and perturbs the optimal similarity transform between source + and target by adding white noise to its parameters. + Parameters + ---------- + source: :class:`menpo.shape.PointCloud` + The source pointcloud instance used in the alignment + target: :class:`menpo.shape.PointCloud` + The target pointcloud instance used in the alignment + noise_std: float or triplet of floats, optional + The standard deviation of the white noise. If float the same amount + of noise is applied to the scale, rotation and translation + parameters of the true similarity transform. If triplet of + floats, the first, second and third elements denote the amount of + noise to be applied to the scale, rotation and translation + parameters respectively. + Returns + ------- + noisy_transform : :class: `menpo.transform.Similarity` + The noisy Similarity Transform + """ + if isinstance(noise_std, float): + noise_std = [noise_std] * 3 + elif len(noise_std) == 1: + noise_std *= 3 + + transform = AlignmentSimilarity(source, target, rotation=True) + parameters = transform.as_vector() + + scale = noise_std[0] * parameters[0] + rotation = noise_std[1] * parameters[1] + translation = noise_std[2] * target.range() + + noise = (([scale, rotation] + list(translation)) * + np.random.randn(transform.n_parameters)) + return Similarity.init_identity(source.n_dims).from_vector( + parameters + noise) + + +def noisy_target_alignment_transform(source, target, + alignment_transform_cls=AlignmentAffine, + noise_std=0.1, **kwargs): + r""" + Constructs and the optimal alignment transform between the source and + a noisy version of the target obtained by adding white noise to each of + its points. + + Parameters + ---------- + source: :class:`menpo.shape.PointCloud` + The source pointcloud instance used in the alignment + target: :class:`menpo.shape.PointCloud` + The target pointcloud instance used in the alignment + alignment_transform_cls: :class:`menpo.transform.Alignment`, optional + The alignment transform class used to perform the alignment. + noise_std: float or triplet of floats, optional + The standard deviation of the white noise to be added to each one of + the target points. + + Returns + ------- + noisy_transform : :class: `menpo.transform.Alignment` + The noisy Similarity Transform """ noise = noise_std * target.range() * np.random.randn(target.n_points, target.n_dims) @@ -341,11 +399,27 @@ def noisy_align(alignment_transform_cls, source, target, noise_std=0.1, return alignment_transform_cls(source, noisy_target, **kwargs) -# TODO: document me! -def align_shape_with_bounding_box(alignment_transform_cls, shape, - bounding_box, **kwargs): +def align_shape_with_bounding_box(shape, bounding_box, + alignment_transform_cls=AlignmentSimilarity, + **kwargs): r""" + Aligns the shape with the bounding box using a particular ali . + + Parameters + ---------- + source: :class:`menpo.shape.PointCloud` + The shape instance used in the alignment. + bounding_box: :class:`menpo.shape.PointCloud` + The bounding box instance used in the alignment. + alignment_transform_cls: :class:`menpo.transform.Alignment`, optional + The class of the alignment transform used to perform the alignment. + + Returns + ------- + noisy_transform : :class: `menpo.transform.Alignment` + The noisy Alignment Transform """ shape_bb = shape.bounding_box() - return alignment_transform_cls(shape_bb, bounding_box, **kwargs) + transform = alignment_transform_cls(shape_bb, bounding_box, **kwargs) + return transform.apply(shape) From 55cd5a0c7d79fa6c54748dfb50ca64f9f9a21fd7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 23 Jun 2015 13:42:19 +0100 Subject: [PATCH 068/423] Make Linear(AAM, ATM) fitting results return point graph shapes --- menpofit/aam/fitter.py | 4 ++-- menpofit/atm/fitter.py | 2 +- menpofit/transform/modeldriven.py | 11 ++++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index de7bed1..c6c477b 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -83,7 +83,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self.aam.n_landmarks) + sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( LinearLKAAMInterface, am, md_transform, sampling=s, @@ -140,7 +140,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self.aam.n_landmarks) + sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = cr_algorithm_cls( CRLinearAAMInterface, am, md_transform, sampling=s, diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 53f8820..702a008 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -36,7 +36,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): type(self.atm) is LinearPatchATM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self.atm.n_landmarks) + sm, self.atm.reference_shape) # set up algorithm using linear aam interface algorithm = algorithm_cls(LKLinearATMInterface, wt, md_transform, sampling=sampling, diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index fc5fd8a..0238db5 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -529,21 +529,26 @@ def __init__(self, model, transform_cls, source=None): class LinearOrthoMDTransform(OrthoPDM, Transform): r""" """ - def __init__(self, model, n_landmarks): + def __init__(self, model, sparse_instance): super(LinearOrthoMDTransform, self).__init__(model) - self.n_landmarks = n_landmarks + self._sparse_instance = sparse_instance self.W = np.vstack((self.similarity_model.components, self.model.components)) V = self.W[:, :self.n_dims*self.n_landmarks] self.pinv_V = np.linalg.pinv(V) + @property + def n_landmarks(self): + return self._sparse_instance.n_points + @property def dense_target(self): return PointCloud(self.target.points[self.n_landmarks:]) @property def sparse_target(self): - return PointCloud(self.target.points[:self.n_landmarks]) + sparse_target = PointCloud(self.target.points[:self.n_landmarks]) + return self._sparse_instance.from_vector(sparse_target.as_vector()) def set_target(self, target): if target.n_points == self.n_landmarks: From 64379e1de658bc3f3dffb60c85f2dd3429060b85 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 29 Jun 2015 12:09:12 +0100 Subject: [PATCH 069/423] Update checks for AAMs, ATMs and LK --- menpofit/aam/builder.py | 22 ++++---- menpofit/aam/fitter.py | 31 +++++------ menpofit/atm/builder.py | 11 ++-- menpofit/atm/fitter.py | 18 +++---- menpofit/checks.py | 116 +++++++++++++++------------------------- menpofit/fitter.py | 34 +----------- menpofit/lk/fitter.py | 52 +++++++----------- 7 files changed, 106 insertions(+), 178 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index ff8a783..9ef763b 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -13,8 +13,6 @@ DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -# TODO: fix features checker -# TODO: implement checker for conflict between features and scale_features # TODO: document me! class AAMBuilder(object): r""" @@ -129,8 +127,9 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -190,7 +189,7 @@ def build(self, images, group=None, label=None, verbose=False): # obtain image representation if j == 0: # compute features at highest level - feature_images = compute_features(images, self.features, + feature_images = compute_features(images, self.features[j], level_str=level_str, verbose=verbose) level_images = feature_images @@ -203,7 +202,8 @@ def build(self, images, group=None, label=None, verbose=False): # scale images and compute features at other levels scaled_images = scale_images(images, s, level_str=level_str, verbose=verbose) - level_images = compute_features(scaled_images, self.features, + level_images = compute_features(scaled_images, + self.features[j], level_str=level_str, verbose=verbose) @@ -381,8 +381,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -523,8 +524,9 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -675,8 +677,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -830,8 +833,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index c6c477b..a4ee35e 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -18,20 +18,10 @@ class AAMFitter(ModelFitter): r""" """ - def __init__(self, aam, n_shape=None, n_appearance=None): - super(AAMFitter, self).__init__(aam) - self._algorithms = [] - self._check_n_shape(n_shape) - self._check_n_appearance(n_appearance) - @property def aam(self): return self._model - @property - def algorithms(self): - return self._algorithms - def _check_n_appearance(self, n_appearance): if n_appearance is not None: if type(n_appearance) is int or type(n_appearance) is float: @@ -60,8 +50,10 @@ class LKAAMFitter(AAMFitter): """ def __init__(self, aam, n_shape=None, n_appearance=None, lk_algorithm_cls=AIC, sampling=None, **kwargs): - super(LKAAMFitter, self).__init__( - aam, n_shape=n_shape, n_appearance=n_appearance) + self._model = aam + self.algorithms = [] + self._check_n_shape(n_shape) + self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self._set_up(lk_algorithm_cls, sampling, **kwargs) @@ -105,7 +97,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) # TODO: document me! @@ -115,8 +107,10 @@ class CRAAMFitter(AAMFitter): def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, max_iters=6, **kwargs): - super(CRAAMFitter, self).__init__( - aam, n_shape=n_shape, n_appearance=n_appearance) + self._model = aam + self.algorithms = [] + self._check_n_shape(n_shape) + self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self.n_perturbations = n_perturbations self.max_iters = checks.check_max_iters(max_iters, self.n_levels) @@ -163,7 +157,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) def train(self, images, group=None, label=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam @@ -172,7 +166,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): if self.scale_features: # compute features at highest level - feature_images = compute_features(images, self.features, + feature_images = compute_features(images, self.features[0], verbose=verbose) # for each pyramid level (low --> high) @@ -195,7 +189,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): # scale images and compute features at other levels scaled_images = scale_images(images, s, level_str=level_str, verbose=verbose) - level_images = compute_features(scaled_images, self.features, + level_images = compute_features(scaled_images, + self.features[j], level_str=level_str, verbose=verbose) diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 523ecdb..42c95f9 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -111,6 +111,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -180,7 +181,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): if j == 0 or self.scale_shapes: if j == 0: level_shapes = shapes - level_reference_shape= reference_shape + level_reference_shape = reference_shape else: scale_transform = Scale(scale_factor=s, n_dims=2) level_shapes = [scale_transform.apply(s) for s in shapes] @@ -202,7 +203,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): # obtain template representation if j == 0: # compute features at highest level - feature_template = self.features(template) + feature_template = self.features[j](template) level_template = feature_template elif self.scale_features: # scale features at other levels @@ -210,7 +211,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): else: # scale template and compute features at other levels scaled_template = template.rescale(s) - level_template = self.features(scaled_template) + level_template = self.features[j](scaled_template) # extract potentially rescaled template shape level_template_shape = level_template.landmarks[group][label] @@ -349,6 +350,7 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -478,6 +480,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -613,6 +616,7 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -742,6 +746,7 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 702a008..3cd7d97 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -14,11 +14,15 @@ class LKATMFitter(ModelFitter): """ def __init__(self, atm, algorithm_cls=IC, n_shape=None, sampling=None, **kwargs): - super(LKATMFitter, self).__init__(atm) - self._algorithms = [] + self._model = atm + self.algorithms = [] self._check_n_shape(n_shape) self._set_up(algorithm_cls, sampling, **kwargs) + @property + def atm(self): + return self._model + def _set_up(self, algorithm_cls, sampling, **kwargs): for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, self.atm.shape_models)): @@ -58,15 +62,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): LinearPatchATM, PartsATM)) # append algorithms to list - self._algorithms.append(algorithm) - - @property - def atm(self): - return self._model - - @property - def algorithms(self): - return self._algorithms + self.algorithms.append(algorithm) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): diff --git a/menpofit/checks.py b/menpofit/checks.py index 2ac6e66..0c5dc0c 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,52 +1,5 @@ import numpy as np -from menpofit.base import is_pyramid_on_features - - -def check_features(features, n_levels): - r""" - Checks the feature type per level. - - Parameters - ---------- - features : callable or list of callables - The features to apply to the images. - n_levels : int - The number of pyramid levels. - - Returns - ------- - feature_list : list - A list of feature function. - """ - # Firstly, make sure we have a list of callables of the right length - if is_pyramid_on_features(features): - return features - else: - try: - all_callables = check_list_callables(features, n_levels, - allow_single=False) - except ValueError: - raise ValueError("features must be a callable or a list of " - "{} callables".format(n_levels)) - return all_callables - - -def check_list_callables(callables, n_callables, allow_single=True): - if not isinstance(callables, list): - if allow_single: - # expand to a list of callables for them - callables = [callables] * n_callables - else: - raise ValueError("Expected a list of callables " - "(allow_single=False)") - # must have a list by now - for c in callables: - if not callable(c): - raise ValueError("All items must be callables") - if len(callables) != n_callables: - raise ValueError("List of callables must be {} " - "long".format(n_callables)) - return callables +import warnings def check_diagonal(diagonal): @@ -73,6 +26,47 @@ def check_scales(scales): "int/float") +def check_features(features, n_levels): + r""" + Checks the feature type per level. + + Parameters + ---------- + features : callable or list of callables + The features to apply to the images. + n_levels : int + The number of pyramid levels. + + Returns + ------- + feature_list : list + A list of feature function. + """ + if callable(features): + return [features] * n_levels + elif len(features) == 1 and np.alltrue([callable(f) for f in features]): + return list(features) * n_levels + elif len(features) == n_levels and np.alltrue([callable(f) + for f in features]): + return list(features) + else: + raise ValueError("features must be a callable or a list/tuple of " + "callables with the same length as scales") + + +# TODO: document me! +def check_scale_features(scale_features, features): + r""" + """ + if np.alltrue([f == features[0] for f in features]): + return scale_features + else: + warnings.warn('scale_features has been automatically set to False ' + 'because different types of features are used at each ' + 'level.') + return False + + # TODO: document me! def check_patch_shape(patch_shape, n_levels): if len(patch_shape) == 2 and isinstance(patch_shape[0], int): @@ -98,7 +92,7 @@ def check_max_components(max_components, n_levels, var_name): str_error = ("{} must be None or an int > 0 or a 0 <= float <= 1 or " "a list of those containing 1 or {} elements").format( var_name, n_levels) - if not isinstance(max_components, list): + if not isinstance(max_components, (list, tuple)): max_components_list = [max_components] * n_levels elif len(max_components) == 1: max_components_list = [max_components[0]] * n_levels @@ -146,27 +140,3 @@ def check_sampling(sampling, n_levels): 'None'.format(n_levels)) return sampling - -# def check_n_levels(n_levels): -# r""" -# Checks the number of pyramid levels - must be int > 0. -# """ -# if not isinstance(n_levels, int) or n_levels < 1: -# raise ValueError("n_levels must be int > 0") -# -# -# def check_downscale(downscale): -# r""" -# Checks the downscale factor of the pyramid that must be >= 1. -# """ -# if downscale < 1: -# raise ValueError("downscale must be >= 1") -# -# -# def check_boundary(boundary): -# r""" -# Checks the boundary added around the reference shape that must be -# int >= 0. -# """ -# if not isinstance(boundary, int) or boundary < 0: -# raise ValueError("boundary must be >= 0") diff --git a/menpofit/fitter.py b/menpofit/fitter.py index b230e2b..4ec63ef 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.shape import PointCloud from menpo.transform import ( @@ -20,26 +19,6 @@ def n_levels(self): """ return len(self.scales) - @abc.abstractproperty - def algorithms(self): - pass - - # @abc.abstractproperty - # def reference_shape(self): - # pass - # - # @abc.abstractproperty - # def features(self): - # pass - # - # @abc.abstractproperty - # def scales(self): - # pass - # - # @abc.abstractproperty - # def scale_features(self): - # pass - def fit(self, image, initial_shape, max_iters=50, gt_shape=None, crop_image=0.5, **kwargs): r""" @@ -174,14 +153,14 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, for j, s in enumerate(scales): if j == 0: # compute features at highest level - feature_image = self.features(image) + feature_image = self.features[j](image) elif self.scale_features: # scale features at other levels feature_image = images[0].rescale(s) else: # scale image and compute features at other levels scaled_image = image.rescale(s) - feature_image = self.features(scaled_image) + feature_image = self.features[j](scaled_image) images.append(feature_image) images.reverse() @@ -247,20 +226,11 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, return algorithm_results - @abc.abstractmethod - def _fitter_result(self, image, algorithm_results, affine_correction, - gt_shape=None): - pass - -# TODO: correctly implement initialization from bounding box # TODO: document me! class ModelFitter(MultiFitter): r""" """ - def __init__(self, model): - self._model = model - @property def reference_shape(self): r""" diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 68f692b..e2a91e0 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -1,7 +1,8 @@ from __future__ import division from menpo.feature import no_op from menpofit.transform import DifferentiableAlignmentAffine -from menpofit.fitter import MultiFitter, noisy_align +from menpofit.fitter import MultiFitter, noisy_target_alignment_transform +from menpofit import checks from .algorithm import IC from .residual import SSD, FourierSSD from .result import LKFitterResult @@ -15,19 +16,25 @@ def __init__(self, template, group=None, label=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, scales=(1, .5), scale_features=True, algorithm_cls=IC, residual_cls=SSD, **kwargs): - self._features = features + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) + # set parameters + self.features = features self.transform_cls = transform_cls self.diagonal = diagonal - self._scales = list(scales) - self._scales.reverse() - self._scale_features = scale_features + self.scales = list(scales) + self.scales.reverse() + self.scale_features = scale_features self.templates, self.sources = self._prepare_template( template, group=group, label=label) - self._reference_shape = self.sources[0] + self.reference_shape = self.sources[0] - self._algorithms = [] + self.algorithms = [] for j, (t, s) in enumerate(zip(self.templates, self.sources)): transform = self.transform_cls(s, s) if ('kernel_func' in kwargs and @@ -39,27 +46,7 @@ def __init__(self, template, group=None, label=None, features=no_op, else: residual = residual_cls() algorithm = algorithm_cls(t, transform, residual, **kwargs) - self._algorithms.append(algorithm) - - @property - def algorithms(self): - return self._algorithms - - @property - def reference_shape(self): - return self._reference_shape - - @property - def features(self): - return self._features - - @property - def scales(self): - return self._scales - - @property - def scale_features(self): - return self._scale_features + self.algorithms.append(algorithm) def _prepare_template(self, template, group=None, label=None): template = template.crop_to_landmarks(group=group, label=label) @@ -78,14 +65,14 @@ def _prepare_template(self, template, group=None, label=None): for j, s in enumerate(scales): if j == 0: # compute features at highest level - feature_template = self.features(template) + feature_template = self.features[j](template) elif self.scale_features: # scale features at other levels feature_template = templates[0].rescale(s) else: # scale image and compute features at other levels scaled_template = template.rescale(s) - feature_template = self.features(scaled_template) + feature_template = self.features[j](scaled_template) templates.append(feature_template) templates.reverse() @@ -95,8 +82,9 @@ def _prepare_template(self, template, group=None, label=None): return templates, sources def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): - transform = noisy_align(self.transform_cls, self.reference_shape, - gt_shape, noise_std=noise_std) + transform = noisy_target_alignment_transform( + self.transform_cls, self.reference_shape, gt_shape, + noise_std=noise_std) return transform.apply(self.reference_shape) def _fitter_result(self, image, algorithm_results, affine_correction, From a3ab4a1719b5285aa744b76e61adae18ba645592 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 29 Jun 2015 16:18:59 +0100 Subject: [PATCH 070/423] Fixed noise_std default value - Allow nise_std to be set in CRAAMs --- menpofit/aam/fitter.py | 5 +++-- menpofit/fitter.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a4ee35e..a9e0cad 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -106,13 +106,14 @@ class CRAAMFitter(AAMFitter): """ def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, - max_iters=6, **kwargs): + noise_std=0.05, max_iters=6, **kwargs): self._model = aam self.algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self.n_perturbations = n_perturbations + self.noise_std = noise_std self.max_iters = checks.check_max_iters(max_iters, self.n_levels) self._set_up(cr_algorithm_cls, sampling, **kwargs) @@ -203,7 +204,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape(gt_s) + p_s = self.noisy_shape_from_shape(gt_s, self.noise_std) perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 4ec63ef..06209da 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -288,17 +288,17 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.5): + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): transform = noisy_params_alignment_similarity( self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.5): + def noisy_shape_from_shape(self, shape, noise_std=0.05): return self.noisy_shape_from_bounding_box( shape.bounding_box(), noise_std=noise_std) -def noisy_params_alignment_similarity(source, target, noise_std=0.5): +def noisy_params_alignment_similarity(source, target, noise_std=0.05): r""" Constructs and perturbs the optimal similarity transform between source and target by adding white noise to its parameters. From 37dbdeecbcec081b7c61d6ddf874c3005a4f8a47 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 29 Jun 2015 23:33:11 +0100 Subject: [PATCH 071/423] Add psi parameter to CRAAM --- menpofit/aam/algorithm/cr.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/cr.py index f9c5d7c..26f78a7 100644 --- a/menpofit/aam/algorithm/cr.py +++ b/menpofit/aam/algorithm/cr.py @@ -326,28 +326,35 @@ def _compute_features2(self, image): class PSD(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None): - return _supervised_descent(features, deltas, gamma=gamma) + def _perform_regression(self, features, deltas, gamma=None, + dtype=np.float64): + regressor = _supervised_newton(features, deltas, gamma=gamma, + dtype=dtype) + regressor.R = self.project_out(regressor.R) + return regressor # TODO: document me! class PAJ(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None): - return _average_jacobian(features, deltas, gamma=gamma) + def _perform_regression(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + return _supervised_gauss_newton(features, deltas, gamma=gamma, + psi=psi, dtype=dtype) # TODO: document me! -class _supervised_descent(object): +class _supervised_newton(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression + def __init__(self, features, deltas, gamma=None, dtype=np.float64): + features = features.astype(dtype) + deltas = deltas.astype(dtype) XX = features.T.dot(features) XT = features.T.dot(deltas) if gamma: - XX += gamma * np.eye(features.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # descent direction self.R = np.linalg.solve(XX, XT) @@ -356,19 +363,23 @@ def __call__(self, features): # TODO: document me! -class _average_jacobian(object): +class _supervised_gauss_newton(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression + def __init__(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + features = features.astype(dtype) + deltas = deltas.astype(dtype) XX = deltas.T.dot(deltas) XT = deltas.T.dot(features) if gamma: - XX += gamma * np.eye(deltas.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # average Jacobian self.J = np.linalg.solve(XX, XT) # average Hessian self.H = self.J.dot(self.J.T) + if psi: + np.fill_diagonal(self.H, psi + np.diag(self.H)) # descent direction self.R = np.linalg.solve(self.H, self.J).T @@ -379,4 +390,3 @@ def __call__(self, features): # TODO: document me! def _compute_rmse(x1, x2): return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) - From c7e27a6434c266ae462963b972c72d5d6993c1d7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 02:26:58 +0100 Subject: [PATCH 072/423] Rename AAM, ATM, LK Fitters, Algorithms and Results. - Names are now explicit --- menpofit/aam/__init__.py | 21 ++- menpofit/aam/algorithm/__init__.py | 19 ++- menpofit/aam/algorithm/lk.py | 179 ++++++++++----------- menpofit/aam/algorithm/{cr.py => sd.py} | 200 ++++++++++++++++-------- menpofit/aam/fitter.py | 39 ++--- menpofit/aam/result.py | 8 +- menpofit/atm/__init__.py | 4 +- menpofit/atm/algorithm.py | 68 ++++---- menpofit/atm/fitter.py | 18 ++- menpofit/atm/result.py | 6 +- menpofit/lk/__init__.py | 8 +- menpofit/lk/algorithm.py | 39 ++--- menpofit/lk/fitter.py | 15 +- menpofit/lk/residual.py | 18 ++- menpofit/lk/result.py | 8 +- 15 files changed, 378 insertions(+), 272 deletions(-) rename menpofit/aam/algorithm/{cr.py => sd.py} (71%) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 32a3556..65d6960 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,12 +1,17 @@ from .builder import ( AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, LinearPatchAAMBuilder, PartsAAMBuilder) -from .fitter import LKAAMFitter, CRAAMFitter +from .fitter import LucasKanadeAAMFitter, SupervisedDescentAAMFitter from .algorithm import ( - PFC, PIC, - SFC, SIC, - AFC, AIC, - MAFC, MAIC, - WFC, WIC, - PSD, PAJ) - + ProjectOutForwardCompositional, ProjectOutInverseCompositional, + SimultaneousForwardCompositional, SimultaneousInverseCompositional, + AlternatingForwardCompositional, AlternatingInverseCompositional, + ModifiedAlternatingForwardCompositional, + ModifiedAlternatingInverseCompositional, + WibergForwardCompositional, WibergInverseCompositional, + SumOfSquaresSupervisedNewtonDescent, + SumOfSquaresSupervisedGaussNewtonDescent, + ProjectOutSupervisedNewtonDescent, + ProjectOutSupervisedGaussNewtonDescent, + AppearanceWeightsSupervisedNewtonDescent, + AppearanceWeightsSupervisedDescent) diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py index 3416b8c..40cf345 100644 --- a/menpofit/aam/algorithm/__init__.py +++ b/menpofit/aam/algorithm/__init__.py @@ -1,7 +1,14 @@ from .lk import ( - PFC, PIC, - SFC, SIC, - AFC, AIC, - MAFC, MAIC, - WFC, WIC) -from .cr import PSD, PAJ + ProjectOutForwardCompositional, ProjectOutInverseCompositional, + SimultaneousForwardCompositional, SimultaneousInverseCompositional, + AlternatingForwardCompositional, AlternatingInverseCompositional, + ModifiedAlternatingForwardCompositional, + ModifiedAlternatingInverseCompositional, + WibergForwardCompositional, WibergInverseCompositional) +from .sd import ( + SumOfSquaresSupervisedNewtonDescent, + SumOfSquaresSupervisedGaussNewtonDescent, + ProjectOutSupervisedNewtonDescent, + ProjectOutSupervisedGaussNewtonDescent, + AppearanceWeightsSupervisedNewtonDescent, + AppearanceWeightsSupervisedDescent) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 1241d88..5b3d3fc 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -1,15 +1,15 @@ from __future__ import division -import abc import numpy as np from menpo.image import Image from menpo.feature import gradient as fast_gradient, no_op from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: needs to use interfaces in menpofit.algorithm.py -# TODO: implement more clever sampling? -class LKAAMInterface(object): - +# TODO: implement more clever sampling for the standard interface? +# TODO document me! +class LucasKanadeStandardInterface(object): + r""" + """ def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm @@ -130,8 +130,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class LinearLKAAMInterface(LKAAMInterface): - +# TODO document me! +class LucasKanaddLinearInterface(LucasKanadeStandardInterface): + r""" + """ @property def shape_model(self): return self.transform.model @@ -143,8 +145,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class PartsLKAAMInterface(LKAAMInterface): - +# TODO document me! +class LucasKanadePartsInterface(LucasKanadeStandardInterface): + r""" + """ def __init__(self, aam_algorithm, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): self.algorithm = aam_algorithm @@ -200,10 +204,10 @@ def steepest_descent_images(self, nabla, dw_dp): return sdi.reshape((-1, sdi.shape[-1])) -# TODO: handle costs for all LKAAMAlgorithms # TODO document me! -class LKAAMAlgorithm(object): - +class LucasKanade(object): + r""" + """ def __init__(self, aam_interface, appearance_model, transform, eps=10**-5, **kwargs): # set common state for all AAM algorithms @@ -214,9 +218,9 @@ def __init__(self, aam_interface, appearance_model, transform, # set interface self.interface = aam_interface(self, **kwargs) # perform pre-computations - self.precompute() + self._precompute() - def precompute(self, **kwargs): + def _precompute(self): # grab number of shape and appearance parameters self.n = self.transform.n_parameters self.m = self.appearance_model.n_active_components @@ -245,13 +249,10 @@ def precompute(self, **kwargs): S = self.appearance_model.eigenvalues self.s2_inv_S = s2 / S - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None, - map_inference=False): - pass - -class ProjectOut(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class ProjectOut(LucasKanade): r""" Abstract Interface for Project-out AAM algorithms """ @@ -280,11 +281,11 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self.e_m = i_m - self.a_bar_m # solve for increments on the shape parameters - self.dp = self.solve(map_inference) + self.dp = self._solve(map_inference) # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -297,20 +298,14 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, gt_shape=gt_shape) - @abc.abstractmethod - def solve(self, map_inference): - pass - @abc.abstractmethod - def update_warp(self): - pass - - -class PFC(ProjectOut): +# TODO: handle costs! +# TODO: Document me! +class ProjectOutForwardCompositional(ProjectOut): r""" Project-out Forward Compositional (PFC) Gauss-Newton algorithm """ - def solve(self, map_inference): + def _solve(self, map_inference): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # compute masked forward Jacobian @@ -327,13 +322,15 @@ def solve(self, map_inference): else: return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class PIC(ProjectOut): +# TODO: handle costs! +# TODO: Document me! +class ProjectOutInverseCompositional(ProjectOut): r""" Project-out Inverse Compositional (PIC) Gauss-Newton algorithm """ @@ -351,7 +348,7 @@ def precompute(self): # compute masked Jacobian pseudo-inverse self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) - def solve(self, map_inference): + def _solve(self, map_inference): # solve for increments on the shape parameters if map_inference: return self.interface.solve_shape_map( @@ -360,13 +357,15 @@ def solve(self, map_inference): else: return -self.pinv_QJ_m.dot(self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) -class Simultaneous(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class Simultaneous(LucasKanade): r""" Abstract Interface for Simultaneous AAM algorithms """ @@ -400,7 +399,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # solve for increments on the appearance and shape parameters # simultaneously - dc, self.dp = self.solve(map_inference) + dc, self.dp = self._solve(map_inference) # update appearance parameters self.c += dc @@ -410,7 +409,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -423,11 +422,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - @abc.abstractmethod - def compute_jacobian(self): - pass - - def solve(self, map_inference): + def _solve(self, map_inference): # compute masked Jacobian J_m = self.compute_jacobian() # assemble masked simultaneous Jacobian @@ -443,48 +438,50 @@ def solve(self, map_inference): else: return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) - @abc.abstractmethod - def update_warp(self): - pass - -class SFC(Simultaneous): +# TODO: handle costs! +# TODO: Document me! +class SimultaneousForwardCompositional(Simultaneous): r""" Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class SIC(Simultaneous): +# TODO: handle costs! +# TODO: Document me! +class SimultaneousInverseCompositional(Simultaneous): r""" Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) -class Alternating(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class Alternating(LucasKanade): r""" Abstract Interface for Alternating AAM algorithms """ - def precompute(self, **kwargs): + def _precompute(self, **kwargs): # call super method super(Alternating, self).precompute() # compute MAP appearance Hessian @@ -529,7 +526,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, dc = self.pinv_A_m.dot(e_m + Jdp) # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # compute masked Hessian H_m = J_m.T.dot(J_m) # solve for increments on the shape parameters @@ -549,7 +546,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -562,47 +559,45 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - @abc.abstractmethod - def compute_jacobian(self): - pass - - @abc.abstractmethod - def update_warp(self): - pass - -class AFC(Alternating): +# TODO: handle costs! +# TODO: Document me! +class AlternatingForwardCompositional(Alternating): r""" Alternating Forward Compositional (AFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class AIC(Alternating): +# TODO: handle costs! +# TODO: Document me! +class AlternatingInverseCompositional(Alternating): r""" Alternating Inverse Compositional (AIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) +# TODO: handle costs! +# TODO: Document me! class ModifiedAlternating(Alternating): r""" Abstract Interface for Modified Alternating AAM algorithms @@ -635,7 +630,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, e_m = i_m - a_m # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # compute masked Hessian H_m = J_m.T.dot(J_m) # solve for increments on the shape parameters @@ -647,7 +642,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -661,23 +656,27 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) -class MAFC(ModifiedAlternating): +# TODO: handle costs! +# TODO: Document me! +class ModifiedAlternatingForwardCompositional(ModifiedAlternating): r""" Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class MAIC(ModifiedAlternating): +# TODO: handle costs! +# TODO: Document me! +class ModifiedAlternatingInverseCompositional(ModifiedAlternating): r""" Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm """ @@ -693,7 +692,9 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Wiberg(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class Wiberg(LucasKanade): r""" Abstract Interface for Wiberg AAM algorithms """ @@ -735,7 +736,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, e_m = i_m - self.a_bar_m # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # project out appearance models QJ_m = self.project_out(J_m) # compute masked Hessian @@ -750,7 +751,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -764,33 +765,37 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) -class WFC(Wiberg): +# TODO: handle costs! +# TODO: Document me! +class WibergForwardCompositional(Wiberg): r""" Wiberg Forward Compositional (WFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class WIC(Wiberg): +# TODO: handle costs! +# TODO: Document me! +class WibergInverseCompositional(Wiberg): r""" Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/sd.py similarity index 71% rename from menpofit/aam/algorithm/cr.py rename to menpofit/aam/algorithm/sd.py index 26f78a7..a6a75b2 100644 --- a/menpofit/aam/algorithm/cr.py +++ b/menpofit/aam/algorithm/sd.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.image import Image from menpo.feature import no_op @@ -7,9 +6,11 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: implement more clever sampling? -class CRAAMInterface(object): - +# TODO: implement more clever sampling for the standard interface? +# TODO document me! +class SupervisedDescentStandardInterface(object): + r""" + """ def __init__(self, cr_aam_algorithm, sampling=None): self.algorithm = cr_aam_algorithm @@ -60,8 +61,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class CRLinearAAMInterface(CRAAMInterface): - +# TODO document me! +class SupervisedDescentLinearInterface(SupervisedDescentStandardInterface): + r""" + """ @property def shape_model(self): return self.transform.model @@ -73,8 +76,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class CRPartsAAMInterface(CRAAMInterface): - +# TODO document me! +class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): + r""" + """ def __init__(self, cr_aam_algorithm, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): self.algorithm = cr_aam_algorithm @@ -102,8 +107,9 @@ def warp(self, image): # TODO document me! -class CRAAMAlgorithm(object): - +class SupervisedDescent(object): + r""" + """ def __init__(self, aam_interface, appearance_model, transform, max_iters=3, eps=10**-5, **kwargs): # set common state for all AAM algorithms @@ -111,42 +117,21 @@ def __init__(self, aam_interface, appearance_model, transform, max_iters=3, self.template = appearance_model.mean() self.transform = transform self.max_iters = max_iters + # TODO: Make use of eps in self.train? self.eps = eps # set interface self.interface = aam_interface(self, **kwargs) # perform pre-computations - self.precompute() - - def precompute(self): - # grab number of shape and appearance parameters - self.n = self.transform.n_parameters - self.m = self.appearance_model.n_active_components - - # grab appearance model components - self.A = self.appearance_model.components - # mask them - self.A_m = self.A.T[self.interface.i_mask, :] - # compute their pseudoinverse - self.pinv_A_m = np.linalg.pinv(self.A_m) + self._precompute() + def _precompute(self): # grab appearance model mean - self.a_bar = self.appearance_model.mean() + a_bar = self.appearance_model.mean() # vectorize it and mask it - self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] - - # compute shape model prior - s2 = (self.appearance_model.noise_variance() / - self.interface.shape_model.noise_variance()) - L = self.interface.shape_model.eigenvalues - self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) - # compute appearance model prior - S = self.appearance_model.eigenvalues - self.s2_inv_S = s2 / S - - def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): - # check training data - self._check_training_data(images, gt_shapes, current_shapes) + self.a_bar_m = a_bar.as_vector()[self.interface.i_mask] + def train(self, images, gt_shapes, current_shapes, verbose=False, + **kwargs): n_images = len(images) n_samples_image = len(current_shapes[0]) @@ -173,8 +158,10 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): **kwargs) # add regressor to list self.regressors.append(regressor) + # compute regression rmse estimated_delta_params = regressor(features) + # TODO: Should print a more informative error here? rmse = _compute_rmse(delta_params, estimated_delta_params) if verbose: print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(rmse)) @@ -198,15 +185,6 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): final_shapes.append(current_shapes[k:l]) return final_shapes - @staticmethod - def _check_training_data(images, gt_shapes, current_shapes): - if len(images) != len(gt_shapes): - raise ValueError("The number of shapes must be equal to " - "the number of images.") - elif len(images) != len(current_shapes): - raise ValueError("The number of current shapes must be " - "equal or multiple to the number of images.") - def _generate_params(self, gt_shapes, current_shapes): # initialize current and delta parameters arrays n_samples = len(gt_shapes) * len(current_shapes[0]) @@ -252,7 +230,7 @@ def _generate_features(self, images, current_params, verbose=False): # set transform self.transform.from_vector_inplace(current_params[k]) # compute regression features - f = self._compute_features(i) + f = self._compute_train_features(i) # add to features array features[k] = f # increment counter @@ -260,14 +238,6 @@ def _generate_features(self, images, current_params, verbose=False): return features - @abc.abstractmethod - def _compute_features(self, image): - pass - - @abc.abstractmethod - def _perform_regression(self, features, deltas, gamma=None): - pass - def run(self, image, initial_shape, gt_shape=None, **kwargs): # initialize transform self.transform.set_target(initial_shape) @@ -279,7 +249,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # Cascaded Regression loop while k < self.max_iters: # compute regression features - features = self._compute_features2(image) + features = self._compute_test_features(image) # solve for increments on the shape parameters dp = self.regressors[k](features) @@ -297,14 +267,64 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class ProjectOut(CRAAMAlgorithm): +class SumOfSquaresSupervisedDescent(SupervisedDescent): r""" """ + def _compute_train_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + def _compute_test_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + +# TODO: document me! +class SumOfSquaresSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, + dtype=np.float64): + return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) + + +# TODO: document me! +class SumOfSquaresSupervisedGaussNewtonDescent(SumOfSquaresSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + return _supervised_gauss_newton(features, deltas, gamma=gamma, + psi=psi, dtype=dtype) + + +# TODO: document me! +class ProjectOutSupervisedDescent(SupervisedDescent): + r""" + """ + def _precompute(self): + # call super method + super(ProjectOutSupervisedNewtonDescent)._precompute() + # grab appearance model components + A = self.appearance_model.components + # mask them + self.A_m = A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + def project_out(self, J): # project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) - def _compute_features(self, image): + def _compute_train_features(self, image): # warp image i = self.interface.warp(image) # vectorize it and mask it @@ -313,7 +333,7 @@ def _compute_features(self, image): e_m = i_m - self.a_bar_m return self.project_out(e_m) - def _compute_features2(self, image): + def _compute_test_features(self, image): # warp image i = self.interface.warp(image) # vectorize it and mask it @@ -323,7 +343,7 @@ def _compute_features2(self, image): # TODO: document me! -class PSD(ProjectOut): +class ProjectOutSupervisedNewtonDescent(ProjectOutSupervisedDescent): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -335,7 +355,62 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class PAJ(ProjectOut): +class ProjectOutSupervisedGaussNewtonDescent(ProjectOutSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + return _supervised_gauss_newton(features, deltas, gamma=gamma, + psi=psi, dtype=dtype) + + +# TODO: document me! +class AppearanceWeightsSupervisedDescent(SupervisedDescent): + r""" + """ + def _precompute(self): + # call super method + super(ProjectOutSupervisedNewtonDescent)._precompute() + # grab appearance model components + A = self.appearance_model.components + # mask them + A_m = A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(A_m) + + def project(self, J): + # project a particular vector or matrix onto the appearance bases + return self.pinv_A_m.dot(J - self.a_bar_m) + + def _compute_train_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # project it onto the appearance model + return self.project(i_m) + + def _compute_test_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # project it onto the appearance model + return self.project(i_m) + + +# TODO: document me! +class AppearanceWeightsSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, + dtype=np.float64): + return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) + + +# TODO: document me! +class AppearanceWeightsSupervisedGaussNewtonDescent( + AppearanceWeightsSupervisedDescent): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, @@ -369,6 +444,7 @@ class _supervised_gauss_newton(object): def __init__(self, features, deltas, gamma=None, psi=None, dtype=np.float64): features = features.astype(dtype) + # ridge regression deltas = deltas.astype(dtype) XX = deltas.T.dot(deltas) XT = deltas.T.dot(features) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a9e0cad..ddd46cc 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -8,9 +8,11 @@ import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM from .algorithm.lk import ( - LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) -from .algorithm.cr import ( - CRAAMInterface, CRLinearAAMInterface, CRPartsAAMInterface, PAJ) + LucasKanadeStandardInterface, LucasKanaddLinearInterface, + LucasKanadePartsInterface, WibergInverseCompositional) +from .algorithm.sd import ( + SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, + SupervisedDescentPartsInterface, ProjectOutSupervisedNewtonDescent) from .result import AAMFitterResult @@ -45,11 +47,11 @@ def _fitter_result(self, image, algorithm_results, affine_correction, # TODO: document me! -class LKAAMFitter(AAMFitter): +class LucasKanadeAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, n_shape=None, n_appearance=None, - lk_algorithm_cls=AIC, sampling=None, **kwargs): + def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, + n_shape=None, n_appearance=None, sampling=None, **kwargs): self._model = aam self.algorithms = [] self._check_n_shape(n_shape) @@ -68,7 +70,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = lk_algorithm_cls( - LKAAMInterface, am, md_transform, sampling=s, + LucasKanadeStandardInterface, am, md_transform, sampling=s, **kwargs) elif (type(self.aam) is LinearAAM or @@ -78,7 +80,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( - LinearLKAAMInterface, am, md_transform, sampling=s, + LucasKanaddLinearInterface, am, md_transform, sampling=s, **kwargs) elif type(self.aam) is PartsAAM: @@ -86,7 +88,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = lk_algorithm_cls( - PartsLKAAMInterface, am, pdm, sampling=s, + LucasKanadePartsInterface, am, pdm, sampling=s, patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) @@ -101,12 +103,12 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): # TODO: document me! -class CRAAMFitter(AAMFitter): +class SupervisedDescentAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, - n_appearance=None, sampling=None, n_perturbations=10, - noise_std=0.05, max_iters=6, **kwargs): + def __init__(self, aam, cr_algorithm_cls=ProjectOutSupervisedNewtonDescent, + n_shape=None,n_appearance=None, sampling=None, + n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam self.algorithms = [] self._check_n_shape(n_shape) @@ -128,8 +130,8 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = cr_algorithm_cls( - CRAAMInterface, am, md_transform, sampling=s, - max_iters=self.max_iters[j], **kwargs) + SupervisedDescentStandardInterface, am, md_transform, + sampling=s, max_iters=self.max_iters[j], **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -138,15 +140,15 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = cr_algorithm_cls( - CRLinearAAMInterface, am, md_transform, sampling=s, - max_iters=self.max_iters[j], **kwargs) + SupervisedDescentLinearInterface, am, md_transform, + sampling=s, max_iters=self.max_iters[j], **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = cr_algorithm_cls( - CRPartsAAMInterface, am, pdm, + SupervisedDescentPartsInterface, am, pdm, sampling=s, max_iters=self.max_iters[j], patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) @@ -160,6 +162,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): # append algorithms to list self.algorithms.append(algorithm) + # TODO: Allow training from bounding boxes def train(self, images, group=None, label=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index afd516c..7c57ce0 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -1,10 +1,9 @@ from __future__ import division -from menpofit.result import ( - ParametricAlgorithmResult, MultiFitterResult, SerializableIterativeResult) +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class AAMAlgorithmResult(ParametricAlgorithmResult): r""" """ @@ -15,6 +14,7 @@ def __init__(self, image, fitter, shape_parameters, self.appearance_parameters = appearance_parameters +# TODO: handle costs! # TODO: document me! class LinearAAMAlgorithmResult(AAMAlgorithmResult): r""" @@ -33,8 +33,8 @@ def initial_shape(self): return self.initial_transform.sparse_target +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class AAMFitterResult(MultiFitterResult): r""" """ diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index 12bd84b..cea05ea 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,5 +1,5 @@ from .builder import ( ATMBuilder, PatchATMBuilder, LinearATMBuilder, LinearPatchATMBuilder, PartsATMBuilder) -from .fitter import LKATMFitter -from .algorithm import FC, IC \ No newline at end of file +from .fitter import LucasKanadeATMFitter +from .algorithm import ForwardCompositional, InverseCompositional diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 2193d70..a171c50 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.image import Image from menpo.feature import no_op @@ -7,9 +6,10 @@ from .result import ATMAlgorithmResult, LinearATMAlgorithmResult -# TODO: implement more clever sampling? -class LKATMInterface(object): - +# TODO: implement more clever sampling for the standard interface? +class LucasKanadeStandardInterface(object): + r""" + """ def __init__(self, lk_algorithm, sampling=None): self.algorithm = lk_algorithm @@ -103,8 +103,10 @@ def algorithm_result(self, image, shape_parameters, gt_shape=None): image, self.algorithm, shape_parameters, gt_shape=gt_shape) -class LKLinearATMInterface(LKATMInterface): - +# TODO document me! +class LucasKanadeLinearInterface(LucasKanadeStandardInterface): + r""" + """ @property def shape_model(self): return self.transform.model @@ -114,8 +116,10 @@ def algorithm_result(self, image, shape_parameters, gt_shape=None): image, self.algorithm, shape_parameters, gt_shape=gt_shape) -class LKPartsATMInterface(LKATMInterface): - +# TODO document me! +class LucasKanadePartsInterface(LucasKanadeStandardInterface): + r""" + """ def __init__(self, lk_algorithm, patch_shape=(17, 17), normalize_parts=no_op, sampling=None): self.algorithm = lk_algorithm @@ -174,9 +178,8 @@ def steepest_descent_images(self, nabla, dw_dp): return sdi.reshape((-1, sdi.shape[-1])) -# TODO: handle costs for all LKAAMAlgorithms # TODO document me! -class LKATMAlgorithm(object): +class LucasKanade(object): def __init__(self, lk_atm_interface_cls, template, transform, eps=10**-5, **kwargs): @@ -187,9 +190,9 @@ def __init__(self, lk_atm_interface_cls, template, transform, # set interface self.interface = lk_atm_interface_cls(self, **kwargs) # perform pre-computations - self.precompute() + self._precompute() - def precompute(self, **kwargs): + def _precompute(self, **kwargs): # grab number of shape and appearance parameters self.n = self.transform.n_parameters @@ -204,13 +207,10 @@ def precompute(self, **kwargs): L = self.interface.shape_model.eigenvalues self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None, - map_inference=False): - pass - -class Compositional(LKATMAlgorithm): +# TODO: handle costs! +# TODO document me! +class Compositional(LucasKanade): r""" Abstract Interface for Compositional ATM algorithms """ @@ -235,11 +235,11 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self.e_m = i_m - self.t_m # solve for increments on the shape parameters - self.dp = self.solve(map_inference) + self.dp = self._solve(map_inference) # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -252,20 +252,14 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, gt_shape=gt_shape) - @abc.abstractmethod - def solve(self, map_inference): - pass - @abc.abstractmethod - def update_warp(self): - pass - - -class FC(Compositional): +# TODO: handle costs! +# TODO document me! +class ForwardCompositional(Compositional): r""" Forward Compositional (FC) Gauss-Newton algorithm """ - def solve(self, map_inference): + def _solve(self, map_inference): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # compute masked forward Jacobian @@ -280,19 +274,21 @@ def solve(self, map_inference): else: return self.interface.solve_shape_ml(JJ_m, J_m, self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class IC(Compositional): +# TODO: handle costs! +# TODO document me! +class InverseCompositional(Compositional): r""" Inverse Compositional (IC) Gauss-Newton algorithm """ - def precompute(self): + def _precompute(self): # call super method - super(IC, self).precompute() + super(InverseCompositional, self).precompute() # compute appearance model mean gradient nabla_t = self.interface.gradient(self.template) # compute masked inverse Jacobian @@ -302,7 +298,7 @@ def precompute(self): # compute masked Jacobian pseudo-inverse self.pinv_J_m = np.linalg.solve(self.JJ_m, self.J_m.T) - def solve(self, map_inference): + def _solve(self, map_inference): # solve for increments on the shape parameters if map_inference: return self.interface.solve_shape_map( @@ -311,7 +307,7 @@ def solve(self, map_inference): else: return -self.pinv_J_m.dot(self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 3cd7d97..8dbb453 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -4,16 +4,17 @@ from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM from .algorithm import ( - LKATMInterface, LKLinearATMInterface, LKPartsATMInterface, IC) + LucasKanadeStandardInterface, LucasKanadeLinearInterface, + LucasKanadePartsInterface, InverseCompositional) from .result import ATMFitterResult # TODO: document me! -class LKATMFitter(ModelFitter): +class LucasKanadeATMFitter(ModelFitter): r""" """ - def __init__(self, atm, algorithm_cls=IC, n_shape=None, sampling=None, - **kwargs): + def __init__(self, atm, algorithm_cls=InverseCompositional, + n_shape=None, sampling=None, **kwargs): self._model = atm self.algorithms = [] self._check_n_shape(n_shape) @@ -33,8 +34,9 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): sm, self.atm.transform, source=wt.landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(LKATMInterface, wt, md_transform, - sampling=sampling, **kwargs) + algorithm = algorithm_cls(LucasKanadeStandardInterface, wt, + md_transform, sampling=sampling, + **kwargs) elif (type(self.atm) is LinearATM or type(self.atm) is LinearPatchATM): @@ -42,7 +44,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.atm.reference_shape) # set up algorithm using linear aam interface - algorithm = algorithm_cls(LKLinearATMInterface, wt, + algorithm = algorithm_cls(LucasKanadeLinearInterface, wt, md_transform, sampling=sampling, **kwargs) @@ -51,7 +53,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = algorithm_cls( - LKPartsATMInterface, wt, pdm, sampling=sampling, + LucasKanadePartsInterface, wt, pdm, sampling=sampling, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 4ee4cab..f285f76 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -2,12 +2,14 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class ATMAlgorithmResult(ParametricAlgorithmResult): r""" """ + +# TODO: handle costs! # TODO: document me! class LinearATMAlgorithmResult(ATMAlgorithmResult): r""" @@ -26,8 +28,8 @@ def initial_shape(self): return self.initial_transform.sparse_target +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class ATMFitterResult(MultiFitterResult): r""" """ diff --git a/menpofit/lk/__init__.py b/menpofit/lk/__init__.py index b01bf94..7a1abbc 100644 --- a/menpofit/lk/__init__.py +++ b/menpofit/lk/__init__.py @@ -1,3 +1,5 @@ -from .fitter import LKFitter -from .algorithm import FA, FC, IC -from .residual import SSD, FourierSSD, ECC, GradientImages, GradientCorrelation +from .fitter import LucasKanadeFitter +from .algorithm import ( + ForwardAdditive, ForwardCompositional, InverseCompositional) +from .residual import ( + SSD, FourierSSD, ECC, GradientImages, GradientCorrelation) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 3034c08..9671e94 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -1,15 +1,12 @@ from scipy.linalg import norm -import abc import numpy as np -from .result import LKAlgorithmResult +from .result import LucasKanadeAlgorithmResult # TODO: implement Inverse Additive Algorithm? -# TODO: implement Linear, Parts interfaces? Will they play nice with residuals? # TODO: implement sampling? -# TODO: handle costs for all LKAlgorithms # TODO: document me! -class LKAlgorithm(object): +class LucasKanade(object): r""" """ def __init__(self, template, transform, residual, eps=10**-10): @@ -18,12 +15,10 @@ def __init__(self, template, transform, residual, eps=10**-10): self.residual = residual self.eps = eps - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None): - pass - -class FA(LKAlgorithm): +# TODO: handle costs! +# TODO: document me! +class ForwardAdditive(LucasKanade): r""" Forward Additive Lucas-Kanade algorithm """ @@ -69,18 +64,21 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LKAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, gt_shape=None) -class FC(LKAlgorithm): +# TODO: handle costs! +# TODO: document me! +class ForwardCompositional(LucasKanade): r""" Forward Compositional Lucas-Kanade algorithm """ def __init__(self, template, transform, residual, eps=10**-10): - super(FC, self).__init__(template, transform, residual, eps=eps) - self.precompute() + super(ForwardCompositional, self).__init__( + template, transform, residual, eps=eps) + self._precompute() - def precompute(self): + def _precompute(self): # compute warp jacobian self.dW_dp = np.rollaxis( self.transform.d_dp(self.template.indices()), -1) @@ -126,15 +124,18 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): return LKAlgorithmResult(image, self, p_list, gt_shape=None) -class IC(LKAlgorithm): +# TODO: handle costs! +# TODO: document me! +class InverseCompositional(LucasKanade): r""" Inverse Compositional Lucas-Kanade algorithm """ def __init__(self, template, transform, residual, eps=10**-10): - super(IC, self).__init__(template, transform, residual, eps=eps) - self.precompute() + super(InverseCompositional, self).__init__( + template, transform, residual, eps=eps) + self._precompute() - def precompute(self): + def _precompute(self): # compute warp jacobian dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index e2a91e0..2983299 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -3,19 +3,20 @@ from menpofit.transform import DifferentiableAlignmentAffine from menpofit.fitter import MultiFitter, noisy_target_alignment_transform from menpofit import checks -from .algorithm import IC +from .algorithm import InverseCompositional from .residual import SSD, FourierSSD -from .result import LKFitterResult +from .result import LucasKanadeFitterResult # TODO: document me! -class LKFitter(MultiFitter): +class LucasKanadeFitter(MultiFitter): r""" """ def __init__(self, template, group=None, label=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, - scales=(1, .5), scale_features=True, algorithm_cls=IC, - residual_cls=SSD, **kwargs): + scales=(1, .5), scale_features=True, + algorithm_cls=InverseCompositional, residual_cls=SSD, + **kwargs): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -89,5 +90,5 @@ def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): - return LKFitterResult(image, self, algorithm_results, - affine_correction, gt_shape=gt_shape) \ No newline at end of file + return LucasKanadeFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index 401b1e5..5f808d3 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -4,7 +4,7 @@ import scipy.linalg from menpo.feature import gradient - +# TODO: Do we want residuals to support masked templates? class Residual(object): """ An abstract base class for calculating the residual between two images @@ -132,7 +132,8 @@ def steepest_descent_update(self, sdi, image, template): class SSD(Residual): - + r""" + """ def __init__(self, kernel=None): self.kernel = kernel @@ -188,8 +189,10 @@ def steepest_descent_update(self, sdi, image, template): return sdi.T.dot(error_img) +# TODO: Does not support masked templates at the moment class FourierSSD(Residual): - + r""" + """ def __init__(self, kernel=None): self.kernel = kernel @@ -254,7 +257,8 @@ def steepest_descent_update(self, sdi, image, template): class ECC(Residual): - + r""" + """ def _normalise_images(self, image): # TODO: do we need to copy the image? # TODO: is this supposed to be per channel normalization? @@ -327,7 +331,8 @@ def steepest_descent_update(self, sdi, image, template): class GradientImages(Residual): - + r""" + """ def _regularise_gradients(self, grad): pixels = grad.pixels ab = np.sqrt(np.sum(pixels**2, axis=0)) @@ -391,7 +396,8 @@ def steepest_descent_update(self, sdi, image, template): class GradientCorrelation(Residual): - + r""" + """ def steepest_descent_images(self, image, dW_dp, forward=None): n_dims = image.n_dims n_channels = image.n_channels diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py index 1eddf5d..6674a63 100644 --- a/menpofit/lk/result.py +++ b/menpofit/lk/result.py @@ -2,17 +2,17 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: document me! # TODO: handle costs! -class LKAlgorithmResult(ParametricAlgorithmResult): +# TODO: document me! +class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): r""" """ pass +# TODO: handle costs! # TODO: document me! -# TODO: handle costs -class LKFitterResult(MultiFitterResult): +class LucasKanadeFitterResult(MultiFitterResult): r""" """ pass From c3fef849b5dac8511a99a389868ed9aa5cd8bd03 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 04:27:02 +0100 Subject: [PATCH 073/423] Small fixes related to the previous commit --- menpofit/aam/__init__.py | 9 +++------ menpofit/aam/algorithm/__init__.py | 12 ++++++------ menpofit/aam/algorithm/lk.py | 10 +++++----- menpofit/aam/algorithm/sd.py | 23 +++++++++++------------ menpofit/aam/fitter.py | 4 ++-- menpofit/atm/algorithm.py | 2 +- menpofit/lk/algorithm.py | 9 ++++++--- menpofit/lk/fitter.py | 4 ++-- 8 files changed, 36 insertions(+), 37 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 65d6960..76765db 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -9,9 +9,6 @@ ModifiedAlternatingForwardCompositional, ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional, - SumOfSquaresSupervisedNewtonDescent, - SumOfSquaresSupervisedGaussNewtonDescent, - ProjectOutSupervisedNewtonDescent, - ProjectOutSupervisedGaussNewtonDescent, - AppearanceWeightsSupervisedNewtonDescent, - AppearanceWeightsSupervisedDescent) + SumOfSquaresNewton, SumOfSquaresGaussNewton, + ProjectOutNewton, ProjectOutGaussNewton, + AppearanceWeightsNewton, AppearanceWeightsGaussNewton) diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py index 40cf345..4c054af 100644 --- a/menpofit/aam/algorithm/__init__.py +++ b/menpofit/aam/algorithm/__init__.py @@ -6,9 +6,9 @@ ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional) from .sd import ( - SumOfSquaresSupervisedNewtonDescent, - SumOfSquaresSupervisedGaussNewtonDescent, - ProjectOutSupervisedNewtonDescent, - ProjectOutSupervisedGaussNewtonDescent, - AppearanceWeightsSupervisedNewtonDescent, - AppearanceWeightsSupervisedDescent) + SumOfSquaresNewton, + SumOfSquaresGaussNewton, + ProjectOutNewton, + ProjectOutGaussNewton, + AppearanceWeightsNewton, + AppearanceWeightsGaussNewton) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 5b3d3fc..33cd8c9 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -334,9 +334,9 @@ class ProjectOutInverseCompositional(ProjectOut): r""" Project-out Inverse Compositional (PIC) Gauss-Newton algorithm """ - def precompute(self): + def _precompute(self): # call super method - super(PIC, self).precompute() + super(ProjectOutInverseCompositional, self)._precompute() # compute appearance model mean gradient nabla_a = self.interface.gradient(self.a_bar) # compute masked inverse Jacobian @@ -483,7 +483,7 @@ class Alternating(LucasKanade): """ def _precompute(self, **kwargs): # call super method - super(Alternating, self).precompute() + super(Alternating, self)._precompute() # compute MAP appearance Hessian self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) @@ -680,13 +680,13 @@ class ModifiedAlternatingInverseCompositional(ModifiedAlternating): r""" Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index a6a75b2..e8934f7 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -267,7 +267,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class SumOfSquaresSupervisedDescent(SupervisedDescent): +class SumOfSquares(SupervisedDescent): r""" """ def _compute_train_features(self, image): @@ -288,7 +288,7 @@ def _compute_test_features(self, image): # TODO: document me! -class SumOfSquaresSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): +class SumOfSquaresNewton(SumOfSquares): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -297,7 +297,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class SumOfSquaresSupervisedGaussNewtonDescent(SumOfSquaresSupervisedDescent): +class SumOfSquaresGaussNewton(SumOfSquares): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, @@ -307,12 +307,12 @@ def _perform_regression(self, features, deltas, gamma=None, psi=None, # TODO: document me! -class ProjectOutSupervisedDescent(SupervisedDescent): +class ProjectOut(SupervisedDescent): r""" """ def _precompute(self): # call super method - super(ProjectOutSupervisedNewtonDescent)._precompute() + super(ProjectOut, self)._precompute() # grab appearance model components A = self.appearance_model.components # mask them @@ -343,7 +343,7 @@ def _compute_test_features(self, image): # TODO: document me! -class ProjectOutSupervisedNewtonDescent(ProjectOutSupervisedDescent): +class ProjectOutNewton(ProjectOut): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -355,7 +355,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class ProjectOutSupervisedGaussNewtonDescent(ProjectOutSupervisedDescent): +class ProjectOutGaussNewton(ProjectOut): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, @@ -365,12 +365,12 @@ def _perform_regression(self, features, deltas, gamma=None, psi=None, # TODO: document me! -class AppearanceWeightsSupervisedDescent(SupervisedDescent): +class AppearanceWeights(SupervisedDescent): r""" """ def _precompute(self): # call super method - super(ProjectOutSupervisedNewtonDescent)._precompute() + super(AppearanceWeights, self)._precompute() # grab appearance model components A = self.appearance_model.components # mask them @@ -400,7 +400,7 @@ def _compute_test_features(self, image): # TODO: document me! -class AppearanceWeightsSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): +class AppearanceWeightsNewton(AppearanceWeights): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -409,8 +409,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class AppearanceWeightsSupervisedGaussNewtonDescent( - AppearanceWeightsSupervisedDescent): +class AppearanceWeightsGaussNewton(AppearanceWeights): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index ddd46cc..232f6d4 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -12,7 +12,7 @@ LucasKanadePartsInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, - SupervisedDescentPartsInterface, ProjectOutSupervisedNewtonDescent) + SupervisedDescentPartsInterface, ProjectOutNewton) from .result import AAMFitterResult @@ -106,7 +106,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): class SupervisedDescentAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, cr_algorithm_cls=ProjectOutSupervisedNewtonDescent, + def __init__(self, aam, cr_algorithm_cls=ProjectOutNewton, n_shape=None,n_appearance=None, sampling=None, n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index a171c50..5deda18 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -288,7 +288,7 @@ class InverseCompositional(Compositional): """ def _precompute(self): # call super method - super(InverseCompositional, self).precompute() + super(InverseCompositional, self)._precompute() # compute appearance model mean gradient nabla_t = self.interface.gradient(self.template) # compute masked inverse Jacobian diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 9671e94..37325f6 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -64,7 +64,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LucasKanadeAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, + gt_shape=gt_shape) # TODO: handle costs! @@ -121,7 +122,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LKAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, + gt_shape=gt_shape) # TODO: handle costs! @@ -178,4 +180,5 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LKAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, + gt_shape=gt_shape) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 2983299..593af6c 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -84,8 +84,8 @@ def _prepare_template(self, template, group=None, label=None): def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): transform = noisy_target_alignment_transform( - self.transform_cls, self.reference_shape, gt_shape, - noise_std=noise_std) + self.reference_shape, gt_shape, + alignment_transform_cls=self.transform_cls, noise_std=noise_std) return transform.apply(self.reference_shape) def _fitter_result(self, image, algorithm_results, affine_correction, From 81373def13761e583bdaf0fd5d22133ed3ed2dfa Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 14:05:29 +0100 Subject: [PATCH 074/423] Tidy up scales in prepare_image and prepare_template --- menpofit/fitter.py | 5 +---- menpofit/lk/fitter.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 06209da..c8fb9c8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -146,11 +146,8 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, group='initial_shape') # obtain image representation - from copy import deepcopy - scales = deepcopy(self.scales) - scales.reverse() images = [] - for j, s in enumerate(scales): + for j, s in enumerate(self.scales[::-1]): if j == 0: # compute features at highest level feature_image = self.features[j](image) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 593af6c..5e1cc72 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -59,11 +59,8 @@ def _prepare_template(self, template, group=None, label=None): self.diagonal, group=group, label=label) # obtain image representation - from copy import deepcopy - scales = deepcopy(self.scales) - scales.reverse() templates = [] - for j, s in enumerate(scales): + for j, s in enumerate(self.scales[::-1]): if j == 0: # compute features at highest level feature_template = self.features[j](template) From 28b8a6d8bf9287ae5fd1394796d4d2dc2355f3b4 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 18:37:56 +0100 Subject: [PATCH 075/423] Add new holistic sampling methods --- menpofit/aam/__init__.py | 4 +++- menpofit/aam/algorithm/lk.py | 8 +++++--- menpofit/aam/fitter.py | 39 +++++++++++++++++++++++++++++++++++- menpofit/checks.py | 19 +++++++++++------- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 76765db..c06a2d2 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,7 +1,9 @@ from .builder import ( AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, LinearPatchAAMBuilder, PartsAAMBuilder) -from .fitter import LucasKanadeAAMFitter, SupervisedDescentAAMFitter +from .fitter import ( + LucasKanadeAAMFitter, SupervisedDescentAAMFitter, + holistic_sampling_from_scale, holistic_sampling_from_step) from .algorithm import ( ProjectOutForwardCompositional, ProjectOutInverseCompositional, SimultaneousForwardCompositional, SimultaneousInverseCompositional, diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 33cd8c9..73fe696 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -19,9 +19,11 @@ def __init__(self, aam_algorithm, sampling=None): sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 + sampling = xrange(0, n_true_pixels, 1) + elif isinstance(sampling, np.int): + sampling = xrange(0, n_true_pixels, sampling) + + sampling_mask[sampling] = 1 self.i_mask = np.nonzero(np.tile( sampling_mask[None, ...], (n_channels, 1)).flatten())[0] diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 232f6d4..e95b967 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,5 +1,8 @@ from __future__ import division -from menpo.transform import Scale +import numpy as np +from copy import deepcopy +from menpo.transform import Scale, AlignmentUniformScale +from menpo.image import BooleanImage from menpofit.builder import ( rescale_images_to_reference_shape, compute_features, scale_images) from menpofit.fitter import ModelFitter @@ -224,3 +227,37 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): transform.apply_inplace(shape) +# TODO: Document me! +def holistic_sampling_from_scale(aam, scale=0.35): + reference = aam.appearance_models[0].mean() + scaled_reference = reference.rescale(scale) + + t = AlignmentUniformScale(scaled_reference.landmarks['source'].lms, + reference.landmarks['source'].lms) + new_indices = np.require(np.round(t.apply( + scaled_reference.mask.true_indices())), dtype=np.int) + + modified_mask = deepcopy(reference.mask.pixels) + modified_mask[:] = False + modified_mask[:, new_indices[:, 0], new_indices[:, 1]] = True + + true_positions = np.nonzero( + modified_mask[:, reference.mask.mask].ravel())[0] + + return true_positions, BooleanImage(modified_mask[0]) + + +def holistic_sampling_from_step(aam, step=8): + reference = aam.appearance_models[0].mean() + + n_true_pixels = reference.n_true_pixels() + true_positions = np.zeros(n_true_pixels, dtype=np.bool) + sampling = xrange(0, n_true_pixels, step) + true_positions[sampling] = True + + modified_mask = reference.mask.copy() + new_indices = modified_mask.true_indices()[sampling, :] + modified_mask.mask[:] = False + modified_mask.mask[new_indices[:, 0], new_indices[:, 1]] = True + + return true_positions, modified_mask diff --git a/menpofit/checks.py b/menpofit/checks.py index 0c5dc0c..4d06498 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -125,18 +125,23 @@ def check_max_iters(max_iters, n_levels): # TODO: document me! def check_sampling(sampling, n_levels): - if isinstance(sampling, (list, tuple)): + if (isinstance(sampling, (list, tuple)) and + np.alltrue([isinstance(s, (np.ndarray, np.int, None)) + for s in sampling])): if len(sampling) == 1: - sampling = sampling * n_levels - elif len(sampling) != n_levels: + return sampling * n_levels + elif len(sampling) == n_levels: + return sampling + else: raise ValueError('A sampling list can only ' 'contain 1 element or {} ' 'elements'.format(n_levels)) - elif isinstance(sampling, np.ndarray): - sampling = [sampling] * n_levels + elif isinstance(sampling, (np.ndarray, np.int, None)): + return [sampling] * n_levels else: - raise ValueError('sampling can be a ndarray, a ndarray list ' + raise ValueError('sampling can be an integer or ndarray, ' + 'a integer or ndarray list ' 'containing 1 or {} elements or ' 'None'.format(n_levels)) - return sampling + From 2fd02345af4ca9d16570be56d7d31c5d7fa4b989 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 19:09:51 +0100 Subject: [PATCH 076/423] Delete TODOs regarding sampling --- menpofit/aam/algorithm/lk.py | 1 - menpofit/aam/algorithm/sd.py | 1 - menpofit/atm/algorithm.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 73fe696..2144e14 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -5,7 +5,6 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: implement more clever sampling for the standard interface? # TODO document me! class LucasKanadeStandardInterface(object): r""" diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index e8934f7..e1f8e09 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -6,7 +6,6 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: implement more clever sampling for the standard interface? # TODO document me! class SupervisedDescentStandardInterface(object): r""" diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 5deda18..3fb1098 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -6,7 +6,7 @@ from .result import ATMAlgorithmResult, LinearATMAlgorithmResult -# TODO: implement more clever sampling for the standard interface? +# TODO document me! class LucasKanadeStandardInterface(object): r""" """ From 4881f243c51f9ce76001a075ab83a04f44dcf8fa Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 19:13:09 +0100 Subject: [PATCH 077/423] Update checks --- menpofit/checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/checks.py b/menpofit/checks.py index 4d06498..82f4978 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -126,7 +126,7 @@ def check_max_iters(max_iters, n_levels): # TODO: document me! def check_sampling(sampling, n_levels): if (isinstance(sampling, (list, tuple)) and - np.alltrue([isinstance(s, (np.ndarray, np.int, None)) + np.alltrue([isinstance(s, (np.ndarray, np.int)) or sampling is None for s in sampling])): if len(sampling) == 1: return sampling * n_levels @@ -136,7 +136,7 @@ def check_sampling(sampling, n_levels): raise ValueError('A sampling list can only ' 'contain 1 element or {} ' 'elements'.format(n_levels)) - elif isinstance(sampling, (np.ndarray, np.int, None)): + elif isinstance(sampling, (np.ndarray, np.int)) or sampling is None: return [sampling] * n_levels else: raise ValueError('sampling can be an integer or ndarray, ' From 09f7acc69cd6f5ddfec8318c04ad3ab0c75b2e12 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 1 Jul 2015 11:17:54 +0100 Subject: [PATCH 078/423] Correct typo in LucasKanadeLinearInterface --- menpofit/aam/algorithm/lk.py | 2 +- menpofit/aam/fitter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 2144e14..9a20c46 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -132,7 +132,7 @@ def algorithm_result(self, image, shape_parameters, # TODO document me! -class LucasKanaddLinearInterface(LucasKanadeStandardInterface): +class LucasKanadeLinearInterface(LucasKanadeStandardInterface): r""" """ @property diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index e95b967..e8039fe 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -11,7 +11,7 @@ import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM from .algorithm.lk import ( - LucasKanadeStandardInterface, LucasKanaddLinearInterface, + LucasKanadeStandardInterface, LucasKanadeLinearInterface, LucasKanadePartsInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, @@ -83,7 +83,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( - LucasKanaddLinearInterface, am, md_transform, sampling=s, + LucasKanadeLinearInterface, am, md_transform, sampling=s, **kwargs) elif type(self.aam) is PartsAAM: From cb44a935c80c6c4e62d65aac7df1a03cf4220589 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 1 Jul 2015 17:20:15 +0100 Subject: [PATCH 079/423] Add costs to AAMs - Little renaming as well --- menpofit/aam/__init__.py | 2 +- menpofit/aam/algorithm/__init__.py | 4 +- menpofit/aam/algorithm/lk.py | 272 ++++++++++++++++++++--------- menpofit/aam/algorithm/sd.py | 6 +- menpofit/aam/fitter.py | 14 +- menpofit/aam/result.py | 80 ++++++++- menpofit/atm/result.py | 2 +- menpofit/result.py | 16 +- 8 files changed, 288 insertions(+), 108 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index c06a2d2..bb37752 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -11,6 +11,6 @@ ModifiedAlternatingForwardCompositional, ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional, - SumOfSquaresNewton, SumOfSquaresGaussNewton, + MeanTemplateNewton, MeanTemplateGaussNewton, ProjectOutNewton, ProjectOutGaussNewton, AppearanceWeightsNewton, AppearanceWeightsGaussNewton) diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py index 4c054af..636c758 100644 --- a/menpofit/aam/algorithm/__init__.py +++ b/menpofit/aam/algorithm/__init__.py @@ -6,8 +6,8 @@ ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional) from .sd import ( - SumOfSquaresNewton, - SumOfSquaresGaussNewton, + MeanTemplateNewton, + MeanTemplateGaussNewton, ProjectOutNewton, ProjectOutGaussNewton, AppearanceWeightsNewton, diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 9a20c46..d970387 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -124,10 +124,11 @@ def solve_all_ml(self, H, J, e): dq = - np.linalg.solve(H, J.T.dot(e)) return dq[:self.m], dq[self.m:] - def algorithm_result(self, image, shape_parameters, + def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( image, self.algorithm, shape_parameters, + cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -263,6 +264,10 @@ def project_out(self, J): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x, f): + return lambda: x.T.dot(f(x)) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -271,16 +276,20 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # vectorize it and mask it - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - # compute masked error - self.e_m = i_m - self.a_bar_m + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # update cost_functions + cost_functions = [cost_closure(self.e_m, self.project_out)] + while k < max_iters and eps > self.eps: # solve for increments on the shape parameters self.dp = self._solve(map_inference) @@ -289,6 +298,17 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # update cost + cost_functions.append(cost_closure(self.e_m, self.project_out)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -297,7 +317,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) # TODO: handle costs! @@ -372,6 +392,10 @@ class Simultaneous(LucasKanade): """ def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x): + return lambda: x.T.dot(x) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -380,30 +404,33 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(self.c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [self.c] + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - self.e_m = i_m - a_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [self.c] + # compute masked error + self.e_m = i_m - a_m + + # update cost + cost_functions = [cost_closure(self.e_m)] + + while k < max_iters and eps > self.eps: # solve for increments on the appearance and shape parameters # simultaneously dc, self.dp = self._solve(map_inference) # update appearance parameters - self.c += dc + self.c = self.c + dc self.a = self.appearance_model.instance(self.c) a_m = self.a.as_vector()[self.interface.i_mask] c_list.append(self.c) @@ -413,6 +440,17 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - a_m + + # update cost + cost_functions.append(cost_closure(self.e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -421,11 +459,12 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) def _solve(self, map_inference): # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # assemble masked simultaneous Jacobian J_sim_m = np.hstack((-self.A_m, J_m)) # compute masked Hessian @@ -490,6 +529,10 @@ def _precompute(self, **kwargs): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x): + return lambda: x.T.dot(x) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -498,27 +541,28 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - Jdp = 0 - else: - Jdp = J_m.dot(self.dp) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - a_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + Jdp = 0 + + # compute masked error + e_m = i_m - a_m + # update cost + cost_functions = [cost_closure(e_m)] + + while k < max_iters and eps > self.eps: # solve for increment on the appearance parameters if map_inference: Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) @@ -540,7 +584,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, e_m - self.A_m.dot(dc)) # update appearance parameters - c += dc + c = c + dc self.a = self.appearance_model.instance(c) a_m = self.a.as_vector()[self.interface.i_mask] c_list.append(c) @@ -550,6 +594,20 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute Jdp + Jdp = J_m.dot(self.dp) + + # compute masked error + e_m = i_m - a_m + + # update cost + cost_functions.append(cost_closure(e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -558,7 +616,8 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) # TODO: handle costs! @@ -605,6 +664,10 @@ class ModifiedAlternating(Alternating): """ def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x): + return lambda: x.T.dot(x) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -615,21 +678,27 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - c = self.pinv_A_m.dot(i_m - a_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - a_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - a_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + # update cost + cost_functions = [cost_closure(e_m)] + + while k < max_iters and eps > self.eps: # compute masked Jacobian J_m = self._compute_jacobian() # compute masked Hessian @@ -646,6 +715,23 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # update appearance parameters + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + + # update cost + cost_functions.append(cost_closure(e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -654,7 +740,8 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) # TODO: handle costs! @@ -705,6 +792,10 @@ def project_out(self, J): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x, f): + return lambda: x.T.dot(f(x)) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -713,29 +804,27 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - else: - c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - self.a_bar_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + # compute masked error + e_m = i_m - self.a_bar_m + + # update cost + cost_functions = [cost_closure(e_m, self.project_out)] + + while k < max_iters and eps > self.eps: # compute masked Jacobian J_m = self._compute_jacobian() # project out appearance models @@ -755,6 +844,24 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # update appearance parameters + dc = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) + c = c + dc + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - self.a_bar_m + + # update cost + cost_functions.append(cost_closure(e_m, self.project_out)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -763,7 +870,8 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) # TODO: handle costs! diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index e1f8e09..903f4bf 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -266,7 +266,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class SumOfSquares(SupervisedDescent): +class MeanTemplate(SupervisedDescent): r""" """ def _compute_train_features(self, image): @@ -287,7 +287,7 @@ def _compute_test_features(self, image): # TODO: document me! -class SumOfSquaresNewton(SumOfSquares): +class MeanTemplateNewton(MeanTemplate): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -296,7 +296,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class SumOfSquaresGaussNewton(SumOfSquares): +class MeanTemplateGaussNewton(MeanTemplate): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index e8039fe..a985331 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -109,8 +109,8 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): class SupervisedDescentAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, cr_algorithm_cls=ProjectOutNewton, - n_shape=None,n_appearance=None, sampling=None, + def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, + n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam self.algorithms = [] @@ -120,9 +120,9 @@ def __init__(self, aam, cr_algorithm_cls=ProjectOutNewton, self.n_perturbations = n_perturbations self.noise_std = noise_std self.max_iters = checks.check_max_iters(max_iters, self.n_levels) - self._set_up(cr_algorithm_cls, sampling, **kwargs) + self._set_up(sd_algorithm_cls, sampling, **kwargs) - def _set_up(self, cr_algorithm_cls, sampling, **kwargs): + def _set_up(self, sd_algorithm_cls, sampling, **kwargs): for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, sampling)): @@ -132,7 +132,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( SupervisedDescentStandardInterface, am, md_transform, sampling=s, max_iters=self.max_iters[j], **kwargs) @@ -142,7 +142,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) # set up algorithm using linear aam interface - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( SupervisedDescentLinearInterface, am, md_transform, sampling=s, max_iters=self.max_iters[j], **kwargs) @@ -150,7 +150,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( SupervisedDescentPartsInterface, am, pdm, sampling=s, max_iters=self.max_iters[j], patch_shape=self.aam.patch_shape[j], diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 7c57ce0..38fabff 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -7,11 +7,50 @@ class AAMAlgorithmResult(ParametricAlgorithmResult): r""" """ - def __init__(self, image, fitter, shape_parameters, + def __init__(self, image, algorithm, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): super(AAMAlgorithmResult, self).__init__( - image, fitter, shape_parameters, gt_shape=gt_shape) + image, algorithm, shape_parameters, gt_shape=gt_shape) + self._cost_functions = cost_functions self.appearance_parameters = appearance_parameters + self._warped_images = None + self._appearance_reconstructions = None + self._costs = None + + @property + def warped_images(self): + if self._warped_images is None: + self._warped_images = [] + for p in self.shape_parameters: + self.algorithm.transform.from_vector_inplace(p) + self._warped_images.append( + self.algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def appearance_reconstructions(self): + if self.appearance_parameters is not None: + if self._appearance_reconstructions is None: + self._appearance_reconstructions = [] + for c in self.appearance_parameters: + instance = self.algorithm.appearance_model.instance(c) + self._appearance_reconstructions.append(instance) + return self._appearance_reconstructions + else: + raise ValueError('appearance_reconstructions is not well ' + 'defined for the chosen AAM algorithm: ' + '{}'.format(self.algorithm.__class__)) + + @property + def costs(self): + if self._cost_functions is not None: + if self._costs is None: + self._costs = [f() for f in self._cost_functions] + return self._costs + else: + raise ValueError('costs is not well ' + 'defined for the chosen AAM algorithm: ' + '{}'.format(self.algorithm.__class__)) # TODO: handle costs! @@ -21,7 +60,7 @@ class LinearAAMAlgorithmResult(AAMAlgorithmResult): """ @property def shapes(self, as_points=False): - return [self.fitter.transform.from_vector(p).sparse_target + return [self.algorithm.transform.from_vector(p).sparse_target for p in self.shape_parameters] @property @@ -38,4 +77,37 @@ def initial_shape(self): class AAMFitterResult(MultiFitterResult): r""" """ - pass + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(AAMFitterResult, self).__init__( + image, fitter, algorithm_results, affine_correction, + gt_shape=gt_shape) + self._warped_images = None + + @property + def warped_images(self): + if self._warped_images is None: + algorithm = self.algorithm_results[-1].algorithm + self._warped_images = [] + for s in self.shapes: + algorithm.transform.set_target(s) + self._warped_images.append( + algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def appearance_reconstructions(self): + reconstructions = self.algorithm_results[0].appearance_reconstructions + if reconstructions is not None: + for a in self.algorithm_results[1:]: + reconstructions = (reconstructions + + a.appearance_reconstructions) + return reconstructions + + @property + def costs(self): + costs = [] + for a in self.algorithm_results: + costs += a.costs + return costs + diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index f285f76..56b698e 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -16,7 +16,7 @@ class LinearATMAlgorithmResult(ATMAlgorithmResult): """ @property def shapes(self): - return [self.fitter.transform.from_vector(p).sparse_target + return [self.algorithm.transform.from_vector(p).sparse_target for p in self.shape_parameters] @property diff --git a/menpofit/result.py b/menpofit/result.py index 6fbe755..335a51b 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -412,9 +412,9 @@ def as_serializableresult(self): class ParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, fitter, shape_parameters, gt_shape=None): + def __init__(self, image, algorithm, shape_parameters, gt_shape=None): self.image = image - self.fitter = fitter + self.algorithm = algorithm self.shape_parameters = shape_parameters self._gt_shape = gt_shape @@ -428,7 +428,7 @@ def transforms(self): Generates a list containing the transforms obtained at each fitting iteration. """ - return [self.fitter.transform.from_vector(p) + return [self.algorithm.transform.from_vector(p) for p in self.shape_parameters] @property @@ -436,18 +436,18 @@ def final_transform(self): r""" Returns the final transform. """ - return self.fitter.transform.from_vector(self.shape_parameters[-1]) + return self.algorithm.transform.from_vector(self.shape_parameters[-1]) @property def initial_transform(self): r""" Returns the initial transform from which the fitting started. """ - return self.fitter.transform.from_vector(self.shape_parameters[0]) + return self.algorithm.transform.from_vector(self.shape_parameters[0]) @property def shapes(self): - return [self.fitter.transform.from_vector(p).target + return [self.algorithm.transform.from_vector(p).target for p in self.shape_parameters] @property @@ -463,9 +463,9 @@ def initial_shape(self): class NonParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, fitter, shapes, gt_shape=None): + def __init__(self, image, algorithm, shapes, gt_shape=None): self.image = image - self.fitter = fitter + self.algorithm = algorithm self._shapes = shapes self._gt_shape = gt_shape From ef81fec301da2c03798e6aeb03df64237ce6fce9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 1 Jul 2015 18:11:53 +0100 Subject: [PATCH 080/423] Add cost for ATMs --- menpofit/aam/algorithm/lk.py | 3 ++- menpofit/atm/algorithm.py | 49 ++++++++++++++++++++++++++---------- menpofit/atm/result.py | 48 ++++++++++++++++++++++++++++++++++- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index d970387..98eaa6d 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -140,10 +140,11 @@ class LucasKanadeLinearInterface(LucasKanadeStandardInterface): def shape_model(self): return self.transform.model - def algorithm_result(self, image, shape_parameters, + def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( image, self.algorithm, shape_parameters, + cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 3fb1098..423afab 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -98,9 +98,11 @@ def solve_shape_ml(cls, H, J, e): # compute and return ML solution return -np.linalg.solve(H, J.T.dot(e)) - def algorithm_result(self, image, shape_parameters, gt_shape=None): + def algorithm_result(self, image, shape_parameters, cost_functions=None, + gt_shape=None): return ATMAlgorithmResult( - image, self.algorithm, shape_parameters, gt_shape=gt_shape) + image, self.algorithm, shape_parameters, + cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! @@ -111,9 +113,11 @@ class LucasKanadeLinearInterface(LucasKanadeStandardInterface): def shape_model(self): return self.transform.model - def algorithm_result(self, image, shape_parameters, gt_shape=None): + def algorithm_result(self, image, shape_parameters, cost_functions=None, + gt_shape=None): return LinearATMAlgorithmResult( - image, self.algorithm, shape_parameters, gt_shape=gt_shape) + image, self.algorithm, shape_parameters, + cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! @@ -216,6 +220,10 @@ class Compositional(LucasKanade): """ def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x, f): + return lambda: x.T.dot(f(x)) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -224,16 +232,20 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # vectorize it and mask it - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - # compute masked error - self.e_m = i_m - self.t_m + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.t_m + + # update cost + cost_functions = [cost_closure(self.e_m)] + while k < max_iters and eps > self.eps: # solve for increments on the shape parameters self.dp = self._solve(map_inference) @@ -242,6 +254,17 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.t_m + + # update cost + cost_functions.append(cost_closure(self.e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -250,7 +273,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) # TODO: handle costs! diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 56b698e..68571ac 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -7,6 +7,29 @@ class ATMAlgorithmResult(ParametricAlgorithmResult): r""" """ + def __init__(self, image, algorithm, shape_parameters, cost_functions=None, + gt_shape=None): + super(ATMAlgorithmResult, self).__init__( + image, algorithm, shape_parameters, gt_shape=gt_shape) + self._cost_functions = cost_functions + self._warped_images = None + self._costs = None + + @property + def warped_images(self): + if self._warped_images is None: + self._warped_images = [] + for p in self.shape_parameters: + self.algorithm.transform.from_vector_inplace(p) + self._warped_images.append( + self.algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def costs(self): + if self._costs is None: + self._costs = [f() for f in self._cost_functions] + return self._costs # TODO: handle costs! @@ -33,4 +56,27 @@ def initial_shape(self): class ATMFitterResult(MultiFitterResult): r""" """ - pass + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(ATMFitterResult, self).__init__( + image, fitter, algorithm_results, affine_correction, + gt_shape=gt_shape) + self._warped_images = None + + @property + def warped_images(self): + if self._warped_images is None: + algorithm = self.algorithm_results[-1].algorithm + self._warped_images = [] + for s in self.shapes: + algorithm.transform.set_target(s) + self._warped_images.append( + algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def costs(self): + costs = [] + for a in self.algorithm_results: + costs += a.costs + return costs From c12624536e90ddeeac45185f7e7328348a5e5614 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 2 Jul 2015 13:19:27 +0100 Subject: [PATCH 081/423] Add cost for LK - Fixes a couple of bugs in the GradientImages and ECC residuals. --- menpofit/lk/algorithm.py | 18 +++++ menpofit/lk/residual.py | 138 +++++++++++++++++++++++++++------------ menpofit/lk/result.py | 53 +++++++++++++-- 3 files changed, 163 insertions(+), 46 deletions(-) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 37325f6..7a02a63 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -27,6 +27,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] + cost_functions = [] + # initialize iteration counter and epsilon k = 0 eps = np.Inf @@ -58,6 +60,9 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.from_vector_inplace(self.transform.as_vector() + dp) p_list.append(self.transform.as_vector()) + # update cost + cost_functions.append(self.residual.cost_closure()) + # test convergence eps = np.abs(norm(dp)) @@ -65,6 +70,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): k += 1 return LucasKanadeAlgorithmResult(image, self, p_list, + cost_functions=cost_functions, gt_shape=gt_shape) @@ -89,6 +95,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] + cost_functions = [] + # initialize iteration counter and epsilon k = 0 eps = np.Inf @@ -116,6 +124,9 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.compose_after_from_vector_inplace(dp) p_list.append(self.transform.as_vector()) + # update cost + cost_functions.append(self.residual.cost_closure()) + # test convergence eps = np.abs(norm(dp)) @@ -123,6 +134,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): k += 1 return LucasKanadeAlgorithmResult(image, self, p_list, + cost_functions=cost_functions, gt_shape=gt_shape) @@ -153,6 +165,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] + cost_functions = [] + # initialize iteration counter and epsilon k = 0 eps = np.Inf @@ -174,6 +188,9 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.compose_after_from_vector_inplace(inv_dp) p_list.append(self.transform.as_vector()) + # update cost + cost_functions.append(self.residual.cost_closure()) + # test convergence eps = np.abs(norm(dp)) @@ -181,4 +198,5 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): k += 1 return LucasKanadeAlgorithmResult(image, self, p_list, + cost_functions=cost_functions, gt_shape=gt_shape) diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index 5f808d3..57b386b 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -1,3 +1,4 @@ +from __future__ import division import abc import numpy as np from numpy.fft import fftn, ifftn, fft2 @@ -130,12 +131,16 @@ def steepest_descent_update(self, sdi, image, template): """ pass + @abc.abstractmethod + def cost_closure(self): + pass + class SSD(Residual): r""" """ def __init__(self, kernel=None): - self.kernel = kernel + self._kernel = kernel def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient @@ -153,23 +158,23 @@ def steepest_descent_images(self, image, dW_dp, forward=None): for d in a: sdi += d - if self.kernel is not None: - # if required, filter steepest descent images - # fft_sdi: ch x h x w x params - filtered_sdi = ifftn(self.kernel[..., None] * - fftn(sdi, axes=(-3, -2)), - axes=(-3, -2)) + if self._kernel is None: # reshape steepest descent images # sdi: (ch x h x w) x params # filtered_sdi: (ch x h x w) x params sdi = sdi.reshape((-1, sdi.shape[-1])) - filtered_sdi = filtered_sdi.reshape(sdi.shape) + filtered_sdi = sdi else: + # if required, filter steepest descent images + # fft_sdi: ch x h x w x params + filtered_sdi = ifftn(self._kernel[..., None] * + fftn(sdi, axes=(-3, -2)), + axes=(-3, -2)) # reshape steepest descent images # sdi: (ch x h x w) x params # filtered_sdi: (ch x h x w) x params sdi = sdi.reshape((-1, sdi.shape[-1])) - filtered_sdi = sdi + filtered_sdi = filtered_sdi.reshape(sdi.shape) return filtered_sdi, sdi @@ -185,8 +190,19 @@ def hessian(self, sdi, sdi2=None): return H def steepest_descent_update(self, sdi, image, template): - error_img = image.as_vector() - template.as_vector() - return sdi.T.dot(error_img) + self._error_img = image.as_vector() - template.as_vector() + return sdi.T.dot(self._error_img) + + def cost_closure(self): + def cost_closure(x, k): + if k is None: + return lambda: x.T.dot(x) + else: + x = x.reshape((-1,) + k.shape[-2:]) + kx = ifftn(k[..., None] * fftn(x, axes=(-2, -1)), + axes=(-2, -1)) + return lambda: x.ravel().T.dot(kx.ravel()) + return cost_closure(self._error_img, self._kernel) # TODO: Does not support masked templates at the moment @@ -194,7 +210,7 @@ class FourierSSD(Residual): r""" """ def __init__(self, kernel=None): - self.kernel = kernel + self._kernel = kernel def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient @@ -216,20 +232,20 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # fft_sdi: ch x h x w x params fft_sdi = fftn(sdi, axes=(-3, -2)) - if self.kernel is not None: - # if required, filter steepest descent images - filtered_fft_sdi = self.kernel[..., None] * fft_sdi + if self._kernel is None: # reshape steepest descent images # fft_sdi: (ch x h x w) x params # filtered_fft_sdi: (ch x h x w) x params fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) - filtered_fft_sdi = filtered_fft_sdi.reshape(fft_sdi.shape) + filtered_fft_sdi = fft_sdi else: + # if required, filter steepest descent images + filtered_fft_sdi = self._kernel[..., None] * fft_sdi # reshape steepest descent images # fft_sdi: (ch x h x w) x params # filtered_fft_sdi: (ch x h x w) x params fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) - filtered_fft_sdi = fft_sdi + filtered_fft_sdi = filtered_fft_sdi.reshape(fft_sdi.shape) return filtered_fft_sdi, fft_sdi @@ -243,11 +259,11 @@ def hessian(self, sdi, sdi2=None): def steepest_descent_update(self, sdi, image, template): # compute error image # error_img: ch x h x w - error_img = image.pixels - template.pixels + self._error_img = image.pixels - template.pixels # compute error image fft # fft_error_img: ch x (h x w) - fft_error_img = fft2(error_img) + fft_error_img = fft2(self._error_img) # compute steepest descent update # fft_sdi: params x (ch x h x w) @@ -255,6 +271,16 @@ def steepest_descent_update(self, sdi, image, template): # fft_sdu: params return sdi.conjugate().T.dot(fft_error_img.ravel()) + def cost_closure(self): + def cost_closure(x, k): + if k is None: + return lambda: x.ravel().T.dot(x.ravel()) + else: + kx = ifftn(k[..., None] * fftn(x, axes=(-2, -1)), + axes=(-2, -1)) + return lambda: x.ravel().T.dot(kx.ravel()) + return cost_closure(self._error_img, self._kernel) + class ECC(Residual): r""" @@ -273,7 +299,8 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient # gradient: dims x ch x pixels grad = self.gradient(norm_image, forward=forward) - grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) + grad = grad.as_vector().reshape((image.n_dims, image.n_channels) + + image.shape) # compute steepest descent images # gradient: dims x ch x pixels @@ -286,32 +313,38 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # reshape steepest descent images # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) + sdi = sdi.reshape((-1, sdi.shape[-1])) - def hessian(self, sdi): + return sdi, sdi + + def hessian(self, sdi, sdi2=None): # compute hessian - # sdi.T: params x (ch x pixels) - # sdi: (ch x pixels) x params + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params # hessian: params x x params - H = sdi.T.dot(sdi) + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) self._H_inv = scipy.linalg.inv(H) return H def steepest_descent_update(self, sdi, image, template): - normalised_IWxp = self._normalise_images(image).as_vector() - normalised_template = self._normalise_images(template).as_vector() + self._normalised_IWxp = self._normalise_images(image).as_vector() + self._normalised_template = self._normalise_images( + template).as_vector() - Gt = sdi.T.dot(normalised_template) - Gw = sdi.T.dot(normalised_IWxp) + Gt = sdi.T.dot(self._normalised_template) + Gw = sdi.T.dot(self._normalised_IWxp) # Calculate the numerator - IWxp_norm = scipy.linalg.norm(normalised_IWxp) + IWxp_norm = scipy.linalg.norm(self._normalised_IWxp) num1 = IWxp_norm ** 2 num2 = np.dot(Gw.T, np.dot(self._H_inv, Gw)) num = num1 - num2 # Calculate the denominator - den1 = np.dot(normalised_template, normalised_IWxp) + den1 = np.dot(self._normalised_template, self._normalised_IWxp) den2 = np.dot(Gt.T, np.dot(self._H_inv, Gw)) den = den1 - den2 @@ -325,10 +358,15 @@ def steepest_descent_update(self, sdi, image, template): l2 = - den / den3 l = np.maximum(l1, l2) - self._error_img = l * normalised_IWxp - normalised_template + self._error_img = l * self._normalised_IWxp - self._normalised_template return sdi.T.dot(self._error_img) + def cost_closure(self): + def cost_closure(x, y): + return lambda: x.T.dot(y) + return cost_closure(self._normalised_IWxp, self._normalised_template) + class GradientImages(Residual): r""" @@ -353,7 +391,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # second_grad: dims x dims x ch x pixels second_grad = self.gradient(self._template_grad) second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels, -1)) + (n_dims, n_dims, n_channels) + image.shape) # Fix crossed derivatives: dydx = dxdy second_grad[1, 0, ...] = second_grad[0, 1, ...] @@ -368,15 +406,21 @@ def steepest_descent_images(self, image, dW_dp, forward=None): sdi += d # reshape steepest descent images - # sdi: (dims x ch x h x w) x params - return sdi.reshape((-1, sdi.shape[-1])) + # sdi: (ch x pixels) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) - def hessian(self, sdi): + return sdi, sdi + + def hessian(self, sdi, sdi2=None): # compute hessian - # sdi.T: params x (dims x ch x pixels) - # sdi: (dims x ch x pixels) x params - # hessian: params x x params - return sdi.T.dot(sdi) + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params + # hessian: params x x params + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) + return H def steepest_descent_update(self, sdi, image, template): # compute image regularized gradient @@ -394,6 +438,11 @@ def steepest_descent_update(self, sdi, image, template): # sdu: params return sdi.T.dot(self._error_img) + def cost_closure(self): + def cost_closure(x): + return lambda: x.T.dot(x) + return cost_closure(self._error_img) + class GradientCorrelation(Residual): r""" @@ -502,5 +551,10 @@ def steepest_descent_update(self, sdi, image, template): # compute step size qp = np.sum(self._cos_phi * IWxp_cos_phi + self._sin_phi * IWxp_sin_phi) - l = self._N / qp - return l * sdu + self._l = self._N / qp + return self._l * sdu + + def cost_closure(self): + def cost_closure(x): + return lambda: 1/x + return cost_closure(self._l) diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py index 6674a63..9d6e4c5 100644 --- a/menpofit/lk/result.py +++ b/menpofit/lk/result.py @@ -5,9 +5,30 @@ # TODO: handle costs! # TODO: document me! class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): - r""" - """ - pass + def __init__(self, image, algorithm, shape_parameters, + cost_functions=None, gt_shape=None): + super(LucasKanadeAlgorithmResult, self).__init__( + image, algorithm, shape_parameters, gt_shape=gt_shape) + self._cost_functions = cost_functions + self._warped_images = None + self._costs = None + + @property + def warped_images(self): + if self._warped_images is None: + self._warped_images = [] + for p in self.shape_parameters: + self.algorithm.transform.from_vector_inplace(p) + self._warped_images.append( + self.image.warp_to_mask(self.algorithm.template.mask, + self.algorithm.transform)) + return self._warped_images + + @property + def costs(self): + if self._costs is None: + self._costs = [f() for f in self._cost_functions] + return self._costs # TODO: handle costs! @@ -15,4 +36,28 @@ class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): class LucasKanadeFitterResult(MultiFitterResult): r""" """ - pass + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(LucasKanadeFitterResult, self).__init__( + image, fitter, algorithm_results, affine_correction, + gt_shape=gt_shape) + self._warped_images = None + + @property + def warped_images(self): + if self._warped_images is None: + algorithm = self.algorithm_results[-1].algorithm + self._warped_images = [] + for s in self.shapes: + algorithm.transform.set_target(s) + self._warped_images.append( + self.image.warp_to_mask(algorithm.template.mask, + algorithm.transform)) + return self._warped_images + + @property + def costs(self): + costs = [] + for a in self.algorithm_results: + costs += a.costs + return costs From 1f3095b5d13f45066505b228cb3fdbb8afb4a762 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 2 Jul 2015 19:58:48 +0100 Subject: [PATCH 082/423] Remove addressed TODOs --- menpofit/aam/algorithm/lk.py | 15 --------------- menpofit/aam/result.py | 3 --- menpofit/atm/algorithm.py | 3 --- menpofit/atm/result.py | 3 --- menpofit/lk/algorithm.py | 3 --- menpofit/lk/result.py | 2 -- 6 files changed, 29 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 98eaa6d..756fe2a 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -253,7 +253,6 @@ def _precompute(self): self.s2_inv_S = s2 / S -# TODO: handle costs! # TODO: Document me! class ProjectOut(LucasKanade): r""" @@ -321,7 +320,6 @@ def cost_closure(x, f): image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class ProjectOutForwardCompositional(ProjectOut): r""" @@ -350,7 +348,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class ProjectOutInverseCompositional(ProjectOut): r""" @@ -385,7 +382,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class Simultaneous(LucasKanade): r""" @@ -480,7 +476,6 @@ def _solve(self, map_inference): return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) -# TODO: handle costs! # TODO: Document me! class SimultaneousForwardCompositional(Simultaneous): r""" @@ -498,7 +493,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class SimultaneousInverseCompositional(Simultaneous): r""" @@ -516,7 +510,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class Alternating(LucasKanade): r""" @@ -621,7 +614,6 @@ def cost_closure(x): appearance_parameters=c_list, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class AlternatingForwardCompositional(Alternating): r""" @@ -639,7 +631,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class AlternatingInverseCompositional(Alternating): r""" @@ -657,7 +648,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class ModifiedAlternating(Alternating): r""" @@ -745,7 +735,6 @@ def cost_closure(x): appearance_parameters=c_list, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class ModifiedAlternatingForwardCompositional(ModifiedAlternating): r""" @@ -763,7 +752,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class ModifiedAlternatingInverseCompositional(ModifiedAlternating): r""" @@ -781,7 +769,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class Wiberg(LucasKanade): r""" @@ -875,7 +862,6 @@ def cost_closure(x, f): appearance_parameters=c_list, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class WibergForwardCompositional(Wiberg): r""" @@ -893,7 +879,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class WibergInverseCompositional(Wiberg): r""" diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 38fabff..00a500a 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -2,7 +2,6 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: handle costs! # TODO: document me! class AAMAlgorithmResult(ParametricAlgorithmResult): r""" @@ -53,7 +52,6 @@ def costs(self): '{}'.format(self.algorithm.__class__)) -# TODO: handle costs! # TODO: document me! class LinearAAMAlgorithmResult(AAMAlgorithmResult): r""" @@ -72,7 +70,6 @@ def initial_shape(self): return self.initial_transform.sparse_target -# TODO: handle costs! # TODO: document me! class AAMFitterResult(MultiFitterResult): r""" diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 423afab..43e27a1 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -212,7 +212,6 @@ def _precompute(self, **kwargs): self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) -# TODO: handle costs! # TODO document me! class Compositional(LucasKanade): r""" @@ -276,7 +275,6 @@ def cost_closure(x, f): image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) -# TODO: handle costs! # TODO document me! class ForwardCompositional(Compositional): r""" @@ -303,7 +301,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO document me! class InverseCompositional(Compositional): r""" diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 68571ac..b7aec3e 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -2,7 +2,6 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: handle costs! # TODO: document me! class ATMAlgorithmResult(ParametricAlgorithmResult): r""" @@ -32,7 +31,6 @@ def costs(self): return self._costs -# TODO: handle costs! # TODO: document me! class LinearATMAlgorithmResult(ATMAlgorithmResult): r""" @@ -51,7 +49,6 @@ def initial_shape(self): return self.initial_transform.sparse_target -# TODO: handle costs! # TODO: document me! class ATMFitterResult(MultiFitterResult): r""" diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 7a02a63..b296300 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -16,7 +16,6 @@ def __init__(self, template, transform, residual, eps=10**-10): self.eps = eps -# TODO: handle costs! # TODO: document me! class ForwardAdditive(LucasKanade): r""" @@ -74,7 +73,6 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): gt_shape=gt_shape) -# TODO: handle costs! # TODO: document me! class ForwardCompositional(LucasKanade): r""" @@ -138,7 +136,6 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): gt_shape=gt_shape) -# TODO: handle costs! # TODO: document me! class InverseCompositional(LucasKanade): r""" diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py index 9d6e4c5..874fbda 100644 --- a/menpofit/lk/result.py +++ b/menpofit/lk/result.py @@ -2,7 +2,6 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: handle costs! # TODO: document me! class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): def __init__(self, image, algorithm, shape_parameters, @@ -31,7 +30,6 @@ def costs(self): return self._costs -# TODO: handle costs! # TODO: document me! class LucasKanadeFitterResult(MultiFitterResult): r""" From ecacbe8c81d7ea1acdce53c65f4d862298382848 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 3 Jul 2015 14:47:59 +0100 Subject: [PATCH 083/423] Fix noisy_align method removed Replaced with new methods - simple refactoring bug. --- menpofit/sdm/fitter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2c844e5..28af7f0 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -3,7 +3,7 @@ from menpo.transform import Scale, AlignmentSimilarity from menpo.feature import no_op from menpofit.builder import normalization_wrt_reference_shape, scale_images -from menpofit.fitter import MultiFitter, noisy_align +from menpofit.fitter import MultiFitter, noisy_target_alignment_transform from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import SN @@ -174,8 +174,9 @@ def _fitter_result(self, image, algorithm_results, affine_correction, def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, rotation=False): - transform = noisy_align(AlignmentSimilarity, + transform = noisy_target_alignment_transform( self.reference_bounding_box, bounding_box, + alignment_transform_cls=AlignmentSimilarity, noise_std=noise_std, rotation=rotation) return transform.apply(self.reference_shape) @@ -408,4 +409,4 @@ def __str__(self): # # feat_str = [feat_str] # # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out \ No newline at end of file +# # return out From 6a4ed771a7dc21ac632c9fa33f2eeffe32cbb5fa Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 3 Jul 2015 16:45:16 +0100 Subject: [PATCH 084/423] Add incremental sdm --- menpofit/sdm/__init__.py | 4 +- menpofit/sdm/algorithm.py | 162 +++++++++++++++++++++----------------- menpofit/sdm/fitter.py | 115 ++++++++++++++++++++------- 3 files changed, 178 insertions(+), 103 deletions(-) diff --git a/menpofit/sdm/__init__.py b/menpofit/sdm/__init__.py index 9180e28..16e88b4 100644 --- a/menpofit/sdm/__init__.py +++ b/menpofit/sdm/__init__.py @@ -1,2 +1,2 @@ -from .algorithm import SN, SGN -from .fitter import CRFitter, SDMFitter +from .algorithm import Newton, GaussNewton +from .fitter import SupervisedDescentFitter diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index db5a195..951cd97 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -5,8 +5,9 @@ from menpofit.result import NonParametricAlgorithmResult -# TODO document me! -class CRAlgorithm(object): +# TODO: compute more meaningful error +# TODO: document me! +class SupervisedDescentAlgorithm(object): r""" """ def train(self, images, gt_shapes, current_shapes, verbose=False, @@ -31,13 +32,14 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, # perform regression if verbose: - print_dynamic('- Performing regression...') - regressor = self._perform_regression(features, delta_x, **kwargs) + print_dynamic('- Performing regression.') + r = self._regressor_cls(**kwargs) + r.train(features, delta_x) # add regressor to list - self.regressors.append(regressor) + self.regressors.append(r) # estimate delta_points - estimated_delta_x = regressor(features) + estimated_delta_x = r.predict(features) if verbose: error = _compute_rmse(delta_x, estimated_delta_x) print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) @@ -59,6 +61,44 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, # rearrange current shapes into their original list of list form return current_shapes + def increment(self, images, gt_shapes, current_shapes, verbose=False, + **kwarg): + # obtain delta_x and gt_x + delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) + + # Cascaded Regression loop + for r in self.regressors: + # generate regression data + features = obtain_patch_features( + images, current_shapes, self.patch_shape, self.features, + features_patch_length=self._features_patch_length) + + # update regression + if verbose: + print_dynamic('- Updating regression') + r.increment(features, delta_x) + + # estimate delta_points + estimated_delta_x = r.predict(features) + if verbose: + error = _compute_rmse(delta_x, estimated_delta_x) + print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + + j = 0 + for shapes in current_shapes: + for s in shapes: + # update current x + current_x = s.as_vector() + estimated_delta_x[j] + # update current shape inplace + s.from_vector_inplace(current_x) + # update delta_x + delta_x[j] = gt_x[j] - current_x + # increase index + j += 1 + + # rearrange current shapes into their original list of list form + return current_shapes + def run(self, image, initial_shape, gt_shape=None, **kwargs): # set current shape and initialize list of shapes current_shape = initial_shape @@ -72,7 +112,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): features_patch_length=self._features_patch_length) # solve for increments on the shape vector - dx = r(features) + dx = r.predict(features) # update current shape current_shape = current_shape.from_vector( @@ -85,100 +125,80 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class SN(CRAlgorithm): +class Newton(SupervisedDescentAlgorithm): r""" - Supervised Newton. - - This class implements the Supervised Descent Method technique, proposed - by Xiong and De la Torre in [XiongD13]. - - References - ---------- - .. [XiongD13] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10 ** -5): + eps=10**-5): self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - # wire regression callable - self._perform_regression = _supervised_newton + self._regressor_cls = _incremental_least_squares # TODO: document me! -class SGN(CRAlgorithm): +class GaussNewton(SupervisedDescentAlgorithm): r""" - Supervised Gauss-Newton - - This class implements a variation of the Supervised Descent Method - [XiongD13] by some of the ideas incorporating ideas... - - References - ---------- - .. [XiongD13] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 - .. [Tzimiropoulos15] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10 ** -5): + eps=10**-5): self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - # wire regression callable - self._perform_regression = _supervised_gauss_newton + self._perform_regression = _incremental_indirect_least_squares # TODO: document me! -class _supervised_newton(object): +class _incremental_least_squares(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression - XX = features.T.dot(features) - XT = features.T.dot(deltas) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # descent direction - self.R = np.linalg.solve(XX, XT) + def __init__(self, l=0): + self.l = l + + def train(self, X, Y): + # regularized least squares + XX = X.T.dot(X) + np.fill_diagonal(XX, self.l + np.diag(XX)) + self.V = np.linalg.inv(XX) + self.W = self.V.dot(X.T.dot(Y)) + + def increment(self, X, Y): + # incremental regularized least squares + U = X.dot(self.V).dot(X.T) + np.fill_diagonal(U, 1 + np.diag(U)) + U = np.linalg.inv(U) + Q = self.V.dot(X.T).dot(U).dot(X) + self.V = self.V - Q.dot(self.V) + self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) - def __call__(self, features): - return np.dot(features, self.R) + def predict(self, x): + return np.dot(x, self.W) # TODO: document me! -class _supervised_gauss_newton(object): +class _incremental_indirect_least_squares(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression - XX = deltas.T.dot(deltas) - XT = deltas.T.dot(features) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # average Jacobian - self.J = np.linalg.solve(XX, XT) - # average Hessian - self.H = self.J.dot(self.J.T) - # descent direction - self.R = np.linalg.solve(self.H, self.J).T - - def __call__(self, features): - return np.dot(features, self.R) + def __init__(self, l=0, d=0): + self._ils = _incremental_least_squares(l) + self.d = d + + def train(self, X, Y): + # regularized least squares exchanging the roles of X and Y + self._ils.train(Y, X) + J = self._ils.W + # solve the original problem by computing the pseudo-inverse of the + # previous solution + H = J.T.dot(J) + np.fill_diagonal(H, self.d + np.diag(H)) + self.W = np.linalg.solve(H, J.T) + + def predict(self, x): + return np.dot(x, self.W) # TODO: document me! diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2c844e5..5dd2994 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,49 +1,48 @@ from __future__ import division -from functools import partial +import numpy as np from menpo.transform import Scale, AlignmentSimilarity from menpo.feature import no_op -from menpofit.builder import normalization_wrt_reference_shape, scale_images -from menpofit.fitter import MultiFitter, noisy_align +from menpofit.builder import ( + normalization_wrt_reference_shape, rescale_images_to_reference_shape, + scale_images) +from menpofit.fitter import MultiFitter, noisy_params_alignment_similarity from menpofit.result import MultiFitterResult import menpofit.checks as checks -from .algorithm import SN +from .algorithm import Newton # TODO: document me! -class CRFitter(MultiFitter): +class SupervisedDescentFitter(MultiFitter): r""" """ - def __init__(self, cr_algorithm_cls=SN, features=no_op, + def __init__(self, sd_algorithm_cls=Newton, features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), - iterations=6, n_perturbations=10, **kwargs): + iterations=6, n_perturbations=10, noise_std=0.05, **kwargs): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters - self._algorithms = [] self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations + self.noise_std = noise_std self.iterations = checks.check_max_iters(iterations, n_levels) # set up algorithms - self._set_up(cr_algorithm_cls, features, patch_shape, **kwargs) - - @property - def algorithms(self): - return self._algorithms + self._set_up(sd_algorithm_cls, features, patch_shape, **kwargs) @property def reference_bounding_box(self): return self.reference_shape.bounding_box() - def _set_up(self, cr_algorithm_cls, features, patch_shape, **kwargs): + def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): + self.algorithms = [] for j in range(self.n_levels): - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( features=features[j], patch_shape=patch_shape[j], iterations=self.iterations[j], **kwargs) - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) def train(self, images, group=None, label=None, verbose=False, **kwargs): # normalize images and compute reference shape @@ -71,7 +70,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape(gt_s) + p_s = self.noisy_shape_from_shape( + gt_s, noise_std=self.noise_std) perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) @@ -87,6 +87,68 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for shape in image_shapes: transform.apply_inplace(shape) + def increment(self, images, group=None, label=None, verbose=False, + **kwargs): + # normalize images with respect to reference shape of aam + images = rescale_images_to_reference_shape( + images, group, label, self.reference_shape, verbose=verbose) + + # for each pyramid level (low --> high) + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # scale images and compute features at other levels + level_images = scale_images(images, self.scales[j], + level_str=level_str, verbose=verbose) + + # extract ground truth shapes for current level + level_gt_shapes = [i.landmarks[group][label] for i in level_images] + + if j == 0: + # generate perturbed shapes + current_shapes = [] + for gt_s in level_gt_shapes: + perturbed_shapes = [] + for _ in range(self.n_perturbations): + p_s = self.noisy_shape_from_shape( + gt_s, noise_std=self.noise_std) + perturbed_shapes.append(p_s) + current_shapes.append(perturbed_shapes) + + # train cascaded regression algorithm + current_shapes = self.algorithms[j].increment( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + + # scale current shapes to next level resolution + if self.scales[j] != (1 or self.scales[-1]): + transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + + def train_incrementally(self, images, group=None, label=None, + batch_size=100, verbose=False, **kwargs): + n_batches = np.int(np.ceil(len(images) / batch_size)) + + # train first batch + print 'Training batch 1.' + self.train(images[:batch_size], group=group, label=label, + verbose=verbose, **kwargs) + + # train all other batches + start = batch_size + for j in range(1, n_batches): + print 'Training batch {}.'.format(j+1) + end = start + batch_size + self.increment(images[start:end], group=group, label=label, + verbose=verbose, **kwargs) + start = end + def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): r""" @@ -137,8 +199,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # if specified, crop the image if crop_image: - image = image.copy() - image.crop_to_landmarks_proportion_inplace(crop_image, + image = image.crop_to_landmarks_proportion(crop_image, group='initial_shape') # rescale image wrt the scale factor between reference_shape and @@ -172,16 +233,14 @@ def _fitter_result(self, image, algorithm_results, affine_correction, return MultiFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, - rotation=False): - transform = noisy_align(AlignmentSimilarity, - self.reference_bounding_box, bounding_box, - noise_std=noise_std, rotation=rotation) + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): + transform = noisy_params_alignment_similarity( + self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): + def noisy_shape_from_shape(self, shape, noise_std=0.05): return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std, rotation=rotation) + shape.bounding_box(), noise_std=noise_std) # TODO: fix me! def __str__(self): @@ -256,10 +315,6 @@ def __str__(self): # return out -# TODO: document me! -SDMFitter = partial(CRFitter, cr_algorithm_cls=SN) - - # class CRFitter(MultiFitter): # r""" # """ From 4a528ddad87c3cb1a8159ff924e5c9822d2fa8f9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 10:03:46 +0100 Subject: [PATCH 085/423] Add incremental AAM --- menpofit/aam/builder.py | 109 +++++++++++++++++++++++++++++++++++++++- menpofit/aam/fitter.py | 4 +- menpofit/atm/fitter.py | 2 +- 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 9ef763b..373236a 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,4 +1,5 @@ from __future__ import division +import numpy as np from copy import deepcopy from menpo.model import PCAModel from menpo.shape import mean_pointcloud @@ -8,7 +9,8 @@ from menpofit.builder import ( normalization_wrt_reference_shape, compute_features, scale_images, warp_images, extract_patches, build_shape_model, align_shapes, - build_reference_frame, build_patch_reference_frame, densify_shapes) + build_reference_frame, build_patch_reference_frame, densify_shapes, + rescale_images_to_reference_shape) from menpofit.transform import ( DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) @@ -253,6 +255,111 @@ def build(self, images, group=None, label=None, verbose=False): return aam + def increment(self, aam, images, group=None, label=None, + forgetting_factor=1.0, verbose=False): + # normalize images with respect to reference shape of aam + images = rescale_images_to_reference_shape( + images, group, label, aam.reference_shape, verbose=verbose) + + # increment models at each scale + if verbose: + print_dynamic('- Incrementing models\n') + + # for each pyramid level (high --> low) + for j, s in enumerate(self.scales[::-1]): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if j == 0: + # compute features at highest level + feature_images = compute_features(images, self.features[j], + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, + self.features[j], + level_str=level_str, + verbose=verbose) + + # extract potentially rescaled shapes + level_shapes = [i.landmarks[group][label] + for i in level_images] + + # obtain shape representation + if j == 0 or self.scale_shapes: + if verbose: + print_dynamic('{}Incrementing shape model'.format( + level_str)) + # compute aligned shapes + aligned_shapes = align_shapes(level_shapes) + # increment shape model + aam.shape_models[j].increment( + aligned_shapes, forgetting_factor=forgetting_factor) + if self.max_shape_components is not None: + aam.shape_models[j].trim_components( + self.max_appearance_components[j]) + else: + # copy previous shape model + aam.shape_models[j] = deepcopy(aam.shape_models[j-1]) + + mean_shape = aam.appearance_models[j].mean().landmarks[ + 'source'].lms + + # obtain warped images + warped_images = self._warp_images(level_images, level_shapes, + mean_shape, j, + level_str, verbose) + + # obtain appearance representation + if verbose: + print_dynamic('{}Incrementing appearance model'.format( + level_str)) + # increment appearance model + aam.appearance_models[j].increment(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + aam.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + def build_incrementally(self, images, group=None, label=None, + forgetting_factor=1.0, batch_size=100, + verbose=False): + # number of batches + n_batches = np.int(np.ceil(len(images) / batch_size)) + + # train first batch + print 'Training batch 1.' + aam = self.build(images[:batch_size], group=group, label=label, + verbose=verbose) + + # train all other batches + start = batch_size + for j in range(1, n_batches): + print 'Training batch {}.'.format(j+1) + end = start + batch_size + self.increment(aam, images[start:end], group=group, label=label, + forgetting_factor=forgetting_factor, + verbose=verbose) + start = end + + return aam + @classmethod def _build_shape_model(cls, shapes, max_components, level): return build_shape_model(shapes, max_components=max_components) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a985331..a8fbc9c 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -56,13 +56,13 @@ class LucasKanadeAAMFitter(AAMFitter): def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=None, n_appearance=None, sampling=None, **kwargs): self._model = aam - self.algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): + self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, sampling)): @@ -113,7 +113,6 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam - self.algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) @@ -123,6 +122,7 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, self._set_up(sd_algorithm_cls, sampling, **kwargs) def _set_up(self, sd_algorithm_cls, sampling, **kwargs): + self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, sampling)): diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 8dbb453..4a5ce71 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -16,7 +16,6 @@ class LucasKanadeATMFitter(ModelFitter): def __init__(self, atm, algorithm_cls=InverseCompositional, n_shape=None, sampling=None, **kwargs): self._model = atm - self.algorithms = [] self._check_n_shape(n_shape) self._set_up(algorithm_cls, sampling, **kwargs) @@ -25,6 +24,7 @@ def atm(self): return self._model def _set_up(self, algorithm_cls, sampling, **kwargs): + self.algorithms = [] for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, self.atm.shape_models)): From a6da810376b93c43919345f37f20a97765c59970 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 10:05:05 +0100 Subject: [PATCH 086/423] Small change in SDMFiter --- menpofit/sdm/fitter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 5dd2994..99a6530 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -133,6 +133,7 @@ def increment(self, images, group=None, label=None, verbose=False, def train_incrementally(self, images, group=None, label=None, batch_size=100, verbose=False, **kwargs): + # number of batches n_batches = np.int(np.ceil(len(images) / batch_size)) # train first batch From 4227537d1d60c43be6d7a5945e4cad261db82d92 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 16:09:21 +0100 Subject: [PATCH 087/423] Add initialization from user provided shapes and bounding boxes. - Allows users to define their own perturb_from_shape and perturb_from_bounding_box functions - Fixes typo in GaussNewton --- menpofit/fitter.py | 12 ++++ menpofit/sdm/algorithm.py | 3 +- menpofit/sdm/fitter.py | 131 +++++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 38 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index c8fb9c8..848203d 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -366,6 +366,18 @@ def noisy_target_alignment_transform(source, target, return alignment_transform_cls(source, noisy_target, **kwargs) +def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05): + transform = noisy_params_alignment_similarity( + shape.bounding_box(), bounding_box, noise_std=noise_std) + return transform.apply(shape) + + +def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05): + transform = noisy_params_alignment_similarity( + reference_shape, shape, noise_std=noise_std) + return transform.apply(reference_shape) + + def align_shape_with_bounding_box(shape, bounding_box, alignment_transform_cls=AlignmentSimilarity, **kwargs): diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 951cd97..fe76cd4 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -149,7 +149,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._perform_regression = _incremental_indirect_least_squares + self._regressor_cls = _incremental_indirect_least_squares # TODO: document me! @@ -331,6 +331,7 @@ def compute_features_info(image, shape, features_callable, return (features_patch_shape, features_patch_length, features_shape, features_length) + # def initialize_sampling(self, image, group=None, label=None): # if self._sampling is None: # sampling = np.ones(self.patch_shape, dtype=np.bool) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 99a6530..3e9bb1b 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,11 +1,14 @@ from __future__ import division import numpy as np -from menpo.transform import Scale, AlignmentSimilarity +import warnings +from menpo.transform import Scale from menpo.feature import no_op from menpofit.builder import ( normalization_wrt_reference_shape, rescale_images_to_reference_shape, scale_images) -from menpofit.fitter import MultiFitter, noisy_params_alignment_similarity +from menpofit.fitter import ( + MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, + align_shape_with_bounding_box) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -17,7 +20,10 @@ class SupervisedDescentFitter(MultiFitter): """ def __init__(self, sd_algorithm_cls=Newton, features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), - iterations=6, n_perturbations=10, noise_std=0.05, **kwargs): + iterations=6, n_perturbations=30, + perturb_from_shape=noisy_shape_from_shape, + perturb_from_bounding_box=noisy_shape_from_bounding_box, + **kwargs): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -27,15 +33,12 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations - self.noise_std = noise_std self.iterations = checks.check_max_iters(iterations, n_levels) + self._perturb_from_shape = perturb_from_shape + self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._set_up(sd_algorithm_cls, features, patch_shape, **kwargs) - @property - def reference_bounding_box(self): - return self.reference_shape.bounding_box() - def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): self.algorithms = [] for j in range(self.n_levels): @@ -44,11 +47,42 @@ def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): iterations=self.iterations[j], **kwargs) self.algorithms.append(algorithm) - def train(self, images, group=None, label=None, verbose=False, **kwargs): + def perturb_from_shape(self, shape, **kwargs): + return self._perturb_from_shape(self.reference_shape, shape, **kwargs) + + def perturb_from_bounding_box(self, bounding_box, **kwargs): + return self._perturb_from_bounding_box(self.reference_shape, + bounding_box, **kwargs) + + def train(self, images, group=None, label=None, + perturbation_group=None, verbose=False, **kwargs): # normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) + # handle perturbations + if perturbation_group is None: + perturbation_group = 'perturbed_' + # generate perturbations by perturbing ground truth shapes + for i in images: + gt_s = i.landmarks[group][label] + for j in range(self.n_perturbations): + p_s = self.perturb_from_shape(gt_s) + p_group = perturbation_group + '{}'.format(j) + i.landmarks[p_group] = p_s + else: + # reset number of perturbations + n_perturbations = 0 + for k in images[0].landmarks.keys(): + if perturbation_group in k: + n_perturbations += 1 + if n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided initialization_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + # for each pyramid level (low --> high) for j in range(self.n_levels): if verbose: @@ -65,17 +99,21 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - # generate perturbed shapes + # extract perturbations at the very bottom level current_shapes = [] - for gt_s in level_gt_shapes: - perturbed_shapes = [] - for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape( - gt_s, noise_std=self.noise_std) - perturbed_shapes.append(p_s) - current_shapes.append(perturbed_shapes) - - # train cascaded regression algorithm + for i in level_images: + c_shapes = [] + for k in range(self.n_perturbations): + p_group = perturbation_group + '{}'.format(k) + c_s = i.landmarks[p_group].lms + if c_s.n_points != level_gt_shapes[0].n_points: + # assume c_s is bounding box + c_s = align_shape_with_bounding_box( + self.reference_shape, c_s) + c_shapes.append(c_s) + current_shapes.append(c_shapes) + + # train supervised descent algorithm current_shapes = self.algorithms[j].train( level_images, level_gt_shapes, current_shapes, verbose=verbose, **kwargs) @@ -87,12 +125,36 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for shape in image_shapes: transform.apply_inplace(shape) - def increment(self, images, group=None, label=None, verbose=False, + def increment(self, images, group=None, label=None, + perturbation_group=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( images, group, label, self.reference_shape, verbose=verbose) + # handle perturbations + if perturbation_group is None: + perturbation_group = 'perturbed_' + # generate perturbations by perturbing ground truth shapes + for i in images: + gt_s = i.landmarks[group][label] + for j in range(self.n_perturbations): + p_s = self.perturb_from_shape(gt_s) + p_group = perturbation_group + '{}'.format(j) + i.landmarks[p_group] = p_s + else: + # reset number of perturbations + n_perturbations = 0 + for k in images[0].landmarks.keys(): + if perturbation_group in k: + n_perturbations += 1 + if n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided initialization_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + # for each pyramid level (low --> high) for j in range(self.n_levels): if verbose: @@ -109,15 +171,19 @@ def increment(self, images, group=None, label=None, verbose=False, level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - # generate perturbed shapes + # extract perturbations at the very bottom level current_shapes = [] - for gt_s in level_gt_shapes: - perturbed_shapes = [] - for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape( - gt_s, noise_std=self.noise_std) - perturbed_shapes.append(p_s) - current_shapes.append(perturbed_shapes) + for i in level_images: + c_shapes = [] + for k in range(self.n_perturbations): + p_group = perturbation_group + '{}'.format(k) + c_s = i.landmarks[p_group].lms + if c_s.n_points != level_gt_shapes[0].n_points: + # assume c_s is bounding box + c_s = align_shape_with_bounding_box( + self.reference_shape, c_s) + c_shapes.append(c_s) + current_shapes.append(c_shapes) # train cascaded regression algorithm current_shapes = self.algorithms[j].increment( @@ -234,15 +300,6 @@ def _fitter_result(self, image, algorithm_results, affine_correction, return MultiFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): - transform = noisy_params_alignment_similarity( - self.reference_bounding_box, bounding_box, noise_std=noise_std) - return transform.apply(self.reference_shape) - - def noisy_shape_from_shape(self, shape, noise_std=0.05): - return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std) - # TODO: fix me! def __str__(self): pass From 835167aa95eabd47978eccb2ceb0ec042ab6dba9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 18:43:23 +0100 Subject: [PATCH 088/423] Add new math subpackage - At the moment this subpackage contains regression techniques used by sdm. - This package might be moved to menpo core in the long run. - Fixes small bugs on fitting results and fitters. --- menpofit/fitter.py | 4 +-- menpofit/math/__init__.py | 2 ++ menpofit/math/least_squares.py | 60 ++++++++++++++++++++++++++++++++++ menpofit/result.py | 10 +++--- menpofit/sdm/algorithm.py | 56 +++---------------------------- 5 files changed, 73 insertions(+), 59 deletions(-) create mode 100644 menpofit/math/__init__.py create mode 100644 menpofit/math/least_squares.py diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 848203d..231dba8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -218,8 +218,8 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, shape = algorithm_result.final_shape if s != self.scales[-1]: - Scale(self.scales[j+1]/s, - n_dims=shape.n_dims).apply_inplace(shape) + shape = Scale(self.scales[j+1]/s, + n_dims=shape.n_dims).apply(shape) return algorithm_results diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py new file mode 100644 index 0000000..1fec9a8 --- /dev/null +++ b/menpofit/math/__init__.py @@ -0,0 +1,2 @@ +from least_squares import ( + incremental_least_squares, incremental_indirect_least_squares) \ No newline at end of file diff --git a/menpofit/math/least_squares.py b/menpofit/math/least_squares.py new file mode 100644 index 0000000..4f6cbf2 --- /dev/null +++ b/menpofit/math/least_squares.py @@ -0,0 +1,60 @@ +import numpy as np + + +# TODO: document me! +class incremental_least_squares(object): + r""" + """ + def __init__(self, l=0): + self.l = l + + def train(self, X, Y): + # regularized least squares + XX = X.T.dot(X) + np.fill_diagonal(XX, self.l + np.diag(XX)) + self.V = np.linalg.inv(XX) + self.W = self.V.dot(X.T.dot(Y)) + + def increment(self, X, Y): + # incremental regularized least squares + U = X.dot(self.V).dot(X.T) + np.fill_diagonal(U, 1 + np.diag(U)) + U = np.linalg.inv(U) + Q = self.V.dot(X.T).dot(U).dot(X) + self.V = self.V - Q.dot(self.V) + self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) + + def predict(self, x): + return np.dot(x, self.W) + + +# TODO: document me! +class incremental_indirect_least_squares(object): + r""" + """ + def __init__(self, l=0, d=0): + self._ils = incremental_least_squares(l) + self.d = d + + def train(self, X, Y): + # regularized least squares exchanging the roles of X and Y + self._ils.train(Y, X) + J = self._ils.W + # solve the original problem by computing the pseudo-inverse of the + # previous solution + H = J.T.dot(J) + np.fill_diagonal(H, self.d + np.diag(H)) + self.W = np.linalg.solve(H, J.T) + + def increment(self, X, Y): + # incremental least squares exchanging the roles of X and Y + self._ils.increment(Y, X) + J = self._ils.W + # solve the original problem by computing the pseudo-inverse of the + # previous solution + H = J.T.dot(J) + np.fill_diagonal(H, self.d + np.diag(H)) + self.W = np.linalg.solve(H, J.T) + + def predict(self, x): + return np.dot(x, self.W) \ No newline at end of file diff --git a/menpofit/result.py b/menpofit/result.py index 335a51b..81d12ba 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -595,11 +595,11 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): r""" """ shapes = [] - for j, (alg, s) in enumerate(zip(algorithm_results, scales)): - transform = Scale(scales[-1]/s, alg.final_shape.n_dims) - for t in alg.shapes: - t = transform.apply(t) - shapes.append(affine_correction.apply(t)) + for j, (alg, scale) in enumerate(zip(algorithm_results, scales)): + transform = Scale(scales[-1]/scale, alg.final_shape.n_dims) + for shape in alg.shapes: + shape = transform.apply(shape) + shapes.append(affine_correction.apply(shape)) return shapes diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index fe76cd4..d9e66c6 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -3,7 +3,8 @@ from menpo.feature import no_op from menpo.visualize import print_dynamic from menpofit.result import NonParametricAlgorithmResult - +from menpofit.math import ( + incremental_least_squares, incremental_indirect_least_squares) # TODO: compute more meaningful error # TODO: document me! @@ -135,7 +136,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = _incremental_least_squares + self._regressor_cls = incremental_least_squares # TODO: document me! @@ -149,56 +150,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = _incremental_indirect_least_squares - - -# TODO: document me! -class _incremental_least_squares(object): - r""" - """ - def __init__(self, l=0): - self.l = l - - def train(self, X, Y): - # regularized least squares - XX = X.T.dot(X) - np.fill_diagonal(XX, self.l + np.diag(XX)) - self.V = np.linalg.inv(XX) - self.W = self.V.dot(X.T.dot(Y)) - - def increment(self, X, Y): - # incremental regularized least squares - U = X.dot(self.V).dot(X.T) - np.fill_diagonal(U, 1 + np.diag(U)) - U = np.linalg.inv(U) - Q = self.V.dot(X.T).dot(U).dot(X) - self.V = self.V - Q.dot(self.V) - self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) - - def predict(self, x): - return np.dot(x, self.W) - - -# TODO: document me! -class _incremental_indirect_least_squares(object): - r""" - """ - def __init__(self, l=0, d=0): - self._ils = _incremental_least_squares(l) - self.d = d - - def train(self, X, Y): - # regularized least squares exchanging the roles of X and Y - self._ils.train(Y, X) - J = self._ils.W - # solve the original problem by computing the pseudo-inverse of the - # previous solution - H = J.T.dot(J) - np.fill_diagonal(H, self.d + np.diag(H)) - self.W = np.linalg.solve(H, J.T) - - def predict(self, x): - return np.dot(x, self.W) + self._regressor_cls = incremental_indirect_least_squares # TODO: document me! From 490841cdc464c38651d0225a059cd5d3a17becd8 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 18:52:00 +0100 Subject: [PATCH 089/423] Small fix in result.py --- menpofit/result.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 81d12ba..2d346f0 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -558,8 +558,8 @@ def final_shape(self): @property def initial_shape(self): initial_shape = self.algorithm_results[0].initial_shape - Scale(self.scales[-1]/self.scales[0], - initial_shape.n_dims).apply_inplace(initial_shape) + initial_shape = Scale(self.scales[-1]/self.scales[0], + initial_shape.n_dims).apply_inplace(initial_shape) return self._affine_correction.apply(initial_shape) From 006d52cc0ae5d33993e9535ea4c2f3650b860fe0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 18:58:41 +0100 Subject: [PATCH 090/423] Fix for results.py --- menpofit/result.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 335a51b..4306185 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -558,8 +558,8 @@ def final_shape(self): @property def initial_shape(self): initial_shape = self.algorithm_results[0].initial_shape - Scale(self.scales[-1]/self.scales[0], - initial_shape.n_dims).apply_inplace(initial_shape) + initial_shape = Scale(self.scales[-1]/self.scales[0], + initial_shape.n_dims).apply(initial_shape) return self._affine_correction.apply(initial_shape) @@ -595,11 +595,11 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): r""" """ shapes = [] - for j, (alg, s) in enumerate(zip(algorithm_results, scales)): - transform = Scale(scales[-1]/s, alg.final_shape.n_dims) - for t in alg.shapes: - t = transform.apply(t) - shapes.append(affine_correction.apply(t)) + for j, (alg, scale) in enumerate(zip(algorithm_results, scales)): + transform = Scale(scales[-1]/scale, alg.final_shape.n_dims) + for shape in alg.shapes: + shape = transform.apply(shape) + shapes.append(affine_correction.apply(shape)) return shapes From 72054c2936f93ec2222dae240c0f475689ed6350 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 19:04:16 +0100 Subject: [PATCH 091/423] Fixing what got undone by previous commit --- menpofit/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/result.py b/menpofit/result.py index 2d346f0..4306185 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -559,7 +559,7 @@ def final_shape(self): def initial_shape(self): initial_shape = self.algorithm_results[0].initial_shape initial_shape = Scale(self.scales[-1]/self.scales[0], - initial_shape.n_dims).apply_inplace(initial_shape) + initial_shape.n_dims).apply(initial_shape) return self._affine_correction.apply(initial_shape) From e9836276deedaad082159ba8f7c543dac3c829a1 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:26:53 +0100 Subject: [PATCH 092/423] Update errors in results. - results now receive a function to compute errors. - previous error funtions are slightly modified. --- menpofit/result.py | 96 +++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 4306185..2f2e1de 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -2,6 +2,7 @@ import abc import numpy as np from menpo.transform import Scale +from menpo.shape import PointCloud from menpo.image import Image @@ -48,44 +49,48 @@ def fitted_image(self): image.landmarks['ground'] = self.gt_shape return image - def final_error(self, error_type='me_norm'): + def final_error(self, compute_error=None): r""" Returns the final fitting error. Parameters ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. + compute_error: `callable`, optional + Callable that computes the error between the fitted and + ground truth shapes. Returns ------- final_error : `float` The final error at the end of the fitting procedure. """ + if compute_error is None: + compute_error = compute_normalise_point_to_point_error if self.gt_shape is not None: - return compute_error(self.final_shape, self.gt_shape, error_type) + return compute_error(self.final_shape, self.gt_shape) else: raise ValueError('Ground truth has not been set, final error ' 'cannot be computed') - def initial_error(self, error_type='me_norm'): + def initial_error(self, compute_error=None): r""" Returns the initial fitting error. Parameters ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. + compute_error: `callable`, optional + Callable that computes the error between the fitted and + ground truth shapes. Returns ------- initial_error : `float` The initial error at the start of the fitting procedure. """ + if compute_error is None: + compute_error = compute_normalise_point_to_point_error if self.gt_shape is not None: - return compute_error(self.initial_shape, self.gt_shape, error_type) + return compute_error(self.initial_shape, self.gt_shape) else: raise ValueError('Ground truth has not been set, final error ' 'cannot be computed') @@ -141,23 +146,25 @@ def iter_image(self): image.landmarks['iter_'+str(j)] = s return image - def errors(self, error_type='me_norm'): + def errors(self, compute_error=None): r""" Returns a list containing the error at each fitting iteration. Parameters ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. + compute_error: `callable`, optional + Callable that computes the error between the fitted and + ground truth shapes. Returns ------- errors : `list` of `float` The errors at each iteration of the fitting process. """ + if compute_error is None: + compute_error = compute_normalise_point_to_point_error if self.gt_shape is not None: - return [compute_error(t, self.gt_shape, error_type) + return [compute_error(t, self.gt_shape) for t in self.shapes] else: raise ValueError('Ground truth has not been set, errors cannot ' @@ -604,48 +611,57 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): # TODO: Document me! -def compute_error(target, ground_truth, error_type='me_norm'): +def pointcloud_to_points(func): + def func_wrapper(*args, **kwargs): + args = list(args) + for index, arg in enumerate(args): + if isinstance(arg, PointCloud): + args[index] = arg.points + for key in kwargs: + if isinstance(kwargs[key], PointCloud): + kwargs[key] = kwargs[key].points + return func(*args, **kwargs) + return func_wrapper + + +# TODO: Document me! +@pointcloud_to_points +def compute_root_mean_square_error(shape, gt_shape): r""" """ - gt_points = ground_truth.points - target_points = target.points - - if error_type == 'me_norm': - return _compute_norm_p2p_error(target_points, gt_points) - elif error_type == 'me': - return _compute_me(target_points, gt_points) - elif error_type == 'rmse': - return _compute_rmse(target_points, gt_points) - else: - raise ValueError("Unknown error_type string selected. Valid options " - "are: me_norm, me, rmse'") + return np.sqrt(np.mean((shape.flatten() - gt_shape.flatten()) ** 2)) # TODO: Document me! -# TODO: rename to more descriptive name -def _compute_me(target, ground_truth): +@pointcloud_to_points +def compute_point_to_point_error(shape, gt_shape): r""" """ - return np.mean(np.sqrt(np.sum((target - ground_truth) ** 2, axis=-1))) + return np.mean(np.sqrt(np.sum((shape - gt_shape) ** 2, axis=-1))) # TODO: Document me! -# TODO: rename to more descriptive name -def _compute_rmse(target, ground_truth): +@pointcloud_to_points +def compute_normalise_root_mean_square_error(shape, gt_shape, norm_shape=None): r""" """ - return np.sqrt(np.mean((target.flatten() - ground_truth.flatten()) ** 2)) + if norm_shape is None: + norm_shape = gt_shape + normalizer = np.mean(np.max(norm_shape, axis=0) - + np.min(norm_shape, axis=0)) + return compute_root_mean_square_error(shape, gt_shape) / normalizer # TODO: Document me! -def _compute_norm_p2p_error(target, source, ground_truth=None): +@pointcloud_to_points +def compute_normalise_point_to_point_error(shape, gt_shape, norm_shape=None): r""" """ - if ground_truth is None: - ground_truth = source - normalizer = np.mean(np.max(ground_truth, axis=0) - - np.min(ground_truth, axis=0)) - return _compute_me(target, source) / normalizer + if norm_shape is None: + norm_shape = gt_shape + normalizer = np.mean(np.max(norm_shape, axis=0) - + np.min(norm_shape, axis=0)) + return compute_point_to_point_error(shape, gt_shape) / normalizer # TODO: Document me! From fc6c33a15539a81ec95340cd8b18b33453946320 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:30:22 +0100 Subject: [PATCH 093/423] Update sdm algorithms to print more informative errors. - algorithms (optionally) receive a function to compute the training error. --- menpofit/sdm/algorithm.py | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index d9e66c6..2be9d1d 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -2,17 +2,21 @@ import numpy as np from menpo.feature import no_op from menpo.visualize import print_dynamic -from menpofit.result import NonParametricAlgorithmResult +from menpofit.result import ( + NonParametricAlgorithmResult, compute_normalise_point_to_point_error) from menpofit.math import ( incremental_least_squares, incremental_indirect_least_squares) -# TODO: compute more meaningful error + # TODO: document me! class SupervisedDescentAlgorithm(object): r""" """ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): + + n_perturbations = len(current_shapes[0]) + template_shape = gt_shapes[0] self._features_patch_length = compute_features_info( images[0], gt_shapes[0], self.features, patch_shape=self.patch_shape)[1] @@ -42,8 +46,18 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, # estimate delta_points estimated_delta_x = r.predict(features) if verbose: - error = _compute_rmse(delta_x, estimated_delta_x) - print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + s1 = template_shape.from_vector(dx) + s2 = template_shape.from_vector(edx) + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('- Training error -> mean: {0:.4f}, ' + 'std: {1:.4f}, median: {2:.4f}.\n'. + format(mean, std, median)) j = 0 for shapes in current_shapes: @@ -64,6 +78,10 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, def increment(self, images, gt_shapes, current_shapes, verbose=False, **kwarg): + + n_perturbations = len(current_shapes[0]) + template_shape = gt_shapes[0] + # obtain delta_x and gt_x delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) @@ -82,8 +100,18 @@ def increment(self, images, gt_shapes, current_shapes, verbose=False, # estimate delta_points estimated_delta_x = r.predict(features) if verbose: - error = _compute_rmse(delta_x, estimated_delta_x) - print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + s1 = template_shape.from_vector(dx) + s2 = template_shape.from_vector(edx) + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('- Training error -> mean: {0:.4f}, ' + 'std: {1:.4f}, median: {2:.4f}.\n'. + format(mean, std, median)) j = 0 for shapes in current_shapes: @@ -130,13 +158,15 @@ class Newton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5): + eps=10**-5, + compute_error=compute_normalise_point_to_point_error): + self._regressor_cls = incremental_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = incremental_least_squares + self._compute_error = compute_error # TODO: document me! @@ -144,18 +174,15 @@ class GaussNewton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5): + eps=10**-5, + compute_error=compute_normalise_point_to_point_error): + self._regressor_cls = incremental_indirect_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = incremental_indirect_least_squares - - -# TODO: document me! -def _compute_rmse(x1, x2): - return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + self._compute_error = compute_error # TODO: docment me! From cdf04e89702f485b53e88f18496889a27ae642d9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:38:30 +0100 Subject: [PATCH 094/423] Add the ability to train sdmfitters using a single bounding box per image. --- menpofit/sdm/algorithm.py | 13 +++++----- menpofit/sdm/fitter.py | 54 +++++++++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 2be9d1d..1f711e9 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -158,31 +158,30 @@ class Newton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5, - compute_error=compute_normalise_point_to_point_error): + compute_error=compute_normalise_point_to_point_error, + eps=10**-5): self._regressor_cls = incremental_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations - self.eps = eps self._compute_error = compute_error - + self.eps = eps # TODO: document me! class GaussNewton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5, - compute_error=compute_normalise_point_to_point_error): + compute_error=compute_normalise_point_to_point_error, + eps=10**-5): self._regressor_cls = incremental_indirect_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations - self.eps = eps self._compute_error = compute_error + self.eps = eps # TODO: docment me! diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 3e9bb1b..9f71488 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -54,32 +54,42 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): return self._perturb_from_bounding_box(self.reference_shape, bounding_box, **kwargs) - def train(self, images, group=None, label=None, - perturbation_group=None, verbose=False, **kwargs): + def train(self, images, group=None, label=None, bounding_box_group=None, + verbose=False, **kwargs): # normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) # handle perturbations - if perturbation_group is None: - perturbation_group = 'perturbed_' + if bounding_box_group is None: + bounding_box_group = 'bb_' # generate perturbations by perturbing ground truth shapes for i in images: gt_s = i.landmarks[group][label] for j in range(self.n_perturbations): p_s = self.perturb_from_shape(gt_s) - p_group = perturbation_group + '{}'.format(j) + p_group = bounding_box_group + '{}'.format(j) i.landmarks[p_group] = p_s else: # reset number of perturbations n_perturbations = 0 for k in images[0].landmarks.keys(): - if perturbation_group in k: + if bounding_box_group in k: n_perturbations += 1 - if n_perturbations != self.n_perturbations: + if n_perturbations == 1: + for i in images: + bb = i.landmarks[bounding_box_group].lms + p_s = align_shape_with_bounding_box( + self.reference_shape, bb) + i.landmarks[bounding_box_group + '0'] = p_s + for j in range(1, self.n_perturbations): + p_s = self.perturb_from_bounding_box(bb) + p_group = bounding_box_group + '{}'.format(j) + i.landmarks[p_group] = p_s + elif n_perturbations != self.n_perturbations: warnings.warn('The original value of n_perturbation {} ' 'will be reset to {} in order to agree with ' - 'the provided initialization_group.'. + 'the provided bounding_box_group.'. format(self.n_perturbations, n_perturbations)) self.n_perturbations = n_perturbations @@ -104,7 +114,7 @@ def train(self, images, group=None, label=None, for i in level_images: c_shapes = [] for k in range(self.n_perturbations): - p_group = perturbation_group + '{}'.format(k) + p_group = bounding_box_group + '{}'.format(k) c_s = i.landmarks[p_group].lms if c_s.n_points != level_gt_shapes[0].n_points: # assume c_s is bounding box @@ -126,32 +136,42 @@ def train(self, images, group=None, label=None, transform.apply_inplace(shape) def increment(self, images, group=None, label=None, - perturbation_group=None, verbose=False, + bounding_box_group=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( images, group, label, self.reference_shape, verbose=verbose) # handle perturbations - if perturbation_group is None: - perturbation_group = 'perturbed_' + if bounding_box_group is None: + bounding_box_group = 'bb_' # generate perturbations by perturbing ground truth shapes for i in images: gt_s = i.landmarks[group][label] for j in range(self.n_perturbations): p_s = self.perturb_from_shape(gt_s) - p_group = perturbation_group + '{}'.format(j) + p_group = bounding_box_group + '{}'.format(j) i.landmarks[p_group] = p_s else: # reset number of perturbations n_perturbations = 0 for k in images[0].landmarks.keys(): - if perturbation_group in k: + if bounding_box_group in k: n_perturbations += 1 - if n_perturbations != self.n_perturbations: + if n_perturbations == 1: + for i in images: + bb = i.landmarks[bounding_box_group].lms + p_s = align_shape_with_bounding_box( + self.reference_shape, bb) + i.landmarks[bounding_box_group + '0'] = p_s + for j in range(1, self.n_perturbations): + p_s = self.perturb_from_bounding_box(bb) + p_group = bounding_box_group + '{}'.format(j) + i.landmarks[p_group] = p_s + elif n_perturbations != self.n_perturbations: warnings.warn('The original value of n_perturbation {} ' 'will be reset to {} in order to agree with ' - 'the provided initialization_group.'. + 'the provided bounding_box_group.'. format(self.n_perturbations, n_perturbations)) self.n_perturbations = n_perturbations @@ -176,7 +196,7 @@ def increment(self, images, group=None, label=None, for i in level_images: c_shapes = [] for k in range(self.n_perturbations): - p_group = perturbation_group + '{}'.format(k) + p_group = bounding_box_group + '{}'.format(k) c_s = i.landmarks[p_group].lms if c_s.n_points != level_gt_shapes[0].n_points: # assume c_s is bounding box From 839335c8b0ea7578dfa7c128001fe501a31029b7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:39:21 +0100 Subject: [PATCH 095/423] Update widget to play well with changes on error computation - This is a temporal fix. --- menpofit/visualize/widgets/base.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 8dd5b22..cb634bf 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -2448,11 +2448,20 @@ def update_info(name, value): # Create output str if fitting_results[im].gt_shape is not None: + from menpofit.result import ( + compute_root_mean_square_error, compute_point_to_point_error, + compute_normalise_point_to_point_error) + if value is 'me_norm': + func = compute_normalise_point_to_point_error + elif value is 'me': + func = compute_point_to_point_error + elif value is 'rmse': + func = compute_root_mean_square_error text_per_line = [ "> Initial error: {:.4f}".format( - fitting_results[im].initial_error(error_type=value)), + fitting_results[im].initial_error(compute_error=func)), "> Final error: {:.4f}".format( - fitting_results[im].final_error(error_type=value)), + fitting_results[im].final_error(compute_error=func)), "> {} iterations".format(fitting_results[im].n_iters)] else: text_per_line = [ @@ -2511,10 +2520,20 @@ def plot_ced_fun(name): # Get error type error_type = error_type_wid.value + from menpofit.result import ( + compute_root_mean_square_error, compute_point_to_point_error, + compute_normalise_point_to_point_error) + if error_type is 'me_norm': + func = compute_normalise_point_to_point_error + elif error_type is 'me': + func = compute_point_to_point_error + elif error_type is 'rmse': + func = compute_root_mean_square_error + # Create errors list - fit_errors = [f.final_error(error_type=error_type) + fit_errors = [f.final_error(compute_error=func) for f in fitting_results] - initial_errors = [f.initial_error(error_type=error_type) + initial_errors = [f.initial_error(compute_error=func) for f in fitting_results] errors = [fit_errors, initial_errors] From cfef43eb5bbe8834609f8a8ba14aa42835eb9d9c Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 8 Jul 2015 15:32:58 +0100 Subject: [PATCH 096/423] Add back ability to not use rotation on initialization. --- menpofit/builder.py | 2 +- menpofit/fitter.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index b4cfa57..a309b4e 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -150,7 +150,7 @@ def scale_images(images, scale, level_str='', verbose=None): for c, i in enumerate(images): if verbose: print_dynamic( - '{}Scaling features: {}'.format( + '{}Scaling images: {}'.format( level_str, progress_bar_str((c + 1.) / len(images), show_bar=False))) scaled_images.append(i.rescale(scale)) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 231dba8..808bceb 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -295,7 +295,8 @@ def noisy_shape_from_shape(self, shape, noise_std=0.05): shape.bounding_box(), noise_std=noise_std) -def noisy_params_alignment_similarity(source, target, noise_std=0.05): +def noisy_params_alignment_similarity(source, target, noise_std=0.05, + rotation=True): r""" Constructs and perturbs the optimal similarity transform between source and target by adding white noise to its parameters. @@ -322,7 +323,7 @@ def noisy_params_alignment_similarity(source, target, noise_std=0.05): elif len(noise_std) == 1: noise_std *= 3 - transform = AlignmentSimilarity(source, target, rotation=True) + transform = AlignmentSimilarity(source, target, rotation=rotation) parameters = transform.as_vector() scale = noise_std[0] * parameters[0] @@ -366,15 +367,18 @@ def noisy_target_alignment_transform(source, target, return alignment_transform_cls(source, noisy_target, **kwargs) -def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05): +def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, + rotation=True): transform = noisy_params_alignment_similarity( - shape.bounding_box(), bounding_box, noise_std=noise_std) + shape.bounding_box(), bounding_box, noise_std=noise_std, + rotation=rotation) return transform.apply(shape) -def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05): +def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05, + rotation=True): transform = noisy_params_alignment_similarity( - reference_shape, shape, noise_std=noise_std) + reference_shape, shape, noise_std=noise_std, rotation=rotation) return transform.apply(reference_shape) From 043812fb4e699506ca2a7637ee3f58de74541c08 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 13 Jul 2015 14:24:09 +0100 Subject: [PATCH 097/423] Add the ability to perturb without rotation --- menpofit/fitter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 808bceb..94f814d 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,5 +1,6 @@ from __future__ import division import numpy as np +from copy import deepcopy from menpo.shape import PointCloud from menpo.transform import ( Scale, Similarity, AlignmentAffine, AlignmentSimilarity) @@ -296,7 +297,7 @@ def noisy_shape_from_shape(self, shape, noise_std=0.05): def noisy_params_alignment_similarity(source, target, noise_std=0.05, - rotation=True): + rotation=False): r""" Constructs and perturbs the optimal similarity transform between source and target by adding white noise to its parameters. @@ -324,7 +325,7 @@ def noisy_params_alignment_similarity(source, target, noise_std=0.05, noise_std *= 3 transform = AlignmentSimilarity(source, target, rotation=rotation) - parameters = transform.as_vector() + parameters = deepcopy(transform.as_vector()) scale = noise_std[0] * parameters[0] rotation = noise_std[1] * parameters[1] @@ -368,7 +369,7 @@ def noisy_target_alignment_transform(source, target, def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, - rotation=True): + rotation=False): transform = noisy_params_alignment_similarity( shape.bounding_box(), bounding_box, noise_std=noise_std, rotation=rotation) @@ -376,7 +377,7 @@ def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05, - rotation=True): + rotation=False): transform = noisy_params_alignment_similarity( reference_shape, shape, noise_std=noise_std, rotation=rotation) return transform.apply(reference_shape) @@ -405,4 +406,3 @@ def align_shape_with_bounding_box(shape, bounding_box, shape_bb = shape.bounding_box() transform = alignment_transform_cls(shape_bb, bounding_box, **kwargs) return transform.apply(shape) - From 7e3cf95fa246ff9865403624cfcef3b5cc56dada Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 13 Jul 2015 14:27:30 +0100 Subject: [PATCH 098/423] Small refactoring of regression classes --- menpofit/math/__init__.py | 3 +- .../math/{least_squares.py => regression.py} | 43 ++++++++++++------- menpofit/sdm/algorithm.py | 8 ++-- 3 files changed, 33 insertions(+), 21 deletions(-) rename menpofit/math/{least_squares.py => regression.py} (54%) diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py index 1fec9a8..d916940 100644 --- a/menpofit/math/__init__.py +++ b/menpofit/math/__init__.py @@ -1,2 +1 @@ -from least_squares import ( - incremental_least_squares, incremental_indirect_least_squares) \ No newline at end of file +from regression import IRLRegression, IIRLRegression \ No newline at end of file diff --git a/menpofit/math/least_squares.py b/menpofit/math/regression.py similarity index 54% rename from menpofit/math/least_squares.py rename to menpofit/math/regression.py index 4f6cbf2..58db19f 100644 --- a/menpofit/math/least_squares.py +++ b/menpofit/math/regression.py @@ -2,21 +2,31 @@ # TODO: document me! -class incremental_least_squares(object): +class IRLRegression(object): r""" + Incremental Regularized Linear Regression """ - def __init__(self, l=0): + def __init__(self, l=0, bias=True): self.l = l + self.bias = bias def train(self, X, Y): - # regularized least squares + if self.bias: + # add bias + X = np.hstack((X, np.ones((X.shape[0], 1)))) + + # regularized linear regression XX = X.T.dot(X) np.fill_diagonal(XX, self.l + np.diag(XX)) self.V = np.linalg.inv(XX) self.W = self.V.dot(X.T.dot(Y)) def increment(self, X, Y): - # incremental regularized least squares + if self.bias: + # add bias + X = np.hstack((X, np.ones((X.shape[0], 1)))) + + # incremental regularized linear regression U = X.dot(self.V).dot(X.T) np.fill_diagonal(U, 1 + np.diag(U)) U = np.linalg.inv(U) @@ -25,21 +35,27 @@ def increment(self, X, Y): self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) def predict(self, x): + if self.bias: + if len(x.shape) == 1: + x = np.hstack((x, np.ones(1))) + else: + x = np.hstack((x, np.ones((x.shape[0], 1)))) return np.dot(x, self.W) # TODO: document me! -class incremental_indirect_least_squares(object): +class IIRLRegression(IRLRegression): r""" + Indirect Incremental Regularized Linear Regression """ - def __init__(self, l=0, d=0): - self._ils = incremental_least_squares(l) + def __init__(self, l=0, bias=True, d=0): + super(IIRLRegression, self).__init__(l=l, bias=bias) self.d = d def train(self, X, Y): - # regularized least squares exchanging the roles of X and Y - self._ils.train(Y, X) - J = self._ils.W + # regularized linear regression exchanging the roles of X and Y + super(IIRLRegression, self).train(Y, X) + J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) @@ -48,13 +64,10 @@ def train(self, X, Y): def increment(self, X, Y): # incremental least squares exchanging the roles of X and Y - self._ils.increment(Y, X) - J = self._ils.W + super(IIRLRegression, self).increment(Y, X) + J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) np.fill_diagonal(H, self.d + np.diag(H)) self.W = np.linalg.solve(H, J.T) - - def predict(self, x): - return np.dot(x, self.W) \ No newline at end of file diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 1f711e9..2c433ad 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -4,8 +4,7 @@ from menpo.visualize import print_dynamic from menpofit.result import ( NonParametricAlgorithmResult, compute_normalise_point_to_point_error) -from menpofit.math import ( - incremental_least_squares, incremental_indirect_least_squares) +from menpofit.math import IRLRegression, IIRLRegression # TODO: document me! @@ -160,7 +159,7 @@ class Newton(SupervisedDescentAlgorithm): def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5): - self._regressor_cls = incremental_least_squares + self._regressor_cls = IRLRegression self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape @@ -168,6 +167,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self._compute_error = compute_error self.eps = eps + # TODO: document me! class GaussNewton(SupervisedDescentAlgorithm): r""" @@ -175,7 +175,7 @@ class GaussNewton(SupervisedDescentAlgorithm): def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5): - self._regressor_cls = incremental_indirect_least_squares + self._regressor_cls = IIRLRegression self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape From c4a7cd331baa62ac70c1aaccda54257e471c9496 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 15 Jul 2015 17:15:44 +0100 Subject: [PATCH 099/423] Fix LK --- menpofit/lk/algorithm.py | 10 +++++++--- menpofit/lk/residual.py | 12 ++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index b296300..49bcd5b 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -40,6 +40,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # compute warp jacobian dW_dp = np.rollaxis( self.transform.d_dp(self.template.indices()), -1) + dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + + dW_dp.shape[-1:]) # compute steepest descent images filtered_J, J = self.residual.steepest_descent_images( @@ -53,7 +55,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): filtered_J, IWxp, self.template) # compute gradient descent parameter updates - dp = np.real(np.linalg.solve(H, sd_dp)) + dp = -np.real(np.linalg.solve(H, sd_dp)) # Update warp weights self.transform.from_vector_inplace(self.transform.as_vector() + dp) @@ -85,8 +87,10 @@ def __init__(self, template, transform, residual, eps=10**-10): def _precompute(self): # compute warp jacobian - self.dW_dp = np.rollaxis( + dW_dp = np.rollaxis( self.transform.d_dp(self.template.indices()), -1) + self.dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + + dW_dp.shape[-1:]) def run(self, image, initial_shape, max_iters=20, gt_shape=None): # initialize transform @@ -116,7 +120,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): filtered_J, IWxp, self.template) # compute gradient descent parameter updates - dp = np.real(np.linalg.solve(H, sd_dp)) + dp = -np.real(np.linalg.solve(H, sd_dp)) # Update warp weights self.transform.compose_after_from_vector_inplace(dp) diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index 57b386b..d406c0f 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -147,7 +147,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # grad: dims x ch x h x w nabla = self.gradient(image, forward=forward) nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + - image.shape) + nabla.shape) # compute steepest descent images # gradient: dims x ch x h x w @@ -217,7 +217,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # grad: dims x ch x h x w nabla = self.gradient(image, forward=forward) nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + - image.shape) + nabla.shape) # compute steepest descent images # gradient: dims x ch x h x w @@ -300,7 +300,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # gradient: dims x ch x pixels grad = self.gradient(norm_image, forward=forward) grad = grad.as_vector().reshape((image.n_dims, image.n_channels) + - image.shape) + grad.shape) # compute steepest descent images # gradient: dims x ch x pixels @@ -391,7 +391,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # second_grad: dims x dims x ch x pixels second_grad = self.gradient(self._template_grad) second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels) + image.shape) + (n_dims, n_dims, n_channels) + second_grad.shape) # Fix crossed derivatives: dydx = dxdy second_grad[1, 0, ...] = second_grad[0, 1, ...] @@ -454,7 +454,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient # grad: dims x ch x pixels grad = self.gradient(image, forward=forward) - grad2 = grad.as_vector().reshape((n_dims, n_channels) + image.shape) + grad2 = grad.as_vector().reshape((n_dims, n_channels) + grad.shape) # compute IGOs (remember axis 0 is y, axis 1 is x) # grad: dims x ch x pixels @@ -479,7 +479,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # second_grad: dims x dims x ch x pixels second_grad = self.gradient(grad) second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels) + image.shape) + (n_dims, n_dims, n_channels) + second_grad.shape) # Fix crossed derivatives: dydx = dxdy second_grad[1, 0, ...] = second_grad[0, 1, ...] From bfdafac10d9b7260843880f191f1862374301d30 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 20 Jul 2015 08:28:37 +0100 Subject: [PATCH 100/423] Add new perturbation procedure. -Perturbations are can be now generated from uniform and gaussian distributions. --- menpofit/fitter.py | 90 +++++++++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 94f814d..ab69006 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -3,6 +3,7 @@ from copy import deepcopy from menpo.shape import PointCloud from menpo.transform import ( + scale_about_centre, rotate_ccw_about_centre, Translation, Scale, Similarity, AlignmentAffine, AlignmentSimilarity) import menpofit.checks as checks @@ -287,7 +288,7 @@ def _check_n_shape(self, n_shape): 'those'.format(self._model.n_levels)) def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): - transform = noisy_params_alignment_similarity( + transform = noisy_alignment_similarity_transform( self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) @@ -296,45 +297,67 @@ def noisy_shape_from_shape(self, shape, noise_std=0.05): shape.bounding_box(), noise_std=noise_std) -def noisy_params_alignment_similarity(source, target, noise_std=0.05, - rotation=False): +def noisy_alignment_similarity_transform(source, target, noise_type='uniform', + noise_percentage=0.1, rotation=False): r""" Constructs and perturbs the optimal similarity transform between source - and target by adding white noise to its parameters. + and target by adding noise to its parameters. + Parameters ---------- source: :class:`menpo.shape.PointCloud` The source pointcloud instance used in the alignment target: :class:`menpo.shape.PointCloud` The target pointcloud instance used in the alignment - noise_std: float or triplet of floats, optional - The standard deviation of the white noise. If float the same amount + noise_type: str, optional + The type of noise to be added, 'uniform' or 'gaussian'. + noise_percentage: 0 < float < 1 or triplet of 0 < float < 1, optional + The standard percentage of noise to be added. If float the same amount of noise is applied to the scale, rotation and translation parameters of the true similarity transform. If triplet of floats, the first, second and third elements denote the amount of noise to be applied to the scale, rotation and translation parameters respectively. + rotation: boolean, optional + If False rotation is not considered when computing the optimal + similarity transform between source and target. + Returns ------- - noisy_transform : :class: `menpo.transform.Similarity` - The noisy Similarity Transform + noisy_alignment_similarity_transform : :class: `menpo.transform.Similarity` + The noisy Similarity Transform between source and target. """ - if isinstance(noise_std, float): - noise_std = [noise_std] * 3 - elif len(noise_std) == 1: - noise_std *= 3 - - transform = AlignmentSimilarity(source, target, rotation=rotation) - parameters = deepcopy(transform.as_vector()) - - scale = noise_std[0] * parameters[0] - rotation = noise_std[1] * parameters[1] - translation = noise_std[2] * target.range() - - noise = (([scale, rotation] + list(translation)) * - np.random.randn(transform.n_parameters)) - return Similarity.init_identity(source.n_dims).from_vector( - parameters + noise) + if isinstance(noise_percentage, float): + noise_percentage= [noise_percentage] * 3 + elif len(noise_percentage) == 1: + noise_percentage *= 3 + + similarity = AlignmentSimilarity(source, target, rotation=rotation) + + if noise_type is 'normal': + # + s = noise_percentage[0] * (0.5 / 3) * np.asscalar(np.random.randn(1)) + # + r = noise_percentage[1] * (180 / 3) * np.asscalar(np.random.randn(1)) + # + t = noise_percentage[2] * (target.range() / 3) * np.random.randn(2) + + s = scale_about_centre(target, 1 + s) + r = rotate_ccw_about_centre(target, r) + t = Translation(t, source.n_dims) + elif noise_type is 'uniform': + # + s = noise_percentage[0] * 0.5 * (2 * np.asscalar(np.random.randn(1)) - 1) + # + r = noise_percentage[1] * 180 * (2 * np.asscalar(np.random.rand(1)) - 1) + # + t = noise_percentage[2] * target.range() * (2 * np.random.rand(2) - 1) + + s = scale_about_centre(target, 1. + s) + r = rotate_ccw_about_centre(target, r) + t = Translation(t, source.n_dims) + + return similarity.compose_after(t.compose_after(s.compose_after(r))) def noisy_target_alignment_transform(source, target, @@ -368,18 +391,19 @@ def noisy_target_alignment_transform(source, target, return alignment_transform_cls(source, noisy_target, **kwargs) -def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, - rotation=False): - transform = noisy_params_alignment_similarity( - shape.bounding_box(), bounding_box, noise_std=noise_std, - rotation=rotation) +def noisy_shape_from_bounding_box(shape, bounding_box, noise_type='uniform', + noise_percentage=0.1, rotation=False): + transform = noisy_alignment_similarity_transform( + shape.bounding_box(), bounding_box, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) return transform.apply(shape) -def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05, - rotation=False): - transform = noisy_params_alignment_similarity( - reference_shape, shape, noise_std=noise_std, rotation=rotation) +def noisy_shape_from_shape(reference_shape, shape, noise_type='uniform', + noise_percentage=0.1, rotation=False): + transform = noisy_alignment_similarity_transform( + reference_shape, shape, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) return transform.apply(reference_shape) From 810497c4d2cea07e516c91e62282fce1d3941823 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 16:19:43 +0100 Subject: [PATCH 101/423] Fix pickling of SDM --- menpofit/result.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 2f2e1de..f19c516 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -1,5 +1,6 @@ from __future__ import division import abc +from functools import wraps import numpy as np from menpo.transform import Scale from menpo.shape import PointCloud @@ -611,8 +612,10 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): # TODO: Document me! -def pointcloud_to_points(func): - def func_wrapper(*args, **kwargs): +def pointcloud_to_points(wrapped): + + @wraps(wrapped) + def wrapper(*args, **kwargs): args = list(args) for index, arg in enumerate(args): if isinstance(arg, PointCloud): @@ -620,8 +623,8 @@ def func_wrapper(*args, **kwargs): for key in kwargs: if isinstance(kwargs[key], PointCloud): kwargs[key] = kwargs[key].points - return func(*args, **kwargs) - return func_wrapper + return wrapped(*args, **kwargs) + return wrapper # TODO: Document me! From 317d418f20ee2479515c51f1141df36c9ac0a3b7 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 16:21:26 +0100 Subject: [PATCH 102/423] Fix the None bounding box key error --- menpofit/sdm/fitter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 9f71488..07287fa 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -56,11 +56,17 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, **kwargs): - # normalize images and compute reference shape + # Normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) - # handle perturbations + # In the case where group is None, we need to get the only key so that + # we can add landmarks below and not get a complaint about using None + first_image = images[0] + if group is None: + group = first_image.landmarks.group_labels[0] + + # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: bounding_box_group = 'bb_' # generate perturbations by perturbing ground truth shapes From cec7e5aeaaac756ae3255bbe48c1b7833077ffc5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:25:38 +0100 Subject: [PATCH 103/423] Fixing verbose mode of helper functions Use the print_progress method --- menpofit/builder.py | 85 +++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index a309b4e..291f46d 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -1,11 +1,12 @@ from __future__ import division +from functools import partial import numpy as np from menpo.shape import mean_pointcloud, PointCloud, TriMesh from menpo.image import Image, MaskedImage from menpo.feature import no_op from menpo.transform import Scale, Translation, GeneralizedProcrustesAnalysis from menpo.model.pca import PCAModel -from menpo.visualize import print_dynamic, progress_bar_str +from menpo.visualize import print_dynamic, print_progress def compute_reference_shape(shapes, normalization_diagonal, verbose=False): @@ -51,18 +52,15 @@ def rescale_images_to_reference_shape(images, group, label, reference_shape, verbose=False): r""" """ - # normalize the scaling of all images wrt the reference_shape size - normalized_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic('- Normalizing images size: {}'.format( - progress_bar_str((c + 1.) / len(images), - show_bar=False))) - normalized_images.append(i.rescale_to_reference_shape( - reference_shape, group=group, label=label)) - if verbose: - print_dynamic('- Normalizing images size: Done\n') + wrap = partial(print_progress, prefix='- Normalizing images size') + else: + wrap = lambda x: x + + # Normalize the scaling of all images wrt the reference_shape size + normalized_images = [i.rescale_to_reference_shape(reference_shape, + group=group, label=label) + for i in wrap(images)] return normalized_images @@ -119,42 +117,37 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, shapes = [i.landmarks[group][label] for i in images] # compute the reference shape and fix its diagonal length - reference_shape = compute_reference_shape(shapes, diagonal, - verbose=verbose) + reference_shape = compute_reference_shape(shapes, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size normalized_images = rescale_images_to_reference_shape( - images, group, label, reference_shape, verbose=False) + images, group, label, reference_shape, verbose=verbose) return reference_shape, normalized_images # TODO: document me! -def compute_features(images, features, level_str='', verbose=None): - feature_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic( - '{}Computing feature space: {}'.format( - level_str, progress_bar_str((c + 1.) / len(images), - show_bar=False))) - i = features(i) - feature_images.append(i) +def compute_features(images, features, level_str='', verbose=False): + if verbose: + wrap = partial(print_progress, + prefix='{}Computing feature space'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x - return feature_images + return [features(i) for i in wrap(images)] # TODO: document me! -def scale_images(images, scale, level_str='', verbose=None): - if scale != 1: - scaled_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic( - '{}Scaling images: {}'.format( - level_str, progress_bar_str((c + 1.) / len(images), - show_bar=False))) - scaled_images.append(i.rescale(scale)) - return scaled_images +def scale_images(images, scale, level_str='', verbose=False): + if verbose: + wrap = partial(print_progress, + prefix='{}Scaling images'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + + if not np.allclose(scale, 1): + return [i.rescale(scale) for i in wrap(images)] else: return images @@ -182,14 +175,16 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! def extract_patches(images, shapes, patch_shape, normalize_function=no_op, - level_str='', verbose=None): + level_str='', verbose=False): + if verbose: + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + parts_images = [] - for c, (i, s) in enumerate(zip(images, shapes)): - if verbose: - print_dynamic('{}Warping images - {}'.format( - level_str, - progress_bar_str(float(c + 1) / len(images), - show_bar=False))) + for i, s in wrap(zip(images, shapes)): parts = i.extract_patches(s, patch_size=patch_shape, as_single_array=True) parts = normalize_function(parts) @@ -341,4 +336,4 @@ def build_shape_model(shapes, max_components=None): # trim shape model if required shape_model.trim_components(max_components) - return shape_model \ No newline at end of file + return shape_model From 29be9c2bb471fd7bcb9026f2218ea800b18040de Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:26:03 +0100 Subject: [PATCH 104/423] Update aam warp_images set_target and improve printing --- menpofit/builder.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 291f46d..3ae8322 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -152,21 +152,25 @@ def scale_images(images, scale, level_str='', verbose=False): return images -# TODO: Can be done more efficiently for PWA defining a dummy transform # TODO: document me! def warp_images(images, shapes, reference_frame, transform, level_str='', verbose=None): + if verbose: + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + warped_images = [] - for c, (i, s) in enumerate(zip(images, shapes)): - if verbose: - print_dynamic('{}Warping images - {}'.format( - level_str, - progress_bar_str(float(c + 1) / len(images), - show_bar=False))) - # compute transforms - t = transform(reference_frame.landmarks['source'].lms, s) + # Build a dummy transform, use set_target for efficiency + warp_transform = transform(reference_frame.landmarks['source'].lms, + reference_frame.landmarks['source'].lms) + for i, s in wrap(zip(images, shapes)): + # Update Transform Target + warp_transform.set_target(s) # warp images - warped_i = i.warp_to_mask(reference_frame.mask, t) + warped_i = i.warp_to_mask(reference_frame.mask, warp_transform) # attach reference frame landmarks to images warped_i.landmarks['source'] = reference_frame.landmarks['source'] warped_images.append(warped_i) From 60582ed10a521cb889caf9c289edd76301bcb05c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:26:46 +0100 Subject: [PATCH 105/423] Update regression - better printing Also, use a range rather than a while --- menpofit/sdm/algorithm.py | 52 +++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 2c433ad..24719bb 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -1,7 +1,8 @@ from __future__ import division +from functools import partial import numpy as np from menpo.feature import no_op -from menpo.visualize import print_dynamic +from menpo.visualize import print_dynamic, print_progress from menpofit.result import ( NonParametricAlgorithmResult, compute_normalise_point_to_point_error) from menpofit.math import IRLRegression, IIRLRegression @@ -11,8 +12,8 @@ class SupervisedDescentAlgorithm(object): r""" """ - def train(self, images, gt_shapes, current_shapes, verbose=False, - **kwargs): + def train(self, images, gt_shapes, current_shapes, level_str='', + verbose=False, **kwargs): n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] @@ -24,27 +25,32 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) # initialize iteration counter and list of regressors - k = 0 self.regressors = [] # Cascaded Regression loop - while k < self.iterations: + for k in range(self.iterations): # generate regression data features = obtain_patch_features( images, current_shapes, self.patch_shape, self.features, - features_patch_length=self._features_patch_length) + features_patch_length=self._features_patch_length, + level_str='{}(Iteration {}) - '.format(level_str, k), + verbose=verbose) - # perform regression + # Perform regression if verbose: - print_dynamic('- Performing regression.') + print_dynamic('{}(Iteration {}) - Performing regression'.format( + level_str, k)) r = self._regressor_cls(**kwargs) r.train(features, delta_x) # add regressor to list self.regressors.append(r) - # estimate delta_points + # Estimate delta_points estimated_delta_x = r.predict(features) + if verbose: + print_dynamic('{}(Iteration {}) - Calculating errors'.format( + level_str, k)) errors = [] for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): s1 = template_shape.from_vector(dx) @@ -54,9 +60,9 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, mean = np.mean(errors) std = np.std(errors) median = np.median(errors) - print_dynamic('- Training error -> mean: {0:.4f}, ' - 'std: {1:.4f}, median: {2:.4f}.\n'. - format(mean, std, median)) + print_dynamic('{}(Iteration {}) - Training error -> ' + 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. + format(level_str, k, mean, std, median)) j = 0 for shapes in current_shapes: @@ -69,10 +75,7 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, delta_x[j] = gt_x[j] - current_x # increase index j += 1 - # increase iteration counter - k += 1 - # rearrange current shapes into their original list of list form return current_shapes def increment(self, images, gt_shapes, current_shapes, verbose=False, @@ -198,7 +201,7 @@ def compute_patch_features(image, shape, patch_shape, features_callable, patch_features[j] = features_callable(p[0]).ravel() else: patch_features = [] - for j, p in enumerate(patches): + for p in patches: patch_features.append(features_callable(p[0]).ravel()) patch_features = np.asarray(patch_features) @@ -219,7 +222,7 @@ def generate_patch_features(image, shapes, patch_shape, features_callable, features_patch_length=features_patch_length) else: patch_features = [] - for j, s in enumerate(shapes): + for s in shapes: patch_features.append(compute_patch_features( image, s, patch_shape, features_callable, features_patch_length=features_patch_length)) @@ -230,24 +233,31 @@ def generate_patch_features(image, shapes, patch_shape, features_callable, # TODO: docment me! def obtain_patch_features(images, shapes, patch_shape, features_callable, - features_patch_length=None): + features_patch_length=None, level_str='', + verbose=False): """r """ + if verbose: + wrap = partial(print_progress, + prefix='{}Extracting patches'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + n_images = len(images) n_shapes = len(shapes[0]) n_points = shapes[0][0].n_points if features_patch_length: - patch_features = np.empty((n_images, (n_shapes * n_points * features_patch_length))) - for j, i in enumerate(images): + for j, i in enumerate(wrap(images)): patch_features[j] = generate_patch_features( i, shapes[j], patch_shape, features_callable, features_patch_length=features_patch_length) else: patch_features = [] - for j, i in images: + for j, i in enumerate(wrap(images)): patch_features.append(generate_patch_features( i, shapes[j], patch_shape, features_callable, features_patch_length=features_patch_length)) From d2448132dec3eef033c08d3837a2ee9271ab287a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:27:08 +0100 Subject: [PATCH 106/423] Big change to how bboxes are created May need updating a bit, but the logic works for noisy bboxes. Essentially, unify the logic for creating perturbations so that the noisy perturbations are generated based on the difference between the ground truth and the given bounding box. --- menpofit/sdm/fitter.py | 113 +++++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 07287fa..387205f 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,14 +1,16 @@ from __future__ import division +from functools import partial import numpy as np import warnings from menpo.transform import Scale from menpo.feature import no_op +from menpo.visualize import print_progress from menpofit.builder import ( normalization_wrt_reference_shape, rescale_images_to_reference_shape, scale_images) from menpofit.fitter import ( MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, - align_shape_with_bounding_box) + align_shape_with_bounding_box, noisy_params_alignment_similarity) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -68,36 +70,52 @@ def train(self, images, group=None, label=None, bounding_box_group=None, # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: - bounding_box_group = 'bb_' - # generate perturbations by perturbing ground truth shapes + bounding_box_group = '__gt_bb_' for i in images: gt_s = i.landmarks[group][label] - for j in range(self.n_perturbations): - p_s = self.perturb_from_shape(gt_s) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - else: - # reset number of perturbations - n_perturbations = 0 - for k in images[0].landmarks.keys(): - if bounding_box_group in k: - n_perturbations += 1 - if n_perturbations == 1: - for i in images: - bb = i.landmarks[bounding_box_group].lms - p_s = align_shape_with_bounding_box( - self.reference_shape, bb) - i.landmarks[bounding_box_group + '0'] = p_s - for j in range(1, self.n_perturbations): - p_s = self.perturb_from_bounding_box(bb) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) - self.n_perturbations = n_perturbations + perturb_bbox_group = bounding_box_group + '0' + i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + + # Find all bounding boxes on the images with the given bounding box key + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) + n_perturbations = len(all_bb_keys) + + # If there is only one example bounding box, then we will generate + # more perturbations based on the bounding box. + if n_perturbations == 1: + if verbose: + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg) + else: + wrap = lambda x: x + + for i in wrap(images): + # We assume that the first bounding box is a valid perturbation + # thus create n_perturbations - 1 new bounding boxes + for j in range(1, self.n_perturbations): + # TODO: This should use the new logic that @jalabort + # has come up with. Also, would it be good if this was + # customizable? As in, the ability to pass some kind of + # probability distribution to draw from? + gt_s = i.landmarks[group][label].bounding_box() + bb = i.landmarks[all_bb_keys[0]].lms + # TODO: Noisy align given bb to gt_s - is this correct? + p_s = noisy_params_alignment_similarity(bb, gt_s).apply(bb) + perturb_bbox_group = bounding_box_group + '_{}'.format(j) + i.landmarks[perturb_bbox_group] = p_s + elif n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided bounding_box_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + + # Re-grab all the bounding box keys for iterating over when calculating + # perturbations + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) # for each pyramid level (low --> high) for j in range(self.n_levels): @@ -106,37 +124,44 @@ def train(self, images, group=None, label=None, bounding_box_group=None, level_str = ' - Level {}: '.format(j) else: level_str = ' - ' + else: + level_str = None - # scale images and compute features at other levels + # Scale images and compute features at other levels level_images = scale_images(images, self.scales[j], level_str=level_str, verbose=verbose) - # extract ground truth shapes for current level + # Extract scaled ground truth shapes for current level level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - # extract perturbations at the very bottom level + if verbose: + msg = '{}Generating {} perturbations per image'.format( + level_str, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False) + else: + wrap = lambda x: x + + # Extract perturbations at the very bottom level current_shapes = [] - for i in level_images: + for i in wrap(level_images): c_shapes = [] - for k in range(self.n_perturbations): - p_group = bounding_box_group + '{}'.format(k) - c_s = i.landmarks[p_group].lms - if c_s.n_points != level_gt_shapes[0].n_points: - # assume c_s is bounding box - c_s = align_shape_with_bounding_box( - self.reference_shape, c_s) + for perturb_bbox_group in all_bb_keys: + bbox = i.landmarks[perturb_bbox_group].lms + c_s = align_shape_with_bounding_box( + self.reference_shape, bbox) c_shapes.append(c_s) current_shapes.append(c_shapes) # train supervised descent algorithm current_shapes = self.algorithms[j].train( level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) + level_str=level_str, verbose=verbose, **kwargs) - # scale current shapes to next level resolution + # Scale current shapes to next level resolution if self.scales[j] != (1 or self.scales[-1]): - transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) + transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) for image_shapes in current_shapes: for shape in image_shapes: transform.apply_inplace(shape) @@ -547,4 +572,4 @@ def __str__(self): # # feat_str = [feat_str] # # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out \ No newline at end of file +# # return out From 4295084f7fe7c91adbfbad60fcfc484874ca44c1 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 21 Jul 2015 09:02:16 +0100 Subject: [PATCH 107/423] Small changes --- menpofit/fitter.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ab69006..72fd8fd 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -328,29 +328,24 @@ def noisy_alignment_similarity_transform(source, target, noise_type='uniform', The noisy Similarity Transform between source and target. """ if isinstance(noise_percentage, float): - noise_percentage= [noise_percentage] * 3 + noise_percentage = [noise_percentage] * 3 elif len(noise_percentage) == 1: noise_percentage *= 3 similarity = AlignmentSimilarity(source, target, rotation=rotation) - if noise_type is 'normal': - # + if noise_type is 'gaussian': s = noise_percentage[0] * (0.5 / 3) * np.asscalar(np.random.randn(1)) - # r = noise_percentage[1] * (180 / 3) * np.asscalar(np.random.randn(1)) - # t = noise_percentage[2] * (target.range() / 3) * np.random.randn(2) s = scale_about_centre(target, 1 + s) r = rotate_ccw_about_centre(target, r) t = Translation(t, source.n_dims) + elif noise_type is 'uniform': - # s = noise_percentage[0] * 0.5 * (2 * np.asscalar(np.random.randn(1)) - 1) - # r = noise_percentage[1] * 180 * (2 * np.asscalar(np.random.rand(1)) - 1) - # t = noise_percentage[2] * target.range() * (2 * np.random.rand(2) - 1) s = scale_about_centre(target, 1. + s) From 8ef02f4b51bd29703644b9ae0b11e86a3acd3456 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 22 Jul 2015 14:36:38 +0100 Subject: [PATCH 108/423] Call reset_algorithms when recalling train --- menpofit/sdm/fitter.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 387205f..3ff4a4b 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -32,6 +32,11 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, features = checks.check_features(features, n_levels) patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters + self.algorithms = [] + self.reference_shape = None + self._sd_algorithm_cls = sd_algorithm_cls + self._features = features + self._patch_shape = patch_shape self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations @@ -39,15 +44,16 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self._perturb_from_shape = perturb_from_shape self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms - self._set_up(sd_algorithm_cls, features, patch_shape, **kwargs) + self._reset_algorithms(**kwargs) - def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): - self.algorithms = [] + def _reset_algorithms(self, **kwargs): + if len(self.algorithms) > 0: + for j in range(len(self.algorithms) - 1, -1, -1): + del self.algorithms[j] for j in range(self.n_levels): - algorithm = sd_algorithm_cls( - features=features[j], patch_shape=patch_shape[j], - iterations=self.iterations[j], **kwargs) - self.algorithms.append(algorithm) + self.algorithms.append(self._sd_algorithm_cls( + features=self._features[j], patch_shape=self._patch_shape[j], + iterations=self.iterations[j], **kwargs)) def perturb_from_shape(self, shape, **kwargs): return self._perturb_from_shape(self.reference_shape, shape, **kwargs) @@ -58,6 +64,9 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, **kwargs): + # Reset the algorithm classes + self._reset_algorithms() + # Normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) @@ -102,7 +111,8 @@ def train(self, images, group=None, label=None, bounding_box_group=None, gt_s = i.landmarks[group][label].bounding_box() bb = i.landmarks[all_bb_keys[0]].lms # TODO: Noisy align given bb to gt_s - is this correct? - p_s = noisy_params_alignment_similarity(bb, gt_s).apply(bb) + p_s = noisy_params_alignment_similarity( + bb, gt_s, noise_std=0.03).apply(bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: @@ -442,13 +452,13 @@ def __str__(self): # self.n_perturbations = n_perturbations # self.iterations = checks.check_iterations(iterations, n_levels) # # set up algorithms -# self._set_up(cr_algorithm_cls, features, sampling, **kwargs) +# self._reset_algorithms(cr_algorithm_cls, features, sampling, **kwargs) # # @property # def algorithms(self): # return self._algorithms # -# def _set_up(self, cr_algorithm_cls, features, sampling, **kwargs): +# def _reset_algorithms(self, cr_algorithm_cls, features, sampling, **kwargs): # for j, s in range(self.n_levels): # algorithm = cr_algorithm_cls( # features=features[j], sampling=sampling[j], From f66e2e59e33ed7a848c927e80631e4ff982a66cf Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 22 Jul 2015 15:29:35 +0100 Subject: [PATCH 109/423] Properly use the perturb method for SDM --- menpofit/sdm/fitter.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 3ff4a4b..d16c9c1 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -10,7 +10,7 @@ scale_images) from menpofit.fitter import ( MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, - align_shape_with_bounding_box, noisy_params_alignment_similarity) + align_shape_with_bounding_box) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -104,15 +104,11 @@ def train(self, images, group=None, label=None, bounding_box_group=None, # We assume that the first bounding box is a valid perturbation # thus create n_perturbations - 1 new bounding boxes for j in range(1, self.n_perturbations): - # TODO: This should use the new logic that @jalabort - # has come up with. Also, would it be good if this was - # customizable? As in, the ability to pass some kind of - # probability distribution to draw from? gt_s = i.landmarks[group][label].bounding_box() bb = i.landmarks[all_bb_keys[0]].lms - # TODO: Noisy align given bb to gt_s - is this correct? - p_s = noisy_params_alignment_similarity( - bb, gt_s, noise_std=0.03).apply(bb) + + # This is customizable by passing in the correct method + p_s = self._perturb_from_bounding_box(gt_s, bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: From 6eb47c9b6f3b104dd6b908573e853802c5fda2b9 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 22 Jul 2015 15:44:24 +0100 Subject: [PATCH 110/423] Update increment on SDM to same as train Use one function since they were identical. Just call added a kwarg for incrementing. --- menpofit/sdm/fitter.py | 107 ++++++++--------------------------------- 1 file changed, 21 insertions(+), 86 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index d16c9c1..429e0ca 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -63,9 +63,13 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): bounding_box, **kwargs) def train(self, images, group=None, label=None, bounding_box_group=None, - verbose=False, **kwargs): - # Reset the algorithm classes - self._reset_algorithms() + verbose=False, increment=False, **kwargs): + if not increment: + # Reset the algorithm classes + self._reset_algorithms() + else: + if len(self.algorithms) == 0: + raise ValueError('Must train before training incrementally.') # Normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( @@ -161,9 +165,14 @@ def train(self, images, group=None, label=None, bounding_box_group=None, current_shapes.append(c_shapes) # train supervised descent algorithm - current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose, **kwargs) + if increment: + current_shapes = self.algorithms[j].increment( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + else: + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + level_str=level_str, verbose=verbose, **kwargs) # Scale current shapes to next level resolution if self.scales[j] != (1 or self.scales[-1]): @@ -175,84 +184,10 @@ def train(self, images, group=None, label=None, bounding_box_group=None, def increment(self, images, group=None, label=None, bounding_box_group=None, verbose=False, **kwargs): - # normalize images with respect to reference shape of aam - images = rescale_images_to_reference_shape( - images, group, label, self.reference_shape, verbose=verbose) - - # handle perturbations - if bounding_box_group is None: - bounding_box_group = 'bb_' - # generate perturbations by perturbing ground truth shapes - for i in images: - gt_s = i.landmarks[group][label] - for j in range(self.n_perturbations): - p_s = self.perturb_from_shape(gt_s) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - else: - # reset number of perturbations - n_perturbations = 0 - for k in images[0].landmarks.keys(): - if bounding_box_group in k: - n_perturbations += 1 - if n_perturbations == 1: - for i in images: - bb = i.landmarks[bounding_box_group].lms - p_s = align_shape_with_bounding_box( - self.reference_shape, bb) - i.landmarks[bounding_box_group + '0'] = p_s - for j in range(1, self.n_perturbations): - p_s = self.perturb_from_bounding_box(bb) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) - self.n_perturbations = n_perturbations - - # for each pyramid level (low --> high) - for j in range(self.n_levels): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # scale images and compute features at other levels - level_images = scale_images(images, self.scales[j], - level_str=level_str, verbose=verbose) - - # extract ground truth shapes for current level - level_gt_shapes = [i.landmarks[group][label] for i in level_images] - - if j == 0: - # extract perturbations at the very bottom level - current_shapes = [] - for i in level_images: - c_shapes = [] - for k in range(self.n_perturbations): - p_group = bounding_box_group + '{}'.format(k) - c_s = i.landmarks[p_group].lms - if c_s.n_points != level_gt_shapes[0].n_points: - # assume c_s is bounding box - c_s = align_shape_with_bounding_box( - self.reference_shape, c_s) - c_shapes.append(c_s) - current_shapes.append(c_shapes) - - # train cascaded regression algorithm - current_shapes = self.algorithms[j].increment( - level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) - - # scale current shapes to next level resolution - if self.scales[j] != (1 or self.scales[-1]): - transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) + return self.train(images, group=group, label=label, + bounding_box_group=bounding_box_group, + verbose=verbose, + increment=True, **kwargs) def train_incrementally(self, images, group=None, label=None, batch_size=100, verbose=False, **kwargs): @@ -260,14 +195,14 @@ def train_incrementally(self, images, group=None, label=None, n_batches = np.int(np.ceil(len(images) / batch_size)) # train first batch - print 'Training batch 1.' + print('Training batch 1.') self.train(images[:batch_size], group=group, label=label, verbose=verbose, **kwargs) # train all other batches start = batch_size for j in range(1, n_batches): - print 'Training batch {}.'.format(j+1) + print('Training batch {}.'.format(j+1)) end = start + batch_size self.increment(images[start:end], group=group, label=label, verbose=verbose, **kwargs) From fba8aa21119e343d92756c0cf911c5b1eaa58acb Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 23 Jul 2015 10:43:59 +0100 Subject: [PATCH 111/423] Fix rescaling to reference shape for incremental sdm Also, add verbose guard to printing --- menpofit/sdm/fitter.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 429e0ca..231460d 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -5,9 +5,8 @@ from menpo.transform import Scale from menpo.feature import no_op from menpo.visualize import print_progress -from menpofit.builder import ( - normalization_wrt_reference_shape, rescale_images_to_reference_shape, - scale_images) +from menpofit.builder import (normalization_wrt_reference_shape, scale_images, + rescale_images_to_reference_shape) from menpofit.fitter import ( MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, align_shape_with_bounding_box) @@ -64,22 +63,25 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, increment=False, **kwargs): + # In the case where group is None, we need to get the only key so that + # we can add landmarks below and not get a complaint about using None + first_image = images[0] + if group is None: + group = first_image.landmarks.group_labels[0] + if not increment: # Reset the algorithm classes self._reset_algorithms() + # Normalize images and compute reference shape + self.reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) else: if len(self.algorithms) == 0: raise ValueError('Must train before training incrementally.') - - # Normalize images and compute reference shape - self.reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) - - # In the case where group is None, we need to get the only key so that - # we can add landmarks below and not get a complaint about using None - first_image = images[0] - if group is None: - group = first_image.landmarks.group_labels[0] + # We are incrementing, so rescale to existing reference shape + images = rescale_images_to_reference_shape(images, group, label, + self.reference_shape, + verbose=verbose) # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: @@ -195,14 +197,16 @@ def train_incrementally(self, images, group=None, label=None, n_batches = np.int(np.ceil(len(images) / batch_size)) # train first batch - print('Training batch 1.') + if verbose: + print('Training batch 1.') self.train(images[:batch_size], group=group, label=label, verbose=verbose, **kwargs) # train all other batches start = batch_size for j in range(1, n_batches): - print('Training batch {}.'.format(j+1)) + if verbose: + print('Training batch {}.'.format(j + 1)) end = start + batch_size self.increment(images[start:end], group=group, label=label, verbose=verbose, **kwargs) From 5c6ee5afb29d7f0ac0ec18958c600d0bf12dbeed Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 14:06:21 +0100 Subject: [PATCH 112/423] Remove perturb_from_shape We will just let people do that themselves explicitly --- menpofit/sdm/fitter.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 231460d..044709b 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -22,7 +22,6 @@ class SupervisedDescentFitter(MultiFitter): def __init__(self, sd_algorithm_cls=Newton, features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), iterations=6, n_perturbations=30, - perturb_from_shape=noisy_shape_from_shape, perturb_from_bounding_box=noisy_shape_from_bounding_box, **kwargs): # check parameters @@ -40,7 +39,6 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations self.iterations = checks.check_max_iters(iterations, n_levels) - self._perturb_from_shape = perturb_from_shape self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._reset_algorithms(**kwargs) @@ -51,15 +49,12 @@ def _reset_algorithms(self, **kwargs): del self.algorithms[j] for j in range(self.n_levels): self.algorithms.append(self._sd_algorithm_cls( - features=self._features[j], patch_shape=self._patch_shape[j], + features=self._holistic_features[j], patch_shape=self._patch_shape[j], iterations=self.iterations[j], **kwargs)) - def perturb_from_shape(self, shape, **kwargs): - return self._perturb_from_shape(self.reference_shape, shape, **kwargs) - - def perturb_from_bounding_box(self, bounding_box, **kwargs): + def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, - bounding_box, **kwargs) + bounding_box) def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, increment=False, **kwargs): @@ -114,7 +109,7 @@ def train(self, images, group=None, label=None, bounding_box_group=None, bb = i.landmarks[all_bb_keys[0]].lms # This is customizable by passing in the correct method - p_s = self._perturb_from_bounding_box(gt_s, bb) + p_s = self.perturb_from_bounding_box(gt_s, bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: @@ -227,13 +222,10 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, ---------- image : :map:`Image` or subclass The image to be fitted. - initial_shape : :map:`PointCloud` The initial shape from which the fitting will start. - - gt_shape : class : :map:`PointCloud`, optional + gt_shape : :map:`PointCloud`, optional The original ground truth shape associated to the image. - crop_image: `None` or float`, optional If `float`, it specifies the proportion of the border wrt the initial shape to which the image will be internally cropped around @@ -248,10 +240,8 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, ------- images : `list` of :map:`Image` or subclass The list of images that will be fitted by the fitters. - initial_shapes : `list` of :map:`PointCloud` The initial shape for each one of the previous images. - gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ From a1c91f732a214231488018d3fe032800de64307b Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 16:45:12 +0100 Subject: [PATCH 113/423] Make train_incremental work on generators Use a batch method to allow ingesting a generator rather than a list. --- menpofit/base.py | 11 ++++++++++- menpofit/sdm/fitter.py | 40 ++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index 23f683b..5c5c816 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -1,5 +1,5 @@ from __future__ import division -from menpo.transform import AlignmentSimilarity, Similarity +import itertools import numpy as np from menpo.visualize import progress_bar_str, print_dynamic @@ -11,6 +11,15 @@ def name_of_callable(c): return c.__class__.__name__ # callable class +def batch(iterable, n): + it = iter(iterable) + while True: + chunk = tuple(itertools.islice(it, n)) + if not chunk: + return + yield chunk + + def is_pyramid_on_features(features): r""" True if feature extraction happens once and then a gaussian pyramid diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 044709b..109a4be 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,15 +1,14 @@ from __future__ import division from functools import partial -import numpy as np import warnings from menpo.transform import Scale from menpo.feature import no_op from menpo.visualize import print_progress +from menpofit.base import batch from menpofit.builder import (normalization_wrt_reference_shape, scale_images, rescale_images_to_reference_shape) -from menpofit.fitter import ( - MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, - align_shape_with_bounding_box) +from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, + align_shape_with_bounding_box) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -109,7 +108,7 @@ def train(self, images, group=None, label=None, bounding_box_group=None, bb = i.landmarks[all_bb_keys[0]].lms # This is customizable by passing in the correct method - p_s = self.perturb_from_bounding_box(gt_s, bb) + p_s = self._perturb_from_bounding_box(gt_s, bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: @@ -134,7 +133,7 @@ def train(self, images, group=None, label=None, bounding_box_group=None, else: level_str = None - # Scale images and compute features at other levels + # Scale images level_images = scale_images(images, self.scales[j], level_str=level_str, verbose=verbose) @@ -188,24 +187,21 @@ def increment(self, images, group=None, label=None, def train_incrementally(self, images, group=None, label=None, batch_size=100, verbose=False, **kwargs): - # number of batches - n_batches = np.int(np.ceil(len(images) / batch_size)) - - # train first batch - if verbose: - print('Training batch 1.') - self.train(images[:batch_size], group=group, label=label, - verbose=verbose, **kwargs) - - # train all other batches - start = batch_size - for j in range(1, n_batches): + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + + # Train all batches + for k, image_batch in enumerate(image_batches): + n_images = len(image_batch) if verbose: - print('Training batch {}.'.format(j + 1)) - end = start + batch_size - self.increment(images[start:end], group=group, label=label, + print('Training batch {} of {} images.'.format(k, n_images)) + if k == 0: + self.train(image_batch, group=group, label=label, verbose=verbose, **kwargs) - start = end + else: + self.increment(image_batch, group=group, label=label, + verbose=verbose, **kwargs) def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): From b8baeab3068844b0570cf74fdd0b25e75a3e2385 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 16:46:17 +0100 Subject: [PATCH 114/423] Fix non-verbose mode for SDM Add a new little print_progress method that takes a verbose flag and can ignore verbosity. --- menpofit/base.py | 2 +- menpofit/builder.py | 45 ++++++++++++--------------------- menpofit/sdm/algorithm.py | 12 ++++----- menpofit/sdm/fitter.py | 24 +++++++----------- menpofit/visualize/__init__.py | 1 + menpofit/visualize/textutils.py | 24 ++++++++++++++++++ 6 files changed, 56 insertions(+), 52 deletions(-) create mode 100644 menpofit/visualize/textutils.py diff --git a/menpofit/base.py b/menpofit/base.py index 5c5c816..de8d6c8 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -116,4 +116,4 @@ def build_sampling_grid(patch_shape): start = -patch_half_shape end = patch_half_shape + 1 sampling_grid = np.mgrid[start[0]:end[0], start[1]:end[1]] - return sampling_grid.swapaxes(0, 2).swapaxes(0, 1) \ No newline at end of file + return sampling_grid.swapaxes(0, 2).swapaxes(0, 1) diff --git a/menpofit/builder.py b/menpofit/builder.py index 3ae8322..b1f4b2a 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -6,7 +6,8 @@ from menpo.feature import no_op from menpo.transform import Scale, Translation, GeneralizedProcrustesAnalysis from menpo.model.pca import PCAModel -from menpo.visualize import print_dynamic, print_progress +from menpo.visualize import print_dynamic +from menpofit.visualize import print_progress def compute_reference_shape(shapes, normalization_diagonal, verbose=False): @@ -52,10 +53,8 @@ def rescale_images_to_reference_shape(images, group, label, reference_shape, verbose=False): r""" """ - if verbose: - wrap = partial(print_progress, prefix='- Normalizing images size') - else: - wrap = lambda x: x + wrap = partial(print_progress, prefix='- Normalizing images size', + verbose=verbose) # Normalize the scaling of all images wrt the reference_shape size normalized_images = [i.rescale_to_reference_shape(reference_shape, @@ -127,24 +126,18 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, # TODO: document me! def compute_features(images, features, level_str='', verbose=False): - if verbose: - wrap = partial(print_progress, - prefix='{}Computing feature space'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Computing feature space'.format(level_str), + end_with_newline=not level_str, verbose=verbose) return [features(i) for i in wrap(images)] # TODO: document me! def scale_images(images, scale, level_str='', verbose=False): - if verbose: - wrap = partial(print_progress, - prefix='{}Scaling images'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Scaling images'.format(level_str), + end_with_newline=not level_str, verbose=verbose) if not np.allclose(scale, 1): return [i.rescale(scale) for i in wrap(images)] @@ -155,12 +148,9 @@ def scale_images(images, scale, level_str='', verbose=False): # TODO: document me! def warp_images(images, shapes, reference_frame, transform, level_str='', verbose=None): - if verbose: - wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str, verbose=verbose) warped_images = [] # Build a dummy transform, use set_target for efficiency @@ -180,12 +170,9 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! def extract_patches(images, shapes, patch_shape, normalize_function=no_op, level_str='', verbose=False): - if verbose: - wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str, verbose=verbose) parts_images = [] for i, s in wrap(zip(images, shapes)): diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 24719bb..92c54b4 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -2,7 +2,8 @@ from functools import partial import numpy as np from menpo.feature import no_op -from menpo.visualize import print_dynamic, print_progress +from menpo.visualize import print_dynamic +from menpofit.visualize import print_progress from menpofit.result import ( NonParametricAlgorithmResult, compute_normalise_point_to_point_error) from menpofit.math import IRLRegression, IIRLRegression @@ -237,12 +238,9 @@ def obtain_patch_features(images, shapes, patch_shape, features_callable, verbose=False): """r """ - if verbose: - wrap = partial(print_progress, - prefix='{}Extracting patches'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Extracting patches'.format(level_str), + end_with_newline=not level_str, verbose=verbose) n_images = len(images) n_shapes = len(shapes[0]) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 109a4be..0422a28 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -3,8 +3,8 @@ import warnings from menpo.transform import Scale from menpo.feature import no_op -from menpo.visualize import print_progress -from menpofit.base import batch +from menpofit.visualize import print_progress +from menpofit.base import batch, name_of_callable from menpofit.builder import (normalization_wrt_reference_shape, scale_images, rescale_images_to_reference_shape) from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, @@ -93,12 +93,9 @@ def train(self, images, group=None, label=None, bounding_box_group=None, # If there is only one example bounding box, then we will generate # more perturbations based on the bounding box. if n_perturbations == 1: - if verbose: - msg = '- Generating {} new initial bounding boxes ' \ - 'per image'.format(self.n_perturbations) - wrap = partial(print_progress, prefix=msg) - else: - wrap = lambda x: x + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg, verbose=verbose) for i in wrap(images): # We assume that the first bounding box is a valid perturbation @@ -141,13 +138,10 @@ def train(self, images, group=None, label=None, bounding_box_group=None, level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - if verbose: - msg = '{}Generating {} perturbations per image'.format( - level_str, self.n_perturbations) - wrap = partial(print_progress, prefix=msg, - end_with_newline=False) - else: - wrap = lambda x: x + msg = '{}Generating {} perturbations per image'.format( + level_str, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False, verbose=verbose) # Extract perturbations at the very bottom level current_shapes = [] diff --git a/menpofit/visualize/__init__.py b/menpofit/visualize/__init__.py index 6aaea70..30039c3 100644 --- a/menpofit/visualize/__init__.py +++ b/menpofit/visualize/__init__.py @@ -5,3 +5,4 @@ visualize_fitting_result) except ImportError: pass +from .textutils import print_progress diff --git a/menpofit/visualize/textutils.py b/menpofit/visualize/textutils.py new file mode 100644 index 0000000..ce49bad --- /dev/null +++ b/menpofit/visualize/textutils.py @@ -0,0 +1,24 @@ +from menpo.visualize import print_progress as menpo_print_progress + + +def print_progress(iterable, prefix='', n_items=None, offset=0, + show_bar=True, show_count=True, show_eta=True, + end_with_newline=True, verbose=True): + r""" + Please see the menpo ``print_progress`` documentation. + + This method is identical to the print progress method, but adds a verbose + flag which allows the printing to be skipped if necessary. + """ + if verbose: + # Yield the images from the menpo print_progress (yield from would + # be perfect here :( ) + for i in menpo_print_progress(iterable, prefix=prefix, n_items=n_items, + offset=offset, show_bar=show_bar, + show_count=show_count, show_eta=show_eta, + end_with_newline=end_with_newline): + yield i + else: + # Skip the verbosity! + for i in iterable: + yield i From 27b8e798f0b3ec57d8dbbd9503f0f02587e6b40e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 16:47:13 +0100 Subject: [PATCH 115/423] Add the idea of holistic features Seperate features into patch and holistic. Holisitic gets computed on the whole image, before scaling. patch_features get computed inside each patch - and are equivalent to the old features. --- menpofit/sdm/fitter.py | 195 ++++++----------------------------------- 1 file changed, 27 insertions(+), 168 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 0422a28..567d40a 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -18,21 +18,21 @@ class SupervisedDescentFitter(MultiFitter): r""" """ - def __init__(self, sd_algorithm_cls=Newton, features=no_op, - patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), - iterations=6, n_perturbations=30, - perturb_from_bounding_box=noisy_shape_from_bounding_box, - **kwargs): + def __init__(self, sd_algorithm_cls=Newton, holistic_feature=no_op, + patch_features=no_op, patch_shape=(17, 17), diagonal=None, + scales=(1, 0.5), iterations=6, n_perturbations=30, + perturb_from_bounding_box=noisy_shape_from_bounding_box): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + patch_features = checks.check_features(patch_features, n_levels) patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters self.algorithms = [] self.reference_shape = None self._sd_algorithm_cls = sd_algorithm_cls - self._features = features + self._holistic_feature = holistic_feature + self._patch_features = patch_features self._patch_shape = patch_shape self.diagonal = diagonal self.scales = list(scales)[::-1] @@ -40,16 +40,17 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self.iterations = checks.check_max_iters(iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms - self._reset_algorithms(**kwargs) + self._reset_algorithms() - def _reset_algorithms(self, **kwargs): + def _reset_algorithms(self): if len(self.algorithms) > 0: for j in range(len(self.algorithms) - 1, -1, -1): del self.algorithms[j] for j in range(self.n_levels): self.algorithms.append(self._sd_algorithm_cls( - features=self._holistic_features[j], patch_shape=self._patch_shape[j], - iterations=self.iterations[j], **kwargs)) + features=self._patch_features[j], + patch_shape=self._patch_shape[j], + iterations=self.iterations[j])) def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, @@ -120,6 +121,12 @@ def train(self, images, group=None, label=None, bounding_box_group=None, all_bb_keys = list(first_image.landmarks.keys_matching( '*{}*'.format(bounding_box_group))) + # Before scaling, we compute the holistic feature on the whole image + msg = '- Computing holistic features ({})'.format( + name_of_callable(self._holistic_feature)) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + images = [self._holistic_feature(im) for im in wrap(images)] + # for each pyramid level (low --> high) for j in range(self.n_levels): if verbose: @@ -235,22 +242,25 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ - # attach landmarks to the image + # Attach landmarks to the image image.landmarks['initial_shape'] = initial_shape if gt_shape: image.landmarks['gt_shape'] = gt_shape - # if specified, crop the image + # If specified, crop the image if crop_image: image = image.crop_to_landmarks_proportion(crop_image, group='initial_shape') - # rescale image wrt the scale factor between reference_shape and + # Rescale image w.r.t the scale factor between reference_shape and # initial_shape image = image.rescale_to_reference_shape(self.reference_shape, group='initial_shape') - # obtain image representation + # Compute the holistic feature on the normalized image + image = self._holistic_feature(image) + + # Obtain image representation images = [] for s in self.scales: if s != 1: @@ -260,10 +270,10 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, scaled_image = image images.append(scaled_image) - # get initial shapes per level + # Get initial shapes per level initial_shapes = [i.landmarks['initial_shape'].lms for i in images] - # get ground truth shapes per level + # Get ground truth shapes per level if gt_shape: gt_shapes = [i.landmarks['gt_shape'].lms for i in images] else: @@ -347,154 +357,3 @@ def __str__(self): # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # out, feat_str[0], n_channels[0], ch_str[0]) # return out - - -# class CRFitter(MultiFitter): -# r""" -# """ -# def __init__(self, cr_algorithm_cls=SN, features=no_op, diagonal=None, -# scales=(1, 0.5), sampling=None, n_perturbations=10, -# iterations=6, **kwargs): -# # check parameters -# checks.check_diagonal(diagonal) -# scales, n_levels = checks.check_scales(scales) -# features = checks.check_features(features, n_levels) -# sampling = checks.check_sampling(sampling, n_levels) -# # set parameters -# self._algorithms = [] -# self.diagonal = diagonal -# self.scales = list(scales) -# self.n_perturbations = n_perturbations -# self.iterations = checks.check_iterations(iterations, n_levels) -# # set up algorithms -# self._reset_algorithms(cr_algorithm_cls, features, sampling, **kwargs) -# -# @property -# def algorithms(self): -# return self._algorithms -# -# def _reset_algorithms(self, cr_algorithm_cls, features, sampling, **kwargs): -# for j, s in range(self.n_levels): -# algorithm = cr_algorithm_cls( -# features=features[j], sampling=sampling[j], -# max_iters=self.iterations[j], **kwargs) -# self._algorithms.append(algorithm) -# -# def train(self, images, group=None, label=None, verbose=False, **kwargs): -# # normalize images and compute reference shape -# reference_shape, images = normalization_wrt_reference_shape( -# images, group, label, self.diagonal, verbose=verbose) -# -# # for each pyramid level (low --> high) -# for j in range(self.n_levels): -# if verbose: -# if len(self.scales) > 1: -# level_str = ' - Level {}: '.format(j) -# else: -# level_str = ' - ' -# -# # scale images and compute features at other levels -# level_images = scale_images(images, self.scales[j], -# level_str=level_str, verbose=verbose) -# -# # extract ground truth shapes for current level -# level_gt_shapes = [i.landmarks[group][label] for i in level_images] -# -# if j == 0: -# # generate perturbed shapes -# current_shapes = [] -# for gt_s in level_gt_shapes: -# perturbed_shapes = [] -# for _ in range(self.n_perturbations): -# p_s = self.noisy_shape_from_shape(gt_s) -# perturbed_shapes.append(p_s) -# current_shapes.append(perturbed_shapes) -# -# # train cascaded regression algorithm -# current_shapes = self.algorithms[j].train( -# level_images, level_gt_shapes, current_shapes, -# verbose=verbose, **kwargs) -# -# # scale current shapes to next level resolution -# if self.scales[j] != self.scales[-1]: -# transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) -# for image_shapes in current_shapes: -# for shape in image_shapes: -# transform.apply_inplace(shape) -# -# def _fitter_result(self, image, algorithm_results, affine_correction, -# gt_shape=None): -# return MultiFitterResult(image, algorithm_results, affine_correction, -# gt_shape=gt_shape) -# -# # TODO: fix me! -# def __str__(self): -# pass -# # out = "Supervised Descent Method\n" \ -# # " - Non-Parametric '{}' Regressor\n" \ -# # " - {} training images.\n".format( -# # name_of_callable(self._fitters[0].regressor), -# # self._n_training_images) -# # # small strings about number of channels, channels string and downscale -# # down_str = [] -# # for j in range(self.n_levels): -# # if j == self.n_levels - 1: -# # down_str.append('(no downscale)') -# # else: -# # down_str.append('(downscale by {})'.format( -# # self.downscale**(self.n_levels - j - 1))) -# # temp_img = Image(image_data=np.random.rand(40, 40)) -# # if self.pyramid_on_features: -# # temp = self.features(temp_img) -# # n_channels = [temp.n_channels] * self.n_levels -# # else: -# # n_channels = [] -# # for j in range(self.n_levels): -# # temp = self.features[j](temp_img) -# # n_channels.append(temp.n_channels) -# # # string about features and channels -# # if self.pyramid_on_features: -# # feat_str = "- Feature is {} with ".format( -# # name_of_callable(self.features)) -# # if n_channels[0] == 1: -# # ch_str = ["channel"] -# # else: -# # ch_str = ["channels"] -# # else: -# # feat_str = [] -# # ch_str = [] -# # for j in range(self.n_levels): -# # if isinstance(self.features[j], str): -# # feat_str.append("- Feature is {} with ".format( -# # self.features[j])) -# # elif self.features[j] is None: -# # feat_str.append("- No features extracted. ") -# # else: -# # feat_str.append("- Feature is {} with ".format( -# # self.features[j].__name__)) -# # if n_channels[j] == 1: -# # ch_str.append("channel") -# # else: -# # ch_str.append("channels") -# # if self.n_levels > 1: -# # out = "{} - Gaussian pyramid with {} levels and downscale " \ -# # "factor of {}.\n".format(out, self.n_levels, -# # self.downscale) -# # if self.pyramid_on_features: -# # out = "{} - Pyramid was applied on feature space.\n " \ -# # "{}{} {} per image.\n".format(out, feat_str, -# # n_channels[0], ch_str[0]) -# # else: -# # out = "{} - Features were extracted at each pyramid " \ -# # "level.\n".format(out) -# # for i in range(self.n_levels - 1, -1, -1): -# # out = "{} - Level {} {}: \n {}{} {} per " \ -# # "image.\n".format( -# # out, self.n_levels - i, down_str[i], feat_str[i], -# # n_channels[i], ch_str[i]) -# # else: -# # if self.pyramid_on_features: -# # feat_str = [feat_str] -# # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( -# # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out From b1b7d38fa6e5754e1af8c69dffc59338105e2ec1 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:17:08 +0100 Subject: [PATCH 116/423] Try creating a sensible __str__ for sdm --- menpofit/sdm/fitter.py | 108 +++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 70 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 567d40a..5f548d3 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,4 +1,5 @@ from __future__ import division +import numpy as np from functools import partial import warnings from menpo.transform import Scale @@ -286,74 +287,41 @@ def _fitter_result(self, image, algorithm_results, affine_correction, return MultiFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - # TODO: fix me! def __str__(self): - pass - # out = "Supervised Descent Method\n" \ - # " - Non-Parametric '{}' Regressor\n" \ - # " - {} training images.\n".format( - # name_of_callable(self._fitters[0].regressor), - # self._n_training_images) - # # small strings about number of channels, channels string and downscale - # down_str = [] - # for j in range(self.n_levels): - # if j == self.n_levels - 1: - # down_str.append('(no downscale)') - # else: - # down_str.append('(downscale by {})'.format( - # self.downscale**(self.n_levels - j - 1))) - # temp_img = Image(image_data=np.random.rand(40, 40)) - # if self.pyramid_on_features: - # temp = self.features(temp_img) - # n_channels = [temp.n_channels] * self.n_levels - # else: - # n_channels = [] - # for j in range(self.n_levels): - # temp = self.features[j](temp_img) - # n_channels.append(temp.n_channels) - # # string about features and channels - # if self.pyramid_on_features: - # feat_str = "- Feature is {} with ".format( - # name_of_callable(self.features)) - # if n_channels[0] == 1: - # ch_str = ["channel"] - # else: - # ch_str = ["channels"] - # else: - # feat_str = [] - # ch_str = [] - # for j in range(self.n_levels): - # if isinstance(self.features[j], str): - # feat_str.append("- Feature is {} with ".format( - # self.features[j])) - # elif self.features[j] is None: - # feat_str.append("- No features extracted. ") - # else: - # feat_str.append("- Feature is {} with ".format( - # self.features[j].__name__)) - # if n_channels[j] == 1: - # ch_str.append("channel") - # else: - # ch_str.append("channels") - # if self.n_levels > 1: - # out = "{} - Gaussian pyramid with {} levels and downscale " \ - # "factor of {}.\n".format(out, self.n_levels, - # self.downscale) - # if self.pyramid_on_features: - # out = "{} - Pyramid was applied on feature space.\n " \ - # "{}{} {} per image.\n".format(out, feat_str, - # n_channels[0], ch_str[0]) - # else: - # out = "{} - Features were extracted at each pyramid " \ - # "level.\n".format(out) - # for i in range(self.n_levels - 1, -1, -1): - # out = "{} - Level {} {}: \n {}{} {} per " \ - # "image.\n".format( - # out, self.n_levels - i, down_str[i], feat_str[i], - # n_channels[i], ch_str[i]) - # else: - # if self.pyramid_on_features: - # feat_str = [feat_str] - # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( - # out, feat_str[0], n_channels[0], ch_str[0]) - # return out + if self.diagonal is not None: + diagonal = self.diagonal + else: + diagonal = np.sqrt(np.sum(np.asarray(self.reference_shape.bounds()) + ** 2)) + is_custom_perturb_func = (self._perturb_from_bounding_box != + noisy_shape_from_bounding_box) + regressor_cls = self.algorithms[0]._regressor_cls + + # Compute level info strings + level_info = [] + lvl_str_tmplt = r""" - Level {} (Scale {}) + - {} iterations + - Patch shape: {}""" + for k, s in enumerate(self.scales): + level_info.append(lvl_str_tmplt.format(k, s, + self.iterations[k], + self._patch_shape[k])) + level_info = '\n'.join(level_info) + + cls_str = r"""Supervised Descent Method + - Regression performed using the {reg_alg} algorithm + - Regression class: {reg_cls} + - Levels: {levels} +{level_info} + - Perturbations generated per shape: {n_perturbations} + - Images scaled to diagonal: {diagonal:.2f} + - Custom perturbation scheme used: {is_custom_perturb_func}""".format( + reg_alg=name_of_callable(self._sd_algorithm_cls), + reg_cls=name_of_callable(regressor_cls), + n_levels=len(self.scales), + levels=self.scales, + level_info=level_info, + n_perturbations=self.n_perturbations, + diagonal=diagonal, + is_custom_perturb_func=is_custom_perturb_func) + return cls_str From 8d1d301e5495a2539f497a92705b867b9c04d396 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:20:20 +0100 Subject: [PATCH 117/423] Calculate diagonal correctly. --- menpofit/sdm/fitter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 5f548d3..2b3fc9a 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -291,8 +291,8 @@ def __str__(self): if self.diagonal is not None: diagonal = self.diagonal else: - diagonal = np.sqrt(np.sum(np.asarray(self.reference_shape.bounds()) - ** 2)) + y, x = self.reference_shape.range() + diagonal = np.sqrt(x ** 2 + y ** 2) is_custom_perturb_func = (self._perturb_from_bounding_box != noisy_shape_from_bounding_box) regressor_cls = self.algorithms[0]._regressor_cls @@ -317,11 +317,11 @@ def __str__(self): - Images scaled to diagonal: {diagonal:.2f} - Custom perturbation scheme used: {is_custom_perturb_func}""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), - reg_cls=name_of_callable(regressor_cls), - n_levels=len(self.scales), - levels=self.scales, - level_info=level_info, - n_perturbations=self.n_perturbations, - diagonal=diagonal, - is_custom_perturb_func=is_custom_perturb_func) + reg_cls=name_of_callable(regressor_cls), + n_levels=len(self.scales), + levels=self.scales, + level_info=level_info, + n_perturbations=self.n_perturbations, + diagonal=diagonal, + is_custom_perturb_func=is_custom_perturb_func) return cls_str From 88a2af9a7016ac8152cb597a92e5489a075f6ad3 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:41:04 +0100 Subject: [PATCH 118/423] Update name_of_callable to support partial Return the name of the partially wrapped function rather than 'partial' --- menpofit/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index de8d6c8..fd65647 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -1,4 +1,5 @@ from __future__ import division +from functools import partial import itertools import numpy as np from menpo.visualize import progress_bar_str, print_dynamic @@ -6,7 +7,10 @@ def name_of_callable(c): try: - return c.__name__ # function + if isinstance(c, partial): # partial + return c.func.__name__ + else: + return c.__name__ # function except AttributeError: return c.__class__.__name__ # callable class From caca793e82d9db0b814ca2af22907d6a6216b929 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:42:57 +0100 Subject: [PATCH 119/423] Partial commit before squash Have to leave to play squash, need to turn the PC off for the weekend due to power outage. Was working on removing the kwargs. --- menpofit/math/regression.py | 18 ++++++++++-------- menpofit/sdm/algorithm.py | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/menpofit/math/regression.py b/menpofit/math/regression.py index 58db19f..82dcce0 100644 --- a/menpofit/math/regression.py +++ b/menpofit/math/regression.py @@ -6,9 +6,11 @@ class IRLRegression(object): r""" Incremental Regularized Linear Regression """ - def __init__(self, l=0, bias=True): - self.l = l + def __init__(self, alpha=0, bias=True): + self.alpha = alpha self.bias = bias + self.V = None + self.W = None def train(self, X, Y): if self.bias: @@ -17,7 +19,7 @@ def train(self, X, Y): # regularized linear regression XX = X.T.dot(X) - np.fill_diagonal(XX, self.l + np.diag(XX)) + np.fill_diagonal(XX, self.alpha + np.diag(XX)) self.V = np.linalg.inv(XX) self.W = self.V.dot(X.T.dot(Y)) @@ -48,9 +50,9 @@ class IIRLRegression(IRLRegression): r""" Indirect Incremental Regularized Linear Regression """ - def __init__(self, l=0, bias=True, d=0): - super(IIRLRegression, self).__init__(l=l, bias=bias) - self.d = d + def __init__(self, alpha=0, bias=True, alpha2=0): + super(IIRLRegression, self).__init__(alpha=alpha, bias=bias) + self.alpha2 = alpha2 def train(self, X, Y): # regularized linear regression exchanging the roles of X and Y @@ -59,7 +61,7 @@ def train(self, X, Y): # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) - np.fill_diagonal(H, self.d + np.diag(H)) + np.fill_diagonal(H, self.alpha2 + np.diag(H)) self.W = np.linalg.solve(H, J.T) def increment(self, X, Y): @@ -69,5 +71,5 @@ def increment(self, X, Y): # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) - np.fill_diagonal(H, self.d + np.diag(H)) + np.fill_diagonal(H, self.alpha2 + np.diag(H)) self.W = np.linalg.solve(H, J.T) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 92c54b4..e25fd3d 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -162,8 +162,8 @@ class Newton(SupervisedDescentAlgorithm): """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, - eps=10**-5): - self._regressor_cls = IRLRegression + eps=10**-5, alpha=0, bias=True): + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape @@ -178,8 +178,9 @@ class GaussNewton(SupervisedDescentAlgorithm): """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, - eps=10**-5): - self._regressor_cls = IIRLRegression + eps=10**-5, alpha=0, bias=True, alpha2=0): + self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, + alpha2=alpha2) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape From a9408ff1b500cc42e9343799cac55c60d454c863 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Sun, 26 Jul 2015 15:53:15 +0100 Subject: [PATCH 120/423] Add recursive call to name_of_callable --- menpofit/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index fd65647..a20147c 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -8,7 +8,9 @@ def name_of_callable(c): try: if isinstance(c, partial): # partial - return c.func.__name__ + # Recursively call as partial may be wrapping either a callable + # or a function (or another partial for some reason!) + return name_of_callable(c) else: return c.__name__ # function except AttributeError: From 699adefd4c407a602496289e37137f2ce4d0a819 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:54:41 +0100 Subject: [PATCH 121/423] Fix error in name_of_callable Meant to walk down the partial functions, forgot to do that and so was an infinite recursion. --- menpofit/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index a20147c..0382bf1 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -10,7 +10,7 @@ def name_of_callable(c): if isinstance(c, partial): # partial # Recursively call as partial may be wrapping either a callable # or a function (or another partial for some reason!) - return name_of_callable(c) + return name_of_callable(c.func) else: return c.__name__ # function except AttributeError: From 20ed78c1c695db122e364d96ad5300ea93e9650f Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:55:14 +0100 Subject: [PATCH 122/423] Incorrect indent in batch --- menpofit/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index 0382bf1..1883721 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -22,7 +22,7 @@ def batch(iterable, n): while True: chunk = tuple(itertools.islice(it, n)) if not chunk: - return + return yield chunk From 46fdb132d0a1e963c03cb4ce9689a99f22f0e0f2 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:55:49 +0100 Subject: [PATCH 123/423] SDM refactor training Move training into the constructor. This makes it easier to serialize SDMs. Now, you build a single model which gets trained inside the constructor. Therefore, I also refactored the train_incrementally function to inside the train method - so all training can be done incrementally. --- menpofit/sdm/fitter.py | 302 +++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 147 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2b3fc9a..f79d2ab 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,4 +1,5 @@ from __future__ import division +from itertools import chain import numpy as np from functools import partial import warnings @@ -19,10 +20,12 @@ class SupervisedDescentFitter(MultiFitter): r""" """ - def __init__(self, sd_algorithm_cls=Newton, holistic_feature=no_op, + def __init__(self, images, group=None, bounding_box_group=None, + sd_algorithm_cls=Newton, holistic_feature=no_op, patch_features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), iterations=6, n_perturbations=30, - perturb_from_bounding_box=noisy_shape_from_bounding_box): + perturb_from_bounding_box=noisy_shape_from_bounding_box, + batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -41,12 +44,13 @@ def __init__(self, sd_algorithm_cls=Newton, holistic_feature=no_op, self.iterations = checks.check_max_iters(iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms - self._reset_algorithms() + self._setup_algorithms() - def _reset_algorithms(self): - if len(self.algorithms) > 0: - for j in range(len(self.algorithms) - 1, -1, -1): - del self.algorithms[j] + # Now, train the model! + self._train(images, group=group, bounding_box_group=bounding_box_group, + verbose=verbose, increment=False, batch_size=batch_size) + + def _setup_algorithms(self): for j in range(self.n_levels): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], @@ -57,153 +61,157 @@ def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, bounding_box) - def train(self, images, group=None, label=None, bounding_box_group=None, - verbose=False, increment=False, **kwargs): + def _train(self, images, group=None, bounding_box_group=None, + verbose=False, increment=False, batch_size=None): + + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + first_batch = next(image_batches) + else: + image_batches = [] + first_batch = list(images) + # In the case where group is None, we need to get the only key so that - # we can add landmarks below and not get a complaint about using None - first_image = images[0] + # we can attach landmarks below and not get a complaint about using None + first_image = first_batch[0] if group is None: group = first_image.landmarks.group_labels[0] - if not increment: - # Reset the algorithm classes - self._reset_algorithms() - # Normalize images and compute reference shape - self.reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) - else: - if len(self.algorithms) == 0: - raise ValueError('Must train before training incrementally.') - # We are incrementing, so rescale to existing reference shape - images = rescale_images_to_reference_shape(images, group, label, - self.reference_shape, - verbose=verbose) - - # No bounding box is given, so we will use the ground truth box - if bounding_box_group is None: - bounding_box_group = '__gt_bb_' - for i in images: - gt_s = i.landmarks[group][label] - perturb_bbox_group = bounding_box_group + '0' - i.landmarks[perturb_bbox_group] = gt_s.bounding_box() - - # Find all bounding boxes on the images with the given bounding box key - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) - n_perturbations = len(all_bb_keys) - - # If there is only one example bounding box, then we will generate - # more perturbations based on the bounding box. - if n_perturbations == 1: - msg = '- Generating {} new initial bounding boxes ' \ - 'per image'.format(self.n_perturbations) - wrap = partial(print_progress, prefix=msg, verbose=verbose) + for k, image_batch in enumerate(chain([first_batch], image_batches)): + # After the first batch, we are incrementing the model + if k > 0: + increment = True - for i in wrap(images): - # We assume that the first bounding box is a valid perturbation - # thus create n_perturbations - 1 new bounding boxes - for j in range(1, self.n_perturbations): - gt_s = i.landmarks[group][label].bounding_box() - bb = i.landmarks[all_bb_keys[0]].lms - - # This is customizable by passing in the correct method - p_s = self._perturb_from_bounding_box(gt_s, bb) - perturb_bbox_group = bounding_box_group + '_{}'.format(j) - i.landmarks[perturb_bbox_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) - self.n_perturbations = n_perturbations - - # Re-grab all the bounding box keys for iterating over when calculating - # perturbations - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) - - # Before scaling, we compute the holistic feature on the whole image - msg = '- Computing holistic features ({})'.format( - name_of_callable(self._holistic_feature)) - wrap = partial(print_progress, prefix=msg, verbose=verbose) - images = [self._holistic_feature(im) for im in wrap(images)] - - # for each pyramid level (low --> high) - for j in range(self.n_levels): if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - else: - level_str = None - - # Scale images - level_images = scale_images(images, self.scales[j], - level_str=level_str, verbose=verbose) - - # Extract scaled ground truth shapes for current level - level_gt_shapes = [i.landmarks[group][label] for i in level_images] - - if j == 0: - msg = '{}Generating {} perturbations per image'.format( - level_str, self.n_perturbations) - wrap = partial(print_progress, prefix=msg, - end_with_newline=False, verbose=verbose) - - # Extract perturbations at the very bottom level - current_shapes = [] - for i in wrap(level_images): - c_shapes = [] - for perturb_bbox_group in all_bb_keys: - bbox = i.landmarks[perturb_bbox_group].lms - c_s = align_shape_with_bounding_box( - self.reference_shape, bbox) - c_shapes.append(c_s) - current_shapes.append(c_shapes) - - # train supervised descent algorithm - if increment: - current_shapes = self.algorithms[j].increment( - level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) - else: - current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose, **kwargs) - - # Scale current shapes to next level resolution - if self.scales[j] != (1 or self.scales[-1]): - transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) - - def increment(self, images, group=None, label=None, - bounding_box_group=None, verbose=False, - **kwargs): - return self.train(images, group=group, label=label, - bounding_box_group=bounding_box_group, - verbose=verbose, - increment=True, **kwargs) - - def train_incrementally(self, images, group=None, label=None, - batch_size=100, verbose=False, **kwargs): - # Create a generator of fixed sized batches. Will still work even - # on an infinite list. - image_batches = batch(images, batch_size) - - # Train all batches - for k, image_batch in enumerate(image_batches): - n_images = len(image_batch) - if verbose: - print('Training batch {} of {} images.'.format(k, n_images)) - if k == 0: - self.train(image_batch, group=group, label=label, - verbose=verbose, **kwargs) + print('Computing batch {}'.format(k)) + + if not increment: + # Normalize images and compute reference shape + self.reference_shape, image_batch = normalization_wrt_reference_shape( + image_batch, group, self.diagonal, verbose=verbose) else: - self.increment(image_batch, group=group, label=label, - verbose=verbose, **kwargs) + # We are incrementing, so rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # No bounding box is given, so we will use the ground truth box + if bounding_box_group is None: + bounding_box_group = '__gt_bb_' + for i in image_batch: + gt_s = i.landmarks[group].lms + perturb_bbox_group = bounding_box_group + '0' + i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + + # Find all bounding boxes on the images with the given bounding + # box key + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) + n_perturbations = len(all_bb_keys) + + # If there is only one example bounding box, then we will generate + # more perturbations based on the bounding box. + if n_perturbations == 1: + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + + for i in wrap(image_batch): + # We assume that the first bounding box is a valid + # perturbation thus create n_perturbations - 1 new bounding + # boxes + for j in range(1, self.n_perturbations): + gt_s = i.landmarks[group].lms.bounding_box() + bb = i.landmarks[all_bb_keys[0]].lms + + # This is customizable by passing in the correct method + p_s = self._perturb_from_bounding_box(gt_s, bb) + perturb_bbox_group = '{}_{}'.format(bounding_box_group, + j) + i.landmarks[perturb_bbox_group] = p_s + elif n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided bounding_box_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + + # Re-grab all the bounding box keys for iterating over when + # calculating perturbations + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) + + # Before scaling, we compute the holistic feature on the whole image + msg = '- Computing holistic features ({})'.format( + name_of_callable(self._holistic_feature)) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + image_batch = [self._holistic_feature(im) + for im in wrap(image_batch)] + + # for each pyramid level (low --> high) + current_shapes = [] + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + else: + level_str = None + + # Scale images + level_images = scale_images(image_batch, self.scales[j], + level_str=level_str, + verbose=verbose) + + # Extract scaled ground truth shapes for current level + level_gt_shapes = [i.landmarks[group].lms + for i in level_images] + + if j == 0: + msg = '{}Generating {} perturbations per image'.format( + level_str, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False, verbose=verbose) + + # Extract perturbations at the very bottom level + for i in wrap(level_images): + c_shapes = [] + for perturb_bbox_group in all_bb_keys: + bbox = i.landmarks[perturb_bbox_group].lms + c_s = align_shape_with_bounding_box( + self.reference_shape, bbox) + c_shapes.append(c_s) + current_shapes.append(c_shapes) + + # train supervised descent algorithm + if increment: + current_shapes = self.algorithms[j].increment( + level_images, level_gt_shapes, current_shapes, + verbose=verbose) + else: + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + level_str=level_str, verbose=verbose) + + # Scale current shapes to next level resolution + if self.scales[j] != (1 or self.scales[-1]): + transform = Scale(self.scales[j + 1] / self.scales[j], + n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + + def increment(self, images, group=None, bounding_box_group=None, + verbose=False): + return self._train(images, group=group, + bounding_box_group=bounding_box_group, + verbose=verbose, + increment=True) def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): From 742ffb90ee86958af262193bbef2058c37b245e8 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:58:22 +0100 Subject: [PATCH 124/423] Allow batch training of incremental SDM --- menpofit/sdm/fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index f79d2ab..2dcf5e4 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -207,11 +207,11 @@ def _train(self, images, group=None, bounding_box_group=None, transform.apply_inplace(shape) def increment(self, images, group=None, bounding_box_group=None, - verbose=False): + verbose=False, batch_size=None): return self._train(images, group=group, bounding_box_group=bounding_box_group, verbose=verbose, - increment=True) + increment=True, batch_size=batch_size) def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): From ba79f463fad1dec074ab933e0348baf6bf2bb6cc Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:59:39 +0100 Subject: [PATCH 125/423] Remove label kwarg from builder functions --- menpofit/builder.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index b1f4b2a..ae2fb48 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -49,7 +49,7 @@ def compute_reference_shape(shapes, normalization_diagonal, verbose=False): # TODO: document me! -def rescale_images_to_reference_shape(images, group, label, reference_shape, +def rescale_images_to_reference_shape(images, group, reference_shape, verbose=False): r""" """ @@ -58,13 +58,12 @@ def rescale_images_to_reference_shape(images, group, label, reference_shape, # Normalize the scaling of all images wrt the reference_shape size normalized_images = [i.rescale_to_reference_shape(reference_shape, - group=group, label=label) + group=group) for i in wrap(images)] return normalized_images -def normalization_wrt_reference_shape(images, group, label, diagonal, - verbose=False): +def normalization_wrt_reference_shape(images, group, diagonal, verbose=False): r""" Function that normalizes the images sizes with respect to the reference shape (mean shape) scaling. This step is essential before building a @@ -81,15 +80,9 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, ---------- images : list of :class:`menpo.image.MaskedImage` The set of landmarked images to normalize. - group : `str` The key of the landmark set that should be used. If None, and if there is only one set of landmarks, this set will be used. - - label : `str` - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - diagonal: `int` If int, it ensures that the mean shape is scaled so that the diagonal of the bounding box containing it matches the @@ -100,7 +93,6 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, landmarks, this kwarg also specifies the diagonal length of the reference frame (provided that features computation does not change the image size). - verbose : `bool`, Optional Flag that controls information and progress printing. @@ -113,14 +105,14 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, A list with the normalized images. """ # get shapes - shapes = [i.landmarks[group][label] for i in images] + shapes = [i.landmarks[group].lms for i in images] # compute the reference shape and fix its diagonal length reference_shape = compute_reference_shape(shapes, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size normalized_images = rescale_images_to_reference_shape( - images, group, label, reference_shape, verbose=verbose) + images, group, reference_shape, verbose=verbose) return reference_shape, normalized_images From c86bade7bea156d00633999b256a58918175d781 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:01:24 +0100 Subject: [PATCH 126/423] Remove label kwarg from AAMs --- menpofit/aam/builder.py | 25 +++++++++---------------- menpofit/aam/fitter.py | 6 +++--- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 373236a..717a813 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -145,7 +145,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - def build(self, images, group=None, label=None, verbose=False): + def build(self, images, group=None, verbose=False): r""" Builds an Active Appearance Model from a list of landmarked images. @@ -153,15 +153,9 @@ def build(self, images, group=None, label=None, verbose=False): ---------- images : list of :map:`MaskedImage` The set of landmarked images from which to build the AAM. - group : `string`, optional The key of the landmark set that should be used. If ``None``, and if there is only one set of landmarks, this set will be used. - - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - verbose : `boolean`, optional Flag that controls information and progress printing. @@ -173,7 +167,7 @@ def build(self, images, group=None, label=None, verbose=False): """ # normalize images and compute reference shape reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) + images, group, self.diagonal, verbose=verbose) # build models at each scale if verbose: @@ -210,7 +204,7 @@ def build(self, images, group=None, label=None, verbose=False): verbose=verbose) # extract potentially rescaled shapes - level_shapes = [i.landmarks[group][label] + level_shapes = [i.landmarks[group].lms for i in level_images] # obtain shape representation @@ -255,11 +249,11 @@ def build(self, images, group=None, label=None, verbose=False): return aam - def increment(self, aam, images, group=None, label=None, + def increment(self, aam, images, group=None, forgetting_factor=1.0, verbose=False): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( - images, group, label, aam.reference_shape, verbose=verbose) + images, group, aam.reference_shape, verbose=verbose) # increment models at each scale if verbose: @@ -295,7 +289,7 @@ def increment(self, aam, images, group=None, label=None, verbose=verbose) # extract potentially rescaled shapes - level_shapes = [i.landmarks[group][label] + level_shapes = [i.landmarks[group].lms for i in level_images] # obtain shape representation @@ -337,7 +331,7 @@ def increment(self, aam, images, group=None, label=None, if verbose: print_dynamic('{}Done\n'.format(level_str)) - def build_incrementally(self, images, group=None, label=None, + def build_incrementally(self, images, group=None, forgetting_factor=1.0, batch_size=100, verbose=False): # number of batches @@ -345,15 +339,14 @@ def build_incrementally(self, images, group=None, label=None, # train first batch print 'Training batch 1.' - aam = self.build(images[:batch_size], group=group, label=label, - verbose=verbose) + aam = self.build(images[:batch_size], group=group, verbose=verbose) # train all other batches start = batch_size for j in range(1, n_batches): print 'Training batch {}.'.format(j+1) end = start + batch_size - self.increment(aam, images[start:end], group=group, label=label, + self.increment(aam, images[start:end], group=group, forgetting_factor=forgetting_factor, verbose=verbose) start = end diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a8fbc9c..24fe85c 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -166,10 +166,10 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): self.algorithms.append(algorithm) # TODO: Allow training from bounding boxes - def train(self, images, group=None, label=None, verbose=False, **kwargs): + def train(self, images, group=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( - images, group, label, self.reference_shape, verbose=verbose) + images, group, self.reference_shape, verbose=verbose) if self.scale_features: # compute features at highest level @@ -202,7 +202,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): verbose=verbose) # extract ground truth shapes for current level - level_gt_shapes = [i.landmarks[group][label] for i in level_images] + level_gt_shapes = [i.landmarks[group].lms for i in level_images] if j == 0: # generate perturbed shapes From 81b7e1c9cd8fa682377ddb6542f99283c2d1e7f2 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:02:09 +0100 Subject: [PATCH 127/423] Remove label kwarg from ATM --- menpofit/atm/builder.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 42c95f9..0f44545 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -124,7 +124,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, self.scale_features = scale_features self.max_shape_components = max_shape_components - def build(self, shapes, template, group=None, label=None, verbose=False): + def build(self, shapes, template, group=None, verbose=False): r""" Builds a Multilevel Active Template Model given a list of shapes and a template image. @@ -133,21 +133,13 @@ def build(self, shapes, template, group=None, label=None, verbose=False): ---------- shapes : list of :map:`PointCloud` The set of shapes from which to build the shape model of the ATM. - template : :map:`Image` or subclass The image to be used as template. - - group : `string`, optional + group : `str`, optional The key of the landmark set of the template that should be used. If ``None``, and if there is only one set of landmarks, this set will be used. - - label : `string`, optional - The label of the landmark manager of the template that you wish to - use. If ``None`` is passed, the convex hull of all landmarks is - used. - - verbose : `boolean`, optional + verbose : `bool`, optional Flag that controls information and progress printing. Returns @@ -162,7 +154,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): # normalize the template size using the reference_shape scaling template = template.rescale_to_reference_shape( - reference_shape, group=group, label=label) + reference_shape, group=group) # build models at each scale if verbose: @@ -214,7 +206,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): level_template = self.features[j](scaled_template) # extract potentially rescaled template shape - level_template_shape = level_template.landmarks[group][label] + level_template_shape = level_template.landmarks[group].lms # obtain warped template warped_template = self._warp_template(level_template, From a5144b45b55b450083ebf91e91dee315e5e16e39 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:02:42 +0100 Subject: [PATCH 128/423] Remove label kwarg from LK package --- menpofit/lk/fitter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 5e1cc72..e00501c 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -12,7 +12,7 @@ class LucasKanadeFitter(MultiFitter): r""" """ - def __init__(self, template, group=None, label=None, features=no_op, + def __init__(self, template, group=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, scales=(1, .5), scale_features=True, algorithm_cls=InverseCompositional, residual_cls=SSD, @@ -31,7 +31,7 @@ def __init__(self, template, group=None, label=None, features=no_op, self.scale_features = scale_features self.templates, self.sources = self._prepare_template( - template, group=group, label=label) + template, group=group) self.reference_shape = self.sources[0] @@ -49,14 +49,14 @@ def __init__(self, template, group=None, label=None, features=no_op, algorithm = algorithm_cls(t, transform, residual, **kwargs) self.algorithms.append(algorithm) - def _prepare_template(self, template, group=None, label=None): - template = template.crop_to_landmarks(group=group, label=label) + def _prepare_template(self, template, group=None): + template = template.crop_to_landmarks(group=group) template = template.as_masked() # rescale template to diagonal range if self.diagonal: template = template.rescale_landmarks_to_diagonal_range( - self.diagonal, group=group, label=label) + self.diagonal, group=group) # obtain image representation templates = [] @@ -75,7 +75,7 @@ def _prepare_template(self, template, group=None, label=None): templates.reverse() # get sources per level - sources = [i.landmarks[group][label] for i in templates] + sources = [i.landmarks[group].lms for i in templates] return templates, sources From c9bf9b443b7415e26f06a3e6044b7eaa4ef4ea39 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:04:01 +0100 Subject: [PATCH 129/423] Remove label kwarg from visualize package --- menpofit/visualize/widgets/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index cb634bf..302a5e2 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -213,7 +213,7 @@ def render_function(name, value): axes_font_weight=tmp3['axes_font_weight'], axes_x_limits=tmp3['axes_x_limits'], axes_y_limits=tmp3['axes_y_limits'], - figure_size=new_figure_size, label=None) + figure_size=new_figure_size) # Invert y axis if needed if mean_wid.value and axes_mode_wid.value == 1: @@ -247,7 +247,7 @@ def render_function(name, value): axes_font_weight=tmp3['axes_font_weight'], axes_x_limits=tmp3['axes_x_limits'], axes_y_limits=tmp3['axes_y_limits'], - figure_size=new_figure_size, label=None) + figure_size=new_figure_size) # Render vectors ax = plt.gca() From b0274bee487b5944ca6ee4d0df3b1f1a7ccb7d88 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:54:38 +0100 Subject: [PATCH 130/423] Cleanup the feature/patch extraction code Now just uses the asarray codepath since the performance difference is negligable. Also, renamed those methods. --- menpofit/sdm/algorithm.py | 92 ++++++++++----------------------------- 1 file changed, 23 insertions(+), 69 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index e25fd3d..29081af 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -14,13 +14,10 @@ class SupervisedDescentAlgorithm(object): r""" """ def train(self, images, gt_shapes, current_shapes, level_str='', - verbose=False, **kwargs): + verbose=False): n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] - self._features_patch_length = compute_features_info( - images[0], gt_shapes[0], self.features, - patch_shape=self.patch_shape)[1] # obtain delta_x and gt_x delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) @@ -31,9 +28,8 @@ def train(self, images, gt_shapes, current_shapes, level_str='', # Cascaded Regression loop for k in range(self.iterations): # generate regression data - features = obtain_patch_features( + features = features_per_image( images, current_shapes, self.patch_shape, self.features, - features_patch_length=self._features_patch_length, level_str='{}(Iteration {}) - '.format(level_str, k), verbose=verbose) @@ -41,7 +37,7 @@ def train(self, images, gt_shapes, current_shapes, level_str='', if verbose: print_dynamic('{}(Iteration {}) - Performing regression'.format( level_str, k)) - r = self._regressor_cls(**kwargs) + r = self._regressor_cls() r.train(features, delta_x) # add regressor to list self.regressors.append(r) @@ -79,8 +75,7 @@ def train(self, images, gt_shapes, current_shapes, level_str='', return current_shapes - def increment(self, images, gt_shapes, current_shapes, verbose=False, - **kwarg): + def increment(self, images, gt_shapes, current_shapes, verbose=False): n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] @@ -91,9 +86,8 @@ def increment(self, images, gt_shapes, current_shapes, verbose=False, # Cascaded Regression loop for r in self.regressors: # generate regression data - features = obtain_patch_features( - images, current_shapes, self.patch_shape, self.features, - features_patch_length=self._features_patch_length) + features = features_per_image(images, current_shapes, + self.patch_shape, self.features) # update regression if verbose: @@ -139,9 +133,8 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # Cascaded Regression loop for r in self.regressors: # compute regression features - features = compute_patch_features( - image, current_shape, self.patch_shape, self.features, - features_patch_length=self._features_patch_length) + features = features_per_patch(image, current_shape, + self.patch_shape, self.features) # solve for increments on the shape vector dx = r.predict(features) @@ -190,79 +183,40 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, # TODO: docment me! -def compute_patch_features(image, shape, patch_shape, features_callable, - features_patch_length=None): +def features_per_patch(image, shape, patch_shape, features_callable): """r """ patches = image.extract_patches(shape, patch_size=patch_shape, as_single_array=True) - if features_patch_length: - patch_features = np.empty((shape.n_points, features_patch_length)) - for j, p in enumerate(patches): - patch_features[j] = features_callable(p[0]).ravel() - else: - patch_features = [] - for p in patches: - patch_features.append(features_callable(p[0]).ravel()) - patch_features = np.asarray(patch_features) - - return patch_features.ravel() + patch_features = [features_callable(p[0]).ravel() for p in patches] + return np.asarray(patch_features).ravel() # TODO: docment me! -def generate_patch_features(image, shapes, patch_shape, features_callable, - features_patch_length=None): +def features_per_shape(image, shapes, patch_shape, features_callable): """r """ - if features_patch_length: - patch_features = np.empty((len(shapes), - shapes[0].n_points * features_patch_length)) - for j, s in enumerate(shapes): - patch_features[j] = compute_patch_features( - image, s, patch_shape, features_callable, - features_patch_length=features_patch_length) - else: - patch_features = [] - for s in shapes: - patch_features.append(compute_patch_features( - image, s, patch_shape, features_callable, - features_patch_length=features_patch_length)) - patch_features = np.asarray(patch_features) - - return patch_features.ravel() + patch_features = [features_per_patch(image, s, patch_shape, + features_callable) + for s in shapes] + + return np.asarray(patch_features).ravel() # TODO: docment me! -def obtain_patch_features(images, shapes, patch_shape, features_callable, - features_patch_length=None, level_str='', - verbose=False): +def features_per_image(images, shapes, patch_shape, features_callable, + level_str='', verbose=False): """r """ wrap = partial(print_progress, prefix='{}Extracting patches'.format(level_str), end_with_newline=not level_str, verbose=verbose) - n_images = len(images) - n_shapes = len(shapes[0]) - n_points = shapes[0][0].n_points - - if features_patch_length: - patch_features = np.empty((n_images, (n_shapes * n_points * - features_patch_length))) - for j, i in enumerate(wrap(images)): - patch_features[j] = generate_patch_features( - i, shapes[j], patch_shape, features_callable, - features_patch_length=features_patch_length) - else: - patch_features = [] - for j, i in enumerate(wrap(images)): - patch_features.append(generate_patch_features( - i, shapes[j], patch_shape, features_callable, - features_patch_length=features_patch_length)) - patch_features = np.asarray(patch_features) - - return patch_features.reshape((-1, n_points * features_patch_length)) + patch_features = [features_per_shape(i, shapes[j], patch_shape, + features_callable) + for j, i in enumerate(wrap(images))] + return np.asarray(patch_features) def compute_delta_x(gt_shape, current_shapes): From f24f642185165b71c521b09f288ce56b52c22111 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 27 Jul 2015 16:25:55 +0100 Subject: [PATCH 131/423] Fixes some errors introduced by sdm_refactor - Also define SDM alias --- menpofit/sdm/__init__.py | 2 +- menpofit/sdm/algorithm.py | 5 +++-- menpofit/sdm/fitter.py | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/menpofit/sdm/__init__.py b/menpofit/sdm/__init__.py index 16e88b4..8a616c7 100644 --- a/menpofit/sdm/__init__.py +++ b/menpofit/sdm/__init__.py @@ -1,2 +1,2 @@ from .algorithm import Newton, GaussNewton -from .fitter import SupervisedDescentFitter +from .fitter import SupervisedDescentFitter, SDM diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 29081af..b4fc7e7 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -201,7 +201,7 @@ def features_per_shape(image, shapes, patch_shape, features_callable): features_callable) for s in shapes] - return np.asarray(patch_features).ravel() + return np.asarray(patch_features) # TODO: docment me! @@ -216,7 +216,8 @@ def features_per_image(images, shapes, patch_shape, features_callable, patch_features = [features_per_shape(i, shapes[j], patch_shape, features_callable) for j, i in enumerate(wrap(images))] - return np.asarray(patch_features) + patch_features = np.asarray(patch_features) + return patch_features.reshape((-1, patch_features.shape[-1])) def compute_delta_x(gt_shape, current_shapes): diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2dcf5e4..ab42b8c 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -77,9 +77,8 @@ def _train(self, images, group=None, bounding_box_group=None, # In the case where group is None, we need to get the only key so that # we can attach landmarks below and not get a complaint about using None - first_image = first_batch[0] if group is None: - group = first_image.landmarks.group_labels[0] + group = first_batch[0].landmarks.group_labels[0] for k, image_batch in enumerate(chain([first_batch], image_batches)): # After the first batch, we are incrementing the model @@ -109,7 +108,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Find all bounding boxes on the images with the given bounding # box key - all_bb_keys = list(first_image.landmarks.keys_matching( + all_bb_keys = list(image_batch[0].landmarks.keys_matching( '*{}*'.format(bounding_box_group))) n_perturbations = len(all_bb_keys) @@ -142,7 +141,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Re-grab all the bounding box keys for iterating over when # calculating perturbations - all_bb_keys = list(first_image.landmarks.keys_matching( + all_bb_keys = list(image_batch[0].landmarks.keys_matching( '*{}*'.format(bounding_box_group))) # Before scaling, we compute the holistic feature on the whole image @@ -333,3 +332,6 @@ def __str__(self): diagonal=diagonal, is_custom_perturb_func=is_custom_perturb_func) return cls_str + + +SDM = partial(SupervisedDescentFitter, sd_algorithm_cls=Newton) From 06c1e1b0ccee39e0b4d3aa38ca6509f0d81151c0 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 10:44:22 +0100 Subject: [PATCH 132/423] Get rid of the first_batch thing That was a bit confusing and @jalabort rightly pointed out that it didn't work correctly anyway. --- menpofit/sdm/fitter.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index ab42b8c..72a09e8 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -70,17 +70,10 @@ def _train(self, images, group=None, bounding_box_group=None, # Create a generator of fixed sized batches. Will still work even # on an infinite list. image_batches = batch(images, batch_size) - first_batch = next(image_batches) else: - image_batches = [] - first_batch = list(images) + image_batches = [list(images)] - # In the case where group is None, we need to get the only key so that - # we can attach landmarks below and not get a complaint about using None - if group is None: - group = first_batch[0].landmarks.group_labels[0] - - for k, image_batch in enumerate(chain([first_batch], image_batches)): + for k, image_batch in enumerate(image_batches): # After the first batch, we are incrementing the model if k > 0: increment = True @@ -88,6 +81,13 @@ def _train(self, images, group=None, bounding_box_group=None, if verbose: print('Computing batch {}'.format(k)) + # In the case where group is None, we need to get the only key so + # that we can attach landmarks below and not get a complaint about + # using None + first_image = image_batch[0] + if group is None: + group = first_image.landmarks.group_labels[0] + if not increment: # Normalize images and compute reference shape self.reference_shape, image_batch = normalization_wrt_reference_shape( @@ -108,7 +108,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Find all bounding boxes on the images with the given bounding # box key - all_bb_keys = list(image_batch[0].landmarks.keys_matching( + all_bb_keys = list(first_image.landmarks.keys_matching( '*{}*'.format(bounding_box_group))) n_perturbations = len(all_bb_keys) @@ -141,7 +141,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Re-grab all the bounding box keys for iterating over when # calculating perturbations - all_bb_keys = list(image_batch[0].landmarks.keys_matching( + all_bb_keys = list(first_image.landmarks.keys_matching( '*{}*'.format(bounding_box_group))) # Before scaling, we compute the holistic feature on the whole image From 2151476c84fd2379e597d65a9d046637a3a429ba Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 10:49:17 +0100 Subject: [PATCH 133/423] SDM, flip the scales logic so that its increasing Go from smallest to largest scales rather than reversing the list. --- menpofit/sdm/fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 72a09e8..ec60492 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -23,7 +23,7 @@ class SupervisedDescentFitter(MultiFitter): def __init__(self, images, group=None, bounding_box_group=None, sd_algorithm_cls=Newton, holistic_feature=no_op, patch_features=no_op, patch_shape=(17, 17), diagonal=None, - scales=(1, 0.5), iterations=6, n_perturbations=30, + scales=(0.5, 1.0), iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): # check parameters @@ -39,7 +39,7 @@ def __init__(self, images, group=None, bounding_box_group=None, self._patch_features = patch_features self._patch_shape = patch_shape self.diagonal = diagonal - self.scales = list(scales)[::-1] + self.scales = scales self.n_perturbations = n_perturbations self.iterations = checks.check_max_iters(iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box From 9cee234620f72a1af75e313d988779595482ab1e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 10:50:45 +0100 Subject: [PATCH 134/423] Remove useless tests and benchmark packages Benchmark has been totally superseded by menpobench and the failing tests were annoying and pointless. --- menpofit/benchmark/__init__.py | 1 - menpofit/benchmark/base.py | 699 ---------------- menpofit/benchmark/io.py | 111 --- menpofit/benchmark/predefined.py | 779 ------------------ menpofit/test/__init__.py | 0 menpofit/test/aam_builder_test.py | 196 ----- menpofit/test/aam_fitter_test.py | 448 ---------- menpofit/test/atm_builder_test.py | 175 ---- menpofit/test/atm_fitter_test.py | 438 ---------- menpofit/test/clm_builder_test.py | 194 ----- menpofit/test/clm_fitter_test.py | 370 --------- menpofit/test/fitmulitlevel_base_test.py | 29 - menpofit/test/fittingresult_test.py | 108 --- .../test/multilevel_fittingresult_test.py | 18 - menpofit/test/sdm_test.py | 111 --- 15 files changed, 3677 deletions(-) delete mode 100644 menpofit/benchmark/__init__.py delete mode 100644 menpofit/benchmark/base.py delete mode 100644 menpofit/benchmark/io.py delete mode 100644 menpofit/benchmark/predefined.py delete mode 100644 menpofit/test/__init__.py delete mode 100644 menpofit/test/aam_builder_test.py delete mode 100644 menpofit/test/aam_fitter_test.py delete mode 100644 menpofit/test/atm_builder_test.py delete mode 100644 menpofit/test/atm_fitter_test.py delete mode 100644 menpofit/test/clm_builder_test.py delete mode 100644 menpofit/test/clm_fitter_test.py delete mode 100644 menpofit/test/fitmulitlevel_base_test.py delete mode 100644 menpofit/test/fittingresult_test.py delete mode 100644 menpofit/test/multilevel_fittingresult_test.py delete mode 100644 menpofit/test/sdm_test.py diff --git a/menpofit/benchmark/__init__.py b/menpofit/benchmark/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/menpofit/benchmark/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/menpofit/benchmark/base.py b/menpofit/benchmark/base.py deleted file mode 100644 index c87ef2d..0000000 --- a/menpofit/benchmark/base.py +++ /dev/null @@ -1,699 +0,0 @@ -import os - -import numpy as np - -import menpo.io as mio -from menpo.visualize.text_utils import print_dynamic, progress_bar_str -from menpo.landmark import labeller -from menpo.visualize.base import GraphPlotter -from menpofit.aam import AAMBuilder, LucasKanadeAAMFitter -from menpofit.clm import CLMBuilder, GradientDescentCLMFitter -from menpofit.sdm import SDMTrainer - - -def aam_fit_benchmark(fitting_images, aam, fitting_options=None, - perturb_options=None, verbose=False): - r""" - Fits a trained AAM model to a database. - - Parameters - ---------- - fitting_images: list of :class:MaskedImage objects - A list of the fitting images. - aam: :class:menpo.fitmultilevel.aam.AAM object - The trained AAM object. It can be generated from the - aam_build_benchmark() method. - fitting_options: dictionary, optional - A dictionary with the parameters that will be passed in the - LucasKanadeAAMFitter (:class:menpo.fitmultilevel.aam.base). - If None, the default options will be used. - This is an example of the dictionary with the default options: - fitting_options = {'algorithm': AlternatingInverseCompositional, - 'md_transform': OrthoMDTransform, - 'global_transform': AlignmentSimilarity, - 'n_shape': None, - 'n_appearance': None, - 'max_iters': 50, - 'error_type': 'me_norm' - } - For an explanation of the options, please refer to the - LucasKanadeAAMFitter documentation. - - Default: None - bounding_boxes: list of (2, 2) ndarray, optional - If provided, fits will be initialized from a bounding box. If - None, perturbation of ground truth will be used instead. - can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. - perturb_options: dictionary, optional - A dictionary with parameters that control the perturbation on the - ground truth shape with noise of specified std. Note that if - bounding_box is provided perturb_options is ignored and not used. - If None, the default options will be used. - This is an example of the dictionary with the default options: - initialization_options = {'noise_std': 0.04, - 'rotation': False - } - For an explanation of the options, please refer to the perturb_shape() - method documentation of :map:`MultilevelFitter`. - verbose: bool, optional - If True, it prints information regarding the AAM fitting including - progress bar, current image error and percentage of images with errors - less or equal than a value. - - Default: False - - Returns - ------- - fitting_results: :map:`FittingResultList` - A list with the :map:`FittingResult` object per image. - """ - if verbose: - print('AAM Fitting:') - perc1 = 0. - perc2 = 0. - - # parse options - if fitting_options is None: - fitting_options = {} - if perturb_options is None: - perturb_options = {} - - # extract some options - group = fitting_options.pop('gt_group', 'PTS') - max_iters = fitting_options.pop('max_iters', 50) - error_type = fitting_options.pop('error_type', 'me_norm') - - # create fitter - fitter = LucasKanadeAAMFitter(aam, **fitting_options) - - # fit images - n_images = len(fitting_images) - fitting_results = [] - for j, i in enumerate(fitting_images): - # perturb shape - gt_s = i.landmarks[group].lms - if 'bbox' in i.landmarks: - # shape from bounding box - s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) - else: - # shape from perturbation - s = fitter.perturb_shape(gt_s, **perturb_options) - # fit - fr = fitter.fit(i, s, gt_shape=gt_s, max_iters=max_iters) - fitting_results.append(fr) - - # print - final_error = fr.final_error(error_type=error_type) - initial_error = fr.initial_error(error_type=error_type) - if verbose: - if error_type == 'me_norm': - if final_error <= 0.03: - perc1 += 1. - if final_error <= 0.04: - perc2 += 1. - elif error_type == 'rmse': - if final_error <= 0.05: - perc1 += 1. - if final_error <= 0.06: - perc2 += 1. - print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' - 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( - progress_bar_str(float(j + 1) / n_images, show_bar=False), - perc1 * 100. / n_images, perc2 * 100. / n_images, j + 1, - n_images, initial_error, final_error)) - if verbose: - print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' - '{1:.1f}%]\n'.format(perc1 * 100. / n_images, - perc2 * 100. / n_images)) - - return fitting_results - - -def aam_build_benchmark(training_images, training_options=None, verbose=False): - r""" - Builds an AAM model. - - Parameters - ---------- - training_images: list of :class:MaskedImage objects - A list of the training images. - training_options: dictionary, optional - A dictionary with the parameters that will be passed in the AAMBuilder - (:class:menpo.fitmultilevel.aam.AAMBuilder). - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'group': 'PTS', - 'features': 'igo', - 'transform': PiecewiseAffine, - 'trilist': None, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'max_appearance_components': None, - 'boundary': 3 - } - For an explanation of the options, please refer to the AAMBuilder - documentation. - - Default: None - verbose: boolean, optional - If True, it prints information regarding the AAM training. - - Default: False - - Returns - ------- - aam: :class:menpo.fitmultilevel.aam.AAM object - The trained AAM model. - """ - if verbose: - print('AAM Training:') - - # parse options - if training_options is None: - training_options = {} - - # group option - group = training_options.pop('group', None) - - # trilist option - trilist = training_options.pop('trilist', None) - if trilist is not None: - labeller(training_images[0], 'PTS', trilist) - training_options['trilist'] = \ - training_images[0].landmarks[trilist.__name__].lms.trilist - - # build aam - aam = AAMBuilder(**training_options).build(training_images, group=group, - verbose=verbose) - - return aam - - -def clm_fit_benchmark(fitting_images, clm, fitting_options=None, - perturb_options=None, verbose=False): - r""" - Fits a trained CLM model to a database. - - Parameters - ---------- - fitting_images: list of :class:MaskedImage objects - A list of the fitting images. - clm: :class:menpo.fitmultilevel.clm.CLM object - The trained CLM object. It can be generated from the - clm_build_benchmark() method. - fitting_options: dictionary, optional - A dictionary with the parameters that will be passed in the - GradientDescentCLMFitter (:class:menpo.fitmultilevel.clm.base). - If None, the default options will be used. - This is an example of the dictionary with the default options: - fitting_options = {'algorithm': RegularizedLandmarkMeanShift, - 'pdm_transform': OrthoPDM, - 'global_transform': AlignmentSimilarity, - 'n_shape': None, - 'max_iters': 50, - 'error_type': 'me_norm' - } - For an explanation of the options, please refer to the - GradientDescentCLMFitter documentation. - - Default: None - bounding_boxes: list of (2, 2) ndarray, optional - If provided, fits will be initialized from a bounding box. If - None, perturbation of ground truth will be used instead. - can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. - perturb_options: dictionary, optional - A dictionary with parameters that control the perturbation on the - ground truth shape with noise of specified std. Note that if - bounding_box is provided perturb_options is ignored and not used. - verbose: boolean, optional - If True, it prints information regarding the AAM fitting including - progress bar, current image error and percentage of images with errors - less or equal than a value. - - Default: False - - Returns - ------- - fitting_results: :class:menpo.fit.fittingresult.FittingResultList object - A list with the FittingResult object per image. - """ - if verbose: - print('CLM Fitting:') - perc1 = 0. - perc2 = 0. - - # parse options - if fitting_options is None: - fitting_options = {} - - # extract some options - group = fitting_options.pop('gt_group', 'PTS') - max_iters = fitting_options.pop('max_iters', 50) - error_type = fitting_options.pop('error_type', 'me_norm') - - # create fitter - fitter = GradientDescentCLMFitter(clm, **fitting_options) - - # fit images - n_images = len(fitting_images) - fitting_results = [] - for j, i in enumerate(fitting_images): - # perturb shape - gt_s = i.landmarks[group].lms - if 'bbox' in i.landmarks: - # shape from bounding box - s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) - else: - # shape from perturbation - s = fitter.perturb_shape(gt_s, **perturb_options) - # fit - fr = fitter.fit(i, s, gt_shape=gt_s, max_iters=max_iters) - fitting_results.append(fr) - - # print - final_error = fr.final_error(error_type=error_type) - initial_error = fr.initial_error(error_type=error_type) - if verbose: - if error_type == 'me_norm': - if final_error <= 0.03: - perc1 += 1. - if final_error <= 0.04: - perc2 += 1. - elif error_type == 'rmse': - if final_error <= 0.05: - perc1 += 1. - if final_error <= 0.06: - perc2 += 1. - print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' - 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( - progress_bar_str(float(j + 1) / n_images, - show_bar=False), - perc1 * 100. / n_images, perc2 * 100. / n_images, - j + 1, n_images, initial_error, final_error)) - if verbose: - print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' - '{1:.1f}%]\n'.format(perc1 * 100. / n_images, - perc2 * 100. / n_images)) - - return fitting_results - - -def clm_build_benchmark(training_images, training_options=None, verbose=False): - r""" - Builds an CLM model. - - Parameters - ---------- - training_images: list of :class:MaskedImage objects - A list of the training images. - training_options: dictionary, optional - A dictionary with the parameters that will be passed in the CLMBuilder - (:class:menpo.fitmultilevel.clm.CLMBuilder). - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': sparse_hog, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'boundary': 3 - } - For an explanation of the options, please refer to the CLMBuilder - documentation. - - Default: None - verbose: boolean, optional - If True, it prints information regarding the CLM training. - - Default: False - - Returns - ------- - clm: :class:menpo.fitmultilevel.clm.CLM object - The trained CLM model. - """ - if verbose: - print('CLM Training:') - - # parse options - if training_options is None: - training_options = {} - - # group option - group = training_options.pop('group', None) - - # build aam - aam = CLMBuilder(**training_options).build(training_images, group=group, - verbose=verbose) - - return aam - - -def sdm_fit_benchmark(fitting_images, fitter, perturb_options=None, - fitting_options=None, verbose=False): - r""" - Fits a trained SDM to a database. - - Parameters - ---------- - fitting_images: list of :class:MaskedImage objects - A list of the fitting images. - fitter: :map:`SDMFitter` - The trained AAM object. It can be generated from the - aam_build_benchmark() method. - fitting_options: dictionary, optional - A dictionary with the parameters that will be passed in the - LucasKanadeAAMFitter (:class:menpo.fitmultilevel.sdm.base). - If None, the default options will be used. - This is an example of the dictionary with the default options: - fitting_options = {'algorithm': AlternatingInverseCompositional, - 'md_transform': OrthoMDTransform, - 'global_transform': AlignmentSimilarity, - 'n_shape': None, - 'n_appearance': None, - 'max_iters': 50, - 'error_type': 'me_norm' - } - For an explanation of the options, please refer to the - LucasKanadeAAMFitter documentation. - - Default: None - bounding_boxes: list of (2, 2) ndarray, optional - If provided, fits will be initialized from a bounding box. If - None, perturbation of ground truth will be used instead. - can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. - perturb_options: dictionary, optional - A dictionary with parameters that control the perturbation on the - ground truth shape with noise of specified std. Note that if - bounding_box is provided perturb_options is ignored and not used. - If None, the default options will be used. - This is an example of the dictionary with the default options: - initialization_options = {'noise_std': 0.04, - 'rotation': False - } - For an explanation of the options, please refer to the perturb_shape() - method documentation of :map:`MultilevelFitter`. - verbose: bool, optional - If True, it prints information regarding the AAM fitting including - progress bar, current image error and percentage of images with errors - less or equal than a value. - - Default: False - - Returns - ------- - fitting_results: :map:`FittingResultList` - A list with the :map:`FittingResult` object per image. - """ - if verbose: - print('SDM Fitting:') - perc1 = 0. - perc2 = 0. - - # parse options - if fitting_options is None: - fitting_options = {} - if perturb_options is None: - perturb_options = {} - - # extract some options - group = fitting_options.pop('gt_group', 'PTS') - error_type = fitting_options.pop('error_type', 'me_norm') - - # fit images - n_images = len(fitting_images) - fitting_results = [] - for j, i in enumerate(fitting_images): - # perturb shape - gt_s = i.landmarks[group].lms - if 'bbox' in i.landmarks: - # shape from bounding box - s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) - else: - # shape from perturbation - s = fitter.perturb_shape(gt_s, **perturb_options) - # fit - fr = fitter.fit(i, s, gt_shape=gt_s) - fitting_results.append(fr) - - # print - final_error = fr.final_error(error_type=error_type) - initial_error = fr.initial_error(error_type=error_type) - if verbose: - if error_type == 'me_norm': - if final_error <= 0.03: - perc1 += 1. - if final_error <= 0.04: - perc2 += 1. - elif error_type == 'rmse': - if final_error <= 0.05: - perc1 += 1. - if final_error <= 0.06: - perc2 += 1. - print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' - 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( - progress_bar_str(float(j + 1) / n_images, show_bar=False), - perc1 * 100. / n_images, perc2 * 100. / n_images, j + 1, - n_images, initial_error, final_error)) - if verbose: - print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' - '{1:.1f}%]\n'.format(perc1 * 100. / n_images, - perc2 * 100. / n_images)) - - return fitting_results - - -def sdm_build_benchmark(training_images, training_options=None, verbose=False): - r""" - Builds an SDM model. - - Parameters - ---------- - training_images: list of :class:MaskedImage objects - A list of the training images. - training_options: dictionary, optional - A dictionary with the parameters that will be passed in the AAMBuilder - (:class:menpo.fitmultilevel.aam.AAMBuilder). - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'group': 'PTS', - 'features': 'igo', - 'transform': PiecewiseAffine, - 'trilist': None, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'max_appearance_components': None, - 'boundary': 3 - } - For an explanation of the options, please refer to the AAMBuilder - documentation. - - Default: None - verbose: boolean, optional - If True, it prints information regarding the AAM training. - - Default: False - - Returns - ------- - aam: :class:menpo.fitmultilevel.aam.AAM object - The trained AAM model. - """ - if verbose: - print('SDM Training:') - - # parse options - if training_options is None: - training_options = {} - - # group option - group = training_options.pop('group', None) - - # build sdm - sdm = SDMTrainer(**training_options).train(training_images, group=group, - verbose=verbose) - return sdm - - -def load_database(database_path, bounding_boxes=None, - db_loading_options=None, verbose=False): - r""" - Loads the database images, crops them and converts them. - - Parameters - ---------- - database_path: str - The path of the database images. - db_loading_options: dictionary, optional - A dictionary with options related to image loading. - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'crop_proportion': 0.1, - 'convert_to_grey': True, - } - - crop_proportion (float) defines the additional padding to be added all - around the landmarks bounds when the images are cropped. It is defined - as a proportion of the landmarks' range. - - convert_to_grey (boolean)defines whether the images will be converted - to greyscale. - - Default: None - verbose: boolean, optional - If True, it prints a progress percentage bar. - - Default: False - - Returns - ------- - images: list of :class:MaskedImage objects - A list of the loaded images. - - Raises - ------ - ValueError - Invalid path given - ValueError - No {files_extension} files in given path - """ - # check input options - if db_loading_options is None: - db_loading_options = {} - - # check given path - database_path = os.path.abspath(os.path.expanduser(database_path)) - if os.path.isdir(database_path) is not True: - raise ValueError('Invalid path given') - - # create final path - final_path = os.path.join(database_path, '*') - - # get options - crop_proportion = db_loading_options.pop('crop_proportion', 0.5) - convert_to_grey = db_loading_options.pop('convert_to_grey', True) - - # load images - images = [] - for i in mio.import_images(final_path, verbose=verbose): - # If we have bounding boxes then we need to make sure we crop to them! - # If we don't crop to the bounding box then we might crop out part of - # the image the bounding box belongs to. - landmark_group_label = None - if bounding_boxes is not None: - fname = i.path.name - landmark_group_label = 'bbox' - i.landmarks[landmark_group_label] = bounding_boxes[fname].detector - - # crop image - i.crop_to_landmarks_proportion_inplace(crop_proportion, - group=landmark_group_label) - - # convert it to greyscale if needed - if convert_to_grey and i.n_channels == 3: - i = i.as_greyscale(mode='luminosity') - - # append it to the list - images.append(i) - if verbose: - print("\nAssets loaded.") - return images - - -def convert_fitting_results_to_ced(fitting_results, max_error_bin=0.05, - bins_error_step=0.005, error_type='me_norm'): - r""" - Method that given a fitting_result object, it converts it to the - cumulative error distribution values that can be used for plotting. - - Parameters - ---------- - fitting_results: :class:menpo.fit.fittingresult.FittingResultList object - A list with the FittingResult object per image. - max_error_bin: float, Optional - The maximum error of the distribution. - - Default: 0.05 - bins_error_step: float, Optional - The sampling step of the distribution values. - - Default: 0.005 - - Returns - ------- - final_error_dist: list - Cumulative distribution values of the final errors. - initial_error_dist: list - Cumulative distribution values of the initial errors. - """ - error_bins = np.arange(0., max_error_bin + bins_error_step, - bins_error_step) - final_error = [f.final_error(error_type=error_type) - for f in fitting_results] - initial_error = [f.initial_error(error_type=error_type) - for f in fitting_results] - - final_error_dist = np.array( - [float(np.sum(final_error <= k)) / - len(final_error) for k in error_bins]) - initial_error_dist = np.array( - [float(np.sum(initial_error <= k)) / - len(final_error) for k in error_bins]) - return final_error_dist, initial_error_dist, error_bins - - -def plot_fitting_curves(x_axis, ceds, title, figure_id=None, new_figure=False, - x_label='Point-to-Point Normalized RMS Error', - y_limit=1, x_limit=0.05, legend_entries=None, **kwargs): - r""" - Method that plots Cumulative Error Distributions in a single figure. - - Parameters - ---------- - x_axis : ndarray - The horizontal axis values (errors). - ceds : list of ndarrays - The vertical axis values (percentages). - title : string - The plot title. - figure_id : Optional - A figure handle. - new_figure : boolean, Optional - If True, a new figure window will be created. - y_limit : float, Optional - The maximum value of the vertical axis. - x_limit : float, Optional - The maximum value of the vertical axis. - x_label : string - The label of the horizontal axis. - legend_entries : list of strings or None - The legend of the plot. If None, the legend will include an incremental - number per curve. - - Returns - ------- - final_error_dist : list - Cumulative distribution values of the final errors. - initial_error_dist : list - Cumulative distribution values of the initial errors. - """ - if legend_entries is None: - legend_entries = [str(i + 1) for i in range(len(ceds))] - y_label = 'Proportion of images' - x_axis_limits = [0, x_limit] - y_axis_limits = [0, y_limit] - return GraphPlotter(figure_id, new_figure, x_axis, ceds, title=title, - legend_entries=legend_entries, x_label=x_label, - y_label=y_label, - x_axis_limits=x_axis_limits, - y_axis_limits=y_axis_limits).render(**kwargs) diff --git a/menpofit/benchmark/io.py b/menpofit/benchmark/io.py deleted file mode 100644 index 5043e1d..0000000 --- a/menpofit/benchmark/io.py +++ /dev/null @@ -1,111 +0,0 @@ -import urllib2 -import cStringIO -import os -import scipy.io as sio -import glob -import tempfile -import shutil -import zipfile -from collections import namedtuple - -# Container for bounding box -from menpo.shape import PointCloud - -BoundingBox = namedtuple('BoundingBox', ['detector', 'groundtruth']) -# Where the bounding boxes should be fetched from -bboxes_url = 'http://ibug.doc.ic.ac.uk/media/uploads/competitions/bounding_boxes.zip' - - -def download_ibug_bounding_boxes(path=None, verbose=False): - r"""Downloads the bounding box information provided on the iBUG website - and unzips it to the path. - - Parameters - ---------- - path : `str`, optional - The path that the bounding box files should be extracted to. - If None, the current directory will be used. - """ - if path is None: - path = os.getcwd() - else: - path = os.path.abspath(os.path.expanduser(path)) - if verbose: - print('Acquiring bounding box information from iBUG website...') - try: - remotezip = urllib2.urlopen(bboxes_url) - zipinmemory = cStringIO.StringIO(remotezip.read()) - ziplocal = zipfile.ZipFile(zipinmemory) - except Exception as e: - print('Unable to grab bounding boxes (are you online?)') - raise e - if verbose: - print('Extracting to {}'.format(os.path.join(path, 'Bounding Boxes'))) - try: - ziplocal.extractall(path=path) - if verbose: - print('Done.') - except Exception as e: - if verbose: - print('Unable to save.'.format(e)) - raise e - - -def import_bounding_boxes(boxes_path): - r""" - Imports the bounding boxes at boxes_path, returning a dict - where the key is a filename and the value is a BoundingBox. - - Parameters - ---------- - boxes_path : str - A path to a bounding box .mat file downloaded from the - iBUG website. - - Returns - ------- - dict: - Mapping of filenames to bounding boxes - - """ - bboxes_mat = sio.loadmat(boxes_path) - bboxes = {} - for bb in bboxes_mat['bounding_boxes'][0, :]: - fname, detector_bb, gt_bb = bb[0, 0] - bboxes[str(fname[0])] = BoundingBox( - PointCloud(detector_bb.reshape([2, 2])[:, ::-1]), - PointCloud(gt_bb.reshape([2, 2])[:, ::-1])) - return bboxes - - -def import_all_bounding_boxes(boxes_dir_path=None, verbose=True): - r""" - Imports all the bounding boxes contained in boxes_dir_path. - If the path is False, the bounding boxes are downloaded from the - iBUG website directly. - - - """ - temp_path = None - if boxes_dir_path is None: - print('No path provided - acuqiring zip to tmp dir...') - temp_path = tempfile.mkdtemp() - download_ibug_bounding_boxes(path=temp_path, verbose=verbose) - boxes_dir_path = os.path.join(temp_path, 'Bounding Boxes') - prefix = 'bounding_boxes_' - bbox_paths = glob.glob(os.path.join(boxes_dir_path, prefix + '*.mat')) - bboxes = {} - for bbox_path in bbox_paths: - db = os.path.splitext(os.path.split(bbox_path)[-1])[0][len(prefix):] - if verbose: - print('Importing {}'.format(db)) - bboxes[db] = import_bounding_boxes(bbox_path) - if verbose: - print('Cleaning up...') - if temp_path: - # If we downloaded, clean it up! - shutil.rmtree(temp_path) - if verbose: - print('Done.') - return bboxes - diff --git a/menpofit/benchmark/predefined.py b/menpofit/benchmark/predefined.py deleted file mode 100644 index 3780f41..0000000 --- a/menpofit/benchmark/predefined.py +++ /dev/null @@ -1,779 +0,0 @@ -from menpo.landmark import ibug_face_68_trimesh -from menpo.feature import sparse_hog, igo - -from menpofit.lucaskanade import AIC -from menpofit.transform import OrthoMDTransform, DifferentiablePiecewiseAffine -from menpofit.modelinstance import OrthoPDM -from menpofit.gradientdescent import RLMS -from menpofit.clm.classifier import linear_svm_lr - -from .io import import_bounding_boxes -from .base import (aam_build_benchmark, aam_fit_benchmark, - clm_build_benchmark, clm_fit_benchmark, - sdm_build_benchmark, sdm_fit_benchmark, - load_database, convert_fitting_results_to_ced, - plot_fitting_curves) - - -def aam_fastest_alternating_noise(training_db_path, fitting_db_path, - features=igo, noise_std=0.04, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.2, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': 'me_norm' - } - perturb_options = {'noise_std': 0.04, - 'rotation': False} - - # set passed parameters - training_options['features'] = features - perturb_options['noise_std'] = noise_std - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_results = aam_fit_benchmark(fitting_images, aam, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_fastest_alternating_bbox(training_db_path, fitting_db_path, - fitting_bboxes_path, features=igo, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.1, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': [igo] * 3, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': 'me_norm' - } - - # set passed parameters - training_options['features'] = features - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = aam_fit_benchmark(fitting_images, aam, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_best_performance_alternating_noise(training_db_path, fitting_db_path, - features=igo, noise_std=0.04, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.2, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.2, - 'scaled_shape_models': False, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': error_type - } - perturb_options = {'noise_std': 0.04, - 'rotation': False} - - # set passed parameters - training_options['features'] = features - perturb_options['noise_std'] = noise_std - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_results = aam_fit_benchmark(fitting_images, aam, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_best_performance_alternating_bbox(training_db_path, fitting_db_path, - fitting_bboxes_path, - features=igo, verbose=False, - plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.5, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': 200, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 100, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': error_type - } - - # set passed parameters - training_options['features'] = features - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = aam_fit_benchmark(fitting_images, aam, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def clm_basic_noise(training_db_path, fitting_db_path, - features=sparse_hog, classifier_trainers=linear_svm_lr, - noise_std=0.04, verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.4, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': [sparse_hog] * 3, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'boundary': 3 - } - fitting_options = {'algorithm': RLMS, - 'pdm_transform': OrthoPDM, - 'n_shape': [3, 6, 12], - 'max_iters': 50, - 'error_type': error_type - } - perturb_options = {'noise_std': 0.01, - 'rotation': False} - - # set passed parameters - training_options['features'] = features - training_options['classifier_trainers'] = classifier_trainers - perturb_options['noise_std'] = noise_std - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - clm = clm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_results = clm_fit_benchmark(fitting_images, clm, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "CLMs with {} and {} classifier using RLMS".format( - training_options['features'].__name__, - training_options['classifier_trainers']) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def clm_basic_bbox(training_db_path, fitting_db_path, fitting_bboxes_path, - features=sparse_hog, classifier_trainers=linear_svm_lr, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.5, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': [sparse_hog] * 3, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'boundary': 3 - } - fitting_options = {'algorithm': RLMS, - 'pdm_transform': OrthoPDM, - 'n_shape': [3, 6, 12], - 'max_iters': 50, - 'error_type': error_type - } - - # set passed parameters - training_options['features'] = features - training_options['classifier_trainers'] = classifier_trainers - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - - clm = clm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = clm_fit_benchmark(fitting_images, clm, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "CLMs with {} and {} classifier using RLMS".format( - training_options['features'].__name__, - training_options['classifier_trainers']) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def sdm_fastest_bbox(training_db_path, fitting_db_path, - fitting_bboxes_path, features=None, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.8, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'normalization_diagonal': 200, - 'n_levels': 4, - 'downscale': 1.01, - 'noise_std': 0.08, - 'patch_shape': (16, 16), - 'n_perturbations': 15, - } - fitting_options = { - 'error_type': error_type - } - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - sdm = sdm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = sdm_fit_benchmark(fitting_images, sdm, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "SDMs using default (sparse hogs)".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_params_combinations_noise(training_db_path, fitting_db_path, - n_experiments=1, features=None, - scaled_shape_models=None, - n_shape=None, - n_appearance=None, noise_std=None, - rotation=None, verbose=False, plot=False): - - # parse input - if features is None: - features = [igo] * n_experiments - elif len(features) is not n_experiments: - raise ValueError("features has wrong length") - if scaled_shape_models is None: - scaled_shape_models = [True] * n_experiments - elif len(scaled_shape_models) is not n_experiments: - raise ValueError("scaled_shape_models has wrong length") - if n_shape is None: - n_shape = [[3, 6, 12]] * n_experiments - elif len(n_shape) is not n_experiments: - raise ValueError("n_shape has wrong length") - if n_appearance is None: - n_appearance = [50] * n_experiments - elif len(n_appearance) is not n_experiments: - raise ValueError("n_appearance has wrong length") - if noise_std is None: - noise_std = [0.04] * n_experiments - elif len(noise_std) is not n_experiments: - raise ValueError("noise_std has wrong length") - if rotation is None: - rotation = [False] * n_experiments - elif len(rotation) is not n_experiments: - raise ValueError("rotation has wrong length") - - # load images - db_loading_options = {'crop_proportion': 0.1, - 'convert_to_grey': True - } - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - - # run experiments - max_error_bin = 0.05 - bins_error_step = 0.005 - curves_to_plot = [] - all_fitting_results = [] - for i in range(n_experiments): - if verbose: - print("\nEXPERIMENT {}/{}:".format(i + 1, n_experiments)) - print("- features: {}\n- scaled_shape_models: {}\n" - "- n_shape: {}\n" - "- n_appearance: {}\n- noise_std: {}\n" - "- rotation: {}".format( - features[i], scaled_shape_models[i], - n_shape[i], n_appearance[i], noise_std[i], rotation[i])) - - # predefined option dictionaries - error_type = 'me_norm' - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': error_type - } - pertrub_options = {'noise_std': 0.04, - 'rotation': False} - - # training - training_options['features'] = features[i] - training_options['scaled_shape_models'] = scaled_shape_models[i] - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # fitting - fitting_options['n_shape'] = n_shape[i] - fitting_options['n_appearance'] = n_appearance[i] - pertrub_options['noise_std'] = noise_std[i] - pertrub_options['rotation'] = rotation[i] - fitting_results = aam_fit_benchmark(fitting_images, aam, - perturb_options=pertrub_options, - fitting_options=fitting_options, - verbose=verbose) - all_fitting_results.append(fitting_results) - - # convert results - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced( - fitting_results, max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - curves_to_plot.append(final_error_curve) - if i == n_experiments - 1: - curves_to_plot.append(initial_error_curve) - - # plot results - if plot: - title = "AAMs using Alternating IC" - colour_list = ['r', 'b', 'g', 'y', 'c'] * n_experiments - marker_list = ['o', 'x', 'v', 'd'] * n_experiments - plot_fitting_curves(error_bins, curves_to_plot, title, new_figure=True, - x_limit=max_error_bin, line_colour=colour_list, - marker_face_colour=colour_list, - marker_style=marker_list) - return all_fitting_results - - -def clm_params_combinations_noise(training_db_path, fitting_db_path, - n_experiments=1, classifier_trainers=None, - patch_shape=None, features=None, - scaled_shape_models=None, n_shape=None, - noise_std=None, rotation=None, verbose=False, - plot=False): - - # parse input - if classifier_trainers is None: - classifier_trainers = [linear_svm_lr] * n_experiments - elif len(classifier_trainers) is not n_experiments: - raise ValueError("classifier_trainers has wrong length") - if patch_shape is None: - patch_shape = [(5, 5)] * n_experiments - elif len(patch_shape) is not n_experiments: - raise ValueError("patch_shape has wrong length") - if features is None: - features = [igo] * n_experiments - elif len(features) is not n_experiments: - raise ValueError("features has wrong length") - if scaled_shape_models is None: - scaled_shape_models = [True] * n_experiments - elif len(scaled_shape_models) is not n_experiments: - raise ValueError("scaled_shape_models has wrong length") - if n_shape is None: - n_shape = [[3, 6, 12]] * n_experiments - elif len(n_shape) is not n_experiments: - raise ValueError("n_shape has wrong length") - if noise_std is None: - noise_std = [0.04] * n_experiments - elif len(noise_std) is not n_experiments: - raise ValueError("noise_std has wrong length") - if rotation is None: - rotation = [False] * n_experiments - elif len(rotation) is not n_experiments: - raise ValueError("rotation has wrong length") - - # load images - db_loading_options = {'crop_proportion': 0.4, - 'convert_to_grey': True - } - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - - # run experiments - max_error_bin = 0.05 - bins_error_step = 0.005 - curves_to_plot = [] - all_fitting_results = [] - for i in range(n_experiments): - if verbose: - print("\nEXPERIMENT {}/{}:".format(i + 1, n_experiments)) - print("- classifiers: {}\n- patch_shape: {}\n" - "- features: {}\n- scaled_shape_models: {}\n" - "- n_shape: {}\n" - "- noise_std: {}\n- rotation: {}".format( - classifier_trainers[i], patch_shape[i], features[i], - scaled_shape_models[i], n_shape[i], - noise_std[i], rotation[i])) - - # predefined option dictionaries - error_type = 'me_norm' - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': sparse_hog, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': False, - 'max_shape_components': None, - 'boundary': 3 - } - fitting_options = {'algorithm': RLMS, - 'pdm_transform': OrthoPDM, - 'n_shape': [3, 6, 12], - 'max_iters': 50, - 'error_type': error_type - } - perturb_options = {'noise_std': 0.01, - 'rotation': False} - - # training - training_options['classifier_trainers'] = classifier_trainers[i] - training_options['patch_shape'] = patch_shape[i] - training_options['features'] = features[i] - training_options['scaled_shape_models'] = scaled_shape_models[i] - clm = clm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # fitting - fitting_options['n_shape'] = n_shape[i] - perturb_options['noise_std'] = noise_std[i] - perturb_options['rotation'] = rotation[i] - fitting_results = clm_fit_benchmark(fitting_images, clm, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - all_fitting_results.append(fitting_results) - - # convert results - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced( - fitting_results, max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - curves_to_plot.append(final_error_curve) - if i == n_experiments - 1: - curves_to_plot.append(initial_error_curve) - - # plot results - if plot: - title = "CLMs using RLMS" - colour_list = ['r', 'b', 'g', 'y', 'c'] * n_experiments - marker_list = ['o', 'x', 'v', 'd'] * n_experiments - plot_fitting_curves(error_bins, curves_to_plot, title, new_figure=True, - x_limit=max_error_bin, line_colour=colour_list, - marker_face_colour=colour_list, - marker_style=marker_list) - return all_fitting_results diff --git a/menpofit/test/__init__.py b/menpofit/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/menpofit/test/aam_builder_test.py b/menpofit/test/aam_builder_test.py deleted file mode 100644 index a8ce97d..0000000 --- a/menpofit/test/aam_builder_test.py +++ /dev/null @@ -1,196 +0,0 @@ -from StringIO import StringIO -import platform - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.transform import PiecewiseAffine, ThinPlateSplines -from menpo.feature import sparse_hog, igo, lbp, no_op - -import menpo.io as mio -from menpo.landmark import ibug_face_68_trimesh -from menpofit.aam import AAMBuilder, PatchBasedAAMBuilder - - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training.append(im) - -# build aams -template_trilist_image = training[0].landmarks[None] -trilist = ibug_face_68_trimesh(template_trilist_image)[1].lms.trilist -aam1 = AAMBuilder(features=[igo, sparse_hog, no_op], - transform=PiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=False, - max_shape_components=[1, 2, 3], - max_appearance_components=[3, 3, 3], - boundary=3).build(training) - -aam2 = AAMBuilder(features=[no_op, no_op], - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=None, - max_appearance_components=1, - boundary=0).build(training) - -aam3 = AAMBuilder(features=igo, - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=1, - downscale=3, - scaled_shape_models=True, - max_shape_components=[2], - max_appearance_components=10, - boundary=2).build(training) - -aam4 = PatchBasedAAMBuilder(features=lbp, - patch_shape=(10, 13), - normalization_diagonal=200, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=1, - max_appearance_components=None, - boundary=2).build(training) - - -@raises(ValueError) -def test_features_exception(): - AAMBuilder(features=[igo, sparse_hog]).build(training) - - -@raises(ValueError) -def test_n_levels_exception(): - AAMBuilder(n_levels=0).build(training) - - -@raises(ValueError) -def test_downscale_exception(): - aam = AAMBuilder(downscale=1).build(training) - assert (aam.downscale == 1) - AAMBuilder(downscale=0).build(training) - - -@raises(ValueError) -def test_normalization_diagonal_exception(): - aam = AAMBuilder(normalization_diagonal=100).build(training) - assert (aam.appearance_models[0].n_features == 382) - AAMBuilder(normalization_diagonal=10).build(training) - - -@raises(ValueError) -def test_max_shape_components_exception(): - AAMBuilder(max_shape_components=[1, 0.2, 'a']).build(training) - - -@raises(ValueError) -def test_max_appearance_components_exception(): - AAMBuilder(max_appearance_components=[1, 2]).build(training) - - -@raises(ValueError) -def test_boundary_exception(): - AAMBuilder(boundary=-1).build(training) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - AAMBuilder().build(training, verbose=True) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(aam1) - print(aam2) - print(aam3) - print(aam4) - - -def test_aam_1(): - assert(aam1.n_training_images == 4) - assert(aam1.n_levels == 3) - assert(aam1.downscale == 2) - #assert(aam1.features[0] == igo and aam1.features[2] == no_op) - assert_allclose(np.around(aam1.reference_shape.range()), (109., 103.)) - assert(not aam1.scaled_shape_models) - assert(not aam1.pyramid_on_features) - assert_allclose([aam1.shape_models[j].n_components - for j in range(aam1.n_levels)], (1, 2, 3)) - assert (np.all([aam1.appearance_models[j].n_components == 3 - for j in range(aam1.n_levels)])) - assert_allclose([aam1.appearance_models[j].template_instance.n_channels - for j in range(aam1.n_levels)], (2, 36, 1)) - assert_allclose([aam1.appearance_models[j].components.shape[1] - for j in range(aam1.n_levels)], (14892, 268056, 7446)) - - -def test_aam_2(): - assert (aam2.n_training_images == 4) - assert (aam2.n_levels == 2) - assert (aam2.downscale == 1.2) - #assert (aam2.features[0] == no_op and aam2.features[1] == no_op) - assert_allclose(np.around(aam2.reference_shape.range()), (169., 161.)) - assert aam2.scaled_shape_models - assert (not aam2.pyramid_on_features) - assert (np.all([aam2.shape_models[j].n_components == 3 - for j in range(aam2.n_levels)])) - assert (np.all([aam2.appearance_models[j].n_components == 1 - for j in range(aam2.n_levels)])) - assert (np.all([aam2.appearance_models[j].template_instance.n_channels == 1 - for j in range(aam2.n_levels)])) - assert_allclose([aam2.appearance_models[j].components.shape[1] - for j in range(aam2.n_levels)], (12827, 18518)) - - -def test_aam_3(): - assert (aam3.n_training_images == 4) - assert (aam3.n_levels == 1) - assert (aam3.downscale == 3) - #assert (aam3.features[0] == igo and len(aam3.features) == 1) - assert_allclose(np.around(aam3.reference_shape.range()), (169., 161.)) - assert aam3.scaled_shape_models - assert aam3.pyramid_on_features - assert (np.all([aam3.shape_models[j].n_components == 2 - for j in range(aam3.n_levels)])) - assert (np.all([aam3.appearance_models[j].n_components == 3 - for j in range(aam3.n_levels)])) - assert (np.all([aam3.appearance_models[j].template_instance.n_channels == 2 - for j in range(aam3.n_levels)])) - assert_allclose([aam3.appearance_models[j].components.shape[1] - for j in range(aam3.n_levels)], 37036) - - -def test_aam_4(): - assert (aam4.n_training_images == 4) - assert (aam4.n_levels == 2) - assert (aam4.downscale == 1.2) - #assert (aam4.features[0] == lbp) - assert_allclose(np.around(aam4.reference_shape.range()), (145., 138.)) - assert aam4.scaled_shape_models - assert aam4.pyramid_on_features - assert (np.all([aam4.shape_models[j].n_components == 1 - for j in range(aam4.n_levels)])) - assert (np.all([aam4.appearance_models[j].n_components == 3 - for j in range(aam4.n_levels)])) - assert (np.all([aam4.appearance_models[j].template_instance.n_channels == 4 - for j in range(aam4.n_levels)])) - if platform.system() != 'Windows': - # https://github.com/menpo/menpo/issues/450 - assert_allclose([aam4.appearance_models[j].components.shape[1] - for j in range(aam4.n_levels)], (23656, 25988)) diff --git a/menpofit/test/aam_fitter_test.py b/menpofit/test/aam_fitter_test.py deleted file mode 100644 index 11052c6..0000000 --- a/menpofit/test/aam_fitter_test.py +++ /dev/null @@ -1,448 +0,0 @@ -from StringIO import StringIO - -from mock import patch -from nose.plugins.attrib import attr -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import igo -from menpofit.transform import DifferentiablePiecewiseAffine - - -import menpo.io as mio -from menpo.shape.pointcloud import PointCloud -from menpo.landmark import ibug_face_68_trimesh -from menpofit.aam import AAMBuilder, LucasKanadeAAMFitter -from menpofit.lucaskanade.appearance import (AFA, AFC, AIC, - SFA, SFC, SIC, - PIC) - - -initial_shape = [] -initial_shape.append(PointCloud(np.array([[150.9737801, 1.85331141], - [191.20452708, 1.86714624], - [237.5088486, 7.16836457], - [280.68439528, 19.1356864], - [319.00988383, 36.18921029], - [351.31395982, 61.11002727], - [375.83681819, 86.68264647], - [401.50706656, 117.12858347], - [408.46977018, 156.72258055], - [398.49810436, 197.95690492], - [375.44584527, 234.437902], - [342.35427495, 267.96920594], - [299.04149064, 309.66693535], - [250.84207113, 331.07734674], - [198.46150259, 339.47188196], - [144.62222804, 337.84178783], - [89.92321435, 327.81734317], - [101.22474793, 26.90269773], - [89.23456877, 44.52571118], - [84.04683242, 66.6369272], - [86.36993557, 88.61559027], - [94.88123162, 108.04971327], - [88.08448274, 152.88439191], - [68.71150917, 176.94681489], - [55.7165906, 204.86028035], - [53.9169657, 232.87050281], - [69.08534014, 259.8486207], - [121.82883888, 130.79001073], - [152.30894887, 128.91266055], - [183.36381228, 128.04534764], - [216.59234031, 125.86784329], - [235.18182671, 93.18819461], - [242.46006172, 117.24575711], - [246.52987701, 142.46262589], - [240.51603561, 160.38006297], - [232.61083444, 175.36132625], - [137.35714406, 56.53012228], - [124.42060774, 67.0342585], - [121.98869265, 87.71006061], - [130.4421354, 105.16741493], - [139.32511836, 89.65144616], - [144.17935107, 69.97931719], - [125.04221953, 174.72789706], - [103.0127825, 188.96555839], - [97.38196408, 210.70911033], - [107.31622619, 232.4487582], - [119.12835959, 215.57040617], - [124.80355957, 193.64317941], - [304.3174261, 101.83559243], - [293.08249678, 116.76961123], - [287.11523488, 132.55435452], - [289.39839945, 148.49971074], - [283.59574087, 162.33458018], - [286.76478391, 187.30470094], - [292.65033117, 211.98694428], - [310.75841097, 187.33036207], - [319.06250309, 165.27131484], - [321.3339324, 148.86793045], - [321.82844973, 133.03866904], - [316.60228316, 115.15885333], - [303.45716953, 109.59946563], - [301.58563675, 135.32572565], - [298.16531481, 148.240518], - [295.39615418, 162.35992687], - [293.63384823, 201.35617245], - [301.95207707, 163.05299135], - [305.27555828, 148.48478086], - [306.41382116, 133.02994058]]))) - -initial_shape.append(PointCloud(np.array([[33.08569962, 26.2373455], - [43.88613611, 26.24105964], - [56.31709803, 27.66423659], - [67.90810205, 30.87701063], - [78.19704859, 35.45523787], - [86.86947323, 42.14553624], - [93.45293474, 49.0108189], - [100.34442715, 57.18440338], - [102.21365016, 67.81389656], - [99.53663441, 78.88375569], - [93.34797327, 88.67752592], - [84.46413615, 97.67941492], - [72.83628901, 108.8736808], - [59.89656483, 114.62156782], - [45.83436002, 116.87518356], - [31.38054772, 116.43756484], - [16.69592792, 113.74637996], - [19.72996295, 32.96215989], - [16.51105259, 37.69327358], - [15.11834126, 43.62930018], - [15.74200674, 49.52974132], - [18.02696835, 54.74706954], - [16.20229791, 66.78348784], - [11.00138601, 73.24333984], - [7.51274105, 80.73705133], - [7.02960972, 88.25673842], - [11.10174551, 95.4993444], - [25.26138338, 60.85198075], - [33.44414202, 60.34798312], - [41.78120024, 60.11514235], - [50.70180534, 59.53056465], - [55.69238052, 50.75731293], - [57.6463118, 57.21586007], - [58.73890353, 63.98563718], - [57.12441419, 68.79579249], - [55.00216617, 72.817696], - [29.43014699, 40.91600468], - [25.95717546, 43.73596863], - [25.30429808, 49.2866408], - [27.57372827, 53.97328126], - [29.95847378, 49.80782952], - [31.26165197, 44.52660569], - [26.12405475, 72.64764418], - [20.20998272, 76.46991865], - [18.69832059, 82.30724133], - [21.36529486, 88.14351591], - [24.53640666, 83.6123157], - [26.05998356, 77.72568327], - [74.25267847, 53.07881273], - [71.23652416, 57.08803288], - [69.63453966, 61.32564044], - [70.24748314, 65.6063665], - [68.68968841, 69.32050656], - [69.54045681, 76.02404113], - [71.12050401, 82.6502915], - [75.9818397, 76.03093018], - [78.21117488, 70.10890893], - [78.82096788, 65.70521959], - [78.95372711, 61.4556606], - [77.55069872, 56.65560521], - [74.02173206, 55.16311953], - [73.51929617, 62.06964895], - [72.60106888, 65.53678304], - [71.85765381, 69.32731119], - [71.38454121, 79.79633067], - [73.61767156, 69.51337283], - [74.50990078, 65.60235839], - [74.81548138, 61.45331734]]))) - -initial_shape.append(PointCloud(np.array([[46.63369884, 44.08764686], - [65.31491309, 44.09407109], - [86.81640178, 46.55570064], - [106.86503868, 52.11274643], - [124.66154301, 60.0315786], - [139.66199441, 71.6036014], - [151.04922447, 83.47828965], - [162.96924699, 97.61591112], - [166.20238999, 116.0014495], - [161.57203038, 135.14867658], - [150.86767554, 152.08868824], - [135.50154984, 167.65900498], - [115.38918643, 187.02141497], - [93.00770583, 196.9633751], - [68.68470174, 200.86139148], - [43.68434508, 200.10445456], - [18.28476712, 195.44958702], - [23.53265303, 55.71937105], - [17.9649934, 63.90264665], - [15.55605939, 74.17002657], - [16.63479621, 84.37585532], - [20.58703068, 93.40012265], - [17.43094904, 114.21918023], - [8.43507654, 125.39260635], - [2.4008645, 138.35427044], - [1.56520568, 151.36086382], - [8.60866558, 163.88819772], - [33.10019692, 103.95961759], - [47.25368667, 103.08786691], - [61.67406413, 102.68512872], - [77.10378638, 101.67400095], - [85.7358453, 86.49915174], - [89.11550583, 97.67032089], - [91.00533132, 109.37981584], - [88.21279407, 117.69980754], - [84.54200076, 124.65638206], - [40.31079125, 69.47691491], - [34.3036891, 74.35452803], - [33.17442528, 83.95537112], - [37.09979548, 92.06172262], - [41.22462339, 84.85685672], - [43.47869442, 75.72207092], - [34.59233557, 124.36224816], - [24.36292985, 130.97352987], - [21.74824996, 141.07018437], - [26.36124109, 151.16502601], - [31.84622487, 143.32753518], - [34.48151342, 133.14559097], - [117.83907583, 90.5145853], - [112.62211772, 97.44922176], - [109.85120974, 104.77889356], - [110.911401, 112.18314623], - [108.21692684, 118.60739086], - [109.68847724, 130.20230795], - [112.4214409, 141.66354869], - [120.82995787, 130.21422374], - [124.68597685, 119.97106848], - [125.74071883, 112.35412967], - [125.97034877, 105.00378581], - [123.54356964, 96.70126365], - [117.43961426, 94.11975273], - [116.5705649, 106.06578435], - [114.98233273, 112.06278965], - [113.69646838, 118.61916064], - [112.87813868, 136.72713211], - [116.74072208, 118.94098628], - [118.2839861, 112.17621352], - [118.81254036, 104.99973274]]))) - -initial_shape.append(PointCloud(np.array([[29.30459178, 27.24534074], - [39.47004743, 24.38292299], - [51.54667438, 22.42372272], - [63.30767547, 22.37162616], - [74.20561385, 23.95008332], - [84.14265809, 27.94519239], - [92.16017681, 32.65929179], - [100.81474852, 38.52291926], - [105.39445843, 48.03051044], - [105.81247938, 59.1588891], - [102.5870203, 70.01814005], - [96.6149594, 80.84730771], - [88.64221584, 94.46788512], - [77.98963764, 103.31089364], - [65.35346377, 109.16323748], - [51.63461821, 112.58672956], - [37.10056847, 113.95059826], - [18.51972657, 37.11814141], - [16.7457652 , 42.42481409], - [17.01019564, 48.38086547], - [19.16282912, 53.76837796], - [22.69767086, 58.07217393], - [24.17432616, 69.88402627], - [20.99379373, 77.34357057], - [19.69904043, 85.32174442], - [21.23971857, 92.52684647], - [26.99391031, 98.26243543], - [31.12604697, 61.89794357], - [38.69324039, 59.25231487], - [46.47759964, 56.82093276], - [54.71781058, 53.90368008], - [57.08652729, 44.32277008], - [60.63919033, 49.88253722], - [63.46381778, 55.96376588], - [63.2207775 , 60.91909025], - [62.29071322, 65.26731234], - [29.75929632, 42.02967737], - [27.23910711, 45.60515084], - [28.09755316, 51.00222264], - [31.47695917, 54.81070084], - [32.61597345, 50.25772899], - [32.44103485, 44.94168113], - [35.06791957, 72.77012704], - [30.51633486, 77.93664152], - [30.64262749, 83.83136479], - [34.70122609, 88.61629379], - [36.4832508 , 83.51044643], - [36.35508694, 77.56615533], - [75.16994555, 41.58256719], - [73.39524567, 46.15605223], - [73.01204743, 50.56922423], - [74.72479626, 54.43524106], - [74.24428281, 58.34404327], - [76.82374875, 64.42709819], - [80.0690436 , 70.24390436], - [82.88766915, 62.72435028], - [83.41431565, 56.55948008], - [82.81967592, 52.25328539], - [81.81699053, 48.21872699], - [79.2228748 , 44.073611], - [75.50567221, 43.60542492], - [76.86548014, 50.2385966], - [76.9213308 , 53.74522715], - [77.22751327, 57.5098225], - [79.56023029, 67.48793174], - [78.93326695, 57.21790467], - [78.73516471, 53.30042959], - [77.92179698, 49.31461186]]))) - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# build aam -template_trilist_image = training_images[0].landmarks[None] -trilist = ibug_face_68_trimesh(template_trilist_image)[1].lms.trilist -aam = AAMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1, 2, 3], - max_appearance_components=[3, 2, 1], - boundary=3).build(training_images) - -aam2 = AAMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=1, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1], - max_appearance_components=[1], - boundary=3).build(training_images) - - -def test_aam(): - assert (aam.n_training_images == 4) - assert (aam.n_levels == 3) - assert (aam.downscale == 2) - #assert (aam.features[0] == igo and len(aam.features) == 1) - assert_allclose(np.around(aam.reference_shape.range()), (109., 103.)) - assert aam.scaled_shape_models - assert aam.pyramid_on_features - assert_allclose([aam.shape_models[j].n_components - for j in range(aam.n_levels)], (1, 2, 3)) - assert_allclose([aam.appearance_models[j].n_components - for j in range(aam.n_levels)], (3, 2, 1)) - assert_allclose([aam.appearance_models[j].template_instance.n_channels - for j in range(aam.n_levels)], (2, 2, 2)) - assert_allclose([aam.appearance_models[j].components.shape[1] - for j in range(aam.n_levels)], (884, 3652, 14892)) - - -@raises(ValueError) -def test_n_shape_exception(): - fitter = LucasKanadeAAMFitter(aam, n_shape=[3, 6, 'a']) - - -@raises(ValueError) -def test_n_appearance_exception(): - fitter = LucasKanadeAAMFitter(aam, n_appearance=[10, 20]) - - -def test_pertrurb_shape(): - fitter = LucasKanadeAAMFitter(aam) - s = fitter.perturb_shape(training_images[0].landmarks[None].lms, - noise_std=0.08, rotation=False) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -def test_obtain_shape_from_bb(): - fitter = LucasKanadeAAMFitter(aam) - s = fitter.obtain_shape_from_bb(np.array([[53.916, 1.853], - [408.469, 339.471]])) - assert ((np.around(s.points) == np.around(initial_shape[0].points)).all()) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -@raises(ValueError) -def test_max_iters_exception(): - fitter = LucasKanadeAAMFitter(aam, - algorithm=AIC) - fitter.fit(training_images[0], initial_shape[0], - max_iters=[10, 20, 30, 40]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(aam) - fitter = LucasKanadeAAMFitter(aam, - algorithm=AIC) - print(fitter) - print(aam2) - fitter = LucasKanadeAAMFitter(aam2, - algorithm=SFA) - print(fitter) - - -def aam_helper(aam, algorithm, im_number, max_iters, initial_error, - final_error, error_type): - fitter = LucasKanadeAAMFitter(aam, algorithm=algorithm) - fitting_result = fitter.fit( - training_images[im_number], initial_shape[im_number], - gt_shape=training_images[im_number].landmarks[None].lms, - max_iters=max_iters) - assert_allclose( - np.around(fitting_result.initial_error(error_type=error_type), 5), - initial_error) - assert_allclose( - np.around(fitting_result.final_error(error_type=error_type), 5), - final_error) - - -@attr('fuzzy') -def test_alternating_ic(): - aam_helper(aam, AIC, 0, 6, 0.09062, 0.05607, 'me_norm') - - -@attr('fuzzy') -def test_simultaneous_ic(): - aam_helper(aam, SIC, 2, 7, 0.12616, 0.11152, 'me_norm') - - -@attr('fuzzy') -def test_projectout_ic(): - aam_helper(aam, PIC, 3, 6, 0.10796, 0.07346, 'me_norm') - - -@attr('fuzzy') -def test_alternating_fa(): - aam_helper(aam, AFA, 0, 8, 0.09062, 0.07225, 'me_norm') - - -@attr('fuzzy') -def test_simultaneous_fa(): - aam_helper(aam, SFA, 2, 5, 0.12616, 0.11151, 'me_norm') - - -@attr('fuzzy') -def test_alternating_fc(): - aam_helper(aam, AFC, 0, 6, 0.09062, 0.07129, 'me_norm') - - -@attr('fuzzy') -def test_simultaneous_fc(): - aam_helper(aam, SFC, 2, 5, 0.12616, 0.11738, 'me_norm') diff --git a/menpofit/test/atm_builder_test.py b/menpofit/test/atm_builder_test.py deleted file mode 100644 index 437a999..0000000 --- a/menpofit/test/atm_builder_test.py +++ /dev/null @@ -1,175 +0,0 @@ -from StringIO import StringIO - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.transform import PiecewiseAffine, ThinPlateSplines -from menpo.feature import sparse_hog, igo, lbp, no_op - -import menpo.io as mio -from menpofit.atm import ATMBuilder, PatchBasedATMBuilder - - - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training = [] -templates = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training.append(im.landmarks[None].lms) - templates.append(im) - -# build atms -atm1 = ATMBuilder(features=[igo, sparse_hog, no_op], - transform=PiecewiseAffine, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=False, - max_shape_components=[1, 2, 3], - boundary=3).build(training, templates[0]) - -atm2 = ATMBuilder(features=[no_op, no_op], - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=None, - boundary=0).build(training, templates[1]) - -atm3 = ATMBuilder(features=igo, - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=1, - downscale=3, - scaled_shape_models=True, - max_shape_components=[2], - boundary=2).build(training, templates[2]) - -atm4 = PatchBasedATMBuilder(features=lbp, - patch_shape=(10, 13), - normalization_diagonal=200, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=1, - boundary=2).build(training, templates[3]) - - -@raises(ValueError) -def test_features_exception(): - ATMBuilder(features=[igo, sparse_hog]).build(training, templates[0]) - - -@raises(ValueError) -def test_n_levels_exception(): - ATMBuilder(n_levels=0).build(training, templates[1]) - - -@raises(ValueError) -def test_downscale_exception(): - atm = ATMBuilder(downscale=1).build(training, templates[2]) - assert (atm.downscale == 1) - ATMBuilder(downscale=0).build(training, templates[2]) - - -@raises(ValueError) -def test_normalization_diagonal_exception(): - atm = ATMBuilder(normalization_diagonal=100).build(training, templates[3]) - assert (atm.warped_templates[0].n_true_pixels() == 1246) - ATMBuilder(normalization_diagonal=10).build(training, templates[3]) - - -@raises(ValueError) -def test_max_shape_components_exception(): - ATMBuilder(max_shape_components=[1, 0.2, 'a']).build(training, templates[0]) - - -@raises(ValueError) -def test_max_shape_components_exception_2(): - ATMBuilder(max_shape_components=[1, 2]).build(training, templates[0]) - - -@raises(ValueError) -def test_boundary_exception(): - ATMBuilder(boundary=-1).build(training, templates[1]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - ATMBuilder().build(training, templates[2], verbose=True) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(atm1) - print(atm2) - print(atm3) - print(atm4) - - -def test_atm_1(): - assert(atm1.n_training_shapes == 4) - assert(atm1.n_levels == 3) - assert(atm1.downscale == 2) - assert_allclose(np.around(atm1.reference_shape.range()), (109., 103.)) - assert(not atm1.scaled_shape_models) - assert(not atm1.pyramid_on_features) - assert_allclose([atm1.shape_models[j].n_components - for j in range(atm1.n_levels)], (1, 2, 3)) - assert_allclose([atm1.warped_templates[j].n_channels - for j in range(atm1.n_levels)], (2, 36, 1)) - assert_allclose([atm1.warped_templates[j].shape[1] - for j in range(atm1.n_levels)], (164, 164, 164)) - - -def test_atm_2(): - assert (atm2.n_training_shapes == 4) - assert (atm2.n_levels == 2) - assert (atm2.downscale == 1.2) - assert_allclose(np.around(atm2.reference_shape.range()), (169., 161.)) - assert atm2.scaled_shape_models - assert (not atm2.pyramid_on_features) - assert (np.all([atm2.shape_models[j].n_components == 3 - for j in range(atm2.n_levels)])) - assert (np.all([atm2.warped_templates[j].n_channels == 1 - for j in range(atm2.n_levels)])) - assert_allclose([atm2.warped_templates[j].shape[1] - for j in range(atm2.n_levels)], (132, 158)) - - -def test_atm_3(): - assert (atm3.n_training_shapes == 4) - assert (atm3.n_levels == 1) - assert (atm3.downscale == 3) - assert_allclose(np.around(atm3.reference_shape.range()), (169., 161.)) - assert atm3.scaled_shape_models - assert atm3.pyramid_on_features - assert (np.all([atm3.shape_models[j].n_components == 2 - for j in range(atm3.n_levels)])) - assert (np.all([atm3.warped_templates[j].n_channels == 2 - for j in range(atm3.n_levels)])) - assert_allclose([atm3.warped_templates[j].shape[1] - for j in range(atm3.n_levels)], 162) - - -def test_atm_4(): - assert (atm4.n_training_shapes == 4) - assert (atm4.n_levels == 2) - assert (atm4.downscale == 1.2) - assert_allclose(np.around(atm4.reference_shape.range()), (145., 138.)) - assert atm4.scaled_shape_models - assert atm4.pyramid_on_features - assert (np.all([atm4.shape_models[j].n_components == 1 - for j in range(atm4.n_levels)])) - assert (np.all([atm4.warped_templates[j].n_channels == 4 - for j in range(atm4.n_levels)])) - assert_allclose([atm4.warped_templates[j].shape[1] - for j in range(atm4.n_levels)], (162, 188)) diff --git a/menpofit/test/atm_fitter_test.py b/menpofit/test/atm_fitter_test.py deleted file mode 100644 index faf5e61..0000000 --- a/menpofit/test/atm_fitter_test.py +++ /dev/null @@ -1,438 +0,0 @@ -from StringIO import StringIO - -from mock import patch -from nose.plugins.attrib import attr -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import igo -from menpofit.transform import DifferentiablePiecewiseAffine - -import menpo.io as mio -from menpo.shape.pointcloud import PointCloud -from menpofit.atm import ATMBuilder, LucasKanadeATMFitter -from menpofit.lucaskanade.image import FA, FC, IC - - -initial_shape = [] -initial_shape.append(PointCloud(np.array([[150.9737801, 1.85331141], - [191.20452708, 1.86714624], - [237.5088486, 7.16836457], - [280.68439528, 19.1356864], - [319.00988383, 36.18921029], - [351.31395982, 61.11002727], - [375.83681819, 86.68264647], - [401.50706656, 117.12858347], - [408.46977018, 156.72258055], - [398.49810436, 197.95690492], - [375.44584527, 234.437902], - [342.35427495, 267.96920594], - [299.04149064, 309.66693535], - [250.84207113, 331.07734674], - [198.46150259, 339.47188196], - [144.62222804, 337.84178783], - [89.92321435, 327.81734317], - [101.22474793, 26.90269773], - [89.23456877, 44.52571118], - [84.04683242, 66.6369272], - [86.36993557, 88.61559027], - [94.88123162, 108.04971327], - [88.08448274, 152.88439191], - [68.71150917, 176.94681489], - [55.7165906, 204.86028035], - [53.9169657, 232.87050281], - [69.08534014, 259.8486207], - [121.82883888, 130.79001073], - [152.30894887, 128.91266055], - [183.36381228, 128.04534764], - [216.59234031, 125.86784329], - [235.18182671, 93.18819461], - [242.46006172, 117.24575711], - [246.52987701, 142.46262589], - [240.51603561, 160.38006297], - [232.61083444, 175.36132625], - [137.35714406, 56.53012228], - [124.42060774, 67.0342585], - [121.98869265, 87.71006061], - [130.4421354, 105.16741493], - [139.32511836, 89.65144616], - [144.17935107, 69.97931719], - [125.04221953, 174.72789706], - [103.0127825, 188.96555839], - [97.38196408, 210.70911033], - [107.31622619, 232.4487582], - [119.12835959, 215.57040617], - [124.80355957, 193.64317941], - [304.3174261, 101.83559243], - [293.08249678, 116.76961123], - [287.11523488, 132.55435452], - [289.39839945, 148.49971074], - [283.59574087, 162.33458018], - [286.76478391, 187.30470094], - [292.65033117, 211.98694428], - [310.75841097, 187.33036207], - [319.06250309, 165.27131484], - [321.3339324, 148.86793045], - [321.82844973, 133.03866904], - [316.60228316, 115.15885333], - [303.45716953, 109.59946563], - [301.58563675, 135.32572565], - [298.16531481, 148.240518], - [295.39615418, 162.35992687], - [293.63384823, 201.35617245], - [301.95207707, 163.05299135], - [305.27555828, 148.48478086], - [306.41382116, 133.02994058]]))) - -initial_shape.append(PointCloud(np.array([[33.08569962, 26.2373455], - [43.88613611, 26.24105964], - [56.31709803, 27.66423659], - [67.90810205, 30.87701063], - [78.19704859, 35.45523787], - [86.86947323, 42.14553624], - [93.45293474, 49.0108189], - [100.34442715, 57.18440338], - [102.21365016, 67.81389656], - [99.53663441, 78.88375569], - [93.34797327, 88.67752592], - [84.46413615, 97.67941492], - [72.83628901, 108.8736808], - [59.89656483, 114.62156782], - [45.83436002, 116.87518356], - [31.38054772, 116.43756484], - [16.69592792, 113.74637996], - [19.72996295, 32.96215989], - [16.51105259, 37.69327358], - [15.11834126, 43.62930018], - [15.74200674, 49.52974132], - [18.02696835, 54.74706954], - [16.20229791, 66.78348784], - [11.00138601, 73.24333984], - [7.51274105, 80.73705133], - [7.02960972, 88.25673842], - [11.10174551, 95.4993444], - [25.26138338, 60.85198075], - [33.44414202, 60.34798312], - [41.78120024, 60.11514235], - [50.70180534, 59.53056465], - [55.69238052, 50.75731293], - [57.6463118, 57.21586007], - [58.73890353, 63.98563718], - [57.12441419, 68.79579249], - [55.00216617, 72.817696], - [29.43014699, 40.91600468], - [25.95717546, 43.73596863], - [25.30429808, 49.2866408], - [27.57372827, 53.97328126], - [29.95847378, 49.80782952], - [31.26165197, 44.52660569], - [26.12405475, 72.64764418], - [20.20998272, 76.46991865], - [18.69832059, 82.30724133], - [21.36529486, 88.14351591], - [24.53640666, 83.6123157], - [26.05998356, 77.72568327], - [74.25267847, 53.07881273], - [71.23652416, 57.08803288], - [69.63453966, 61.32564044], - [70.24748314, 65.6063665], - [68.68968841, 69.32050656], - [69.54045681, 76.02404113], - [71.12050401, 82.6502915], - [75.9818397, 76.03093018], - [78.21117488, 70.10890893], - [78.82096788, 65.70521959], - [78.95372711, 61.4556606], - [77.55069872, 56.65560521], - [74.02173206, 55.16311953], - [73.51929617, 62.06964895], - [72.60106888, 65.53678304], - [71.85765381, 69.32731119], - [71.38454121, 79.79633067], - [73.61767156, 69.51337283], - [74.50990078, 65.60235839], - [74.81548138, 61.45331734]]))) - -initial_shape.append(PointCloud(np.array([[46.63369884, 44.08764686], - [65.31491309, 44.09407109], - [86.81640178, 46.55570064], - [106.86503868, 52.11274643], - [124.66154301, 60.0315786], - [139.66199441, 71.6036014], - [151.04922447, 83.47828965], - [162.96924699, 97.61591112], - [166.20238999, 116.0014495], - [161.57203038, 135.14867658], - [150.86767554, 152.08868824], - [135.50154984, 167.65900498], - [115.38918643, 187.02141497], - [93.00770583, 196.9633751], - [68.68470174, 200.86139148], - [43.68434508, 200.10445456], - [18.28476712, 195.44958702], - [23.53265303, 55.71937105], - [17.9649934, 63.90264665], - [15.55605939, 74.17002657], - [16.63479621, 84.37585532], - [20.58703068, 93.40012265], - [17.43094904, 114.21918023], - [8.43507654, 125.39260635], - [2.4008645, 138.35427044], - [1.56520568, 151.36086382], - [8.60866558, 163.88819772], - [33.10019692, 103.95961759], - [47.25368667, 103.08786691], - [61.67406413, 102.68512872], - [77.10378638, 101.67400095], - [85.7358453, 86.49915174], - [89.11550583, 97.67032089], - [91.00533132, 109.37981584], - [88.21279407, 117.69980754], - [84.54200076, 124.65638206], - [40.31079125, 69.47691491], - [34.3036891, 74.35452803], - [33.17442528, 83.95537112], - [37.09979548, 92.06172262], - [41.22462339, 84.85685672], - [43.47869442, 75.72207092], - [34.59233557, 124.36224816], - [24.36292985, 130.97352987], - [21.74824996, 141.07018437], - [26.36124109, 151.16502601], - [31.84622487, 143.32753518], - [34.48151342, 133.14559097], - [117.83907583, 90.5145853], - [112.62211772, 97.44922176], - [109.85120974, 104.77889356], - [110.911401, 112.18314623], - [108.21692684, 118.60739086], - [109.68847724, 130.20230795], - [112.4214409, 141.66354869], - [120.82995787, 130.21422374], - [124.68597685, 119.97106848], - [125.74071883, 112.35412967], - [125.97034877, 105.00378581], - [123.54356964, 96.70126365], - [117.43961426, 94.11975273], - [116.5705649, 106.06578435], - [114.98233273, 112.06278965], - [113.69646838, 118.61916064], - [112.87813868, 136.72713211], - [116.74072208, 118.94098628], - [118.2839861, 112.17621352], - [118.81254036, 104.99973274]]))) - -initial_shape.append(PointCloud(np.array([[29.30459178, 27.24534074], - [39.47004743, 24.38292299], - [51.54667438, 22.42372272], - [63.30767547, 22.37162616], - [74.20561385, 23.95008332], - [84.14265809, 27.94519239], - [92.16017681, 32.65929179], - [100.81474852, 38.52291926], - [105.39445843, 48.03051044], - [105.81247938, 59.1588891], - [102.5870203, 70.01814005], - [96.6149594, 80.84730771], - [88.64221584, 94.46788512], - [77.98963764, 103.31089364], - [65.35346377, 109.16323748], - [51.63461821, 112.58672956], - [37.10056847, 113.95059826], - [18.51972657, 37.11814141], - [16.7457652 , 42.42481409], - [17.01019564, 48.38086547], - [19.16282912, 53.76837796], - [22.69767086, 58.07217393], - [24.17432616, 69.88402627], - [20.99379373, 77.34357057], - [19.69904043, 85.32174442], - [21.23971857, 92.52684647], - [26.99391031, 98.26243543], - [31.12604697, 61.89794357], - [38.69324039, 59.25231487], - [46.47759964, 56.82093276], - [54.71781058, 53.90368008], - [57.08652729, 44.32277008], - [60.63919033, 49.88253722], - [63.46381778, 55.96376588], - [63.2207775 , 60.91909025], - [62.29071322, 65.26731234], - [29.75929632, 42.02967737], - [27.23910711, 45.60515084], - [28.09755316, 51.00222264], - [31.47695917, 54.81070084], - [32.61597345, 50.25772899], - [32.44103485, 44.94168113], - [35.06791957, 72.77012704], - [30.51633486, 77.93664152], - [30.64262749, 83.83136479], - [34.70122609, 88.61629379], - [36.4832508 , 83.51044643], - [36.35508694, 77.56615533], - [75.16994555, 41.58256719], - [73.39524567, 46.15605223], - [73.01204743, 50.56922423], - [74.72479626, 54.43524106], - [74.24428281, 58.34404327], - [76.82374875, 64.42709819], - [80.0690436 , 70.24390436], - [82.88766915, 62.72435028], - [83.41431565, 56.55948008], - [82.81967592, 52.25328539], - [81.81699053, 48.21872699], - [79.2228748 , 44.073611], - [75.50567221, 43.60542492], - [76.86548014, 50.2385966], - [76.9213308 , 53.74522715], - [77.22751327, 57.5098225], - [79.56023029, 67.48793174], - [78.93326695, 57.21790467], - [78.73516471, 53.30042959], - [77.92179698, 49.31461186]]))) - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_shapes = [] -templates = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_shapes.append(im.landmarks[None].lms) - templates.append(im) - -# build atm -atm1 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1, 2, 3], - boundary=3).build(training_shapes, templates[0]) - -atm2 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=1, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1], - boundary=3).build(training_shapes, templates[1]) - -atm3 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1, 2, 3], - boundary=3).build(training_shapes, templates[2]) - -atm4 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=1, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1], - boundary=3).build(training_shapes, templates[3]) - - -def test_atm1(): - assert (atm1.n_training_shapes == 4) - assert (atm1.n_levels == 3) - assert (atm1.downscale == 2) - assert_allclose(np.around(atm1.reference_shape.range()), (109., 103.)) - assert atm1.scaled_shape_models - assert atm1.pyramid_on_features - assert_allclose([atm1.shape_models[j].n_components - for j in range(atm1.n_levels)], (1, 2, 3)) - assert_allclose([atm1.warped_templates[j].n_channels - for j in range(atm1.n_levels)], (2, 2, 2)) - assert_allclose([atm1.warped_templates[j].shape[1] - for j in range(atm1.n_levels)], (46, 85, 164)) - - -@raises(ValueError) -def test_n_shape_exception(): - fitter = LucasKanadeATMFitter(atm1, n_shape=[3, 6, 'a']) - - -@raises(ValueError) -def test_n_shape_exception_2(): - fitter = LucasKanadeATMFitter(atm1, n_shape=[10, 20]) - - -def test_pertrurb_shape(): - fitter = LucasKanadeATMFitter(atm1) - s = fitter.perturb_shape(templates[0].landmarks[None].lms, - noise_std=0.08, rotation=False) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -def test_obtain_shape_from_bb(): - fitter = LucasKanadeATMFitter(atm1) - s = fitter.obtain_shape_from_bb(np.array([[53.916, 1.853], - [408.469, 339.471]])) - assert ((np.around(s.points) == np.around(initial_shape[0].points)).all()) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -@raises(ValueError) -def test_max_iters_exception(): - fitter = LucasKanadeATMFitter(atm1, - algorithm=IC) - fitter.fit(templates[0], initial_shape[0], max_iters=[10, 20, 30, 40]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(atm1) - fitter = LucasKanadeATMFitter(atm1, - algorithm=IC) - print(fitter) - print(atm2) - fitter = LucasKanadeATMFitter(atm2, - algorithm=FA) - print(fitter) - - -def atm_helper(atm, algorithm, im_number, max_iters, initial_error, - final_error, error_type): - fitter = LucasKanadeATMFitter(atm, algorithm=algorithm) - fitting_result = fitter.fit( - templates[im_number], initial_shape[im_number], - gt_shape=templates[im_number].landmarks[None].lms, - max_iters=max_iters) - assert_allclose( - np.around(fitting_result.initial_error(error_type=error_type), 5), - initial_error) - assert_allclose( - np.around(fitting_result.final_error(error_type=error_type), 5), - final_error) - - -@attr('fuzzy') -def test_ic(): - atm_helper(atm1, IC, 0, 6, 0.09062, 0.06788, 'me_norm') - - -@attr('fuzzy') -def test_fa(): - atm_helper(atm2, FA, 1, 8, 0.09051, 0.08188, 'me_norm') - - -@attr('fuzzy') -def test_fc(): - atm_helper(atm3, FC, 2, 6, 0.12615, 0.08255, 'me_norm') - -@attr('fuzzy') -def test_ic_2(): - atm_helper(atm4, IC, 3, 7, 0.09748, 0.09511, 'me_norm') diff --git a/menpofit/test/clm_builder_test.py b/menpofit/test/clm_builder_test.py deleted file mode 100644 index 7219c3f..0000000 --- a/menpofit/test/clm_builder_test.py +++ /dev/null @@ -1,194 +0,0 @@ -from StringIO import StringIO -from sklearn import qda - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import sparse_hog, igo, no_op - -import menpo.io as mio -from menpofit.clm import CLMBuilder -from menpofit.clm.classifier import linear_svm_lr -from menpofit.base import name_of_callable - - -def random_forest(X, t): - clf = qda.QDA() - clf.fit(X, t) - - def random_forest_predict(x): - return clf.predict_proba(x)[:, 1] - - return random_forest_predict - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# build clms -clm1 = CLMBuilder(classifier_trainers=[linear_svm_lr], - patch_shape=(5, 5), - features=[igo, sparse_hog, no_op], - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=False, - max_shape_components=[1, 2, 3], - boundary=3).build(training_images) - -clm2 = CLMBuilder(classifier_trainers=[random_forest, linear_svm_lr], - patch_shape=(3, 10), - features=[no_op, no_op], - normalization_diagonal=None, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=None, - boundary=0).build(training_images) - -clm3 = CLMBuilder(classifier_trainers=[linear_svm_lr], - patch_shape=(2, 3), - features=igo, - normalization_diagonal=None, - n_levels=1, - downscale=3, - scaled_shape_models=True, - max_shape_components=[1], - boundary=2).build(training_images) - - -@raises(ValueError) -def test_classifier_type_1_exception(): - CLMBuilder(classifier_trainers=[linear_svm_lr, linear_svm_lr]).build( - training_images) - -@raises(ValueError) -def test_classifier_type_2_exception(): - CLMBuilder(classifier_trainers=['linear_svm_lr']).build(training_images) - -@raises(ValueError) -def test_patch_shape_1_exception(): - CLMBuilder(patch_shape=(5, 1)).build(training_images) - -@raises(ValueError) -def test_patch_shape_2_exception(): - CLMBuilder(patch_shape=(5, 6, 7)).build(training_images) - -@raises(ValueError) -def test_features_exception(): - CLMBuilder(features=[igo, sparse_hog]).build(training_images) - -@raises(ValueError) -def test_n_levels_exception(): - clm = CLMBuilder(n_levels=0).build(training_images) - - -@raises(ValueError) -def test_downscale_exception(): - clm = CLMBuilder(downscale=1).build(training_images) - assert (clm.downscale == 1) - CLMBuilder(downscale=0).build(training_images) - - -@raises(ValueError) -def test_normalization_diagonal_exception(): - CLMBuilder(normalization_diagonal=10).build(training_images) - - -@raises(ValueError) -def test_max_shape_components_1_exception(): - CLMBuilder(max_shape_components=[1, 0.2, 'a']).build(training_images) - - -@raises(ValueError) -def test_max_shape_components_2_exception(): - CLMBuilder(max_shape_components=[1, 2]).build(training_images) - - -@raises(ValueError) -def test_boundary_exception(): - CLMBuilder(boundary=-1).build(training_images) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - CLMBuilder().build(training_images, verbose=True) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(clm1) - print(clm2) - print(clm3) - - -def test_clm_1(): - assert (clm1.n_training_images == 4) - assert (clm1.n_levels == 3) - assert (clm1.downscale == 2) - #assert (clm1.features[0] == igo and clm1.features[2] is no_op) - assert_allclose(np.around(clm1.reference_shape.range()), (109., 103.)) - assert (not clm1.scaled_shape_models) - assert (not clm1.pyramid_on_features) - assert_allclose(clm1.patch_shape, (5, 5)) - assert_allclose([clm1.shape_models[j].n_components - for j in range(clm1.n_levels)], (1, 2, 3)) - assert_allclose(clm1.n_classifiers_per_level, [68, 68, 68]) - - ran_0 = np.random.randint(0, clm1.n_classifiers_per_level[0]) - ran_1 = np.random.randint(0, clm1.n_classifiers_per_level[1]) - ran_2 = np.random.randint(0, clm1.n_classifiers_per_level[2]) - - assert (name_of_callable(clm1.classifiers[0][ran_0]) - == 'linear_svm_lr') - assert (name_of_callable(clm1.classifiers[1][ran_1]) - == 'linear_svm_lr') - assert (name_of_callable(clm1.classifiers[2][ran_2]) - == 'linear_svm_lr') - - -def test_clm_2(): - assert (clm2.n_training_images == 4) - assert (clm2.n_levels == 2) - assert (clm2.downscale == 1.2) - #assert (clm2.features[0] is no_op and clm2.features[1] is no_op) - assert_allclose(np.around(clm2.reference_shape.range()), (169., 161.)) - assert clm2.scaled_shape_models - assert (not clm2.pyramid_on_features) - assert_allclose(clm2.patch_shape, (3, 10)) - assert (np.all([clm2.shape_models[j].n_components == 3 - for j in range(clm2.n_levels)])) - assert_allclose(clm2.n_classifiers_per_level, [68, 68]) - - ran_0 = np.random.randint(0, clm2.n_classifiers_per_level[0]) - ran_1 = np.random.randint(0, clm2.n_classifiers_per_level[1]) - - assert (name_of_callable(clm2.classifiers[0][ran_0]) - == 'random_forest_predict') - assert (name_of_callable(clm2.classifiers[1][ran_1]) - == 'linear_svm_lr') - - -def test_clm_3(): - assert (clm3.n_training_images == 4) - assert (clm3.n_levels == 1) - assert (clm3.downscale == 3) - #assert (clm3.features[0] == igo and len(clm3.features) == 1) - assert_allclose(np.around(clm3.reference_shape.range()), (169., 161.)) - assert clm3.scaled_shape_models - assert clm3.pyramid_on_features - assert_allclose(clm3.patch_shape, (2, 3)) - assert (np.all([clm3.shape_models[j].n_components == 1 - for j in range(clm3.n_levels)])) - assert_allclose(clm3.n_classifiers_per_level, [68]) - ran_0 = np.random.randint(0, clm3.n_classifiers_per_level[0]) - - assert (name_of_callable(clm3.classifiers[0][ran_0]) - == 'linear_svm_lr') diff --git a/menpofit/test/clm_fitter_test.py b/menpofit/test/clm_fitter_test.py deleted file mode 100644 index e69ea0d..0000000 --- a/menpofit/test/clm_fitter_test.py +++ /dev/null @@ -1,370 +0,0 @@ -from StringIO import StringIO - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import sparse_hog - -import menpo.io as mio -from menpo.shape.pointcloud import PointCloud -from menpofit.clm import CLMBuilder -from menpofit.clm import GradientDescentCLMFitter -from menpofit.gradientdescent import RLMS -from menpofit.clm.classifier import linear_svm_lr -from menpofit.base import name_of_callable - - -initial_shape = [] -initial_shape.append(PointCloud(np.array([[150.9737801, 1.85331141], - [191.20452708, 1.86714624], - [237.5088486, 7.16836457], - [280.68439528, 19.1356864], - [319.00988383, 36.18921029], - [351.31395982, 61.11002727], - [375.83681819, 86.68264647], - [401.50706656, 117.12858347], - [408.46977018, 156.72258055], - [398.49810436, 197.95690492], - [375.44584527, 234.437902], - [342.35427495, 267.96920594], - [299.04149064, 309.66693535], - [250.84207113, 331.07734674], - [198.46150259, 339.47188196], - [144.62222804, 337.84178783], - [89.92321435, 327.81734317], - [101.22474793, 26.90269773], - [89.23456877, 44.52571118], - [84.04683242, 66.6369272], - [86.36993557, 88.61559027], - [94.88123162, 108.04971327], - [88.08448274, 152.88439191], - [68.71150917, 176.94681489], - [55.7165906, 204.86028035], - [53.9169657, 232.87050281], - [69.08534014, 259.8486207], - [121.82883888, 130.79001073], - [152.30894887, 128.91266055], - [183.36381228, 128.04534764], - [216.59234031, 125.86784329], - [235.18182671, 93.18819461], - [242.46006172, 117.24575711], - [246.52987701, 142.46262589], - [240.51603561, 160.38006297], - [232.61083444, 175.36132625], - [137.35714406, 56.53012228], - [124.42060774, 67.0342585], - [121.98869265, 87.71006061], - [130.4421354, 105.16741493], - [139.32511836, 89.65144616], - [144.17935107, 69.97931719], - [125.04221953, 174.72789706], - [103.0127825, 188.96555839], - [97.38196408, 210.70911033], - [107.31622619, 232.4487582], - [119.12835959, 215.57040617], - [124.80355957, 193.64317941], - [304.3174261, 101.83559243], - [293.08249678, 116.76961123], - [287.11523488, 132.55435452], - [289.39839945, 148.49971074], - [283.59574087, 162.33458018], - [286.76478391, 187.30470094], - [292.65033117, 211.98694428], - [310.75841097, 187.33036207], - [319.06250309, 165.27131484], - [321.3339324, 148.86793045], - [321.82844973, 133.03866904], - [316.60228316, 115.15885333], - [303.45716953, 109.59946563], - [301.58563675, 135.32572565], - [298.16531481, 148.240518], - [295.39615418, 162.35992687], - [293.63384823, 201.35617245], - [301.95207707, 163.05299135], - [305.27555828, 148.48478086], - [306.41382116, 133.02994058]]))) - -initial_shape.append(PointCloud(np.array([[33.08569962, 26.2373455], - [43.88613611, 26.24105964], - [56.31709803, 27.66423659], - [67.90810205, 30.87701063], - [78.19704859, 35.45523787], - [86.86947323, 42.14553624], - [93.45293474, 49.0108189], - [100.34442715, 57.18440338], - [102.21365016, 67.81389656], - [99.53663441, 78.88375569], - [93.34797327, 88.67752592], - [84.46413615, 97.67941492], - [72.83628901, 108.8736808], - [59.89656483, 114.62156782], - [45.83436002, 116.87518356], - [31.38054772, 116.43756484], - [16.69592792, 113.74637996], - [19.72996295, 32.96215989], - [16.51105259, 37.69327358], - [15.11834126, 43.62930018], - [15.74200674, 49.52974132], - [18.02696835, 54.74706954], - [16.20229791, 66.78348784], - [11.00138601, 73.24333984], - [7.51274105, 80.73705133], - [7.02960972, 88.25673842], - [11.10174551, 95.4993444], - [25.26138338, 60.85198075], - [33.44414202, 60.34798312], - [41.78120024, 60.11514235], - [50.70180534, 59.53056465], - [55.69238052, 50.75731293], - [57.6463118, 57.21586007], - [58.73890353, 63.98563718], - [57.12441419, 68.79579249], - [55.00216617, 72.817696], - [29.43014699, 40.91600468], - [25.95717546, 43.73596863], - [25.30429808, 49.2866408], - [27.57372827, 53.97328126], - [29.95847378, 49.80782952], - [31.26165197, 44.52660569], - [26.12405475, 72.64764418], - [20.20998272, 76.46991865], - [18.69832059, 82.30724133], - [21.36529486, 88.14351591], - [24.53640666, 83.6123157], - [26.05998356, 77.72568327], - [74.25267847, 53.07881273], - [71.23652416, 57.08803288], - [69.63453966, 61.32564044], - [70.24748314, 65.6063665], - [68.68968841, 69.32050656], - [69.54045681, 76.02404113], - [71.12050401, 82.6502915], - [75.9818397, 76.03093018], - [78.21117488, 70.10890893], - [78.82096788, 65.70521959], - [78.95372711, 61.4556606], - [77.55069872, 56.65560521], - [74.02173206, 55.16311953], - [73.51929617, 62.06964895], - [72.60106888, 65.53678304], - [71.85765381, 69.32731119], - [71.38454121, 79.79633067], - [73.61767156, 69.51337283], - [74.50990078, 65.60235839], - [74.81548138, 61.45331734]]))) - -initial_shape.append(PointCloud(np.array([[46.63369884, 44.08764686], - [65.31491309, 44.09407109], - [86.81640178, 46.55570064], - [106.86503868, 52.11274643], - [124.66154301, 60.0315786], - [139.66199441, 71.6036014], - [151.04922447, 83.47828965], - [162.96924699, 97.61591112], - [166.20238999, 116.0014495], - [161.57203038, 135.14867658], - [150.86767554, 152.08868824], - [135.50154984, 167.65900498], - [115.38918643, 187.02141497], - [93.00770583, 196.9633751], - [68.68470174, 200.86139148], - [43.68434508, 200.10445456], - [18.28476712, 195.44958702], - [23.53265303, 55.71937105], - [17.9649934, 63.90264665], - [15.55605939, 74.17002657], - [16.63479621, 84.37585532], - [20.58703068, 93.40012265], - [17.43094904, 114.21918023], - [8.43507654, 125.39260635], - [2.4008645, 138.35427044], - [1.56520568, 151.36086382], - [8.60866558, 163.88819772], - [33.10019692, 103.95961759], - [47.25368667, 103.08786691], - [61.67406413, 102.68512872], - [77.10378638, 101.67400095], - [85.7358453, 86.49915174], - [89.11550583, 97.67032089], - [91.00533132, 109.37981584], - [88.21279407, 117.69980754], - [84.54200076, 124.65638206], - [40.31079125, 69.47691491], - [34.3036891, 74.35452803], - [33.17442528, 83.95537112], - [37.09979548, 92.06172262], - [41.22462339, 84.85685672], - [43.47869442, 75.72207092], - [34.59233557, 124.36224816], - [24.36292985, 130.97352987], - [21.74824996, 141.07018437], - [26.36124109, 151.16502601], - [31.84622487, 143.32753518], - [34.48151342, 133.14559097], - [117.83907583, 90.5145853], - [112.62211772, 97.44922176], - [109.85120974, 104.77889356], - [110.911401, 112.18314623], - [108.21692684, 118.60739086], - [109.68847724, 130.20230795], - [112.4214409, 141.66354869], - [120.82995787, 130.21422374], - [124.68597685, 119.97106848], - [125.74071883, 112.35412967], - [125.97034877, 105.00378581], - [123.54356964, 96.70126365], - [117.43961426, 94.11975273], - [116.5705649, 106.06578435], - [114.98233273, 112.06278965], - [113.69646838, 118.61916064], - [112.87813868, 136.72713211], - [116.74072208, 118.94098628], - [118.2839861, 112.17621352], - [118.81254036, 104.99973274]]))) - -initial_shape.append(PointCloud(np.array([[109.7313602, 59.79617265], - [148.98369157, 59.80967103], - [194.16188757, 64.98196322], - [236.28740084, 76.65823864], - [273.68080984, 93.2970192], - [305.19924763, 117.61175954], - [329.12570774, 142.56245019], - [354.17165322, 172.2679391], - [360.96502322, 210.89900639], - [351.23586926, 251.13050805], - [328.74424331, 286.72428381], - [296.45746314, 319.44010324], - [254.19804988, 360.12373989], - [207.17084485, 381.01344803], - [156.06417675, 389.20382735], - [103.53427849, 387.61337726], - [50.16555004, 377.83272806], - [61.19222924, 84.23635551], - [49.49365238, 101.43077559], - [44.43208227, 123.00424473], - [46.69868733, 144.44838462], - [55.00298785, 163.40986789], - [48.37153655, 207.15416287], - [29.46971554, 230.63138542], - [16.79083463, 257.86599274], - [15.03497678, 285.19500392], - [29.83445491, 311.5170114], - [81.29522673, 185.59711915], - [111.03405753, 183.7654263], - [141.3336637, 182.91920653], - [173.75407076, 180.79465929], - [191.89145908, 148.90978278], - [198.99268671, 172.38226306], - [202.96352371, 196.98585519], - [197.0959395, 214.46753849], - [189.38299358, 229.08445602], - [96.44588204, 113.14323827], - [83.82396352, 123.3919129], - [81.45119284, 143.56487752], - [89.69904705, 160.59766732], - [98.36599502, 145.45904839], - [103.10217233, 126.26534747], - [84.43045765, 228.46643189], - [62.93677864, 242.35783193], - [57.44290226, 263.57257862], - [67.13556217, 284.78351617], - [78.66042335, 268.31564727], - [84.19760192, 246.92169276], - [259.34567409, 157.34687506], - [248.38397932, 171.9176971], - [242.5618418, 187.31855393], - [244.78947959, 202.87611756], - [239.12794222, 216.37452165], - [242.21991383, 240.73736669], - [247.96232402, 264.81933552], - [265.63001359, 240.76240374], - [273.7321494, 219.23983464], - [275.94833733, 203.23538213], - [276.43082796, 187.79108987], - [271.33176225, 170.34611298], - [258.50633904, 164.92193012], - [256.68032211, 190.02252505], - [253.34318274, 202.62322841], - [250.64136836, 216.39925191], - [248.92192186, 254.44710508], - [257.03785057, 217.075461], - [260.28050441, 202.86155077], - [261.39108462, 187.78257369]]))) - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# build clm -clm = CLMBuilder(classifier_trainers=linear_svm_lr, - patch_shape=(8, 8), - features=sparse_hog, - normalization_diagonal=100, - n_levels=2, - downscale=1.1, - scaled_shape_models=True, - max_shape_components=[2, 2], - boundary=3).build(training_images) - - -def test_clm(): - assert (clm.n_training_images == 4) - assert (clm.n_levels == 2) - assert (clm.downscale == 1.1) - #assert (clm.features[0] == sparse_hog and len(clm.features) == 1) - assert_allclose(np.around(clm.reference_shape.range()), (72., 69.)) - assert clm.scaled_shape_models - assert clm.pyramid_on_features - assert_allclose(clm.patch_shape, (8, 8)) - assert_allclose([clm.shape_models[j].n_components - for j in range(clm.n_levels)], (2, 2)) - assert_allclose(clm.n_classifiers_per_level, [68, 68]) - - ran_0 = np.random.randint(0, clm.n_classifiers_per_level[0]) - ran_1 = np.random.randint(0, clm.n_classifiers_per_level[1]) - - assert (name_of_callable(clm.classifiers[0][ran_0]) - == 'linear_svm_lr') - assert (name_of_callable(clm.classifiers[1][ran_1]) - == 'linear_svm_lr') - - -@raises(ValueError) -def test_n_shape_1_exception(): - fitter = GradientDescentCLMFitter(clm, n_shape=[3, 6, 'a']) - - -@raises(ValueError) -def test_n_shape_2_exception(): - fitter = GradientDescentCLMFitter(clm, n_shape=[10, 20, 3]) - - -def test_perturb_shape(): - fitter = GradientDescentCLMFitter(clm) - s = fitter.perturb_shape(training_images[0].landmarks[None].lms, - noise_std=0.08, rotation=False) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -@raises(ValueError) -def test_max_iters_exception(): - fitter = GradientDescentCLMFitter(clm) - fitter.fit(training_images[0], initial_shape[0], - max_iters=[10, 20, 30, 40]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(clm) - fitter = GradientDescentCLMFitter( - clm, algorithm=RLMS) - print(fitter) diff --git a/menpofit/test/fitmulitlevel_base_test.py b/menpofit/test/fitmulitlevel_base_test.py deleted file mode 100644 index 3df3c19..0000000 --- a/menpofit/test/fitmulitlevel_base_test.py +++ /dev/null @@ -1,29 +0,0 @@ -from menpo.feature import sparse_hog, igo - -from menpofit.base import (is_pyramid_on_features, - name_of_callable) - - -class Foo(): - def __call__(self): - pass - - -def test_is_pyramid_on_features_true(): - assert is_pyramid_on_features(igo) - - -def test_is_pyramid_on_features_false(): - assert not is_pyramid_on_features([igo, sparse_hog]) - - -def test_name_of_callable_partial(): - assert name_of_callable(sparse_hog) == 'sparse_hog' - - -def test_name_of_callable_function(): - assert name_of_callable(igo) == 'igo' - - -def test_name_of_callable_object_with_call(): - assert name_of_callable(Foo()) == 'Foo' diff --git a/menpofit/test/fittingresult_test.py b/menpofit/test/fittingresult_test.py deleted file mode 100644 index 311de16..0000000 --- a/menpofit/test/fittingresult_test.py +++ /dev/null @@ -1,108 +0,0 @@ -import numpy as np -from numpy.testing import assert_approx_equal -from nose.plugins.attrib import attr -from nose.tools import raises -from mock import MagicMock -from menpo.shape import PointCloud -from menpo.testing import is_same_array -from menpo.image import MaskedImage - -from menpofit.fittingresult import FittingResult, NonParametricFittingResult - - -class MockedFittingResult(FittingResult): - - def __init__(self, gt_shape=None): - FittingResult.__init__(self, MaskedImage.init_blank((10, 10)), - gt_shape=gt_shape) - @property - def n_iters(self): - return 1 - - @property - def shapes(self): - return [PointCloud(np.ones([3, 2]))] - - @property - def final_shape(self): - return PointCloud(np.ones([3, 2])) - - @property - def initial_shape(self): - return PointCloud(np.ones([3, 2])) - - -@attr('fuzzy') -def test_fittingresult_errors_me_norm(): - pcloud = PointCloud(np.array([[1., 2], [3, 4], [5, 6]])) - fr = MockedFittingResult(gt_shape=pcloud) - - assert_approx_equal(fr.errors()[0], 0.9173896) - - -@raises(ValueError) -def test_fittingresult_errors_no_gt(): - fr = MockedFittingResult() - fr.errors() - - -def test_fittingresult_gt_shape(): - pcloud = PointCloud(np.ones([3, 2])) - fr = MockedFittingResult(gt_shape=pcloud) - assert (is_same_array(fr.gt_shape.points, pcloud.points)) - - -@attr('fuzzy') -def test_fittingresult_final_error_me_norm(): - pcloud = PointCloud(np.array([[1., 2], [3, 4], [5, 6]])) - fr = MockedFittingResult(gt_shape=pcloud) - - assert_approx_equal(fr.final_error(), 0.9173896) - - -@raises(ValueError) -def test_fittingresult_final_error_no_gt(): - fr = MockedFittingResult() - fr.final_error() - - -@attr('fuzzy') -def test_fittingresult_initial_error_me_norm(): - pcloud = PointCloud(np.array([[1., 2], [3, 4], [5, 6]])) - fr = MockedFittingResult(gt_shape=pcloud) - - assert_approx_equal(fr.initial_error(), 0.9173896) - - -@raises(ValueError) -def test_fittingresult_initial_error_no_gt(): - fr = MockedFittingResult() - fr.initial_error() - - -def test_nonpara_fittingresult_as_serialized(): - image = MagicMock() - fitter = MagicMock() - parameters = [MagicMock()] - gt_shape = MagicMock() - fr = NonParametricFittingResult(image, fitter, parameters=parameters, - gt_shape=gt_shape) - s_fr = fr.as_serializable() - - image.copy.assert_called_once() - parameters[0].copy.assert_called_once() - gt_shape.copy.assert_called_once() - - -def test_nonpara_fittingresult_as_serialized(): - image = MagicMock() - fitter = MagicMock() - parameters = [MagicMock()] - gt_shape = MagicMock() - fr = NonParametricFittingResult(image, fitter, parameters=parameters, - gt_shape=gt_shape) - s_fr = fr.as_serializable() - - image.copy.assert_called_once() - parameters[0].copy.assert_called_once() - gt_shape.copy.assert_called_once() \ No newline at end of file diff --git a/menpofit/test/multilevel_fittingresult_test.py b/menpofit/test/multilevel_fittingresult_test.py deleted file mode 100644 index 5bdca7a..0000000 --- a/menpofit/test/multilevel_fittingresult_test.py +++ /dev/null @@ -1,18 +0,0 @@ -from mock import MagicMock -from menpofit.fittingresult import MultilevelFittingResult - - -def test_multilevel_fittingresult_as_serialized(): - image = MagicMock() - multiple_fitter = MagicMock() - fitting_results = [MagicMock()] - affine_correction = MagicMock() - gt_shape = MagicMock() - fr = MultilevelFittingResult(image, multiple_fitter, fitting_results, - affine_correction, gt_shape=gt_shape) - s_fr = fr.as_serializable() - - image.copy.assert_called_once() - fitting_results[0].as_serialized.assert_called_once() - affine_correction.copy.assert_called_once() - gt_shape.copy.assert_called_once() diff --git a/menpofit/test/sdm_test.py b/menpofit/test/sdm_test.py deleted file mode 100644 index 335a880..0000000 --- a/menpofit/test/sdm_test.py +++ /dev/null @@ -1,111 +0,0 @@ -from StringIO import StringIO - -from mock import patch -from nose.tools import raises -import numpy as np -from menpo.feature import sparse_hog, igo, no_op -from menpo.transform import PiecewiseAffine - -import menpo.io as mio -from menpo.landmark import ibug_face_68_trimesh -from menpofit.sdm import SDMTrainer, SDAAMTrainer -from menpofit.clm.classifier import linear_svm_lr -from menpofit.regression.regressors import mlr_svd -from menpofit.aam import AAMBuilder -from menpofit.clm import CLMBuilder - - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# Seed the random number generator -np.random.seed(seed=1000) - -template_trilist_image = training_images[0].landmarks[None] -trilist = ibug_face_68_trimesh(template_trilist_image)[1].lms.trilist -aam = AAMBuilder(features=sparse_hog, - transform=PiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=3, - downscale=1.2, - scaled_shape_models=False, - max_shape_components=None, - max_appearance_components=3, - boundary=3).build(training_images) - -clm = CLMBuilder(classifier_trainers=linear_svm_lr, - features=sparse_hog, - normalization_diagonal=100, - patch_shape=(5, 5), - n_levels=1, - downscale=1.1, - scaled_shape_models=True, - max_shape_components=25, - boundary=3).build(training_images) - -@raises(ValueError) -def test_features_exception(): - sdm = SDMTrainer(features=[igo, sparse_hog], - n_levels=3).train(training_images) - - -@raises(ValueError) -def test_regression_features_sdmtrainer_exception_1(): - sdm = SDMTrainer(n_levels=2, regression_features=[no_op, no_op, no_op]).\ - train(training_images) - - -@raises(ValueError) -def test_regression_features_sdmtrainer_exception_2(): - sdm = SDMTrainer(n_levels=3, regression_features=[no_op, sparse_hog, 1]).\ - train(training_images) - - -@raises(ValueError) -def test_regression_features_sdaamtrainer_exception_1(): - sdm = SDAAMTrainer(aam, regression_features=[no_op, sparse_hog]).\ - train(training_images) - - -@raises(ValueError) -def test_regression_features_sdaamtrainer_exception_2(): - sdm = SDAAMTrainer(aam, regression_features=7).\ - train(training_images) - - -@raises(ValueError) -def test_n_levels_exception(): - sdm = SDMTrainer(n_levels=0).train(training_images) - - -@raises(ValueError) -def test_downscale_exception(): - sdm = SDMTrainer(downscale=0).train(training_images) - - -@raises(ValueError) -def test_n_perturbations_exception(): - sdm = SDAAMTrainer(aam, n_perturbations=-10).train(training_images) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - sdm = SDMTrainer(regression_type=mlr_svd, - regression_features=sparse_hog, - patch_shape=(16, 16), - features=no_op, - normalization_diagonal=150, - n_levels=1, - downscale=1.3, - noise_std=0.04, - rotation=False, - n_perturbations=2).train(training_images, - verbose=True) From e1785eee08a388241c031bb66df119317ab9cc94 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 11:35:36 +0100 Subject: [PATCH 135/423] Remove spaces from docs and add super calls Just beginning to cleanup the aam code, removing some extra spaces from the docstrings (but not fixing them) and adding missing calls to super --- menpofit/aam/base.py | 67 +++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index ee97abe..2ffde95 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -16,18 +16,14 @@ class AAM(object): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -43,11 +39,8 @@ class AAM(object): performing AAMs. scales : `int` or float` or list of those, optional - scale_shapes : `boolean` - scale_features : `boolean` - """ def __init__(self, shape_models, appearance_models, reference_shape, transform, features, scales, scale_shapes, scale_features): @@ -90,12 +83,10 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): Weights of the shape model that will be used to create a novel shape instance. If ``None``, the mean shape ``(shape_weights = [0, 0, ..., 0])`` is used. - appearance_weights : ``(n_weights,)`` `ndarray` or `float` list Weights of the appearance model that will be used to create a novel appearance instance. If ``None``, the mean appearance ``(appearance_weights = [0, 0, ..., 0])`` is used. - level : `int`, optional The pyramidal level to be used. @@ -380,17 +371,13 @@ class PatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -406,14 +393,16 @@ class PatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - """ + def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features): + patch_shape, features, scales, scale_shapes, scale_features, + transform): + super(PatchAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, features, + scales, scale_shapes, scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.transform = DifferentiableThinPlateSplines @@ -469,18 +458,14 @@ class LinearAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -496,15 +481,16 @@ class LinearAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - """ + def __init__(self, shape_models, appearance_models, reference_shape, transform, features, scales, scale_shapes, scale_features, n_landmarks): + super(LinearAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, features, + scales, scale_shapes, scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.transform = transform @@ -545,17 +531,13 @@ class LinearPatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -571,17 +553,18 @@ class LinearPatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - n_landmarks: `int` - """ + def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, - scale_features, n_landmarks): + patch_shape, features, scales, scale_shapes, scale_features, + n_landmarks, transform): + super(LinearPatchAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, + features, scales, scale_shapes, + scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.transform = DifferentiableThinPlateSplines @@ -623,17 +606,13 @@ class PartsAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -649,17 +628,17 @@ class PartsAAM(AAM): performing AAMs. normalize_parts: `callable` - scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - """ + def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, normalize_parts, scales, - scale_shapes, scale_features): + patch_shape, features, normalize_parts, scales, scale_shapes, + scale_features, transform): + super(PartsAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, features, + scales, scale_shapes, scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.patch_shape = patch_shape From 649d50fcccf372b8c9cdba98c6837ec66fbfe2d6 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 19:54:30 +0100 Subject: [PATCH 136/423] Start moving the AAMBuilder into AAM This moves the build function into a train function to match the work I did on the SDM. Also, quite a big change is that the scales are no longer flipped, and are now from lowest to highest, again, to match SDM. This meant also changing some fitting code. I need to check this more thoroughly to make sure I'm doing it correctly. Similar changes need to be done in the ATM/LK code. --- menpofit/aam/__init__.py | 4 +- menpofit/aam/base.py | 208 ++++++++++++++++++++++++++++++-- menpofit/aam/builder.py | 251 +-------------------------------------- menpofit/builder.py | 22 +--- menpofit/checks.py | 13 ++ menpofit/fitter.py | 15 ++- 6 files changed, 225 insertions(+), 288 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index bb37752..e9ac26b 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,6 +1,4 @@ -from .builder import ( - AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, - LinearPatchAAMBuilder, PartsAAMBuilder) +from .base import AAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 2ffde95..8ec98f3 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,10 +1,20 @@ from __future__ import division +from copy import deepcopy import numpy as np from menpo.shape import TriMesh -from menpofit.transform import DifferentiableThinPlateSplines -from menpofit.base import name_of_callable +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpo.model import PCAModel +from menpo.transform import Scale +from menpofit import checks +from menpofit.transform import DifferentiableThinPlateSplines, \ + DifferentiablePiecewiseAffine +from menpofit.base import name_of_callable, batch from menpofit.builder import ( - build_reference_frame, build_patch_reference_frame) + build_reference_frame, build_patch_reference_frame, + normalization_wrt_reference_shape, compute_features, scale_images, + build_shape_model, warp_images, align_shapes, + rescale_images_to_reference_shape) # TODO: document me! @@ -42,16 +52,192 @@ class AAM(object): scale_shapes : `boolean` scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform + def __init__(self, images, group=None, verbose=False, + features=no_op, transform=DifferentiablePiecewiseAffine, + diagonal=None, scales=(0.5, 1.0), scale_features=True, + max_shape_components=None, forgetting_factor=1.0, + max_appearance_components=None, batch_size=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) + max_shape_components = checks.check_max_components( + max_shape_components, n_levels, 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes + self.transform = transform self.scale_features = scale_features + self.diagonal = diagonal + self.scales = scales + self.forgetting_factor = forgetting_factor + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components + self.reference_shape = None + self.shape_models = [] + self.appearance_models = [] + + # Train AAM + self._train(images, group=group, verbose=verbose, increment=False, + batch_size=batch_size) + + def _train(self, images, group=None, verbose=False, increment=False, + batch_size=None): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest level + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + if not increment: + checks.check_trilist(image_batch[0], self.transform, + group=group) + # Normalize images and compute reference shape + self.reference_shape, image_batch = normalization_wrt_reference_shape( + image_batch, group, self.diagonal, verbose=verbose) + else: + # We are incrementing, so rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each pyramid level (low --> high) + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + else: + level_str = None + + # obtain image representation + if self.scale_features: + if j == 0: + # Compute features at highest level + feature_images = compute_features(image_batch, + self.features[0], + level_str=level_str, + verbose=verbose) + # Scale features at other levels + level_images = scale_images(feature_images, + self.scales[j], + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(image_batch, self.scales[j], + level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, + self.features[j], + level_str=level_str, + verbose=verbose) + + # Extract potentially rescaled shapes + level_shapes = [i.landmarks[group].lms for i in level_images] + + # Build the shape model + if not increment: + if j == 0: + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # Store shape model + self.shape_models.append(shape_model) + else: + # Copy shape model + self.shape_models.append(deepcopy(shape_model)) + else: + # Compute aligned shapes + aligned_shapes = align_shapes(level_shapes) + # Increment shape model + self.shape_models[j].increment( + aligned_shapes, + forgetting_factor=self.forgetting_factor) + if self.max_shape_components is not None: + self.shape_models[j].trim_components( + self.max_appearance_components[j]) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(level_images, level_shapes, + scaled_reference_shape, + j, level_str, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format(level_str)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + def _build_shape_model(self, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) @property def n_levels(self): diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 717a813..48c1d6d 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -35,15 +35,12 @@ class AAMBuilder(object): Note that from our experience, this approach of extracting features once and then creating a pyramid on top tends to lead to better performing AAMs. - transform : :map:`PureAlignmentTransform`, optional The :map:`PureAlignmentTransform` that will be used to warp the images. - trilist : ``(t, 3)`` `ndarray`, optional Triangle list that will be used to build the reference frame. If ``None``, defaults to performing Delaunay triangulation on the points. - diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. @@ -57,13 +54,9 @@ class AAMBuilder(object): landmarks, this kwarg also specifies the diagonal length of the reference frame (provided that features computation does not change the image size). - scales : `int` or float` or list of those, optional - scale_shapes : `boolean`, optional - scale_features : `boolean`, optional - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is defined per level. The first element of the list specifies the number @@ -80,7 +73,6 @@ class AAMBuilder(object): If ``None``, all the available components are kept (100% of variance). - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of appearance components is defined per level. The first element of the list specifies the number @@ -125,248 +117,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, trilist=None, diagonal=None, scales=(1, 0.5), scale_shapes=False, scale_features=True, max_shape_components=None, max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def build(self, images, group=None, verbose=False): - r""" - Builds an Active Appearance Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest level - """ - # normalize images and compute reference shape - reference_shape, images = normalization_wrt_reference_shape( - images, group, self.diagonal, verbose=verbose) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - shape_models = [] - appearance_models = [] - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if j == 0: - # compute features at highest level - feature_images = compute_features(images, self.features[j], - level_str=level_str, - verbose=verbose) - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) - - # extract potentially rescaled shapes - level_shapes = [i.landmarks[group].lms - for i in level_images] - - # obtain shape representation - if j == 0 or self.scale_shapes: - # obtain shape model - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) - # add shape model to the list - shape_models.append(shape_model) - else: - # copy precious shape model and add it to the list - shape_models.append(deepcopy(shape_model)) - - # obtain warped images - warped_images = self._warp_images(level_images, level_shapes, - shape_model.mean(), j, - level_str, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - appearance_models.append(appearance_model) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - appearance_models.reverse() - self.scales.reverse() - - aam = self._build_aam(shape_models, appearance_models, reference_shape) - - return aam - - def increment(self, aam, images, group=None, - forgetting_factor=1.0, verbose=False): - # normalize images with respect to reference shape of aam - images = rescale_images_to_reference_shape( - images, group, aam.reference_shape, verbose=verbose) - - # increment models at each scale - if verbose: - print_dynamic('- Incrementing models\n') - - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales[::-1]): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if j == 0: - # compute features at highest level - feature_images = compute_features(images, self.features[j], - level_str=level_str, - verbose=verbose) - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) - - # extract potentially rescaled shapes - level_shapes = [i.landmarks[group].lms - for i in level_images] - - # obtain shape representation - if j == 0 or self.scale_shapes: - if verbose: - print_dynamic('{}Incrementing shape model'.format( - level_str)) - # compute aligned shapes - aligned_shapes = align_shapes(level_shapes) - # increment shape model - aam.shape_models[j].increment( - aligned_shapes, forgetting_factor=forgetting_factor) - if self.max_shape_components is not None: - aam.shape_models[j].trim_components( - self.max_appearance_components[j]) - else: - # copy previous shape model - aam.shape_models[j] = deepcopy(aam.shape_models[j-1]) - - mean_shape = aam.appearance_models[j].mean().landmarks[ - 'source'].lms - - # obtain warped images - warped_images = self._warp_images(level_images, level_shapes, - mean_shape, j, - level_str, verbose) - - # obtain appearance representation - if verbose: - print_dynamic('{}Incrementing appearance model'.format( - level_str)) - # increment appearance model - aam.appearance_models[j].increment(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - aam.appearance_models[j].trim_components( - self.max_appearance_components[j]) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - def build_incrementally(self, images, group=None, - forgetting_factor=1.0, batch_size=100, - verbose=False): - # number of batches - n_batches = np.int(np.ceil(len(images) / batch_size)) - - # train first batch - print 'Training batch 1.' - aam = self.build(images[:batch_size], group=group, verbose=verbose) - - # train all other batches - start = batch_size - for j in range(1, n_batches): - print 'Training batch {}.'.format(j+1) - end = start + batch_size - self.increment(aam, images[start:end], group=group, - forgetting_factor=forgetting_factor, - verbose=verbose) - start = end - - return aam - - @classmethod - def _build_shape_model(cls, shapes, max_components, level): - return build_shape_model(shapes, max_components=max_components) - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return AAM(shape_models, appearance_models, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features) + pass # TODO: document me! diff --git a/menpofit/builder.py b/menpofit/builder.py index ae2fb48..fb750c9 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -174,8 +174,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images.append(Image(parts)) return parts_images -def build_reference_frame(landmarks, boundary=3, group='source', - trilist=None): +def build_reference_frame(landmarks, boundary=3, group='source'): r""" Builds a reference frame from a particular set of landmarks. @@ -183,22 +182,14 @@ def build_reference_frame(landmarks, boundary=3, group='source', ---------- landmarks : :map:`PointCloud` The landmarks that will be used to build the reference frame. - boundary : `int`, optional The number of pixels to be left as a safe margin on the boundaries of the reference frame (has potential effects on the gradient computation). - group : `string`, optional Group that will be assigned to the provided set of landmarks on the reference frame. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. - - If ``None``, defaults to performing Delaunay triangulation on the - points. - Returns ------- reference_frame : :map:`Image` @@ -206,14 +197,13 @@ def build_reference_frame(landmarks, boundary=3, group='source', """ reference_frame = _build_reference_frame(landmarks, boundary=boundary, group=group) - if trilist is not None: - reference_frame.landmarks[group] = TriMesh( - reference_frame.landmarks['source'].lms.points, trilist=trilist) + source_landmarks = reference_frame.landmarks['source'].lms + if isinstance(source_landmarks, TriMesh): + trilist = source_landmarks.trilist + else: + trilist = None - # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, - # perhaps the trilist should be directly obtained from the group landmarks reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) - return reference_frame diff --git a/menpofit/checks.py b/menpofit/checks.py index 82f4978..10a9375 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,5 +1,7 @@ import numpy as np import warnings +from menpo.shape import TriMesh +from menpo.transform import PiecewiseAffine def check_diagonal(diagonal): @@ -11,6 +13,17 @@ def check_diagonal(diagonal): raise ValueError("diagonal must be >= 20") +def check_trilist(image, transform, group=None): + trilist = image.landmarks[group].lms + + if not isinstance(trilist, TriMesh) and isinstance(transform, + PiecewiseAffine): + warnings.warn('The given images do not have an explicit triangulation ' + 'applied. A Delaunay Triangulation will be computed ' + 'and used for warping. This may be suboptimal and cause ' + 'warping artifacts.') + + # TODO: document me! def check_scales(scales): if isinstance(scales, (int, float)): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 72fd8fd..223aa58 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -149,19 +149,18 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # obtain image representation images = [] - for j, s in enumerate(self.scales[::-1]): - if j == 0: - # compute features at highest level - feature_image = self.features[j](image) - elif self.scale_features: + for j in range(self.n_levels): + if self.scale_features: + if j == 0: + # compute features at highest level + feature_image = self.features[j](image) # scale features at other levels - feature_image = images[0].rescale(s) + feature_image = feature_image.rescale(self.scales[j]) else: # scale image and compute features at other levels - scaled_image = image.rescale(s) + scaled_image = image.rescale(self.scales[j]) feature_image = self.features[j](scaled_image) images.append(feature_image) - images.reverse() # get initial shapes per level initial_shapes = [i.landmarks['initial_shape'].lms for i in images] From cf055e2eeb2f590c0f50c3362d396958dbdbc5ea Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 20:28:05 +0100 Subject: [PATCH 137/423] Cleaning more code from the AAMBuider, mostly moving stuff around No real changes, just moving stuff. --- menpofit/aam/algorithm/lk.py | 3 +- menpofit/aam/base.py | 105 ++++++++++++++++++++++++------- menpofit/aam/builder.py | 117 +---------------------------------- menpofit/fitter.py | 10 +-- menpofit/sdm/fitter.py | 1 - 5 files changed, 86 insertions(+), 150 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 756fe2a..3750cba 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -67,8 +67,7 @@ def warp_jacobian(self): dW_dp.shape[2])) def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) + return image.warp_to_mask(self.template.mask, self.transform) def gradient(self, img): nabla = fast_gradient(img) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 8ec98f3..6771faa 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -20,21 +20,11 @@ # TODO: document me! class AAM(object): r""" - Active Appearance Model class. + Active Appearance Models. Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - features : `callable` or ``[callable]``, + ---------- + features : `callable` or ``[callable]``, optional If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -47,10 +37,83 @@ class AAM(object): Note that from our experience, this approach of extracting features once and then creating a pyramid on top tends to lead to better performing AAMs. - + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). scales : `int` or float` or list of those, optional - scale_shapes : `boolean` - scale_features : `boolean` + scale_shapes : `boolean`, optional + scale_features : `boolean`, optional + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ def __init__(self, images, group=None, verbose=False, features=no_op, transform=DifferentiablePiecewiseAffine, @@ -248,7 +311,6 @@ def n_levels(self): """ return len(self.scales) - # TODO: Could we directly use class names instead of this? @property def _str_title(self): r""" @@ -329,17 +391,12 @@ def _instance(self, level, shape_instance, appearance_instance): template = self.appearance_models[level].mean() landmarks = template.landmarks['source'].lms - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) + reference_frame = build_reference_frame(shape_instance) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - return appearance_instance.as_unmasked().warp_to_mask( + return appearance_instance.as_unmasked(copy=False).warp_to_mask( reference_frame.mask, transform, warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 48c1d6d..a0d4d59 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,125 +1,14 @@ from __future__ import division -import numpy as np -from copy import deepcopy -from menpo.model import PCAModel from menpo.shape import mean_pointcloud from menpo.feature import no_op -from menpo.visualize import print_dynamic from menpofit import checks -from menpofit.builder import ( - normalization_wrt_reference_shape, compute_features, scale_images, - warp_images, extract_patches, build_shape_model, align_shapes, - build_reference_frame, build_patch_reference_frame, densify_shapes, - rescale_images_to_reference_shape) +from menpofit.builder import (warp_images, extract_patches, build_shape_model, + align_shapes, build_reference_frame, + build_patch_reference_frame, densify_shapes) from menpofit.transform import ( DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -# TODO: document me! -class AAMBuilder(object): - r""" - Class that builds Active Appearance Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - scales : `int` or float` or list of those, optional - scale_shapes : `boolean`, optional - scale_features : `boolean`, optional - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, 0.5), - scale_shapes=False, scale_features=True, - max_shape_components=None, max_appearance_components=None): - pass - - # TODO: document me! class PatchAAMBuilder(AAMBuilder): r""" diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 223aa58..330186c 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -30,20 +30,16 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, ----------- image: :map:`Image` or subclass The image to be fitted. - initial_shape: :map:`PointCloud` The initial shape estimate from which the fitting procedure will start. - max_iters: `int` or `list` of `int`, optional The maximum number of iterations. If `int`, specifies the overall maximum number of iterations. If `list` of `int`, specifies the maximum number of iterations per level. - gt_shape: :map:`PointCloud` The ground truth shape associated to the image. - crop_image: `None` or float`, optional If `float`, it specifies the proportion of the border wrt the initial shape to which the image will be internally cropped around @@ -53,7 +49,6 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, This will limit the fitting algorithm search region but is likely to speed up its running time, specially when the modeled object occupies a small portion of the image. - **kwargs: Additional keyword arguments that can be passed to specific implementations of ``_fit`` method. @@ -103,13 +98,10 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, ---------- image : :map:`Image` or subclass The image to be fitted. - initial_shape : :map:`PointCloud` The initial shape from which the fitting will start. - gt_shape : class : :map:`PointCloud`, optional The original ground truth shape associated to the image. - crop_image: `None` or float`, optional If `float`, it specifies the proportion of the border wrt the initial shape to which the image will be internally cropped around @@ -219,7 +211,7 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, shape = algorithm_result.final_shape if s != self.scales[-1]: - shape = Scale(self.scales[j+1]/s, + shape = Scale(self.scales[j + 1] / s, n_dims=shape.n_dims).apply(shape) return algorithm_results diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index ec60492..52e72cc 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,5 +1,4 @@ from __future__ import division -from itertools import chain import numpy as np from functools import partial import warnings From 9329b2ca69cd09fb4acc0a43e00bbdeca260796a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 09:33:56 +0100 Subject: [PATCH 138/423] Finish moving build logic into base (AAM) Now building is intimately integrated with the AAM objects. Still need to test that these things actually work though. --- menpofit/aam/__init__.py | 2 +- menpofit/aam/base.py | 185 +++++++----- menpofit/aam/builder.py | 605 --------------------------------------- 3 files changed, 121 insertions(+), 671 deletions(-) delete mode 100644 menpofit/aam/builder.py diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index e9ac26b..aca4250 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,4 +1,4 @@ -from .base import AAM +from .base import AAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 6771faa..4b961e8 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,11 +1,11 @@ from __future__ import division from copy import deepcopy import numpy as np -from menpo.shape import TriMesh from menpo.feature import no_op from menpo.visualize import print_dynamic from menpo.model import PCAModel from menpo.transform import Scale +from menpo.shape import mean_pointcloud from menpofit import checks from menpofit.transform import DifferentiableThinPlateSplines, \ DifferentiablePiecewiseAffine @@ -14,7 +14,7 @@ build_reference_frame, build_patch_reference_frame, normalization_wrt_reference_shape, compute_features, scale_images, build_shape_model, warp_images, align_shapes, - rescale_images_to_reference_shape) + rescale_images_to_reference_shape, densify_shapes, extract_patches) # TODO: document me! @@ -118,8 +118,8 @@ class AAM(object): def __init__(self, images, group=None, verbose=False, features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), scale_features=True, - max_shape_components=None, forgetting_factor=1.0, - max_appearance_components=None, batch_size=None): + max_shape_components=None, max_appearance_components=None, + batch_size=None): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -135,7 +135,6 @@ def __init__(self, images, group=None, verbose=False, self.scale_features = scale_features self.diagonal = diagonal self.scales = scales - self.forgetting_factor = forgetting_factor self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components self.reference_shape = None @@ -147,6 +146,7 @@ def __init__(self, images, group=None, verbose=False, batch_size=batch_size) def _train(self, images, group=None, verbose=False, increment=False, + shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, batch_size=None): r""" Builds an Active Appearance Model from a list of landmarked images. @@ -255,7 +255,7 @@ def _train(self, images, group=None, verbose=False, increment=False, # Increment shape model self.shape_models[j].increment( aligned_shapes, - forgetting_factor=self.forgetting_factor) + forgetting_factor=shape_forgetting_factor) if self.max_shape_components is not None: self.shape_models[j].trim_components( self.max_appearance_components[j]) @@ -284,7 +284,9 @@ def _train(self, images, group=None, verbose=False, increment=False, self.appearance_models.append(appearance_model) else: # increment appearance model - self.appearance_models[j].increment(warped_images) + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) # trim appearance model if required if self.max_appearance_components is not None: self.appearance_models[j].trim_components( @@ -293,6 +295,18 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print_dynamic('{}Done\n'.format(level_str)) + def increment(self, images, group=None, verbose=False, + shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, + batch_size=None): + # Literally just to fit under 80 characters, but maintain the sensible + # parameter name + aff = appearance_forgetting_factor + return self._train(images, group=group, + verbose=verbose, + shape_forgetting_factor=shape_forgetting_factor, + appearance_forgetting_factor=aff, + increment=True, batch_size=batch_size) + def _build_shape_model(self, shapes, max_components, level): return build_shape_model(shapes, max_components=max_components) @@ -640,21 +654,26 @@ class PatchAAM(AAM): scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features, - transform): - super(PatchAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, features, - scales, scale_shapes, scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = DifferentiableThinPlateSplines + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + scale_features=True, max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + + super(PatchAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[level]) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) @property def _str_title(self): @@ -728,21 +747,36 @@ class LinearAAM(AAM): scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features, - n_landmarks): - super(LinearAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, features, - scales, scale_shapes, scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + def __init__(self, images, group=None, verbose=False, features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), scale_features=True, + max_shape_components=None, max_appearance_components=None, + batch_size=None): + + super(LinearAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=transform, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model( + dense_shapes, max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) # TODO: implement me! def _instance(self, level, shape_instance, appearance_instance): @@ -801,23 +835,37 @@ class LinearPatchAAM(AAM): n_landmarks: `int` """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features, - n_landmarks, transform): - super(LinearPatchAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, - features, scales, scale_shapes, - scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = DifferentiableThinPlateSplines + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + scale_features=True, max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + + super(LinearPatchAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[level]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes, + max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) # TODO: implement me! def _instance(self, level, shape_instance, appearance_instance): @@ -841,6 +889,7 @@ def __str__(self): # TODO: document me! +# TODO: implement offsets support? class PartsAAM(AAM): r""" Parts based Active Appearance Model class. @@ -876,21 +925,27 @@ class PartsAAM(AAM): scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, normalize_parts, scales, scale_shapes, - scale_features, transform): - super(PartsAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, features, - scales, scale_shapes, scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models + def __init__(self, images, group=None, verbose=False, features=no_op, + normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), scale_features=True, + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_shape = patch_shape - self.features = features self.normalize_parts = normalize_parts - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + + super(PartsAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return extract_patches(images, shapes, self.patch_shape[level], + normalize_function=self.normalize_parts, + level_str=level_str, verbose=verbose) # TODO: implement me! def _instance(self, level, shape_instance, appearance_instance): diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py deleted file mode 100644 index a0d4d59..0000000 --- a/menpofit/aam/builder.py +++ /dev/null @@ -1,605 +0,0 @@ -from __future__ import division -from menpo.shape import mean_pointcloud -from menpo.feature import no_op -from menpofit import checks -from menpofit.builder import (warp_images, extract_patches, build_shape_model, - align_shapes, build_reference_frame, - build_patch_reference_frame, densify_shapes) -from menpofit.transform import ( - DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) - - -# TODO: document me! -class PatchAAMBuilder(AAMBuilder): - r""" - Class that builds Patch based Active Appearance Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=True, - scale_features=True, max_shape_components=None, - max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[level]) - return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return PatchAAM(shape_models, appearance_models, reference_shape, - self.patch_shape, self.features, self.scales, - self.scale_shapes, self.scale_features) - - -# TODO: document me! -class LinearAAMBuilder(AAMBuilder): - r""" - Class that builds Linear Active Appearance Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None, max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_reference_frame(mean_aligned_shape) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model( - dense_shapes, max_components=max_components) - return shape_model - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return LinearAAM(shape_models, appearance_models, - reference_shape, self.transform, - self.features, self.scales, - self.scale_shapes, self.scale_features, - self.n_landmarks) - - -# TODO: document me! -class LinearPatchAAMBuilder(AAMBuilder): - r""" - Class that builds Linear Patch based Active Appearance Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=False, - scale_features=True, max_shape_components=None, - max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[level]) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model(dense_shapes, - max_components=max_components) - return shape_model - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return LinearPatchAAM(shape_models, appearance_models, - reference_shape, self.patch_shape, - self.features, self.scales, self.scale_shapes, - self.scale_features, self.n_landmarks) - - -# TODO: document me! -# TODO: implement offsets support? -class PartsAAMBuilder(AAMBuilder): - r""" - Class that builds Parts based Active Appearance Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - normalize_parts : `callable`, optional - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - normalize_parts=no_op, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None, max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return extract_patches(images, shapes, self.patch_shape[level], - normalize_function=self.normalize_parts, - level_str=level_str, verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return PartsAAM(shape_models, appearance_models, reference_shape, - self.patch_shape, self.features, - self.normalize_parts, self.scales, - self.scale_shapes, self.scale_features) - - -from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM From 67f25559819418fb4444b3b44e4a7dddd633152a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 11:17:02 +0100 Subject: [PATCH 139/423] Add _increment_shape_model This works the same as _build_shape_model and allows densify to be called first for Linear AAMs. --- menpofit/aam/base.py | 50 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 4b961e8..ad7cf61 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -238,10 +238,11 @@ def _train(self, images, group=None, verbose=False, increment=False, level_shapes = [i.landmarks[group].lms for i in level_images] # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + if not increment: if j == 0: - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) shape_model = self._build_shape_model( level_shapes, self.max_shape_components[j], j) # Store shape model @@ -250,15 +251,10 @@ def _train(self, images, group=None, verbose=False, increment=False, # Copy shape model self.shape_models.append(deepcopy(shape_model)) else: - # Compute aligned shapes - aligned_shapes = align_shapes(level_shapes) - # Increment shape model - self.shape_models[j].increment( - aligned_shapes, - forgetting_factor=shape_forgetting_factor) - if self.max_shape_components is not None: - self.shape_models[j].trim_components( - self.max_appearance_components[j]) + self._increment_shape_model( + level_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor, + max_components=self.max_shape_components[j]) # Obtain warped images - we use a scaled version of the # reference shape, computed here. This is because the mean @@ -310,6 +306,16 @@ def increment(self, images, group=None, verbose=False, def _build_shape_model(self, shapes, max_components, level): return build_shape_model(shapes, max_components=max_components) + def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, + max_components=None): + # Compute aligned shapes + aligned_shapes = align_shapes(shapes) + # Increment shape model + shape_model.increment(aligned_shapes, + forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): reference_frame = build_reference_frame(reference_shape) @@ -772,6 +778,17 @@ def _build_shape_model(self, shapes, max_components, level): dense_shapes, max_components=max_components) return shape_model + def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, + max_components=None): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return warp_images(images, shapes, self.reference_frame, @@ -861,6 +878,17 @@ def _build_shape_model(self, shapes, max_components, level): max_components=max_components) return shape_model + def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, + max_components=None): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return warp_images(images, shapes, self.reference_frame, From b358ad45a239f93ebda8d816a0b1cd0023854a42 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 11:17:29 +0100 Subject: [PATCH 140/423] Missing check for patch features --- menpofit/aam/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index ad7cf61..4292197 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -664,7 +664,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), scale_features=True, max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = patch_shape + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -856,7 +856,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), scale_features=True, max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = patch_shape + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(LinearPatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -958,7 +958,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, patch_shape=(17, 17), scale_features=True, max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = patch_shape + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts super(PartsAAM, self).__init__( From 1e8063bde4fdb6f6f5e4c4f4eacdb0ad0d3be443 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 11:17:49 +0100 Subject: [PATCH 141/423] Only raise warning for scale_features it True --- menpofit/checks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/menpofit/checks.py b/menpofit/checks.py index 10a9375..e248ef4 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -73,11 +73,14 @@ def check_scale_features(scale_features, features): """ if np.alltrue([f == features[0] for f in features]): return scale_features - else: + elif scale_features: + # Only raise warning if True was passed. warnings.warn('scale_features has been automatically set to False ' 'because different types of features are used at each ' 'level.') return False + else: + return scale_features # TODO: document me! From d18d5820168d393668e308fd891216065a853c56 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:49:42 +0100 Subject: [PATCH 142/423] Accidentally worked on too many things... This contains four changes: 1. Rename iterations to n_iterations in both SDM fitter and algorithm 2. Fix a bug with first_image, use image_batch[0] to make sure that generating bounding boxes works properly. 3. Provide the ability to pass a reference shape, otherwise we calculate it from the first batch. Raise a new warning if we do use the first batch. 4. Refactor the SDM algorithms so that they share code more, since the only difference between increment train was the call to increment, we can just have a flag about whether to increment or not. --- menpofit/builder.py | 7 ++ menpofit/sdm/algorithm.py | 136 ++++++++++++++++---------------------- menpofit/sdm/fitter.py | 83 +++++++++++++---------- 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index fb750c9..a55f832 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -310,3 +310,10 @@ def build_shape_model(shapes, max_components=None): shape_model.trim_components(max_components) return shape_model + + +class MenpoFitBuilderWarning(Warning): + r""" + A warning that some part of building the model may cause issues. + """ + pass diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index b4fc7e7..fe713c7 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -13,8 +13,26 @@ class SupervisedDescentAlgorithm(object): r""" """ + + def __init__(self): + self.regressors = [] + def train(self, images, gt_shapes, current_shapes, level_str='', verbose=False): + return self._train(images, gt_shapes, current_shapes, increment=False, + level_str=level_str, verbose=verbose) + + def increment(self, images, gt_shapes, current_shapes, level_str='', + verbose=False): + return self._train(images, gt_shapes, current_shapes, increment=True, + level_str=level_str, verbose=verbose) + + def _train(self, images, gt_shapes, current_shapes, increment=False, + level_str='', verbose=False): + + if not increment: + # Reset the regressors + self.regressors = [] n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] @@ -22,93 +40,32 @@ def train(self, images, gt_shapes, current_shapes, level_str='', # obtain delta_x and gt_x delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) - # initialize iteration counter and list of regressors - self.regressors = [] - # Cascaded Regression loop - for k in range(self.iterations): + for k in range(self.n_iterations): # generate regression data features = features_per_image( images, current_shapes, self.patch_shape, self.features, level_str='{}(Iteration {}) - '.format(level_str, k), verbose=verbose) - # Perform regression if verbose: print_dynamic('{}(Iteration {}) - Performing regression'.format( level_str, k)) - r = self._regressor_cls() - r.train(features, delta_x) - # add regressor to list - self.regressors.append(r) - # Estimate delta_points - estimated_delta_x = r.predict(features) - - if verbose: - print_dynamic('{}(Iteration {}) - Calculating errors'.format( - level_str, k)) - errors = [] - for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): - s1 = template_shape.from_vector(dx) - s2 = template_shape.from_vector(edx) - gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] - errors.append(self._compute_error(s1, s2, gt_s)) - mean = np.mean(errors) - std = np.std(errors) - median = np.median(errors) - print_dynamic('{}(Iteration {}) - Training error -> ' - 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. - format(level_str, k, mean, std, median)) + if not increment: + r = self._regressor_cls() + r.train(features, delta_x) + self.regressors.append(r) + else: + self.regressors[k].increment(features, delta_x) - j = 0 - for shapes in current_shapes: - for s in shapes: - # update current x - current_x = s.as_vector() + estimated_delta_x[j] - # update current shape inplace - s.from_vector_inplace(current_x) - # update delta_x - delta_x[j] = gt_x[j] - current_x - # increase index - j += 1 - - return current_shapes - - def increment(self, images, gt_shapes, current_shapes, verbose=False): - - n_perturbations = len(current_shapes[0]) - template_shape = gt_shapes[0] - - # obtain delta_x and gt_x - delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) - - # Cascaded Regression loop - for r in self.regressors: - # generate regression data - features = features_per_image(images, current_shapes, - self.patch_shape, self.features) - - # update regression - if verbose: - print_dynamic('- Updating regression') - r.increment(features, delta_x) - - # estimate delta_points - estimated_delta_x = r.predict(features) + # Estimate delta_points + estimated_delta_x = self.regressors[k].predict(features) if verbose: - errors = [] - for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): - s1 = template_shape.from_vector(dx) - s2 = template_shape.from_vector(edx) - gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] - errors.append(self._compute_error(s1, s2, gt_s)) - mean = np.mean(errors) - std = np.std(errors) - median = np.median(errors) - print_dynamic('- Training error -> mean: {0:.4f}, ' - 'std: {1:.4f}, median: {2:.4f}.\n'. - format(mean, std, median)) + self._print_regression_info(template_shape, gt_shapes, + n_perturbations, delta_x, + estimated_delta_x, k, + level_str=level_str) j = 0 for shapes in current_shapes: @@ -122,7 +79,6 @@ def increment(self, images, gt_shapes, current_shapes, verbose=False): # increase index j += 1 - # rearrange current shapes into their original list of list form return current_shapes def run(self, image, initial_shape, gt_shape=None, **kwargs): @@ -148,19 +104,39 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): return NonParametricAlgorithmResult(image, self, shapes, gt_shape=gt_shape) + def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, + delta_x, estimated_delta_x, level_index, + level_str=''): + print_dynamic('{}(Iteration {}) - Calculating errors'.format( + level_str, level_index)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + s1 = template_shape.from_vector(dx) + s2 = template_shape.from_vector(edx) + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('{}(Iteration {}) - Training error -> ' + 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. + format(level_str, level_index, mean, std, median)) + # TODO: document me! class Newton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True): + super(Newton, self).__init__() + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape - self.iterations = iterations + self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps @@ -169,15 +145,17 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, class GaussNewton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True, alpha2=0): + super(GaussNewton, self).__init__() + self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, alpha2=alpha2) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape - self.iterations = iterations + self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 52e72cc..23b5237 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -6,8 +6,8 @@ from menpo.feature import no_op from menpofit.visualize import print_progress from menpofit.base import batch, name_of_callable -from menpofit.builder import (normalization_wrt_reference_shape, scale_images, - rescale_images_to_reference_shape) +from menpofit.builder import (scale_images, rescale_images_to_reference_shape, + compute_reference_shape, MenpoFitBuilderWarning) from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, align_shape_with_bounding_box) from menpofit.result import MultiFitterResult @@ -20,9 +20,10 @@ class SupervisedDescentFitter(MultiFitter): r""" """ def __init__(self, images, group=None, bounding_box_group=None, - sd_algorithm_cls=Newton, holistic_feature=no_op, - patch_features=no_op, patch_shape=(17, 17), diagonal=None, - scales=(0.5, 1.0), iterations=6, n_perturbations=30, + reference_shape=None, sd_algorithm_cls=Newton, + holistic_feature=no_op, patch_features=no_op, patch_shape=(17, + 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, + n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): # check parameters @@ -32,7 +33,7 @@ def __init__(self, images, group=None, bounding_box_group=None, patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters self.algorithms = [] - self.reference_shape = None + self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls self._holistic_feature = holistic_feature self._patch_features = patch_features @@ -40,7 +41,7 @@ def __init__(self, images, group=None, bounding_box_group=None, self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations - self.iterations = checks.check_max_iters(iterations, n_levels) + self.n_iterations = checks.check_max_iters(n_iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._setup_algorithms() @@ -54,7 +55,7 @@ def _setup_algorithms(self): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], patch_shape=self._patch_shape[j], - iterations=self.iterations[j])) + n_iterations=self.n_iterations[j])) def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, @@ -78,37 +79,49 @@ def _train(self, images, group=None, bounding_box_group=None, increment = True if verbose: - print('Computing batch {}'.format(k)) + print('Computing batch {} - ({})'.format(k, len(image_batch))) # In the case where group is None, we need to get the only key so # that we can attach landmarks below and not get a complaint about # using None - first_image = image_batch[0] if group is None: - group = first_image.landmarks.group_labels[0] - - if not increment: - # Normalize images and compute reference shape - self.reference_shape, image_batch = normalization_wrt_reference_shape( - image_batch, group, self.diagonal, verbose=verbose) - else: - # We are incrementing, so rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) + group = image_batch[0].landmarks.group_labels[0] + + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: - bounding_box_group = '__gt_bb_' + # It's important to use bb_group for batching, so that we + # generate ground truth bounding boxes for each batch, every + # time + bb_group = '__gt_bb_' for i in image_batch: gt_s = i.landmarks[group].lms - perturb_bbox_group = bounding_box_group + '0' + perturb_bbox_group = bb_group + '0' i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + else: + bb_group = bounding_box_group # Find all bounding boxes on the images with the given bounding # box key - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) n_perturbations = len(all_bb_keys) # If there is only one example bounding box, then we will generate @@ -128,20 +141,20 @@ def _train(self, images, group=None, bounding_box_group=None, # This is customizable by passing in the correct method p_s = self._perturb_from_bounding_box(gt_s, bb) - perturb_bbox_group = '{}_{}'.format(bounding_box_group, - j) + perturb_bbox_group = '{}_{}'.format(bb_group, j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: warnings.warn('The original value of n_perturbation {} ' 'will be reset to {} in order to agree with ' 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) + format(self.n_perturbations, n_perturbations), + MenpoFitBuilderWarning) self.n_perturbations = n_perturbations # Re-grab all the bounding box keys for iterating over when # calculating perturbations - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) # Before scaling, we compute the holistic feature on the whole image msg = '- Computing holistic features ({})'.format( @@ -187,12 +200,12 @@ def _train(self, images, group=None, bounding_box_group=None, current_shapes.append(c_shapes) # train supervised descent algorithm - if increment: - current_shapes = self.algorithms[j].increment( + if not increment: + current_shapes = self.algorithms[j].train( level_images, level_gt_shapes, current_shapes, - verbose=verbose) + level_str=level_str, verbose=verbose) else: - current_shapes = self.algorithms[j].train( + current_shapes = self.algorithms[j].increment( level_images, level_gt_shapes, current_shapes, level_str=level_str, verbose=verbose) @@ -310,7 +323,7 @@ def __str__(self): - Patch shape: {}""" for k, s in enumerate(self.scales): level_info.append(lvl_str_tmplt.format(k, s, - self.iterations[k], + self.n_iterations[k], self._patch_shape[k])) level_info = '\n'.join(level_info) From 17c0a903f20be20913b5576672a62e65ef9fd831 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:52:29 +0100 Subject: [PATCH 143/423] Add a RegularizedSDM alias Just allow passing alpha (the regularization parameter) explicitly. --- menpofit/sdm/fitter.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 23b5237..fd110eb 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -346,4 +346,24 @@ def __str__(self): return cls_str +# Aliases for common combinations of supervised descent fitting SDM = partial(SupervisedDescentFitter, sd_algorithm_cls=Newton) + +class RegularizedSDM(SupervisedDescentFitter): + + def __init__(self, images, group=None, bounding_box_group=None, + alpha=1.0, reference_shape=None, + holistic_feature=no_op, patch_features=no_op, + patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + n_iterations=6, n_perturbations=30, + perturb_from_bounding_box=noisy_shape_from_bounding_box, + batch_size=None, verbose=False): + super(RegularizedSDM, self).__init__( + images, group=group, bounding_box_group=bounding_box_group, + reference_shape=reference_shape, + sd_algorithm_cls=partial(Newton, alpha=alpha), + holistic_feature=holistic_feature, patch_features=patch_features, + patch_shape=patch_shape, diagonal=diagonal, scales=scales, + n_iterations=n_iterations, n_perturbations=n_perturbations, + perturb_from_bounding_box=perturb_from_bounding_box, + batch_size=batch_size, verbose=verbose) From 46ffe8cb60586109db1821ce8cb53cfc6953966f Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:53:39 +0100 Subject: [PATCH 144/423] Don't split patch parameter over two lines --- menpofit/sdm/fitter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index fd110eb..e15bedf 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -21,9 +21,9 @@ class SupervisedDescentFitter(MultiFitter): """ def __init__(self, images, group=None, bounding_box_group=None, reference_shape=None, sd_algorithm_cls=Newton, - holistic_feature=no_op, patch_features=no_op, patch_shape=(17, - 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, - n_perturbations=30, + holistic_feature=no_op, patch_features=no_op, + patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): # check parameters From bb81efcd8ecd626d4d5e5425781cd440e18c3225 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:55:40 +0100 Subject: [PATCH 145/423] Allow passing an explicit reference_shape to AAM --- menpofit/aam/base.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 4292197..5ac9f2d 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,5 +1,6 @@ from __future__ import division from copy import deepcopy +import warnings import numpy as np from menpo.feature import no_op from menpo.visualize import print_dynamic @@ -7,14 +8,14 @@ from menpo.transform import Scale from menpo.shape import mean_pointcloud from menpofit import checks -from menpofit.transform import DifferentiableThinPlateSplines, \ - DifferentiablePiecewiseAffine +from menpofit.transform import (DifferentiableThinPlateSplines, + DifferentiablePiecewiseAffine) from menpofit.base import name_of_callable, batch from menpofit.builder import ( build_reference_frame, build_patch_reference_frame, - normalization_wrt_reference_shape, compute_features, scale_images, - build_shape_model, warp_images, align_shapes, - rescale_images_to_reference_shape, densify_shapes, extract_patches) + compute_features, scale_images, build_shape_model, warp_images, + align_shapes, rescale_images_to_reference_shape, densify_shapes, + extract_patches, MenpoFitBuilderWarning, compute_reference_shape) # TODO: document me! @@ -115,7 +116,7 @@ class AAM(object): ``0`` <= `float` <= ``1`` or a list of those containing 1 or ``len(scales)`` elements """ - def __init__(self, images, group=None, verbose=False, + def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), scale_features=True, max_shape_components=None, max_appearance_components=None, @@ -137,7 +138,7 @@ def __init__(self, images, group=None, verbose=False, self.scales = scales self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - self.reference_shape = None + self.reference_shape = reference_shape self.shape_models = [] self.appearance_models = [] @@ -184,17 +185,25 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print('Computing batch {}'.format(k)) - if not increment: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) checks.check_trilist(image_batch[0], self.transform, group=group) - # Normalize images and compute reference shape - self.reference_shape, image_batch = normalization_wrt_reference_shape( - image_batch, group, self.diagonal, verbose=verbose) - else: - # We are incrementing, so rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) # build models at each scale if verbose: From 598e264e940fa898266955d3003c05966e22be86 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 17:02:55 +0100 Subject: [PATCH 146/423] Rename n_levels to n_scales This is a much more descriptive name. n_levels should be scrapped for n_scales --- menpofit/aam/base.py | 14 +++++++------- menpofit/aam/fitter.py | 12 ++++++------ menpofit/atm/base.py | 10 +++++----- menpofit/atm/builder.py | 20 ++++++++++---------- menpofit/checks.py | 4 ++-- menpofit/fitter.py | 12 ++++++------ menpofit/fittingresult.py | 4 ++-- menpofit/result.py | 4 ++-- menpofit/sdm/fitter.py | 4 ++-- menpofit/visualize/widgets/base.py | 26 +++++++++++++------------- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 5ac9f2d..76cf156 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -26,7 +26,7 @@ class AAM(object): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -61,7 +61,7 @@ class AAM(object): scale_shapes : `boolean`, optional scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -77,7 +77,7 @@ class AAM(object): If ``None``, all the available components are kept (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -651,7 +651,7 @@ class PatchAAM(AAM): patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -744,7 +744,7 @@ class LinearAAM(AAM): The transform used to warp the images from which the AAM was constructed. features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -842,7 +842,7 @@ class LinearPatchAAM(AAM): patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -943,7 +943,7 @@ class PartsAAM(AAM): patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 24fe85c..8fe78c2 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -32,16 +32,16 @@ def _check_n_appearance(self, n_appearance): if type(n_appearance) is int or type(n_appearance) is float: for am in self.aam.appearance_models: am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: + elif len(n_appearance) == 1 and self.aam.n_scales > 1: for am in self.aam.appearance_models: am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: + elif len(n_appearance) == self.aam.n_scales: for am, n in zip(self.aam.appearance_models, n_appearance): am.n_active_components = n else: raise ValueError('n_appearance can be an integer or a float ' 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + 'those'.format(self.aam.n_scales)) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): @@ -58,7 +58,7 @@ def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_levels) + sampling = checks.check_sampling(sampling, self.n_scales) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): @@ -115,10 +115,10 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_levels) + sampling = checks.check_sampling(sampling, self.n_scales) self.n_perturbations = n_perturbations self.noise_std = noise_std - self.max_iters = checks.check_max_iters(max_iters, self.n_levels) + self.max_iters = checks.check_max_iters(max_iters, self.n_scales) self._set_up(sd_algorithm_cls, sampling, **kwargs) def _set_up(self, sd_algorithm_cls, sampling, **kwargs): diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 70e58e1..6071904 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -28,7 +28,7 @@ class ATM(object): constructed. features : `callable` or ``[callable]``, - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -333,7 +333,7 @@ class PatchATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -414,7 +414,7 @@ class LinearATM(ATM): constructed. features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -483,7 +483,7 @@ class LinearPatchATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -555,7 +555,7 @@ class PartsATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 0f44545..3f0c8c4 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -22,7 +22,7 @@ class ATMBuilder(object): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -64,7 +64,7 @@ class ATMBuilder(object): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -258,7 +258,7 @@ class PatchATMBuilder(ATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -292,7 +292,7 @@ class PatchATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -383,7 +383,7 @@ class LinearATMBuilder(ATMBuilder): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -425,7 +425,7 @@ class LinearATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -524,7 +524,7 @@ class LinearPatchATMBuilder(LinearATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -558,7 +558,7 @@ class LinearPatchATMBuilder(LinearATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -651,7 +651,7 @@ class PartsATMBuilder(ATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -687,7 +687,7 @@ class PartsATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. diff --git a/menpofit/checks.py b/menpofit/checks.py index e248ef4..16a7188 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -47,7 +47,7 @@ def check_features(features, n_levels): ---------- features : callable or list of callables The features to apply to the images. - n_levels : int + n_scales : int The number of pyramid levels. Returns @@ -103,7 +103,7 @@ def check_max_components(max_components, n_levels, var_name): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be None or int or float or a list of - those containing 1 or {n_levels} elements. + those containing 1 or {n_scales} elements. """ str_error = ("{} must be None or an int > 0 or a 0 <= float <= 1 or " "a list of those containing 1 or {} elements").format( diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 330186c..22cee87 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -13,7 +13,7 @@ class MultiFitter(object): r""" """ @property - def n_levels(self): + def n_scales(self): r""" The number of pyramidal levels used during alignment. @@ -141,7 +141,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # obtain image representation images = [] - for j in range(self.n_levels): + for j in range(self.n_scales): if self.scale_features: if j == 0: # compute features at highest level @@ -196,7 +196,7 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, The fitting object containing the state of the whole fitting procedure. """ - max_iters = checks.check_max_iters(max_iters, self.n_levels) + max_iters = checks.check_max_iters(max_iters, self.n_scales) shape = initial_shape gt_shape = None algorithm_results = [] @@ -267,16 +267,16 @@ def _check_n_shape(self, n_shape): if type(n_shape) is int or type(n_shape) is float: for sm in self._model.shape_models: sm.n_active_components = n_shape - elif len(n_shape) == 1 and self._model.n_levels > 1: + elif len(n_shape) == 1 and self._model.n_scales > 1: for sm in self._model.shape_models: sm.n_active_components = n_shape[0] - elif len(n_shape) == self._model.n_levels: + elif len(n_shape) == self._model.n_scales: for sm, n in zip(self._model.shape_models, n_shape): sm.n_active_components = n else: raise ValueError('n_shape can be an integer or a float or None' 'or a list containing 1 or {} of ' - 'those'.format(self._model.n_levels)) + 'those'.format(self._model.n_scales)) def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): transform = noisy_alignment_similarity_transform( diff --git a/menpofit/fittingresult.py b/menpofit/fittingresult.py index 7fdedef..3c88302 100644 --- a/menpofit/fittingresult.py +++ b/menpofit/fittingresult.py @@ -719,7 +719,7 @@ def n_levels(self): :type: `int` """ - return self.fitter.n_levels + return self.fitter.n_scales @property def downscale(self): @@ -904,7 +904,7 @@ class SerializableMultilevelFittingResult(FittingResult): The list of fitted shapes per iteration of the fitting procedure. gt_shape : :map:`PointCloud` The ground truth shape associated to the image. - n_levels : `int` + n_scales : `int` Number of levels within the multilevel fitter. downscale : `int` Scale of downscaling applied to the image. diff --git a/menpofit/result.py b/menpofit/result.py index f19c516..a8e7364 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -508,13 +508,13 @@ def __init__(self, image, fitter, algorithm_results, affine_correction, self._gt_shape = gt_shape @property - def n_levels(self): + def n_scales(self): r""" The number of levels of the fitter object. :type: `int` """ - return self.fitter.n_levels + return self.fitter.n_scales @property def scales(self): diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index e15bedf..12f9bec 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -51,7 +51,7 @@ def __init__(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=False, batch_size=batch_size) def _setup_algorithms(self): - for j in range(self.n_levels): + for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], patch_shape=self._patch_shape[j], @@ -165,7 +165,7 @@ def _train(self, images, group=None, bounding_box_group=None, # for each pyramid level (low --> high) current_shapes = [] - for j in range(self.n_levels): + for j in range(self.n_scales): if verbose: if len(self.scales) > 1: level_str = ' - Level {}: '.format(j) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 302a5e2..01dbb30 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -31,7 +31,7 @@ def _check_n_parameters(n_params, n_levels, max_n_params): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be ``None`` or `int` or `float` or a `list` - of those containing ``1`` or ``n_levels`` elements. + of those containing ``1`` or ``n_scales`` elements. """ str_error = ("n_params must be None or 1 <= int <= max_n_params or " "a list of those containing 1 or {} elements").format(n_levels) @@ -128,7 +128,7 @@ def visualize_shape_model(shape_model, n_parameters=5, mode='multiple', max_n_params = [sp.n_active_components for sp in shape_model] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_parameters = _check_n_parameters(n_parameters, n_levels, max_n_params) # Initial options dictionaries @@ -487,7 +487,7 @@ def visualize_appearance_model(appearance_model, n_parameters=5, max_n_params = [ap.n_active_components for ap in appearance_model] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_parameters = _check_n_parameters(n_parameters, n_levels, max_n_params) # Find initial groups and labels that will be passed to the landmark options @@ -790,7 +790,7 @@ def visualize_aam(aam, n_shape_parameters=5, n_appearance_parameters=5, print('Initializing...') # Get the number of levels - n_levels = aam.n_levels + n_levels = aam.n_scales # Define the styling options if style == 'coloured': @@ -829,7 +829,7 @@ def visualize_aam(aam, n_shape_parameters=5, n_appearance_parameters=5, max_n_appearance = [ap.n_active_components for ap in aam.appearance_models] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_shape_parameters = _check_n_parameters(n_shape_parameters, n_levels, max_n_shape) n_appearance_parameters = _check_n_parameters(n_appearance_parameters, @@ -972,7 +972,7 @@ def update_info(aam, instance, level, group): if n_levels == 1: tmp_shape_models = '' tmp_pyramid = '' - else: # n_levels > 1 + else: # n_scales > 1 # shape models info if aam.scaled_shape_models: tmp_shape_models = "Each level has a scaled shape model " \ @@ -993,7 +993,7 @@ def update_info(aam, instance, level, group): "> Warp using {} transform".format(aam.transform.__name__), "> {}".format(tmp_pyramid), "> Level {}/{} (downscale={:.1f})".format( - level + 1, aam.n_levels, aam.downscale), + level + 1, aam.n_scales, aam.downscale), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -1216,7 +1216,7 @@ def visualize_atm(atm, n_shape_parameters=5, mode='multiple', print('Initializing...') # Get the number of levels - n_levels = atm.n_levels + n_levels = atm.n_scales # Define the styling options if style == 'coloured': @@ -1252,7 +1252,7 @@ def visualize_atm(atm, n_shape_parameters=5, mode='multiple', max_n_shape = [sp.n_active_components for sp in atm.shape_models] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_shape_parameters = _check_n_parameters(n_shape_parameters, n_levels, max_n_shape) @@ -1388,7 +1388,7 @@ def update_info(atm, instance, level, group): if n_levels == 1: tmp_shape_models = '' tmp_pyramid = '' - else: # n_levels > 1 + else: # n_scales > 1 # shape models info if atm.scaled_shape_models: tmp_shape_models = "Each level has a scaled shape model " \ @@ -1409,7 +1409,7 @@ def update_info(atm, instance, level, group): "> Warp using {} transform".format(atm.transform.__name__), "> {}".format(tmp_pyramid), "> Level {}/{} (downscale={:.1f})".format( - level + 1, atm.n_levels, atm.downscale), + level + 1, atm.n_scales, atm.downscale), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -2466,9 +2466,9 @@ def update_info(name, value): else: text_per_line = [ "> {} iterations".format(fitting_results[im].n_iters)] - if hasattr(fitting_results[im], 'n_levels'): # Multilevel result + if hasattr(fitting_results[im], 'n_scales'): # Multilevel result text_per_line.append("> {} levels with downscale of {:.1f}".format( - fitting_results[im].n_levels, fitting_results[im].downscale)) + fitting_results[im].n_scales, fitting_results[im].downscale)) info_wid.set_widget_state(n_lines=len(text_per_line), text_per_line=text_per_line) From e60c09a201c48387097df2117740694c80e5da76 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 17:11:37 +0100 Subject: [PATCH 147/423] Take @jalabort _prepare_image method About to copy the flow into the SDM. Will add a commit after that renames to holistic_features --- menpofit/fitter.py | 56 +++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 22cee87..ec3dcf7 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -107,59 +107,59 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, initial shape to which the image will be internally cropped around the initial shape range. If `None`, no cropping is performed. - This will limit the fitting algorithm search region but is likely to speed up its running time, specially when the modeled object occupies a small portion of the image. - Returns ------- images : `list` of :map:`Image` or subclass The list of images that will be fitted by the fitters. - initial_shapes : `list` of :map:`PointCloud` The initial shape for each one of the previous images. - gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ - - # attach landmarks to the image - image.landmarks['initial_shape'] = initial_shape + # Attach landmarks to the image + image.landmarks['__initial_shape'] = initial_shape if gt_shape: - image.landmarks['gt_shape'] = gt_shape + image.landmarks['__gt_shape'] = gt_shape - # if specified, crop the image if crop_image: + # If specified, crop the image image = image.crop_to_landmarks_proportion(crop_image, - group='initial_shape') + group='__initial_shape') - # rescale image wrt the scale factor between reference_shape and + # Rescale image wrt the scale factor between reference_shape and # initial_shape image = image.rescale_to_reference_shape(self.reference_shape, - group='initial_shape') + group='__initial_shape') - # obtain image representation + # Compute image representation images = [] - for j in range(self.n_scales): - if self.scale_features: - if j == 0: - # compute features at highest level - feature_image = self.features[j](image) - # scale features at other levels - feature_image = feature_image.rescale(self.scales[j]) + for i in range(self.n_scales): + # Handle features + if i == 0 or self.features[i] is not self.features[i - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_image = self.features[i](image) + + # Handle scales + if self.scales[i] != 1: + # Scale feature images only if scale is different than 1 + scaled_image = feature_image.rescale(self.scales[i]) else: - # scale image and compute features at other levels - scaled_image = image.rescale(self.scales[j]) - feature_image = self.features[j](scaled_image) - images.append(feature_image) + scaled_image = feature_image + + # Add scaled image to list + images.append(scaled_image) - # get initial shapes per level - initial_shapes = [i.landmarks['initial_shape'].lms for i in images] + # Get initial shapes per level + initial_shapes = [i.landmarks['__initial_shape'].lms for i in images] - # get ground truth shapes per level + # Get ground truth shapes per level if gt_shape: - gt_shapes = [i.landmarks['gt_shape'].lms for i in images] + gt_shapes = [i.landmarks['__gt_shape'].lms for i in images] else: gt_shapes = None From a16bc605e9800eafbc6061ed75eecd5f4c48c2c7 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 11:36:57 +0100 Subject: [PATCH 148/423] Move AAM to scales away from 'levels' Mostly simple refactoring. Also, added the correct feature scaling logic from @jalabort CLM branch --- menpofit/aam/base.py | 280 +++++++++-------------------- menpofit/base.py | 2 +- menpofit/fitter.py | 36 ++-- menpofit/visualize/widgets/base.py | 4 +- 4 files changed, 113 insertions(+), 209 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 76cf156..acb847e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -123,13 +123,13 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, batch_size=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales, n_scales = checks.check_scales(scales) + features = checks.check_features(features, n_scales) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, n_scales, 'max_appearance_components') # set parameters self.features = features self.transform = transform @@ -166,7 +166,7 @@ def _train(self, images, group=None, verbose=False, increment=False, ------- aam : :map:`AAM` The AAM object. Shape and appearance models are stored from - lowest to highest level + lowest to highest scale """ # If batch_size is not None, then we may have a generator, else we # assume we have a list. @@ -210,50 +210,45 @@ def _train(self, images, group=None, verbose=False, increment=False, print_dynamic('- Building models\n') feature_images = [] - # for each pyramid level (low --> high) - for j in range(self.n_levels): + # for each scale (low --> high) + for j in range(self.n_scales): if verbose: if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) + scale_prefix = ' - Scale {}: '.format(j) else: - level_str = ' - ' + scale_prefix = ' - ' else: - level_str = None - - # obtain image representation - if self.scale_features: - if j == 0: - # Compute features at highest level - feature_images = compute_features(image_batch, - self.features[0], - level_str=level_str, - verbose=verbose) - # Scale features at other levels - level_images = scale_images(feature_images, - self.scales[j], - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(image_batch, self.scales[j], - level_str=level_str, + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + level_str=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[k] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + level_str=scale_prefix, verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) + else: + scaled_images = feature_images # Extract potentially rescaled shapes - level_shapes = [i.landmarks[group].lms for i in level_images] + scale_shapes = [i.landmarks[group].lms for i in scaled_images] # Build the shape model if verbose: - print_dynamic('{}Building shape model'.format(level_str)) + print_dynamic('{}Building shape model'.format(scale_prefix)) if not increment: if j == 0: shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) + scale_shapes, self.max_shape_components[j], j) # Store shape model self.shape_models.append(shape_model) else: @@ -261,7 +256,7 @@ def _train(self, images, group=None, verbose=False, increment=False, self.shape_models.append(deepcopy(shape_model)) else: self._increment_shape_model( - level_shapes, self.shape_models[j], + scale_shapes, self.shape_models[j], forgetting_factor=shape_forgetting_factor, max_components=self.max_shape_components[j]) @@ -271,13 +266,14 @@ def _train(self, images, group=None, verbose=False, increment=False, # reference frame. scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( self.reference_shape) - warped_images = self._warp_images(level_images, level_shapes, + warped_images = self._warp_images(scaled_images, scale_shapes, scaled_reference_shape, - j, level_str, verbose) + j, scale_prefix, verbose) # obtain appearance model if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) + print_dynamic('{}Building appearance model'.format( + scale_prefix)) if not increment: appearance_model = PCAModel(warped_images) @@ -298,7 +294,7 @@ def _train(self, images, group=None, verbose=False, increment=False, self.max_appearance_components[j]) if verbose: - print_dynamic('{}Done\n'.format(level_str)) + print_dynamic('{}Done\n'.format(scale_prefix)) def increment(self, images, group=None, verbose=False, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, @@ -312,7 +308,7 @@ def increment(self, images, group=None, verbose=False, appearance_forgetting_factor=aff, increment=True, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, level): + def _build_shape_model(self, shapes, max_components, scale_index): return build_shape_model(shapes, max_components=max_components) def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, @@ -325,16 +321,16 @@ def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, if max_components is not None: shape_model.trim_components(max_components) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): reference_frame = build_reference_frame(reference_shape) return warp_images(images, shapes, reference_frame, self.transform, level_str=level_str, verbose=verbose) @property - def n_levels(self): + def n_scales(self): """ - The number of scale levels of the AAM. + The number of scales of the AAM. :type: `int` """ @@ -348,7 +344,8 @@ def _str_title(self): """ return 'Active Appearance Model' - def instance(self, shape_weights=None, appearance_weights=None, level=-1): + def instance(self, shape_weights=None, appearance_weights=None, + scale_index=-1): r""" Generates a novel AAM instance given a set of shape and appearance weights. If no weights are provided, the mean AAM instance is @@ -364,16 +361,16 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): Weights of the appearance model that will be used to create a novel appearance instance. If ``None``, the mean appearance ``(appearance_weights = [0, 0, ..., 0])`` is used. - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[level] - am = self.appearance_models[level] + sm = self.shape_models[scale_index] + am = self.appearance_models[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel if shape_weights is None: @@ -387,24 +384,24 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): appearance_weights *= am.eigenvalues[:n_appearance_weights] ** 0.5 appearance_instance = am.instance(appearance_weights) - return self._instance(level, shape_instance, appearance_instance) + return self._instance(scale_index, shape_instance, appearance_instance) - def random_instance(self, level=-1): + def random_instance(self, scale_index=-1): r""" Generates a novel random instance of the AAM. Parameters ----------- - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[level] - am = self.appearance_models[level] + sm = self.shape_models[scale_index] + am = self.appearance_models[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel shape_weights = (np.random.randn(sm.n_active_components) * @@ -414,10 +411,10 @@ def random_instance(self, level=-1): am.eigenvalues[:am.n_active_components]**0.5) appearance_instance = am.instance(appearance_weights) - return self._instance(level, shape_instance, appearance_instance) + return self._instance(scale_index, shape_instance, appearance_instance) - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() + def _instance(self, scale_index, shape_instance, appearance_instance): + template = self.appearance_models[scale_index].mean() landmarks = template.landmarks['source'].lms reference_frame = build_reference_frame(shape_instance) @@ -470,11 +467,11 @@ def view_appearance_models_widget(self, n_parameters=5, n_parameters : `int` or `list` of `int` or ``None``, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -502,19 +499,19 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, n_shape_parameters : `int` or `list` of `int` or None, optional The number of shape principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. n_appearance_parameters : `int` or `list` of `int` or None, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -532,106 +529,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: fix me! def __str__(self): - out = "{}\n - {} training images.\n".format(self._str_title, - self.n_training_images) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_levels): - n_channels.append( - self.appearance_models[j].template_instance.n_channels) - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_levels > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.appearance_models[0].n_features, - self.appearance_models[0].template_instance.n_true_pixels(), - n_channels[0], - self.appearance_models[0].template_instance._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self.appearance_models[i].n_features, - self.appearance_models[i].template_instance.n_true_pixels(), - n_channels[i], - self.appearance_models[i].template_instance._str_shape, - n_channels[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n - {3} appearance components " \ - "({4:.2f}% of variance)\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100, - self.appearance_models[i].n_components, - self.appearance_models[i].variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} shape components ({10:.2f}% of " \ - "variance)\n - {11} appearance components ({12:.2f}% of " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.appearance_models[0].n_features, - self.appearance_models[0].template_instance.n_true_pixels(), - n_channels[0], - self.appearance_models[0].template_instance._str_shape, - n_channels[0], self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100, - self.appearance_models[0].n_components, - self.appearance_models[0].variance_ratio() * 100) - return out + return '' # TODO: document me! @@ -652,9 +550,9 @@ class PatchAAM(AAM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. + each scale after downscaling of the image. The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. + the lowest scale and so on. If ``callable`` the specified feature will be applied to the original image and pyramid generation will be performed on top of the feature @@ -683,10 +581,10 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[level]) + reference_shape, patch_shape=self.patch_shape[scale_index]) return warp_images(images, shapes, reference_frame, self.transform, level_str=level_str, verbose=verbose) @@ -694,8 +592,8 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, def _str_title(self): return 'Patch-Based Active Appearance Model' - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean + def _instance(self, scale_index, shape_instance, appearance_instance): + template = self.appearance_models[scale_index].mean landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( @@ -776,7 +674,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, level): + def _build_shape_model(self, shapes, max_components, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_reference_frame(mean_aligned_shape) @@ -798,14 +696,14 @@ def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, if max_components is not None: shape_model.trim_components(max_components) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): return warp_images(images, shapes, self.reference_frame, self.transform, level_str=level_str, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -875,11 +773,11 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, level): + def _build_shape_model(self, shapes, max_components, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[level]) + mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model @@ -898,14 +796,14 @@ def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, if max_components is not None: shape_model.trim_components(max_components) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): return warp_images(images, shapes, self.reference_frame, self.transform, level_str=level_str, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -978,14 +876,14 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return extract_patches(images, shapes, self.patch_shape[level], + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): + return extract_patches(images, shapes, self.patch_shape[scale_index], normalize_function=self.normalize_parts, level_str=level_str, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! diff --git a/menpofit/base.py b/menpofit/base.py index 1883721..16dd3e0 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -46,7 +46,7 @@ def create_pyramid(images, n_levels, downscale, features, verbose=False): images: list of :map:`Image` The set of landmarked images from which to build the AAM. - n_levels: int + n_scales: int The number of multi-resolution pyramidal levels to be used. downscale: float diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ec3dcf7..af4e093 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -63,11 +63,6 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, images, initial_shapes, gt_shapes = self._prepare_image( image, initial_shape, gt_shape=gt_shape, crop_image=crop_image) - # detach added landmarks from image - del image.landmarks['initial_shape'] - if gt_shape: - del image.landmarks['gt_shape'] - # work out the affine transform between the initial shape of the # highest pyramidal level and the initial shape of the original image affine_correction = AlignmentAffine(initial_shapes[-1], initial_shape) @@ -192,28 +187,39 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, Returns ------- - algorithm_results: :class:`menpo.fg2015.fittingresult.FittingResult` list + algorithm_results: :class:`FittingResult` list The fitting object containing the state of the whole fitting procedure. """ + # Perform check max_iters = checks.check_max_iters(max_iters, self.n_scales) + + # Set initial and ground truth shapes shape = initial_shape gt_shape = None - algorithm_results = [] - for j, (i, alg, it, s) in enumerate(zip(images, self.algorithms, - max_iters, self.scales)): - if gt_shapes: - gt_shape = gt_shapes[j] - algorithm_result = alg.run(i, shape, gt_shape=gt_shape, - max_iters=it, **kwargs) + # Initialize list of algorithm results + algorithm_results = [] + for i in range(self.n_scales): + # Handle ground truth shape + if gt_shapes is not None: + gt_shape = gt_shapes[i] + + # Run algorithm + algorithm_result = self.algorithms[i].run(images[i], shape, + gt_shape=gt_shape, + max_iters=max_iters[i], + **kwargs) + # Add algorithm result to the list algorithm_results.append(algorithm_result) + # Prepare this scale's final shape for the next scale shape = algorithm_result.final_shape - if s != self.scales[-1]: - shape = Scale(self.scales[j + 1] / s, + if self.scales[i] != self.scales[-1]: + shape = Scale(self.scales[i + 1] / self.scales[i], n_dims=shape.n_dims).apply(shape) + # Return list of algorithm results return algorithm_results diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 01dbb30..396e7c4 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -897,7 +897,7 @@ def render_function(name, value): # Compute weights and instance shape_weights = shape_model_parameters_wid.parameters appearance_weights = appearance_model_parameters_wid.parameters - instance = aam.instance(level=level, shape_weights=shape_weights, + instance = aam.instance(scale_index=level, shape_weights=shape_weights, appearance_weights=appearance_weights) # Update info @@ -1317,7 +1317,7 @@ def render_function(name, value): # Compute weights and instance shape_weights = shape_model_parameters_wid.parameters - instance = atm.instance(level=level, shape_weights=shape_weights) + instance = atm.instance(scale_index=level, shape_weights=shape_weights) # Update info update_info(atm, instance, level, From ad62016a354b6ea9dd103181cc0e9fe8293dbd8e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 11:38:08 +0100 Subject: [PATCH 149/423] Move SDM to scales away from levels Use correct features scaling logic and also get rid of the custom _prepare_image method. --- menpofit/sdm/fitter.py | 171 ++++++++++++----------------------------- 1 file changed, 50 insertions(+), 121 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 12f9bec..e74a90e 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -7,7 +7,8 @@ from menpofit.visualize import print_progress from menpofit.base import batch, name_of_callable from menpofit.builder import (scale_images, rescale_images_to_reference_shape, - compute_reference_shape, MenpoFitBuilderWarning) + compute_reference_shape, MenpoFitBuilderWarning, + compute_features) from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, align_shape_with_bounding_box) from menpofit.result import MultiFitterResult @@ -28,20 +29,21 @@ def __init__(self, images, group=None, bounding_box_group=None, batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_features = checks.check_features(patch_features, n_levels) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) + scales, n_scales = checks.check_scales(scales) + patch_features = checks.check_features(patch_features, n_scales) + holistic_features = checks.check_features(holistic_feature, n_scales) + patch_shape = checks.check_patch_shape(patch_shape, n_scales) # set parameters self.algorithms = [] self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls - self._holistic_feature = holistic_feature + self.features = holistic_features self._patch_features = patch_features self._patch_shape = patch_shape self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations - self.n_iterations = checks.check_max_iters(n_iterations, n_levels) + self.n_iterations = checks.check_max_iters(n_iterations, n_scales) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._setup_algorithms() @@ -156,41 +158,46 @@ def _train(self, images, group=None, bounding_box_group=None, all_bb_keys = list(image_batch[0].landmarks.keys_matching( '*{}*'.format(bb_group))) - # Before scaling, we compute the holistic feature on the whole image - msg = '- Computing holistic features ({})'.format( - name_of_callable(self._holistic_feature)) - wrap = partial(print_progress, prefix=msg, verbose=verbose) - image_batch = [self._holistic_feature(im) - for im in wrap(image_batch)] - - # for each pyramid level (low --> high) + # for each scale (low --> high) current_shapes = [] for j in range(self.n_scales): if verbose: if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) + scale_prefix = ' - Scale {}: '.format(j) else: - level_str = ' - ' + scale_prefix = ' - ' else: - level_str = None - - # Scale images - level_images = scale_images(image_batch, self.scales[j], - level_str=level_str, - verbose=verbose) + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + level_str=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[k] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + level_str=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images - # Extract scaled ground truth shapes for current level - level_gt_shapes = [i.landmarks[group].lms - for i in level_images] + # Extract scaled ground truth shapes for current scale + scaled_shapes = [i.landmarks[group].lms for i in scaled_images] if j == 0: msg = '{}Generating {} perturbations per image'.format( - level_str, self.n_perturbations) + scale_prefix, self.n_perturbations) wrap = partial(print_progress, prefix=msg, end_with_newline=False, verbose=verbose) # Extract perturbations at the very bottom level - for i in wrap(level_images): + for i in wrap(scaled_images): c_shapes = [] for perturb_bbox_group in all_bb_keys: bbox = i.landmarks[perturb_bbox_group].lms @@ -202,14 +209,14 @@ def _train(self, images, group=None, bounding_box_group=None, # train supervised descent algorithm if not increment: current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose) + scaled_images, scaled_shapes, current_shapes, + level_str=scale_prefix, verbose=verbose) else: current_shapes = self.algorithms[j].increment( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose) + scaled_images, scaled_shapes, current_shapes, + level_str=scale_prefix, verbose=verbose) - # Scale current shapes to next level resolution + # Scale current shapes to next resolution if self.scales[j] != (1 or self.scales[-1]): transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) @@ -224,83 +231,6 @@ def increment(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=True, batch_size=batch_size) - def _prepare_image(self, image, initial_shape, gt_shape=None, - crop_image=0.5): - r""" - Prepares the image to be fitted. - - The image is first rescaled wrt the ``reference_landmarks`` and then - a gaussian pyramid is applied. Depending on the - ``pyramid_on_features`` flag, the pyramid is either applied to the - features image computed from the rescaled imaged or applied to the - rescaled image and features extracted at each pyramidal level. - - Parameters - ---------- - image : :map:`Image` or subclass - The image to be fitted. - initial_shape : :map:`PointCloud` - The initial shape from which the fitting will start. - gt_shape : :map:`PointCloud`, optional - The original ground truth shape associated to the image. - crop_image: `None` or float`, optional - If `float`, it specifies the proportion of the border wrt the - initial shape to which the image will be internally cropped around - the initial shape range. - If `None`, no cropping is performed. - - This will limit the fitting algorithm search region but is - likely to speed up its running time, specially when the - modeled object occupies a small portion of the image. - - Returns - ------- - images : `list` of :map:`Image` or subclass - The list of images that will be fitted by the fitters. - initial_shapes : `list` of :map:`PointCloud` - The initial shape for each one of the previous images. - gt_shapes : `list` of :map:`PointCloud` - The ground truth shape for each one of the previous images. - """ - # Attach landmarks to the image - image.landmarks['initial_shape'] = initial_shape - if gt_shape: - image.landmarks['gt_shape'] = gt_shape - - # If specified, crop the image - if crop_image: - image = image.crop_to_landmarks_proportion(crop_image, - group='initial_shape') - - # Rescale image w.r.t the scale factor between reference_shape and - # initial_shape - image = image.rescale_to_reference_shape(self.reference_shape, - group='initial_shape') - - # Compute the holistic feature on the normalized image - image = self._holistic_feature(image) - - # Obtain image representation - images = [] - for s in self.scales: - if s != 1: - # scale image - scaled_image = image.rescale(s) - else: - scaled_image = image - images.append(scaled_image) - - # Get initial shapes per level - initial_shapes = [i.landmarks['initial_shape'].lms for i in images] - - # Get ground truth shapes per level - if gt_shape: - gt_shapes = [i.landmarks['gt_shape'].lms for i in images] - else: - gt_shapes = None - - return images, initial_shapes, gt_shapes - def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): return MultiFitterResult(image, self, algorithm_results, @@ -316,30 +246,29 @@ def __str__(self): noisy_shape_from_bounding_box) regressor_cls = self.algorithms[0]._regressor_cls - # Compute level info strings - level_info = [] - lvl_str_tmplt = r""" - Level {} (Scale {}) + # Compute scale info strings + scales_info = [] + lvl_str_tmplt = r""" - Scale {} - {} iterations - Patch shape: {}""" for k, s in enumerate(self.scales): - level_info.append(lvl_str_tmplt.format(k, s, - self.n_iterations[k], - self._patch_shape[k])) - level_info = '\n'.join(level_info) + scales_info.append(lvl_str_tmplt.format(s, + self.n_iterations[k], + self._patch_shape[k])) + scales_info = '\n'.join(scales_info) cls_str = r"""Supervised Descent Method - Regression performed using the {reg_alg} algorithm - Regression class: {reg_cls} - - Levels: {levels} -{level_info} + - Scales: {scales} +{scales_info} - Perturbations generated per shape: {n_perturbations} - Images scaled to diagonal: {diagonal:.2f} - Custom perturbation scheme used: {is_custom_perturb_func}""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), reg_cls=name_of_callable(regressor_cls), - n_levels=len(self.scales), - levels=self.scales, - level_info=level_info, + scales=self.scales, + scales_info=scales_info, n_perturbations=self.n_perturbations, diagonal=diagonal, is_custom_perturb_func=is_custom_perturb_func) From f33e253e131b13ef1f399128c75db4dabcd8c2c6 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 11:48:22 +0100 Subject: [PATCH 150/423] Remove scale_features --- menpofit/aam/base.py | 47 +++++++++++++------------------------------- menpofit/checks.py | 16 --------------- menpofit/fitter.py | 14 ------------- 3 files changed, 14 insertions(+), 63 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index acb847e..2f6cfd2 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -58,8 +58,6 @@ class AAM(object): reference frame (provided that features computation does not change the image size). scales : `int` or float` or list of those, optional - scale_shapes : `boolean`, optional - scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number @@ -118,14 +116,12 @@ class AAM(object): """ def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), scale_features=True, - max_shape_components=None, max_appearance_components=None, - batch_size=None): + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + max_appearance_components=None, batch_size=None): # check parameters checks.check_diagonal(diagonal) scales, n_scales = checks.check_scales(scales) features = checks.check_features(features, n_scales) - scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( @@ -133,7 +129,6 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, # set parameters self.features = features self.transform = transform - self.scale_features = scale_features self.diagonal = diagonal self.scales = scales self.max_shape_components = max_shape_components @@ -564,20 +559,18 @@ class PatchAAM(AAM): scales : `int` or float` or list of those scale_shapes : `boolean` - scale_features : `boolean` """ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - scale_features=True, max_shape_components=None, - max_appearance_components=None, batch_size=None): + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, scale_features=scale_features, - max_shape_components=max_shape_components, + scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -656,20 +649,16 @@ class LinearAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` """ def __init__(self, images, group=None, verbose=False, features=no_op, transform=DifferentiableThinPlateSplines, diagonal=None, - scales=(0.5, 1.0), scale_features=True, - max_shape_components=None, max_appearance_components=None, - batch_size=None): + scales=(0.5, 1.0), max_shape_components=None, + max_appearance_components=None, batch_size=None): super(LinearAAM, self).__init__( images, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, - scales=scales, scale_features=scale_features, + transform=transform, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -754,22 +743,18 @@ class LinearPatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - n_landmarks: `int` """ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - scale_features=True, max_shape_components=None, - max_appearance_components=None, batch_size=None): + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(LinearPatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, scale_features=scale_features, - max_shape_components=max_shape_components, + scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -856,23 +841,19 @@ class PartsAAM(AAM): normalize_parts: `callable` scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` """ def __init__(self, images, group=None, verbose=False, features=no_op, normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), scale_features=True, - max_shape_components=None, max_appearance_components=None, - batch_size=None): + patch_shape=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts super(PartsAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, scale_features=scale_features, - max_shape_components=max_shape_components, + scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) diff --git a/menpofit/checks.py b/menpofit/checks.py index 16a7188..0547791 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -67,22 +67,6 @@ def check_features(features, n_levels): "callables with the same length as scales") -# TODO: document me! -def check_scale_features(scale_features, features): - r""" - """ - if np.alltrue([f == features[0] for f in features]): - return scale_features - elif scale_features: - # Only raise warning if True was passed. - warnings.warn('scale_features has been automatically set to False ' - 'because different types of features are used at each ' - 'level.') - return False - else: - return scale_features - - # TODO: document me! def check_patch_shape(patch_shape, n_levels): if len(patch_shape) == 2 and isinstance(patch_shape[0], int): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index af4e093..d963fda 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -254,20 +254,6 @@ def features(self): def scales(self): return self._model.scales - @property - def scale_features(self): - r""" - Flag that defined the nature of Gaussian pyramid used to build the - AAM. - If ``True``, the feature space is computed once at the highest scale - and the Gaussian pyramid is applied to the feature images. - If ``False``, the Gaussian pyramid is applied to the original images - and features are extracted at each level. - - :type: `boolean` - """ - return self._model.scale_features - def _check_n_shape(self, n_shape): if n_shape is not None: if type(n_shape) is int or type(n_shape) is float: From 01edeefcfbd181bc93ede42e6d990d538fe4e522 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 16:56:37 +0100 Subject: [PATCH 151/423] Bug using wrong index Fixed SDM no scaling logic --- menpofit/aam/base.py | 2 +- menpofit/sdm/fitter.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 2f6cfd2..d3a1746 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -225,7 +225,7 @@ def _train(self, images, group=None, verbose=False, increment=False, level_str=scale_prefix, verbose=verbose) # handle scales - if self.scales[k] != 1: + if self.scales[j] != 1: # Scale feature images only if scale is different than 1 scaled_images = scale_images(feature_images, self.scales[j], level_str=scale_prefix, diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index e74a90e..741e1dc 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -179,7 +179,7 @@ def _train(self, images, group=None, bounding_box_group=None, level_str=scale_prefix, verbose=verbose) # handle scales - if self.scales[k] != 1: + if self.scales[j] != 1: # Scale feature images only if scale is different than 1 scaled_images = scale_images(feature_images, self.scales[j], level_str=scale_prefix, @@ -216,8 +216,9 @@ def _train(self, images, group=None, bounding_box_group=None, scaled_images, scaled_shapes, current_shapes, level_str=scale_prefix, verbose=verbose) - # Scale current shapes to next resolution - if self.scales[j] != (1 or self.scales[-1]): + # Scale current shapes to next resolution, don't bother + # scaling final level + if j != (self.n_scales - 1): transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) for image_shapes in current_shapes: From 30da28142776893b46834b2176c09ab3198c6f83 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 11:21:54 +0100 Subject: [PATCH 152/423] Add menpofit.feature package - We may consider moving these feature to menpocore in the long run. --- menpofit/feature/features.py | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 menpofit/feature/features.py diff --git a/menpofit/feature/features.py b/menpofit/feature/features.py new file mode 100644 index 0000000..8344b35 --- /dev/null +++ b/menpofit/feature/features.py @@ -0,0 +1,74 @@ +from __future__ import division +import numpy as np +import warnings +from menpo.feature import ndfeature + + +# TODO: Document me! +@ndfeature +def centralize(x, axes=(-2, -1)): + r""" + """ + mean = np.mean(x, axis=axes, keepdims=True) + return x - mean + + +# TODO: Document me! +@ndfeature +def normalize_norm(x, axes=(-2, -1)): + r""" + """ + x = centralize(x, axes=axes) + norm = np.asarray(np.linalg.norm(x, axis=axes)) + positions = np.asarray(axes) + len(x.shape) + for axis in positions: + norm = np.expand_dims(norm, axis=axis) + return handle_div_by_zero(x, norm) + + +# TODO: document me! +@ndfeature +def normalize_std(x, axes=(-2, -1)): + r""" + """ + x = centralize(x, axes=axes) + std = np.std(x, axis=axes, keepdims=True) + return handle_div_by_zero(x, std) + + +# TODO: document me! +@ndfeature +def normalize_var(x, axes=(-2, -1)): + r""" + """ + x = centralize(x, axes=axes) + var = np.var(x, axis=axes, keepdims=True) + return handle_div_by_zero(x, var) + + +# TODO: document me! +@ndfeature +def probability_map(x, axes=(-2, -1)): + r""" + """ + x = x - np.min(x, axis=axes, keepdims=True) + total = np.sum(x, axis=axes, keepdims=True) + nonzero = total > 0 + if np.any(~nonzero): + warnings.warn("some of x axes have 0 variance - uniform probability " + "maps are used them.") + x[nonzero] /= total[nonzero] + x[~nonzero] = 1 / np.prod(axes) + return x + + +# TODO: document me! +def handle_div_by_zero(x, normalizer): + r""" + """ + nonzero = normalizer > 0 + if np.any(~nonzero): + warnings.warn("some of x axes have 0 variance - they cannot be " + "normalized.") + x[nonzero] /= normalizer[nonzero] + return x From a5043ea2300d22952ee527d55ce852d44a648fb8 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 11:23:09 +0100 Subject: [PATCH 153/423] Add __init__.py --- menpofit/feature/__init__.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 menpofit/feature/__init__.py diff --git a/menpofit/feature/__init__.py b/menpofit/feature/__init__.py new file mode 100644 index 0000000..03a2607 --- /dev/null +++ b/menpofit/feature/__init__.py @@ -0,0 +1,2 @@ +from features import ( + centralize, normalize_norm, normalize_std, normalize_var, probability_map) From 056b817d70dda7ff725b7178ed7ad68a9673feb2 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:19:57 +0100 Subject: [PATCH 154/423] Add ftt_utils.py to menpofit.math --- menpofit/math/fft_utils.py | 244 +++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 menpofit/math/fft_utils.py diff --git a/menpofit/math/fft_utils.py b/menpofit/math/fft_utils.py new file mode 100644 index 0000000..a3e9a85 --- /dev/null +++ b/menpofit/math/fft_utils.py @@ -0,0 +1,244 @@ +from __future__ import division +import warnings +import numpy as np +from functools import wraps +from menpo.feature.base import rebuild_feature_image +try: + # try importing pyfftw + from pyfftw.interfaces.numpy_fft import fft2, ifft2, fftshift, ifftshift + + try: + # try calling fft2 on a 4-dimensional array (this is known to have + # problem in some linux distributions) + fft2(np.zeros((1, 1, 1, 1))) + except RuntimeError: + warnings.warn("pyfftw is known to be buggy on your system, numpy.fft " + "will be used instead. Consequently, all algorithms " + "using ffts will be running at a slower speed.", + RuntimeWarning) + from numpy.fft import fft2, ifft2, fftshift, ifftshift +except ImportError: + warnings.warn("pyfftw is not installed on your system, numpy.fft will be " + "used instead. Consequently, all algorithms using ffts " + "will be running at a slower speed. Consider installing " + "pyfftw (pip install pyfftw) to speed up your ffts.", + ImportWarning) + from numpy.fft import fft2, ifft2, fftshift, ifftshift + + +# TODO: Document me! +def pad(pixels, ext_shape, boundary='constant'): + r""" + """ + h, w = pixels.shape[-2:] + + h_margin = (ext_shape[0] - h) // 2 + w_margin = (ext_shape[1] - w) // 2 + + h_margin2 = h_margin + if h + 2 * h_margin < ext_shape[0]: + h_margin += 1 + + w_margin2 = w_margin + if w + 2 * w_margin < ext_shape[1]: + w_margin += 1 + + pad_width = [] + for _ in pixels.shape[:-2]: + pad_width.append((0, 0)) + pad_width += [(h_margin, h_margin2), (w_margin, w_margin2)] + pad_width = tuple(pad_width) + + return np.lib.pad(pixels, pad_width, mode=boundary) + + +# TODO: Document me! +def crop(pixels, shape): + r""" + """ + h, w = pixels.shape[-2:] + + h_margin = (h - shape[0]) // 2 + w_margin = (w - shape[1]) // 2 + + h_corrector = 1 if np.remainder(h - shape[0], 2) != 0 else 0 + w_corrector = 1 if np.remainder(w - shape[1], 2) != 0 else 0 + + return pixels[..., + h_margin + h_corrector:-h_margin, + w_margin + w_corrector:-w_margin] + + +# TODO: Document me! +def ndconvolution(wrapped): + r""" + """ + @wraps(wrapped) + def wrapper(image, filter, *args, **kwargs): + if not isinstance(image, np.ndarray) and not isinstance(filter, np.ndarray): + # Both image and filter are menpo images + feature = wrapped(image.pixels, filter.pixels, *args, **kwargs) + return rebuild_feature_image(image, feature) + elif not isinstance(image, np.ndarray): + # Image is menpo image + feature = wrapped(image.pixels, filter, *args, **kwargs) + return rebuild_feature_image(image, feature) + elif not isinstance(filter, np.ndarray): + # filter is menpo image + return wrapped(image, filter, *args, **kwargs) + else: + return wrapped(image, filter, *args, **kwargs) + return wrapper + + +# TODO: Document me! +@ndconvolution +def fft_convolve2d(x, f, mode='same', boundary='constant', fft_filter=False): + r""" + Performs fast 2d convolution in the frequency domain convolving each image + channel with its corresponding filter channel. + + Parameters + ---------- + x : ``(channels, height, width)`` `ndarray` + Image. + f : ``(channels, height, width)`` `ndarray` + Filter. + mode : str {`full`, `same`, `valid`}, optional + Determines the shape of the resulting convolution. + boundary: str {`constant`, `symmetric`}, optional + Determines how the image is padded. + fft_filter: `bool`, optional + If `True`, the filter is assumed to be defined on the frequency + domain. If `False` the filter is assumed to be defined on the + spatial domain. + + Returns + ------- + c: ``(channels, height, width)`` `ndarray` + Result of convolving each image channel with its corresponding + filter channel. + """ + if fft_filter: + # extended shape is filter shape + ext_shape = np.asarray(f.shape[-2:]) + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + + # compute ffts of extended image + fft_ext_x = fft2(ext_x) + fft_ext_f = f + else: + # extended shape + x_shape = np.asarray(x.shape[-2:]) + f_shape = np.asarray(f.shape[-2:]) + f_half_shape = (f_shape / 2).astype(int) + ext_shape = x_shape + f_half_shape - 1 + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + ext_f = pad(f, ext_shape) + + # compute ffts of extended image and extended filter + fft_ext_x = fft2(ext_x) + fft_ext_f = fft2(ext_f) + + # compute extended convolution in Fourier domain + fft_ext_c = fft_ext_f * fft_ext_x + + # compute ifft of extended convolution + ext_c = np.real(ifftshift(ifft2(fft_ext_c), axes=(-2, -1))) + + if mode is 'full': + return ext_c + elif mode is 'same': + return crop(ext_c, x_shape) + elif mode is 'valid': + return crop(ext_c, x_shape - f_half_shape + 1) + else: + raise ValueError( + "mode={}, is not supported. The only supported " + "modes are: 'full', 'same' and 'valid'.".format(mode)) + + +# TODO: Document me! +@ndconvolution +def fft_convolve2d_sum(x, f, mode='same', boundary='constant', + fft_filter=False, axis=0, keepdims=True): + r""" + Performs fast 2d convolution in the frequency domain convolving each image + channel with its corresponding filter channel and summing across the + channel axis. + + Parameters + ---------- + x : ``(channels, height, width)`` `ndarray` + Image. + f : ``(channels, height, width)`` `ndarray` + Filter. + mode : str {`full`, `same`, `valid`}, optional + Determines the shape of the resulting convolution. + boundary: str {`constant`, `symmetric`}, optional + Determines how the image is padded. + fft_filter: `bool`, optional + If `True`, the filter is assumed to be defined on the frequency + domain. If `False` the filter is assumed to be defined on the + spatial domain. + axis : `int`, optional + The axis across to which the summation is performed. + keepdims: `boolean`, optional + If `True` the number of dimensions of the result is the same as the + number of dimensions of the filter. If `False` the channel dimension + is lost in the result. + Returns + ------- + c: ``(1, height, width)`` `ndarray` + Result of convolving each image channel with its corresponding + filter channel and summing across the channel axis. + """ + if fft_filter: + fft_ext_f = f + + # extended shape is fft_ext_filter shape + x_shape = np.asarray(x.shape[-2:]) + f_shape = ((np.asarray(fft_ext_f.shape[-2:]) + 1) / 1.5).astype(int) + f_half_shape = (f_shape / 2).astype(int) + ext_shape = np.asarray(f.shape[-2:]) + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + + # compute ffts of extended image + fft_ext_x = fft2(ext_x) + else: + # extended shape + x_shape = np.asarray(x.shape[-2:]) + f_shape = np.asarray(f.shape[-2:]) + f_half_shape = (f_shape / 2).astype(int) + ext_shape = x_shape + f_half_shape - 1 + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + ext_f = pad(f, ext_shape) + + # compute ffts of extended image and extended filter + fft_ext_x = fft2(ext_x) + fft_ext_f = fft2(ext_f) + + # compute extended convolution in Fourier domain + fft_ext_c = np.sum(fft_ext_f * fft_ext_x, axis=axis, keepdims=keepdims) + + # compute ifft of extended convolution + ext_c = np.real(ifftshift(ifft2(fft_ext_c), axes=(-2, -1))) + + if mode is 'full': + return ext_c + elif mode is 'same': + return crop(ext_c, x_shape) + elif mode is 'valid': + return crop(ext_c, x_shape - f_half_shape + 1) + else: + raise ValueError( + "mode={}, is not supported. The only supported " + "modes are: 'full', 'same' and 'valid'.".format(mode)) From b85d254350bd0445a0f53fb401176d95f97e2480 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:20:38 +0100 Subject: [PATCH 155/423] Add correlationfilters.py to menpofit.math - Add MOSSE, MCCF and their respective incremental versions --- menpofit/math/correlationfilter.py | 373 +++++++++++++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100644 menpofit/math/correlationfilter.py diff --git a/menpofit/math/correlationfilter.py b/menpofit/math/correlationfilter.py new file mode 100644 index 0000000..753a55e --- /dev/null +++ b/menpofit/math/correlationfilter.py @@ -0,0 +1,373 @@ +import numpy as np +from numpy.fft import fft2, ifft2, ifftshift +from scipy.sparse import spdiags, eye as speye +from scipy.sparse.linalg import spsolve +from menpofit.math.fft_utils import pad, crop + + +# TODO: Document me! +def mosse(X, y, l=0.01, boundary='constant', crop_filter=True): + r""" + Minimum Output Sum of Squared Errors (MOSSE) filter. + + Parameters + ---------- + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l: `float`, optional + Regularization parameter. + boundary: str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter: `bool`, optional + If ``True``, the shape of the MOSSE filter is the same as the shape + of the desired response. If ``False``, the filter's shape is equal to: + ``X[0].shape + y.shape - 1`` + + Returns + ------- + mosse: ``(1, height, width)`` `ndarray` + Minimum Output Sum od Squared Errors (MOSSE) filter associated to + the training images. + + References + ---------- + .. [1] David S. Bolme, J. Ross Beveridge, Bruce A. Draper and Yui Man Lui. + "Visual Object Tracking using Adaptive Correlation Filters". CVPR, 2010. + """ + # number of images, number of channels, height and width + n, k, hx, wx = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # extended shape + ext_h = hx + hy - 1 + ext_w = wx + wy - 1 + ext_shape = (ext_h, ext_w) + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for x in X: + # extend image + ext_x = pad(x, ext_shape, boundary=boundary) + # fft of extended image + fft_ext_x = fft2(ext_x) + + # update auto and cross spectral energy matrices + sXX += fft_ext_x.conj() * fft_ext_x + sXY += fft_ext_x.conj() * fft_ext_y + + # compute desired correlation filter + fft_ext_f = sXY / (sXX + l) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute extended filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX + + +def imosse(A, B, n_ab, X, y, l=0.01, boundary='constant', + crop_filter=True, f=1.0): + r""" + Incremental Minimum Output Sum od Squared Errors (iMOSSE) filter + + Parameters + ---------- + A : + B : + n_ab : `int` + Total number of samples used to produce A and B. + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l : `float`, optional + Regularization parameter. + boundary : str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter : `bool`, optional + f : ``[0, 1]`` `float`, optional + Forgetting factor that weights the relative contribution of new + samples vs old samples. If 1.0, all samples are weighted equally. + If <1.0, more emphasis is put on the new samples. + + Returns + ------- + mccf : ``(1, height, width)`` `ndarray` + Multi-Channel Correlation Filter (MCCF) filter associated to the + training images. + sXY : + sXX : + + References + ---------- + .. [1] David S. Bolme, J. Ross Beveridge, Bruce A. Draper and Yui Man Lui. + "Visual Object Tracking using Adaptive Correlation Filters". CVPR, 2010. + """ + # number of images; number of channels, height and width + n_x, k, hz, wz = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # multiply the number of samples used to produce the auto and cross + # spectral energy matrices A and B by forgetting factor + n_ab *= f + # total number of samples + n = n_ab + n_x + # compute weighting factors + nu_ab = n_ab / n + nu_x = n_x / n + + # extended shape + ext_h = hz + hy - 1 + ext_w = wz + wy - 1 + ext_shape = (ext_h, ext_w) + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # extend images + ext_X = pad(X, ext_shape, boundary=boundary) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for ext_x in ext_X: + # fft of extended image + fft_ext_x = fft2(ext_x) + + # update auto and cross spectral energy matrices + sXX += fft_ext_x.conj() * fft_ext_x + sXY += fft_ext_x.conj() * fft_ext_y + + # combine old and new auto and cross spectral energy matrices + sXY = nu_ab * A + nu_x * sXY + sXX = nu_ab * B + nu_x * sXX + # compute desired correlation filter + fft_ext_f = sXY / (sXX + l) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX + + +# TODO: Document me! +def mccf(X, y, l=0.01, boundary='constant', crop_filter=True): + r""" + Multi-Channel Correlation Filter (MCCF). + + Parameters + ---------- + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l : `float`, optional + Regularization parameter. + boundary : str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter : `bool`, optional + + Returns + ------- + mccf: ``(1, height, width)`` `ndarray` + Multi-Channel Correlation Filter (MCCF) filter associated to the + training images. + sXY : + sXX : + + References + ---------- + .. [1] Hamed Kiani Galoogahi, Terence Sim, Simon Lucey. "Multi-Channel + Correlation Filters". ICCV, 2013. + """ + # number of images; number of channels, height and width + n, k, hx, wx = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # extended shape + ext_h = hx + hy - 1 + ext_w = wx + wy - 1 + ext_shape = (ext_h, ext_w) + # extended dimensionality + ext_d = ext_h * ext_w + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # extend images + ext_X = pad(X, ext_shape, boundary=boundary) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for ext_x in ext_X: + # fft of extended image + fft_ext_x = fft2(ext_x) + + # store extended image fft as sparse diagonal matrix + diag_fft_x = spdiags(fft_ext_x.reshape((k, -1)), + -np.arange(0, k) * ext_d, ext_d * k, ext_d).T + # vectorize extended desired response fft + diag_fft_y = fft_ext_y.ravel() + + # update auto and cross spectral energy matrices + sXX += diag_fft_x.conj().T.dot(diag_fft_x) + sXY += diag_fft_x.conj().T.dot(diag_fft_y) + + # solve ext_d independent k x k linear systems (with regularization) + # to obtain desired extended multi-channel correlation filter + fft_ext_f = spsolve(sXX + l * speye(sXX.shape[-1]), sXY) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX + + +# TODO: Document me! +def imccf(A, B, n_ab, X, y, l=0.01, boundary='constant', crop_filter=True, + f=1.0): + r""" + Incremental Multi-Channel Correlation Filter (MCCF) + + Parameters + ---------- + A : + B : + n_ab : `int` + Total number of samples used to produce A and B. + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l : `float`, optional + Regularization parameter. + boundary : str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter : `bool`, optional + f : ``[0, 1]`` `float`, optional + Forgetting factor that weights the relative contribution of new + samples vs old samples. If 1.0, all samples are weighted equally. + If <1.0, more emphasis is put on the new samples. + + Returns + ------- + mccf : ``(1, height, width)`` `ndarray` + Multi-Channel Correlation Filter (MCCF) filter associated to the + training images. + sXY : + sXX : + + References + ---------- + .. [1] David S. Bolme, J. Ross Beveridge, Bruce A. Draper and Yui Man Lui. + "Visual Object Tracking using Adaptive Correlation Filters". CVPR, 2010. + .. [2] Hamed Kiani Galoogahi, Terence Sim, Simon Lucey. "Multi-Channel + Correlation Filters". ICCV, 2013. + """ + # number of images; number of channels, height and width + n_x, k, hz, wz = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # multiply the number of samples used to produce the auto and cross + # spectral energy matrices A and B by forgetting factor + n_ab *= f + # total number of samples + n = n_ab + n_x + # compute weighting factors + nu_ab = n_ab / n + nu_x = n_x / n + + # extended shape + ext_h = hz + hy - 1 + ext_w = wz + wy - 1 + ext_shape = (ext_h, ext_w) + # extended dimensionality + ext_d = ext_h * ext_w + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # extend images + ext_X = pad(X, ext_shape, boundary=boundary) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for ext_x in ext_X: + # fft of extended image + fft_ext_x = fft2(ext_x) + + # store extended image fft as sparse diagonal matrix + diag_fft_x = spdiags(fft_ext_x.reshape((k, -1)), + -np.arange(0, k) * ext_d, ext_d * k, ext_d).T + # vectorize extended desired response fft + diag_fft_y = fft_ext_y.ravel() + + # update auto and cross spectral energy matrices + sXX += diag_fft_x.conj().T.dot(diag_fft_x) + sXY += diag_fft_x.conj().T.dot(diag_fft_y) + + # combine old and new auto and cross spectral energy matrices + sXY = nu_ab * A + nu_x * sXY + sXX = nu_ab * B + nu_x * sXX + # solve ext_d independent k x k linear systems (with regularization) + # to obtain desired extended multi-channel correlation filter + fft_ext_f = spsolve(sXX + l * speye(sXX.shape[-1]), sXY) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX From eb94f514474a5b4b403fd454480862a8d0921e83 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:22:58 +0100 Subject: [PATCH 156/423] Update __init__.py from menpofit.math - The math package is an advanced package and nothing should be importable at the first level --- menpofit/math/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py index 1fec9a8..e69de29 100644 --- a/menpofit/math/__init__.py +++ b/menpofit/math/__init__.py @@ -1,2 +0,0 @@ -from least_squares import ( - incremental_least_squares, incremental_indirect_least_squares) \ No newline at end of file From 6d5cd23570db7754630cbd3fd338a5da560bace9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:24:36 +0100 Subject: [PATCH 157/423] Add clm results --- menpofit/clm/result.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 menpofit/clm/result.py diff --git a/menpofit/clm/result.py b/menpofit/clm/result.py new file mode 100644 index 0000000..09e532b --- /dev/null +++ b/menpofit/clm/result.py @@ -0,0 +1,14 @@ +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult + + +# TODO: document me! +class CLMAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + + +# TODO: document me! +class CLMFitterResult(MultiFitterResult): + r""" + """ + From c3cfae79f7b8226ad9eadad77c74e23dacada0d3 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:28:01 +0100 Subject: [PATCH 158/423] Add CLM class - Some things could still be improved; mainly how to properly deal with shape models. --- menpofit/clm/base.py | 465 +++++++++++++++++-------------------------- 1 file changed, 187 insertions(+), 278 deletions(-) diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 3711a5d..7feabf3 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -1,321 +1,230 @@ -import numpy as np -from menpo.image import Image - -from menpofit.base import DeformableModel - - -class CLM(DeformableModel): +from __future__ import division +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpofit import checks +from menpofit.base import batch +from menpofit.builder import ( + normalization_wrt_reference_shape, compute_features, scale_images, + build_shape_model, increment_shape_model) +from expert import ExpertEnsemble, CorrelationFilterExpertEnsemble + + +# TODO: Document me! +# TODO: Introduce shape_model_cls +# TODO: Get rid of max_shape_components and shape_forgetting_factor +class CLM(object): r""" - Constrained Local Model class. + Constrained Local Model (CLM) class. Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the CLM. - - classifiers : ``[[callable]]`` - A list containing the list of classifier callables per each pyramidal - level of the CLM. - - n_training_images : `int` - The number of training images used to build the AAM. - - patch_shape : tuple of `int` - The shape of the patches used to train the classifiers. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. - - scaled_shape_models : `boolean`, Optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. - - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. + ---------- + Returns + ------- + clm : :map:`CLM` + The CLM object """ - def __init__(self, shape_models, classifiers, n_training_images, - patch_shape, features, reference_shape, downscale, - scaled_shape_models): - DeformableModel.__init__(self, features) - self.shape_models = shape_models - self.classifiers = classifiers - self.n_training_images = n_training_images - self.patch_shape = patch_shape - self.reference_shape = reference_shape - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + def __init__(self, images, group=None, verbose=False, batch_size=None, + diagonal=None, scales=(0.5, 1), features=no_op, + # shape_model_cls=build_normalised_pca_shape_model, + expert_ensemble_cls=CorrelationFilterExpertEnsemble, + max_shape_components=None, + shape_forgetting_factor=1.0): + self.diagonal = checks.check_diagonal(diagonal) + self.scales = checks.check_scales(scales) + self.features = checks.check_features(features, self.n_scales) + # self.shape_model_cls = checks.check_algorithm_cls( + # shape_model_cls, self.n_scales, ShapeModel) + self.expert_ensemble_cls = checks.check_algorithm_cls( + expert_ensemble_cls, self.n_scales, ExpertEnsemble) + + self.max_shape_components = checks.check_max_components( + max_shape_components, self.n_scales, 'max_shape_components') + self.shape_forgetting_factor = shape_forgetting_factor + + # Train CLM + self.train(images, group=group, verbose=verbose, batch_size=batch_size) @property - def n_levels(self): - """ - The number of multi-resolution pyramidal levels of the CLM. - - :type: `int` - """ - return len(self.shape_models) - - @property - def n_classifiers_per_level(self): - """ - The number of classifiers per pyramidal level of the CLM. + def n_scales(self): + r""" + The number of scales of the CLM. :type: `int` """ - return [len(clf) for clf in self.classifiers] + return len(self.scales) - def instance(self, shape_weights=None, level=-1): + def _train_batch(self, image_batch, increment, group=None, verbose=False): r""" - Generates a novel CLM instance given a set of shape weights. If no - weights are provided, the mean CLM instance is returned. - - Parameters - ----------- - shape_weights : ``(n_weights,)`` `ndarray` or `float` list - Weights of the shape model that will be used to create - a novel shape instance. If `None`, the mean shape - ``(shape_weights = [0, 0, ..., 0])`` is used. - - level : `int`, optional - The pyramidal level to be used. - - Returns - ------- - shape_instance : :map:`PointCloud` - The novel CLM instance. """ - sm = self.shape_models[level] - # TODO: this bit of logic should to be transferred down to PCAModel - if shape_weights is None: - shape_weights = [0] - n_shape_weights = len(shape_weights) - shape_weights *= sm.eigenvalues[:n_shape_weights] ** 0.5 - shape_instance = sm.instance(shape_weights) - return shape_instance + # If increment is False, we need to initialise/reset both shape models + # and ensembles of experts + if not increment: + self.shape_models = [] + self.expert_ensembles = [] + + # normalize images and compute reference shape + self.reference_shape, image_batch = normalization_wrt_reference_shape( + image_batch, group, self.diagonal, verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Training models\n') + + # for each level (low --> high) + for i in range(self.n_scales): + if verbose: + if self.n_scales > 1: + prefix = ' - Scale {}: '.format(i) + else: + prefix = ' - ' + + # handle features + if i == 0 or self.features[i] is not self.features[i-1]: + # compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[i], + prefix=prefix, + verbose=verbose) + # handle scales + if self.scales[i] != 1: + # scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, + self.scales[i], + prefix=prefix, + verbose=verbose) + else: + scaled_images = feature_images - def random_instance(self, level=-1): - r""" - Generates a novel random CLM instance. + # extract scaled shapes + scaled_shapes = [image.landmarks[group].lms + for image in scaled_images] - Parameters - ----------- - level : `int`, optional - The pyramidal level to be used. + # train shape model + if verbose: + print_dynamic('{}Training shape model'.format(prefix)) - Returns - ------- - shape_instance : :map:`PointCloud` - The novel CLM instance. - """ - sm = self.shape_models[level] - # TODO: this bit of logic should to be transferred down to PCAModel - shape_weights = (np.random.randn(sm.n_active_components) * - sm.eigenvalues[:sm.n_active_components]**0.5) - shape_instance = sm.instance(shape_weights) - return shape_instance + # TODO: This should be cleaned up by defining shape model classes + if increment: + increment_shape_model( + self.shape_models[i], scaled_shapes, + max_components=self.max_shape_components[i], + forgetting_factor=self.shape_forgetting_factor, + prefix=prefix, verbose=verbose) - def response_image(self, image, group=None, label=None, level=-1): - r""" - Generates a response image result of applying the classifiers of a - particular pyramidal level of the CLM to an image. + else: + shape_model = build_shape_model( + scaled_shapes, max_components=self.max_shape_components[i], + prefix=prefix, verbose=verbose) + self.shape_models.append(shape_model) + + # train expert ensemble + if verbose: + print_dynamic('{}Training expert ensemble'.format(prefix)) + + if increment: + self.expert_ensembles[i].increment(scaled_images, + scaled_shapes, + prefix=prefix, + verbose=verbose) + else: + expert_ensemble = self.expert_ensemble_cls[i](scaled_images, + scaled_shapes, + prefix=prefix, + verbose=verbose) + self.expert_ensembles.append(expert_ensemble) - Parameters - ----------- - image: :map:`Image` - The image. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - level: `int`, optional - The pyramidal level to be used. + if verbose: + print_dynamic('{}Done\n'.format(prefix)) - Returns - ------- - image : :map:`Image` - The response image. + def _train(self, images, increment, group=None, verbose=False, + batch_size=None): + r""" """ - # rescale image - image = image.rescale_to_reference_shape(self.reference_shape, - group=group, label=label) - - # apply pyramid - if self.n_levels > 1: - if self.pyramid_on_features: - # compute features at highest level - feature_image = self.features(image) - - # apply pyramid on feature image - pyramid = feature_image.gaussian_pyramid( - n_levels=self.n_levels, downscale=self.downscale) + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] - # get rescaled feature images - images = list(pyramid) - else: - # create pyramid on intensities image - pyramid = image.gaussian_pyramid( - n_levels=self.n_levels, downscale=self.downscale) + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True - # compute features at each level - images = [self.features[self.n_levels - j - 1](i) - for j, i in enumerate(pyramid)] - images.reverse() - else: - images = [self.features(image)] + if verbose: + print('Batch {}'.format(k)) - # initialize responses - image = images[level] - image_pixels = np.reshape(image.pixels, (-1, image.n_channels)) - response_data = np.zeros((image.shape[0], image.shape[1], - self.n_classifiers_per_level[level])) - # Compute responses - for j, clf in enumerate(self.classifiers[level]): - response_data[:, :, j] = np.reshape(clf(image_pixels), - image.shape) - return Image(image_data=response_data) + # Train each batch + self._train_batch(image_batch, increment, group=group, + verbose=verbose) - @property - def _str_title(self): + def train(self, images, group=None, verbose=False, batch_size=None): r""" - Returns a string containing name of the model. + """ + return self._train(images, False, group=group, verbose=verbose, + batch_size=batch_size) - : str + def increment(self, images, group=None, verbose=False, batch_size=None): + r""" """ - return 'Constrained Local Model' + return self._train(images, True, group=group, verbose=verbose, + batch_size=batch_size) - def view_shape_models_widget(self, n_parameters=5, mode='multiple', + def view_shape_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), - figure_size=(10, 8), style='coloured'): + mode='multiple', figure_size=(10, 8)): r""" - Visualizes the shape models of the CLM object using the + Visualizes the shape models of the AAM object using the `menpo.visualize.widgets.visualize_shape_model` widget. Parameters ----------- n_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the parameters - sliders. If `int`, then the number of sliders per level is the - minimum between `n_parameters` and the number of active components - per level. If `list` of `int`, then a number of sliders is defined - per level. If ``None``, all the active components per level will - have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + popup : `bool`, optional + If ``True``, the widget will appear as a popup window. figure_size : (`int`, `int`), optional The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. """ from menpofit.visualize import visualize_shape_model - visualize_shape_model( - self.shape_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, figure_size=figure_size, - mode=mode, style=style) + visualize_shape_model(self.shape_models, n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode,) - def __str__(self): - from menpofit.base import name_of_callable - out = "{}\n - {} training images.\n".format(self._str_title, - self.n_training_images) - # small strings about number of channels, channels string and downscale - down_str = [] - for j in range(self.n_levels): - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - temp_img = Image(image_data=np.random.rand(50, 50)) - if self.pyramid_on_features: - temp = self.features(temp_img) - n_channels = [temp.n_channels] * self.n_levels - else: - n_channels = [] - for j in range(self.n_levels): - temp = self.features[j](temp_img) - n_channels.append(temp.n_channels) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.patch_shape[1], self.patch_shape[0]) + # TODO: Implement me! + def view_expert_ensemble_widget(self): + r""" + """ + raise NotImplementedError - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.patch_shape[1], self.patch_shape[0]) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n - {3} {4} classifiers.\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100, - self.n_classifiers_per_level[i], - name_of_callable(self.classifiers[i][0])) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - {4} shape components ({5:.2f}% of " \ - "variance)\n - {6} {7} classifiers.".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100, - self.n_classifiers_per_level[0], - name_of_callable(self.classifiers[0][0])) - return out + # TODO: Implement me! + def view_clm_widget(self): + r""" + """ + raise NotImplementedError + + # TODO: Implement me! + def __str__(self): + r""" + """ + raise NotImplementedError From 8d08a7caeb4da705b2ab5a6f55528d1beae5bedb Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:31:55 +0100 Subject: [PATCH 159/423] Add dummy wrapper for correlation filters --- menpofit/clm/expert/base.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 menpofit/clm/expert/base.py diff --git a/menpofit/clm/expert/base.py b/menpofit/clm/expert/base.py new file mode 100644 index 0000000..c2f0d38 --- /dev/null +++ b/menpofit/clm/expert/base.py @@ -0,0 +1,28 @@ +import numpy as np +from menpofit.math.correlationfilter import mccf, imccf + + +# TODO: document me! +class IncrementalCorrelationFilterThinWrapper(object): + r""" + """ + def __init__(self, cf_callable=mccf, icf_callable=imccf): + self.cf_callable = cf_callable + self.icf_callable = icf_callable + + def increment(self, A, B, n_x, Z, t): + r""" + """ + # Turn list of X into ndarray + if isinstance(Z, list): + Z = np.asarray(Z) + return self.icf_callable(A, B, n_x, Z, t) + + def train(self, X, t): + r""" + """ + # Turn list of X into ndarray + if isinstance(X, list): + X = np.asarray(X) + # Return linear svm filter and bias + return self.cf_callable(X, t) From c6354b56a5a6c228cbc5e3f17eddf3e4effe4a9c Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:32:45 +0100 Subject: [PATCH 160/423] Add CorrelationFilterExpertEnsamble - Supports multichannel features. - Supports incremental training. --- menpofit/clm/expert/ensemble.py | 262 ++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 menpofit/clm/expert/ensemble.py diff --git a/menpofit/clm/expert/ensemble.py b/menpofit/clm/expert/ensemble.py new file mode 100644 index 0000000..a120345 --- /dev/null +++ b/menpofit/clm/expert/ensemble.py @@ -0,0 +1,262 @@ +from __future__ import division +from functools import partial +import numpy as np +from scipy.stats import multivariate_normal +from menpo.shape import PointCloud +from menpo.image import Image +from menpofit.base import build_grid +from menpofit.feature import normalize_norm, probability_map +from menpofit.math.fft_utils import ( + fft2, ifft2, fftshift, pad, crop, fft_convolve2d_sum) +from menpofit.visualize import print_progress +from .base import IncrementalCorrelationFilterThinWrapper + + +# TODO: Document me! +class ExpertEnsemble(object): + r""" + """ + + +# TODO: Document me! +# TODO: Should convolutional experts of ensembles support patch features? +class ConvolutionBasedExpertEnsemble(ExpertEnsemble): + r""" + """ + @property + def n_experts(self): + r""" + """ + return self.fft_padded_filters.shape[0] + + @property + def n_sample_offsets(self): + r""" + """ + if self.sample_offsets: + return self.sample_offsets.n_points + else: + return 1 + + @property + def padded_size(self): + r""" + """ + pad_size = np.floor(1.5 * np.asarray(self.patch_size) - 1).astype(int) + return tuple(pad_size) + + @property + def search_size(self): + r""" + """ + return self.patch_size + + def increment(self, images, shapes, prefix='', verbose=False): + r""" + """ + self._train(images, shapes, prefix=prefix, verbose=verbose, + increment=True) + + @property + def spatial_filter_images(self): + r""" + """ + filter_images = [] + for fft_padded_filter in self.fft_padded_filters: + spatial_filter = np.real(ifft2(fft_padded_filter)) + spatial_filter = crop(spatial_filter, + self.patch_size)[:, ::-1, ::-1] + filter_images.append(Image(spatial_filter)) + return filter_images + + @property + def frequency_filter_images(self): + r""" + """ + filter_images = [] + for fft_padded_filter in self.fft_padded_filters: + spatial_filter = np.real(ifft2(fft_padded_filter)) + spatial_filter = crop(spatial_filter, + self.patch_size)[:, ::-1, ::-1] + frequency_filter = np.abs(fftshift(fft2(spatial_filter))) + filter_images.append(Image(frequency_filter)) + return filter_images + + def _extract_patch(self, image, landmark): + r""" + """ + # Extract patch from image + patch = image.extract_patches( + landmark, patch_size=self.patch_size, + sample_offsets=self.sample_offsets, as_single_array=True) + # Reshape patch + # patch: (offsets x ch) x h x w + patch = patch.reshape((-1,) + patch.shape[-2:]) + # Normalise patch + return self.normalise_callable(patch) + + def _extract_patches(self, image, shape): + r""" + """ + # Obtain patch ensemble, the whole shape is used to extract patches + # from all landmarks at once + patches = image.extract_patches(shape, patch_size=self.patch_size, + sample_offsets=self.sample_offsets, + as_single_array=True) + # Reshape patches + # patches: n_patches x (n_offsets x n_channels) x height x width + patches = patches.reshape((patches.shape[0], -1) + patches.shape[-2:]) + # Normalise patches + return self.normalise_callable(patches) + + def predict_response(self, image, shape): + r""" + """ + # Extract patches + patches = self._extract_patches(image, shape) + # Predict responses + return fft_convolve2d_sum(patches, self.fft_padded_filters, + fft_filter=True, axis=1) + + def predict_probability(self, image, shape): + r""" + """ + # Predict responses + responses = self.predict_response(image, shape) + # Turn them into proper probability maps + return probability_map(responses) + + +# TODO: Document me! +class CorrelationFilterExpertEnsemble(ConvolutionBasedExpertEnsemble): + r""" + """ + def __init__(self, images, shapes, verbose=False, prefix='', + icf_cls=IncrementalCorrelationFilterThinWrapper, + patch_size=(17, 17), context_size=(34, 34), + response_covariance=3, normalise_callable=normalize_norm, + cosine_mask=True, sample_offsets=None): + # TODO: check parameters? + # Set parameters + self._icf = icf_cls() + self.patch_size = patch_size + self.context_size = context_size + self.response_covariance = response_covariance + self.normalise_callable = normalise_callable + self.cosine_mask = cosine_mask + self.sample_offsets = sample_offsets + + # Generate cosine mask + self._cosine_mask = generate_cosine_mask(self.context_size) + + # Generate desired response, i.e. a Gaussian response with the + # specified covariance centred at the middle of the patch + self.response = generate_gaussian_response( + self.patch_size, self.response_covariance)[None, ...] + + # Train ensemble of correlation filter experts + self._train(images, shapes, verbose=verbose, prefix=prefix) + + def _extract_patch(self, image, landmark): + r""" + """ + # Extract patch from image + patch = image.extract_patches( + landmark, patch_size=self.context_size, + sample_offsets=self.sample_offsets, as_single_array=True) + # Reshape patch + # patch: (offsets x ch) x h x w + patch = patch.reshape((-1,) + patch.shape[-2:]) + # Normalise patch + patch = self.normalise_callable(patch) + if self.cosine_mask: + # Apply cosine mask if require + patch = self._cosine_mask * patch + return patch + + def _train(self, images, shapes, prefix='', verbose=False, + increment=False): + r""" + """ + # Define print_progress partial + wrap = partial(print_progress, + prefix='{}Training experts' + .format(prefix), + end_with_newline=not prefix, + verbose=verbose) + + # If increment is False, we need to initialise/reset the ensemble of + # experts + if not increment: + self.fft_padded_filters = [] + self.auto_correlations = [] + self.cross_correlations = [] + # Set number of images + self.n_images = len(images) + else: + # Update number of images + self.n_images += len(images) + + # Obtain total number of experts + n_experts = shapes[0].n_points + + # Train ensemble of correlation filter experts + fft_padded_filters = [] + auto_correlations = [] + cross_correlations = [] + for i in wrap(range(n_experts)): + patches = [] + for image, shape in zip(images, shapes): + # Select the appropriate landmark + landmark = PointCloud([shape.points[i]]) + # Extract patch + patch = self._extract_patch(image, landmark) + # Add patch to the list + patches.append(patch) + + if increment: + # Increment correlation filter + correlation_filter, auto_correlation, cross_correlation = ( + self._icf.increment(self.auto_correlations[i], + self.cross_correlations[i], + self.n_images, + patches, + self.response)) + else: + # Train correlation filter + correlation_filter, auto_correlation, cross_correlation = ( + self._icf.train(patches, self.response)) + + # Pad filter with zeros + padded_filter = pad(correlation_filter, self.padded_size) + # Compute fft of padded filter + fft_padded_filter = fft2(padded_filter) + # Add fft padded filter to list + fft_padded_filters.append(fft_padded_filter) + auto_correlations.append(auto_correlation) + cross_correlations.append(cross_correlation) + + # Turn list into ndarray + self.fft_padded_filters = np.asarray(fft_padded_filters) + self.auto_correlations = np.asarray(auto_correlations) + self.cross_correlations = np.asarray(cross_correlations) + + +# TODO: Document me! +def generate_gaussian_response(patch_size, response_covariance): + r""" + """ + grid = build_grid(patch_size) + mvn = multivariate_normal(mean=np.zeros(2), cov=response_covariance) + return mvn.pdf(grid) + + +# TODO: Document me! +def generate_cosine_mask(patch_size): + r""" + """ + cy = np.hanning(patch_size[0]) + cx = np.hanning(patch_size[1]) + return cy[..., None].dot(cx[None, ...]) + + From f4821047c2f077038aa632205e356d9834d43ac3 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:34:09 +0100 Subject: [PATCH 161/423] Add __init__.py --- menpofit/clm/expert/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 menpofit/clm/expert/__init__.py diff --git a/menpofit/clm/expert/__init__.py b/menpofit/clm/expert/__init__.py new file mode 100644 index 0000000..393b44d --- /dev/null +++ b/menpofit/clm/expert/__init__.py @@ -0,0 +1 @@ +from ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble From 28fe3d41f54648e12ca7cc45465d0bda9b5bd3e0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:36:03 +0100 Subject: [PATCH 162/423] Add CLMFitters --- menpofit/clm/fitter.py | 297 +++++------------------------------------ 1 file changed, 36 insertions(+), 261 deletions(-) diff --git a/menpofit/clm/fitter.py b/menpofit/clm/fitter.py index 4ad4a56..bf6e499 100644 --- a/menpofit/clm/fitter.py +++ b/menpofit/clm/fitter.py @@ -1,272 +1,47 @@ -from __future__ import division -import numpy as np -from menpo.image import Image +from menpofit import checks +from menpofit.fitter import ModelFitter +from menpofit.modelinstance import OrthoPDM +from .algorithm import CLMAlgorithm, RegularisedLandmarkMeanShift +from .result import CLMFitterResult -from menpofit.transform import DifferentiableAlignmentSimilarity -from menpofit.modelinstance import PDM, OrthoPDM -from menpofit.fitter import MultilevelFitter -from menpofit.gradientdescent import RLMS - -class CLMFitter(MultilevelFitter): +# TODO: Document me! +class CLMFitter(ModelFitter): r""" - Abstract Interface for defining Constrained Local Models Fitters. - - Parameters - ----------- - clm : :map:`CLM` - The Constrained Local Model to be used. """ - def __init__(self, clm): - self.clm = clm - - @property - def reference_shape(self): - r""" - The reference shape of the CLM. - - :type: :map:`PointCloud` - """ - return self.clm.reference_shape - - @property - def features(self): - r""" - The feature extracted at each pyramidal level during CLM building. - Stored in ascending pyramidal order. - - :type: `list` - """ - return self.clm.features - @property - def n_levels(self): - r""" - The number of pyramidal levels used during CLM building. + def clm(self): + return self._model - :type: `int` - """ - return self.clm.n_levels - - @property - def downscale(self): - r""" - The downscale used to generate the final scale factor applied at - each pyramidal level during CLM building. - The scale factor is computed as: - - ``(downscale ** k) for k in range(n_levels)`` - - :type: `float` - """ - return self.clm.downscale + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return CLMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) +# TODO: Document me! +# TODO: Rethink shape model and OrthoPDM relation class GradientDescentCLMFitter(CLMFitter): r""" - Gradient Descent based :map:`Fitter` for Constrained Local Models. - - Parameters - ----------- - clm : :map:`CLM` - The Constrained Local Model to be used. - - algorithm : subclass :map:`GradientDescent`, optional - The :map:`GradientDescent` class to be used. - - pdm_transform : :map:`GlobalPDM` or subclass, optional - The point distribution class to be used. - - .. note:: - - Only :map:`GlobalPDM` and its subclasses are supported. - :map:`PDM` is not supported at the moment. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of shape components - capturing the specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. """ - def __init__(self, clm, algorithm=RLMS, - pdm_transform=OrthoPDM, n_shape=None, **kwargs): - super(GradientDescentCLMFitter, self).__init__(clm) - self._set_up(algorithm=algorithm, pdm_transform=pdm_transform, - n_shape=n_shape, **kwargs) - - @property - def algorithm(self): - r""" - Returns a string containing the name of fitting algorithm. - - :type: `string` - """ - return 'GD-CLM-' + self._fitters[0].algorithm - - def _set_up(self, algorithm=RLMS, - pdm_transform=OrthoPDM, - global_transform=DifferentiableAlignmentSimilarity, - n_shape=None, **kwargs): - r""" - Sets up the Gradient Descent Fitter object. - - Parameters - ----------- - algorithm : :map:`GradientDescent`, optional - The Gradient Descent class to be used. - - pdm_transform : :map:`GlobalPDM` or subclass, optional - The point distribution class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per fitting level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - """ - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.clm.n_levels > 1: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.clm.n_levels: - for sm, n in zip(self.clm.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None' - 'or a list containing 1 or {} of ' - 'those'.format(self.clm.n_levels)) - - self._fitters = [] - for j, (sm, clf) in enumerate(zip(self.clm.shape_models, - self.clm.classifiers)): - - if pdm_transform is not PDM: - pdm_trans = pdm_transform(sm, global_transform) - else: - pdm_trans = pdm_transform(sm) - self._fitters.append(algorithm(clf, self.clm.patch_shape, - pdm_trans, **kwargs)) - - def __str__(self): - from menpofit.base import name_of_callable - out = "{0} Fitter\n" \ - " - Gradient-Descent {1}\n" \ - " - Transform is {2}.\n" \ - " - {3} training images.\n".format( - self.clm._str_title, self._fitters[0].algorithm, - self._fitters[0].transform.__class__.__name__, - self.clm.n_training_images) - # small strings about number of channels, channels string and downscale - down_str = [] - for j in range(self.n_levels): - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - temp_img = Image(image_data=np.random.rand(50, 50)) - if self.pyramid_on_features: - temp = self.features(temp_img) - n_channels = [temp.n_channels] * self.n_levels - else: - n_channels = [] - for j in range(self.n_levels): - temp = self.features[j](temp_img) - n_channels.append(temp.n_channels) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.clm.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.clm.patch_shape[1], - self.clm.patch_shape[0]) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.clm.patch_shape[1], - self.clm.patch_shape[0]) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - out = "{0} - {1} motion components\n - {2} {3} " \ - "classifiers.\n".format( - out, self._fitters[i].transform.n_parameters, - len(self._fitters[i].classifiers), - name_of_callable(self._fitters[i].classifiers[0])) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - {4} motion components\n - {5} {6} " \ - "classifiers.".format( - out, feat_str[0], n_channels[0], ch_str[0], - out, self._fitters[0].transform.n_parameters, - len(self._fitters[0].classifiers), - name_of_callable(self._fitters[0].classifiers[0])) - return out + def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, + n_shape=None): + self._model = clm + self._gd_algorithms_cls = checks.check_algorithm_cls( + gd_algorithm_cls, self.n_scales, CLMAlgorithm) + self._check_n_shape(n_shape) + + # Construct algorithms + self.algorithms = [] + for i in range(self.clm.n_scales): + pdm = OrthoPDM(self.clm.shape_models[i]) + algorithm = self._gd_algorithms_cls[i]( + self.clm.expert_ensembles[i], pdm) + self.algorithms.append(algorithm) + + +# TODO: Implement me! +# TODO: Document me! +class SupervisedDescentCLMFitter(CLMFitter): + r""" + """ From 88054a3f1e05dfe8bd21ce4398c3afad13c97cee Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:38:03 +0100 Subject: [PATCH 163/423] Add __init__.py to menpofit.clm - Also modified initi from expert --- menpofit/clm/__init__.py | 4 +++- menpofit/clm/expert/__init__.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) mode change 100755 => 100644 menpofit/clm/__init__.py diff --git a/menpofit/clm/__init__.py b/menpofit/clm/__init__.py old mode 100755 new mode 100644 index 18186e9..83b81e9 --- a/menpofit/clm/__init__.py +++ b/menpofit/clm/__init__.py @@ -1,3 +1,5 @@ from .base import CLM -from .builder import CLMBuilder from .fitter import GradientDescentCLMFitter +from .algorithm import RegularisedLandmarkMeanShift +from .expert import ( + CorrelationFilterExpertEnsemble, IncrementalCorrelationFilterThinWrapper) diff --git a/menpofit/clm/expert/__init__.py b/menpofit/clm/expert/__init__.py index 393b44d..673e02b 100644 --- a/menpofit/clm/expert/__init__.py +++ b/menpofit/clm/expert/__init__.py @@ -1 +1,2 @@ from ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble +from base import IncrementalCorrelationFilterThinWrapper From fbce3c6aecd61a9bb54681bb7dfb6cab1f800e3e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 31 Jul 2015 16:29:39 +0100 Subject: [PATCH 164/423] Refactoring SD-AAM Derive directly from the SDM - abstract out the check appearance model and shape model methods so that we don't need to derive from the AAMFitter as well. Some more, heavier refactoring is about to happen to pull the SD-AAM in line with the new and improved SDM --- menpofit/aam/fitter.py | 140 ++++++++++++----------------------------- menpofit/checks.py | 16 +++++ menpofit/fitter.py | 25 +++----- 3 files changed, 63 insertions(+), 118 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 8fe78c2..58d3404 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,12 +1,11 @@ from __future__ import division import numpy as np from copy import deepcopy -from menpo.transform import Scale, AlignmentUniformScale +from menpo.transform import AlignmentUniformScale from menpo.image import BooleanImage -from menpofit.builder import ( - rescale_images_to_reference_shape, compute_features, scale_images) -from menpofit.fitter import ModelFitter +from menpofit.fitter import ModelFitter, noisy_shape_from_bounding_box from menpofit.modelinstance import OrthoPDM +from menpofit.sdm import SupervisedDescentFitter from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM @@ -28,20 +27,7 @@ def aam(self): return self._model def _check_n_appearance(self, n_appearance): - if n_appearance is not None: - if type(n_appearance) is int or type(n_appearance) is float: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_scales > 1: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_scales: - for am, n in zip(self.aam.appearance_models, n_appearance): - am.n_active_components = n - else: - raise ValueError('n_appearance can be an integer or a float ' - 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_scales)) + checks.set_models_components(self.aam.appearance_models, n_appearance) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): @@ -106,25 +92,39 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): # TODO: document me! -class SupervisedDescentAAMFitter(AAMFitter): +class SupervisedDescentAAMFitter(SupervisedDescentFitter): r""" """ - def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, + def __init__(self, images, aam, group=None, bounding_box_group=None, n_shape=None, n_appearance=None, sampling=None, - n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): - self._model = aam - self._check_n_shape(n_shape) - self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_scales) - self.n_perturbations = n_perturbations - self.noise_std = noise_std - self.max_iters = checks.check_max_iters(max_iters, self.n_scales) - self._set_up(sd_algorithm_cls, sampling, **kwargs) - - def _set_up(self, sd_algorithm_cls, sampling, **kwargs): + sd_algorithm_cls=ProjectOutNewton, + n_iterations=6, n_perturbations=30, + perturb_from_bounding_box=noisy_shape_from_bounding_box, + batch_size=None, verbose=False): + self.aam = aam + checks.set_models_components(aam.appearance_models, n_appearance) + checks.set_models_components(aam.shape_models, n_shape) + self._sampling = checks.check_sampling(sampling, aam.n_scales) + + # patch_feature and patch_shape are not actually + # used because they are fully defined by the AAM already. Therefore, + # we just leave them as their 'defaults' because they won't be used. + super(SupervisedDescentAAMFitter, self).__init__( + images, group=group, bounding_box_group=bounding_box_group, + reference_shape=self.aam.reference_shape, + sd_algorithm_cls=sd_algorithm_cls, + holistic_feature=self.aam.features, + diagonal=self.aam.diagonal, + scales=self.aam.scales, n_iterations=n_iterations, + n_perturbations=n_perturbations, + perturb_from_bounding_box=perturb_from_bounding_box, + batch_size=batch_size, verbose=verbose) + + def _setup_algorithms(self): self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models, sampling)): + self.aam.shape_models, + self._sampling)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform @@ -132,9 +132,9 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = sd_algorithm_cls( + algorithm = self._sd_algorithm_cls( SupervisedDescentStandardInterface, am, md_transform, - sampling=s, max_iters=self.max_iters[j], **kwargs) + sampling=s, max_iters=self.n_iterations[j]) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -142,19 +142,19 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) # set up algorithm using linear aam interface - algorithm = sd_algorithm_cls( + algorithm = self._sd_algorithm_cls( SupervisedDescentLinearInterface, am, md_transform, - sampling=s, max_iters=self.max_iters[j], **kwargs) + sampling=s, max_iters=self.n_iterations[j]) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - algorithm = sd_algorithm_cls( + algorithm = self._sd_algorithm_cls( SupervisedDescentPartsInterface, am, pdm, - sampling=s, max_iters=self.max_iters[j], + sampling=s, max_iters=self.n_iterations[j], patch_shape=self.aam.patch_shape[j], - normalize_parts=self.aam.normalize_parts, **kwargs) + normalize_parts=self.aam.normalize_parts) else: raise ValueError("AAM object must be of one of the " @@ -165,67 +165,6 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): # append algorithms to list self.algorithms.append(algorithm) - # TODO: Allow training from bounding boxes - def train(self, images, group=None, verbose=False, **kwargs): - # normalize images with respect to reference shape of aam - images = rescale_images_to_reference_shape( - images, group, self.reference_shape, verbose=verbose) - - if self.scale_features: - # compute features at highest level - feature_images = compute_features(images, self.features[0], - verbose=verbose) - - # for each pyramid level (low --> high) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if s == self.scales[-1]: - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) - - # extract ground truth shapes for current level - level_gt_shapes = [i.landmarks[group].lms for i in level_images] - - if j == 0: - # generate perturbed shapes - current_shapes = [] - for gt_s in level_gt_shapes: - perturbed_shapes = [] - for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape(gt_s, self.noise_std) - perturbed_shapes.append(p_s) - current_shapes.append(perturbed_shapes) - - # train cascaded regression algorithm - current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) - - # scale current shapes to next level resolution - if s != self.scales[-1]: - transform = Scale(self.scales[j+1]/s, n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) - # TODO: Document me! def holistic_sampling_from_scale(aam, scale=0.35): @@ -247,6 +186,7 @@ def holistic_sampling_from_scale(aam, scale=0.35): return true_positions, BooleanImage(modified_mask[0]) +# TODO: Document me! def holistic_sampling_from_step(aam, step=8): reference = aam.appearance_models[0].mean() diff --git a/menpofit/checks.py b/menpofit/checks.py index 0547791..3c9a516 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -145,3 +145,19 @@ def check_sampling(sampling, n_levels): 'None'.format(n_levels)) +def set_models_components(models, n_components): + if n_components is not None: + n_scales = len(models) + if type(n_components) is int or type(n_components) is float: + for am in models: + am.n_active_components = n_components + elif len(n_components) == 1 and n_scales > 1: + for am in models: + am.n_active_components = n_components[0] + elif len(n_components) == n_scales: + for am, n in zip(models, n_components): + am.n_active_components = n + else: + raise ValueError('n_components can be an integer or a float ' + 'or None or a list containing 1 or {} of ' + 'those'.format(n_scales)) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index d963fda..07f0ec8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -255,29 +255,18 @@ def scales(self): return self._model.scales def _check_n_shape(self, n_shape): - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self._model.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self._model.n_scales > 1: - for sm in self._model.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self._model.n_scales: - for sm, n in zip(self._model.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None' - 'or a list containing 1 or {} of ' - 'those'.format(self._model.n_scales)) + checks.set_models_components(self._model.shape_models, n_shape) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): + def noisy_shape_from_bounding_box(self, bounding_box, + noise_percentage=0.05): transform = noisy_alignment_similarity_transform( - self.reference_bounding_box, bounding_box, noise_std=noise_std) + self.reference_bounding_box, bounding_box, + noise_percentage=noise_percentage) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.05): + def noisy_shape_from_shape(self, shape, noise_percentage=0.05): return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std) + shape.bounding_box(), noise_percentage=noise_percentage) def noisy_alignment_similarity_transform(source, target, noise_type='uniform', From 1e65502e97aaa4b0033483563c9c0f6e2953afe9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:31:10 +0100 Subject: [PATCH 165/423] Add clm base algorithm --- menpofit/clm/algorithm/base.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 menpofit/clm/algorithm/base.py diff --git a/menpofit/clm/algorithm/base.py b/menpofit/clm/algorithm/base.py new file mode 100644 index 0000000..7966c2a --- /dev/null +++ b/menpofit/clm/algorithm/base.py @@ -0,0 +1,6 @@ + + +# TODO: document me! +class CLMAlgorithm(object): + r""" + """ From 1cbc8a2693a1176c42a6add06899139afae09634 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:37:04 +0100 Subject: [PATCH 166/423] Add clm algorithms - ASM and RLMS --- menpofit/clm/algorithm/base.py | 6 - menpofit/clm/algorithm/gd.py | 237 +++++++++++++++++++++++++++++++++ menpofit/clm/fitter.py | 7 +- 3 files changed, 241 insertions(+), 9 deletions(-) delete mode 100644 menpofit/clm/algorithm/base.py create mode 100644 menpofit/clm/algorithm/gd.py diff --git a/menpofit/clm/algorithm/base.py b/menpofit/clm/algorithm/base.py deleted file mode 100644 index 7966c2a..0000000 --- a/menpofit/clm/algorithm/base.py +++ /dev/null @@ -1,6 +0,0 @@ - - -# TODO: document me! -class CLMAlgorithm(object): - r""" - """ diff --git a/menpofit/clm/algorithm/gd.py b/menpofit/clm/algorithm/gd.py new file mode 100644 index 0000000..6a7539b --- /dev/null +++ b/menpofit/clm/algorithm/gd.py @@ -0,0 +1,237 @@ +from __future__ import division +import numpy as np +from menpofit.base import build_grid +from menpofit.clm.result import CLMAlgorithmResult + +multivariate_normal = None # expensive, from scipy.stats + + +# TODO: document me! +class GradientDescentCLMAlgorithm(object): + r""" + """ + + +# TODO: Document me! +class ActiveShapeModel(GradientDescentCLMAlgorithm): + r""" + Active Shape Model (ASM) algorithm + """ + def __init__(self, expert_ensemble, shape_model, gaussian_covariance=10, + eps=10**-5): + # Set parameters + self.expert_ensemble, = expert_ensemble, + self.transform = shape_model + self.gaussian_covariance = gaussian_covariance + self.eps = eps + # Perform pre-computations + self._precompute() + + def _precompute(self): + r""" + """ + # Import multivariate normal distribution from scipy + global multivariate_normal + if multivariate_normal is None: + from scipy.stats import multivariate_normal # expensive + + # Build grid associated to size of the search space + search_size = self.expert_ensemble.search_size + self.half_search_size = np.round( + np.asarray(self.expert_ensemble.search_size) / 2) + self.search_grid = build_grid(search_size)[None, None] + + # set rho2 + self.rho2 = self.transform.model.noise_variance() + + # Compute Gaussian-KDE grid + self.mvn = multivariate_normal(mean=np.zeros(2), + cov=self.gaussian_covariance) + + # Compute shape model prior + sim_prior = np.zeros((4,)) + pdm_prior = self.rho2 / self.transform.model.eigenvalues + self.rho2_inv_L = np.hstack((sim_prior, pdm_prior)) + + # Compute Jacobian + J = np.rollaxis(self.transform.d_dp(None), -1, 1) + self.J = J.reshape((-1, J.shape[-1])) + # Compute inverse Hessian + self.JJ = self.J.T.dot(self.J) + # Compute Jacobian pseudo-inverse + self.pinv_J = np.linalg.solve(self.JJ, self.J.T) + self.inv_JJ_prior = np.linalg.inv(self.JJ + np.diag(self.rho2_inv_L)) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + r""" + """ + # Initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # Initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Expectation-Maximisation loop + while k < max_iters and eps > self.eps: + + target = self.transform.target + # Obtain all landmark positions l_i = (x_i, y_i) being considered + # ie all pixel positions in each landmark's search space + candidate_landmarks = (target.points[:, None, None, None, :] + + self.search_grid) + + # Compute responses + responses = self.expert_ensemble.predict_probability(image, target) + + # Approximate responses using isotropic Gaussian + max_indices = np.argmax( + responses.reshape(responses.shape[:2] + (-1,)), axis=-1) + max_indices = np.unravel_index(max_indices, responses.shape)[-2:] + max_indices = np.hstack((max_indices[0], max_indices[1])) + max_indices = max_indices[:, None, None, None, ...] + max_indices -= self.half_search_size + gaussian_responses = self.mvn.pdf(max_indices + self.search_grid) + # Normalise smoothed responses + gaussian_responses /= np.sum(gaussian_responses, + axis=(-2, -1))[..., None, None] + + # Compute new target + new_target = np.sum(gaussian_responses[:, None, ..., None] * + candidate_landmarks, axis=(-3, -2)) + + # Compute shape error term + error = target.as_vector() - new_target.ravel() + + # Solve for increments on the shape parameters + if map_inference: + Je = (self.rho2_inv_L * self.transform.as_vector() - + self.J.T.dot(error)) + dp = -self.inv_JJ_prior.dot(Je) + else: + dp = self.pinv_J.dot(error) + + # Update pdm + s_k = self.transform.target.points + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # Test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # Increase iteration counter + k += 1 + + # Return algorithm result + return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) + + +# TODO: Document me! +class RegularisedLandmarkMeanShift(GradientDescentCLMAlgorithm): + r""" + Regularized Landmark Mean-Shift (RLMS) algorithm + """ + def __init__(self, expert_ensemble, shape_model, kernel_covariance=10, + eps=10**-5): + # Set parameters + self.expert_ensemble, = expert_ensemble, + self.transform = shape_model + self.kernel_covariance = kernel_covariance + self.eps = eps + # Perform pre-computations + self._precompute() + + def _precompute(self): + r""" + """ + # Import multivariate normal distribution from scipy + global multivariate_normal + if multivariate_normal is None: + from scipy.stats import multivariate_normal # expensive + + # Build grid associated to size of the search space + search_size = self.expert_ensemble.search_size + self.search_grid = build_grid(search_size) + + # set rho2 + self.rho2 = self.transform.model.noise_variance() + + # Compute Gaussian-KDE grid + mvn = multivariate_normal(mean=np.zeros(2), cov=self.kernel_covariance) + self.kernel_grid = mvn.pdf(self.search_grid)[None, None] + + # Compute shape model prior + sim_prior = np.zeros((4,)) + pdm_prior = self.rho2 / self.transform.model.eigenvalues + self.rho2_inv_L = np.hstack((sim_prior, pdm_prior)) + + # Compute Jacobian + J = np.rollaxis(self.transform.d_dp(None), -1, 1) + self.J = J.reshape((-1, J.shape[-1])) + # Compute inverse Hessian + self.JJ = self.J.T.dot(self.J) + # Compute Jacobian pseudo-inverse + self.pinv_J = np.linalg.solve(self.JJ, self.J.T) + self.inv_JJ_prior = np.linalg.inv(self.JJ + np.diag(self.rho2_inv_L)) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + r""" + """ + # Initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # Initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Expectation-Maximisation loop + while k < max_iters and eps > self.eps: + + target = self.transform.target + # Obtain all landmark positions l_i = (x_i, y_i) being considered + # ie all pixel positions in each landmark's search space + candidate_landmarks = (target.points[:, None, None, None, :] + + self.search_grid) + + # Compute patch responses + patch_responses = self.expert_ensemble.predict_probability(image, + target) + + # Smooth responses using the Gaussian-KDE grid + patch_kernels = patch_responses * self.kernel_grid + # Normalise smoothed responses + patch_kernels /= np.sum(patch_kernels, + axis=(-2, -1))[..., None, None] + + # Compute mean shift target + mean_shift_target = np.sum(patch_kernels[..., None] * + candidate_landmarks, axis=(-3, -2)) + + # Compute shape error term + error = mean_shift_target.ravel() - target.as_vector() + + # Solve for increments on the shape parameters + if map_inference: + Je = (self.rho2_inv_L * self.transform.as_vector() - + self.J.T.dot(error)) + dp = -self.inv_JJ_prior.dot(Je) + else: + dp = self.pinv_J.dot(error) + + # Update pdm + s_k = self.transform.target.points + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # Test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # Increase iteration counter + k += 1 + + # Return algorithm result + return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) \ No newline at end of file diff --git a/menpofit/clm/fitter.py b/menpofit/clm/fitter.py index bf6e499..c83c223 100644 --- a/menpofit/clm/fitter.py +++ b/menpofit/clm/fitter.py @@ -1,7 +1,8 @@ from menpofit import checks from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM -from .algorithm import CLMAlgorithm, RegularisedLandmarkMeanShift +from .algorithm import ( + GradientDescentCLMAlgorithm, RegularisedLandmarkMeanShift) from .result import CLMFitterResult @@ -28,10 +29,9 @@ def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, n_shape=None): self._model = clm self._gd_algorithms_cls = checks.check_algorithm_cls( - gd_algorithm_cls, self.n_scales, CLMAlgorithm) + gd_algorithm_cls, self.n_scales, GradientDescentCLMAlgorithm) self._check_n_shape(n_shape) - # Construct algorithms self.algorithms = [] for i in range(self.clm.n_scales): pdm = OrthoPDM(self.clm.shape_models[i]) @@ -45,3 +45,4 @@ def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, class SupervisedDescentCLMFitter(CLMFitter): r""" """ + raise NotImplementedError From 4d257507d7a6c18a64163d77065ccdb358502edc Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:37:40 +0100 Subject: [PATCH 167/423] Add not implemented supervised descent algorthm for clms - Future template for methods like drmf-clm --- menpofit/clm/algorithm/sd.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 menpofit/clm/algorithm/sd.py diff --git a/menpofit/clm/algorithm/sd.py b/menpofit/clm/algorithm/sd.py new file mode 100644 index 0000000..a77b245 --- /dev/null +++ b/menpofit/clm/algorithm/sd.py @@ -0,0 +1,8 @@ + + +# TODO: implement me! +# TODO: document me! +class SupervisedDescentCLMAlgorithm(object): + r""" + """ + raise NotImplementedError From dcab79c71b6d7eb29b022b2b92889429aa2fff57 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:39:01 +0100 Subject: [PATCH 168/423] Add __init__.py --- menpofit/clm/algorithm/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 menpofit/clm/algorithm/__init__.py diff --git a/menpofit/clm/algorithm/__init__.py b/menpofit/clm/algorithm/__init__.py new file mode 100644 index 0000000..1df3fab --- /dev/null +++ b/menpofit/clm/algorithm/__init__.py @@ -0,0 +1,3 @@ +from gd import ( + GradientDescentCLMAlgorithm, ActiveShapeModel, + RegularisedLandmarkMeanShift) From 4984963831dd4a638dc57e22d63568bfd127cf74 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:48:10 +0100 Subject: [PATCH 169/423] Update __init__.py --- menpofit/clm/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/clm/__init__.py b/menpofit/clm/__init__.py index 83b81e9..9406f54 100644 --- a/menpofit/clm/__init__.py +++ b/menpofit/clm/__init__.py @@ -1,5 +1,5 @@ from .base import CLM from .fitter import GradientDescentCLMFitter -from .algorithm import RegularisedLandmarkMeanShift +from .algorithm import ActiveShapeModel, RegularisedLandmarkMeanShift from .expert import ( CorrelationFilterExpertEnsemble, IncrementalCorrelationFilterThinWrapper) From 3bb215b28a1f0b2ef58ab76db087904bd6bc4633 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:10:31 +0100 Subject: [PATCH 170/423] Change n_levels for n_scales in aam --- menpofit/aam/base.py | 30 ++++++------ menpofit/aam/builder.py | 104 ++++++++++++++++++++-------------------- menpofit/aam/fitter.py | 25 +++++----- 3 files changed, 80 insertions(+), 79 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index ee97abe..3fe1c63 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -29,7 +29,7 @@ class AAM(object): constructed. features : `callable` or ``[callable]``, - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -61,7 +61,7 @@ def __init__(self, shape_models, appearance_models, reference_shape, self.scale_features = scale_features @property - def n_levels(self): + def n_scales(self): """ The number of scale levels of the AAM. @@ -274,14 +274,14 @@ def __str__(self): # small strings about number of channels, channels string and downscale n_channels = [] down_str = [] - for j in range(self.n_levels): + for j in range(self.n_scales): n_channels.append( self.appearance_models[j].template_instance.n_channels) - if j == self.n_levels - 1: + if j == self.n_scales - 1: down_str.append('(no downscale)') else: down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) + self.downscale**(self.n_scales - j - 1))) # string about features and channels if self.pyramid_on_features: feat_str = "- Feature is {} with ".format( @@ -293,7 +293,7 @@ def __str__(self): else: feat_str = [] ch_str = [] - for j in range(self.n_levels): + for j in range(self.n_scales): feat_str.append("- Feature is {} with ".format( name_of_callable(self.features[j]))) if n_channels[j] == 1: @@ -301,17 +301,17 @@ def __str__(self): else: ch_str.append("channels") out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_levels > 1: + if self.n_scales > 1: if self.scaled_shape_models: out = "{} - Gaussian pyramid with {} levels and downscale " \ "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, + "model (reference frame).\n".format(out, self.n_scales, self.downscale) else: out = "{} - Gaussian pyramid with {} levels and downscale " \ "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, + "are not scaled.\n".format(out, self.n_scales, self.downscale) if self.pyramid_on_features: out = "{} - Pyramid was applied on feature space.\n " \ @@ -329,8 +329,8 @@ def __str__(self): else: out = "{} - Features were extracted at each pyramid " \ "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, + for i in range(self.n_scales - 1, -1, -1): + out = "{} - Level {} {}: \n".format(out, self.n_scales - i, down_str[i]) if not self.pyramid_on_features: out = "{} {}{} {} per image.\n".format( @@ -392,7 +392,7 @@ class PatchAAM(AAM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -482,7 +482,7 @@ class LinearAAM(AAM): constructed. features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -557,7 +557,7 @@ class LinearPatchAAM(AAM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -635,7 +635,7 @@ class PartsAAM(AAM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 717a813..a85dfa6 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -23,7 +23,7 @@ class AAMBuilder(object): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -65,7 +65,7 @@ class AAMBuilder(object): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -82,7 +82,7 @@ class AAMBuilder(object): (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -127,13 +127,13 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, max_shape_components=None, max_appearance_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, len(scales), 'max_appearance_components') # set parameters self.features = features self.transform = transform @@ -186,21 +186,21 @@ def build(self, images, group=None, verbose=False): if j == 0: # compute features at highest level feature_images = compute_features(images, self.features[j], - level_str=level_str, + prefix=level_str, verbose=verbose) level_images = feature_images elif self.scale_features: # scale features at other levels level_images = scale_images(feature_images, s, - level_str=level_str, + prefix=level_str, verbose=verbose) else: # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, + scaled_images = scale_images(images, s, prefix=level_str, verbose=verbose) level_images = compute_features(scaled_images, self.features[j], - level_str=level_str, + prefix=level_str, verbose=verbose) # extract potentially rescaled shapes @@ -271,21 +271,21 @@ def increment(self, aam, images, group=None, if j == 0: # compute features at highest level feature_images = compute_features(images, self.features[j], - level_str=level_str, + prefix=level_str, verbose=verbose) level_images = feature_images elif self.scale_features: # scale features at other levels level_images = scale_images(feature_images, s, - level_str=level_str, + prefix=level_str, verbose=verbose) else: # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, + scaled_images = scale_images(images, s, prefix=level_str, verbose=verbose) level_images = compute_features(scaled_images, self.features[j], - level_str=level_str, + prefix=level_str, verbose=verbose) # extract potentially rescaled shapes @@ -361,7 +361,7 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): reference_frame = build_reference_frame(reference_shape) return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) + prefix=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return AAM(shape_models, appearance_models, reference_shape, @@ -379,7 +379,7 @@ class PatchAAMBuilder(AAMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -413,7 +413,7 @@ class PatchAAMBuilder(AAMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -430,7 +430,7 @@ class PatchAAMBuilder(AAMBuilder): (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -478,14 +478,15 @@ def __init__(self, patch_shape=(17, 17), features=no_op, max_appearance_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, len(scales), + 'max_appearance_components') # set parameters self.patch_shape = patch_shape self.features = features @@ -502,7 +503,7 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, reference_frame = build_patch_reference_frame( reference_shape, patch_shape=self.patch_shape[level]) return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) + prefix=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return PatchAAM(shape_models, appearance_models, reference_shape, @@ -518,7 +519,7 @@ class LinearAAMBuilder(AAMBuilder): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -560,7 +561,7 @@ class LinearAAMBuilder(AAMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -577,7 +578,7 @@ class LinearAAMBuilder(AAMBuilder): (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -622,13 +623,13 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, max_shape_components=None, max_appearance_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, len(scales), 'max_appearance_components') # set parameters self.features = features self.transform = transform @@ -654,7 +655,7 @@ def _build_shape_model(self, shapes, max_components, level): def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, + self.transform, prefix=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): @@ -675,7 +676,7 @@ class LinearPatchAAMBuilder(AAMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -709,7 +710,7 @@ class LinearPatchAAMBuilder(AAMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -726,7 +727,7 @@ class LinearPatchAAMBuilder(AAMBuilder): (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -774,14 +775,15 @@ def __init__(self, patch_shape=(17, 17), features=no_op, max_appearance_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, len(scales), + 'max_appearance_components') # set parameters self.patch_shape = patch_shape self.features = features @@ -808,7 +810,7 @@ def _build_shape_model(self, shapes, max_components, level): def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, + self.transform, prefix=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): @@ -829,7 +831,7 @@ class PartsAAMBuilder(AAMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -865,7 +867,7 @@ class PartsAAMBuilder(AAMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -882,7 +884,7 @@ class PartsAAMBuilder(AAMBuilder): (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -930,14 +932,14 @@ def __init__(self, patch_shape=(17, 17), features=no_op, max_shape_components=None, max_appearance_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, len(scales), 'max_appearance_components') # set parameters self.patch_shape = patch_shape self.features = features @@ -953,7 +955,7 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return extract_patches(images, shapes, self.patch_shape[level], normalize_function=self.normalize_parts, - level_str=level_str, verbose=verbose) + prefix=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return PartsAAM(shape_models, appearance_models, reference_shape, diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 24fe85c..46d7e2e 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -32,16 +32,16 @@ def _check_n_appearance(self, n_appearance): if type(n_appearance) is int or type(n_appearance) is float: for am in self.aam.appearance_models: am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: + elif len(n_appearance) == 1 and self.aam.n_scales > 1: for am in self.aam.appearance_models: am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: + elif len(n_appearance) == self.aam.n_scales: for am, n in zip(self.aam.appearance_models, n_appearance): am.n_active_components = n else: raise ValueError('n_appearance can be an integer or a float ' 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + 'those'.format(self.aam.n_scales)) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): @@ -58,7 +58,7 @@ def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_levels) + sampling = checks.check_sampling(sampling, self.n_scales) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): @@ -115,10 +115,10 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_levels) + sampling = checks.check_sampling(sampling, self.n_scales) self.n_perturbations = n_perturbations self.noise_std = noise_std - self.max_iters = checks.check_max_iters(max_iters, self.n_levels) + self.max_iters = checks.check_max_iters(max_iters, self.n_scales) self._set_up(sd_algorithm_cls, sampling, **kwargs) def _set_up(self, sd_algorithm_cls, sampling, **kwargs): @@ -171,10 +171,9 @@ def train(self, images, group=None, verbose=False, **kwargs): images = rescale_images_to_reference_shape( images, group, self.reference_shape, verbose=verbose) - if self.scale_features: - # compute features at highest level - feature_images = compute_features(images, self.features[0], - verbose=verbose) + # compute features at highest level + feature_images = compute_features(images, self.features[0], + verbose=verbose) # for each pyramid level (low --> high) for j, s in enumerate(self.scales): @@ -190,15 +189,15 @@ def train(self, images, group=None, verbose=False, **kwargs): elif self.scale_features: # scale features at other levels level_images = scale_images(feature_images, s, - level_str=level_str, + prefix=level_str, verbose=verbose) else: # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, + scaled_images = scale_images(images, s, prefix=level_str, verbose=verbose) level_images = compute_features(scaled_images, self.features[j], - level_str=level_str, + prefix=level_str, verbose=verbose) # extract ground truth shapes for current level From 30bf692531511baf8597a5c466de52899d1da80d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:26:19 +0100 Subject: [PATCH 171/423] Change n_levels for n_scales in atm --- menpofit/atm/algorithm.py | 4 ++-- menpofit/atm/base.py | 30 ++++++++++++------------ menpofit/atm/builder.py | 48 +++++++++++++++++++-------------------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 43e27a1..d92df72 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -220,8 +220,8 @@ class Compositional(LucasKanade): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): # define cost closure - def cost_closure(x, f): - return lambda: x.T.dot(f(x)) + def cost_closure(x): + return lambda: x.T.dot(x) # initialize transform self.transform.set_target(initial_shape) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 70e58e1..b903075 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -28,7 +28,7 @@ class ATM(object): constructed. features : `callable` or ``[callable]``, - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -60,7 +60,7 @@ def __init__(self, shape_models, warped_templates, reference_shape, self.scale_features = scale_features @property - def n_levels(self): + def n_scales(self): """ The number of scale level of the ATM. @@ -219,14 +219,14 @@ def __str__(self): # small strings about number of channels, channels string and downscale n_channels = [] down_str = [] - for j in range(self.n_levels): + for j in range(self.n_scales): n_channels.append( self.warped_templates[j].n_channels) - if j == self.n_levels - 1: + if j == self.n_scales - 1: down_str.append('(no downscale)') else: down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) + self.downscale**(self.n_scales - j - 1))) # string about features and channels if self.pyramid_on_features: feat_str = "- Feature is {} with ".format( @@ -238,7 +238,7 @@ def __str__(self): else: feat_str = [] ch_str = [] - for j in range(self.n_levels): + for j in range(self.n_scales): feat_str.append("- Feature is {} with ".format( name_of_callable(self.features[j]))) if n_channels[j] == 1: @@ -246,17 +246,17 @@ def __str__(self): else: ch_str.append("channels") out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_levels > 1: + if self.n_scales > 1: if self.scaled_shape_models: out = "{} - Gaussian pyramid with {} levels and downscale " \ "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, + "model (reference frame).\n".format(out, self.n_scales, self.downscale) else: out = "{} - Gaussian pyramid with {} levels and downscale " \ "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, + "are not scaled.\n".format(out, self.n_scales, self.downscale) if self.pyramid_on_features: out = "{} - Pyramid was applied on feature space.\n " \ @@ -275,8 +275,8 @@ def __str__(self): else: out = "{} - Features were extracted at each pyramid " \ "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, + for i in range(self.n_scales - 1, -1, -1): + out = "{} - Level {} {}: \n".format(out, self.n_scales - i, down_str[i]) if not self.pyramid_on_features: out = "{} {}{} {} per image.\n".format( @@ -333,7 +333,7 @@ class PatchATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -414,7 +414,7 @@ class LinearATM(ATM): constructed. features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -483,7 +483,7 @@ class LinearPatchATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -555,7 +555,7 @@ class PartsATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 0f44545..da049f0 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -22,7 +22,7 @@ class ATMBuilder(object): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -64,7 +64,7 @@ class ATMBuilder(object): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -109,8 +109,8 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -153,7 +153,7 @@ def build(self, shapes, template, group=None, verbose=False): verbose=verbose) # normalize the template size using the reference_shape scaling - template = template.rescale_to_reference_shape( + template = template.rescale_to_pointcloud( reference_shape, group=group) # build models at each scale @@ -258,7 +258,7 @@ class PatchATMBuilder(ATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -292,7 +292,7 @@ class PatchATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -339,9 +339,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scale_features=True, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -383,7 +383,7 @@ class LinearATMBuilder(ATMBuilder): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -425,7 +425,7 @@ class LinearATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -470,8 +470,8 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -524,7 +524,7 @@ class LinearPatchATMBuilder(LinearATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -558,7 +558,7 @@ class LinearPatchATMBuilder(LinearATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -605,9 +605,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scale_features=True, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -651,7 +651,7 @@ class PartsATMBuilder(ATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -687,7 +687,7 @@ class PartsATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -735,9 +735,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') From 793d55b9a3f54cec1cc1493499fb80bba5be9460 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:30:08 +0100 Subject: [PATCH 172/423] Change n_levels for n_scales in lk --- menpofit/lk/fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index e00501c..11130c2 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -19,8 +19,8 @@ def __init__(self, template, group=None, features=no_op, **kwargs): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) # set parameters self.features = features From 95ab8077d093d6cdf766f032887cc64b555656b0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:35:02 +0100 Subject: [PATCH 173/423] Change n_levels for n_cales in sdm --- menpofit/sdm/fitter.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index ec60492..788ff39 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -28,9 +28,9 @@ def __init__(self, images, group=None, bounding_box_group=None, batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_features = checks.check_features(patch_features, n_levels) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) + scales = checks.check_scales(scales) + patch_features = checks.check_features(patch_features, len(scales)) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) # set parameters self.algorithms = [] self.reference_shape = None @@ -41,7 +41,7 @@ def __init__(self, images, group=None, bounding_box_group=None, self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations - self.iterations = checks.check_max_iters(iterations, n_levels) + self.iterations = checks.check_max_iters(iterations, len(scales)) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._setup_algorithms() @@ -51,7 +51,7 @@ def __init__(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=False, batch_size=batch_size) def _setup_algorithms(self): - for j in range(self.n_levels): + for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], patch_shape=self._patch_shape[j], @@ -153,7 +153,7 @@ def _train(self, images, group=None, bounding_box_group=None, # for each pyramid level (low --> high) current_shapes = [] - for j in range(self.n_levels): + for j in range(self.n_scales): if verbose: if len(self.scales) > 1: level_str = ' - Level {}: '.format(j) @@ -164,7 +164,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Scale images level_images = scale_images(image_batch, self.scales[j], - level_str=level_str, + prefix=level_str, verbose=verbose) # Extract scaled ground truth shapes for current level @@ -325,7 +325,7 @@ def __str__(self): - Custom perturbation scheme used: {is_custom_perturb_func}""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), reg_cls=name_of_callable(regressor_cls), - n_levels=len(self.scales), + n_scales=len(self.scales), levels=self.scales, level_info=level_info, n_perturbations=self.n_perturbations, From 060747d2c276719cb6d411d3c977e78af7e44696 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:37:58 +0100 Subject: [PATCH 174/423] Final changes - Update builder.py - Update checks.py - Other minor changes --- menpofit/base.py | 16 +- menpofit/builder.py | 49 +- menpofit/checks.py | 96 +- menpofit/clm/algorithm/sd.py | 3 +- menpofit/clm/builder.py | 361 ----- menpofit/clm/classifier.py | 19 - menpofit/clm/fitter.py | 3 +- menpofit/fitter.py | 153 +- menpofit/fittingresult.py | 1258 ----------------- menpofit/gradientdescent/__init__.py | 1 - menpofit/gradientdescent/base.py | 201 --- menpofit/gradientdescent/residual.py | 104 -- menpofit/lucaskanade/__init__.py | 3 - menpofit/lucaskanade/appearance/__init__.py | 3 - .../lucaskanade/appearance/alternating.py | 174 --- menpofit/lucaskanade/appearance/base.py | 21 - menpofit/lucaskanade/appearance/projectout.py | 59 - .../lucaskanade/appearance/simultaneous.py | 184 --- menpofit/lucaskanade/base.py | 91 -- menpofit/lucaskanade/image.py | 184 --- menpofit/lucaskanade/residual.py | 573 -------- menpofit/regression/__init__.py | 0 menpofit/regression/base.py | 295 ---- menpofit/regression/parametricfeatures.py | 122 -- menpofit/regression/regressors.py | 151 -- menpofit/regression/trainer.py | 634 --------- menpofit/result.py | 4 +- menpofit/transform/modeldriven.py | 1 + menpofit/visualize/widgets/base.py | 28 +- 29 files changed, 195 insertions(+), 4596 deletions(-) delete mode 100644 menpofit/clm/builder.py delete mode 100644 menpofit/clm/classifier.py delete mode 100644 menpofit/fittingresult.py delete mode 100755 menpofit/gradientdescent/__init__.py delete mode 100644 menpofit/gradientdescent/base.py delete mode 100755 menpofit/gradientdescent/residual.py delete mode 100755 menpofit/lucaskanade/__init__.py delete mode 100644 menpofit/lucaskanade/appearance/__init__.py delete mode 100644 menpofit/lucaskanade/appearance/alternating.py delete mode 100644 menpofit/lucaskanade/appearance/base.py delete mode 100644 menpofit/lucaskanade/appearance/projectout.py delete mode 100644 menpofit/lucaskanade/appearance/simultaneous.py delete mode 100644 menpofit/lucaskanade/base.py delete mode 100644 menpofit/lucaskanade/image.py delete mode 100755 menpofit/lucaskanade/residual.py delete mode 100644 menpofit/regression/__init__.py delete mode 100644 menpofit/regression/base.py delete mode 100644 menpofit/regression/parametricfeatures.py delete mode 100644 menpofit/regression/regressors.py delete mode 100644 menpofit/regression/trainer.py diff --git a/menpofit/base.py b/menpofit/base.py index 1883721..18b3f0f 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -46,7 +46,7 @@ def create_pyramid(images, n_levels, downscale, features, verbose=False): images: list of :map:`Image` The set of landmarked images from which to build the AAM. - n_levels: int + n_scales: int The number of multi-resolution pyramidal levels to be used. downscale: float @@ -113,13 +113,13 @@ def pyramid_on_features(self): return is_pyramid_on_features(self.features) - -def build_sampling_grid(patch_shape): +def build_grid(shape): r""" """ - patch_shape = np.array(patch_shape) - patch_half_shape = np.require(np.floor(patch_shape / 2), dtype=int) - start = -patch_half_shape - end = patch_half_shape + 1 + shape = np.asarray(shape) + half_shape = np.floor(shape / 2) + half_shape = np.require(half_shape, dtype=int) + start = -half_shape + end = half_shape + shape % 2 sampling_grid = np.mgrid[start[0]:end[0], start[1]:end[1]] - return sampling_grid.swapaxes(0, 2).swapaxes(0, 1) + return np.rollaxis(sampling_grid, 0, 3) diff --git a/menpofit/builder.py b/menpofit/builder.py index ae2fb48..b148916 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -54,11 +54,10 @@ def rescale_images_to_reference_shape(images, group, reference_shape, r""" """ wrap = partial(print_progress, prefix='- Normalizing images size', - verbose=verbose) + end_with_newline=False, verbose=verbose) # Normalize the scaling of all images wrt the reference_shape size - normalized_images = [i.rescale_to_reference_shape(reference_shape, - group=group) + normalized_images = [i.rescale_to_pointcloud(reference_shape, group=group) for i in wrap(images)] return normalized_images @@ -117,19 +116,19 @@ def normalization_wrt_reference_shape(images, group, diagonal, verbose=False): # TODO: document me! -def compute_features(images, features, level_str='', verbose=False): +def compute_features(images, features, prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Computing feature space'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Computing feature space'.format(prefix), + end_with_newline=not prefix, verbose=verbose) return [features(i) for i in wrap(images)] # TODO: document me! -def scale_images(images, scale, level_str='', verbose=False): +def scale_images(images, scale, prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Scaling images'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Scaling images'.format(prefix), + end_with_newline=not prefix, verbose=verbose) if not np.allclose(scale, 1): return [i.rescale(scale) for i in wrap(images)] @@ -138,11 +137,11 @@ def scale_images(images, scale, level_str='', verbose=False): # TODO: document me! -def warp_images(images, shapes, reference_frame, transform, level_str='', +def warp_images(images, shapes, reference_frame, transform, prefix='', verbose=None): wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Warping images'.format(prefix), + end_with_newline=not prefix, verbose=verbose) warped_images = [] # Build a dummy transform, use set_target for efficiency @@ -161,10 +160,10 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! def extract_patches(images, shapes, patch_shape, normalize_function=no_op, - level_str='', verbose=False): + prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Warping images'.format(prefix), + end_with_newline=not prefix, verbose=verbose) parts_images = [] for i, s in wrap(zip(images, shapes)): @@ -174,6 +173,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images.append(Image(parts)) return parts_images + def build_reference_frame(landmarks, boundary=3, group='source', trilist=None): r""" @@ -292,7 +292,8 @@ def align_shapes(shapes): return [s.aligned_source() for s in gpa.transforms] -def build_shape_model(shapes, max_components=None): +# TODO: rethink OrthoPDM, should this function be its constructor? +def build_shape_model(shapes, max_components=None, prefix='', verbose=False): r""" Builds a shape model given a set of shapes. @@ -311,6 +312,8 @@ def build_shape_model(shapes, max_components=None): shape_model: :class:`menpo.model.pca` The PCA shape model. """ + if verbose: + print_dynamic('{}Building shape model'.format(prefix)) # compute aligned shapes aligned_shapes = align_shapes(shapes) # build shape model @@ -318,5 +321,19 @@ def build_shape_model(shapes, max_components=None): if max_components is not None: # trim shape model if required shape_model.trim_components(max_components) + return shape_model + +def increment_shape_model(shape_model, shapes, forgetting_factor=None, + max_components=None, prefix='', verbose=False): + r""" + """ + if verbose: + print_dynamic('{}Incrementing shape model'.format(prefix)) + # compute aligned shapes + aligned_shapes = align_shapes(shapes) + # increment shape model + shape_model.increment(aligned_shapes, forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) return shape_model diff --git a/menpofit/checks.py b/menpofit/checks.py index 82f4978..545b380 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,5 +1,6 @@ -import numpy as np import warnings +from functools import partial +import numpy as np def check_diagonal(diagonal): @@ -9,24 +10,23 @@ def check_diagonal(diagonal): """ if diagonal is not None and diagonal < 20: raise ValueError("diagonal must be >= 20") + return diagonal # TODO: document me! def check_scales(scales): if isinstance(scales, (int, float)): - return [scales], 1 + return [scales] elif len(scales) == 1 and isinstance(scales[0], (int, float)): - return list(scales), 1 + return list(scales) elif len(scales) > 1: - l1, n1 = check_scales(scales[0]) - l2, n2 = check_scales(scales[1:]) - return l1 + l2, n1 + n2 + return check_scales(scales[0]) + check_scales(scales[1:]) else: raise ValueError("scales must be an int/float or a list/tuple of " "int/float") -def check_features(features, n_levels): +def check_features(features, n_scales): r""" Checks the feature type per level. @@ -34,7 +34,7 @@ def check_features(features, n_levels): ---------- features : callable or list of callables The features to apply to the images. - n_levels : int + n_scales : int The number of pyramid levels. Returns @@ -43,15 +43,16 @@ def check_features(features, n_levels): A list of feature function. """ if callable(features): - return [features] * n_levels + return [features] * n_scales elif len(features) == 1 and np.alltrue([callable(f) for f in features]): - return list(features) * n_levels - elif len(features) == n_levels and np.alltrue([callable(f) + return list(features) * n_scales + elif len(features) == n_scales and np.alltrue([callable(f) for f in features]): return list(features) else: raise ValueError("features must be a callable or a list/tuple of " - "callables with the same length as scales") + "callables with the same length as the number " + "of scales") # TODO: document me! @@ -68,35 +69,35 @@ def check_scale_features(scale_features, features): # TODO: document me! -def check_patch_shape(patch_shape, n_levels): +def check_patch_shape(patch_shape, n_scales): if len(patch_shape) == 2 and isinstance(patch_shape[0], int): - return [patch_shape] * n_levels + return [patch_shape] * n_scales elif len(patch_shape) == 1: return check_patch_shape(patch_shape[0], 1) - elif len(patch_shape) == n_levels: + elif len(patch_shape) == n_scales: l1 = check_patch_shape(patch_shape[0], 1) - l2 = check_patch_shape(patch_shape[1:], n_levels-1) + l2 = check_patch_shape(patch_shape[1:], n_scales-1) return l1 + l2 else: raise ValueError("patch_shape must be a list/tuple of int or a " "list/tuple of lit/tuple of int/float with the " - "same length as scales") + "same length as the number of scales") -def check_max_components(max_components, n_levels, var_name): +def check_max_components(max_components, n_scales, var_name): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be None or int or float or a list of - those containing 1 or {n_levels} elements. + those containing 1 or {n_scales} elements. """ str_error = ("{} must be None or an int > 0 or a 0 <= float <= 1 or " "a list of those containing 1 or {} elements").format( - var_name, n_levels) + var_name, n_scales) if not isinstance(max_components, (list, tuple)): - max_components_list = [max_components] * n_levels + max_components_list = [max_components] * n_scales elif len(max_components) == 1: - max_components_list = [max_components[0]] * n_levels - elif len(max_components) == n_levels: + max_components_list = [max_components[0]] * n_scales + elif len(max_components) == n_scales: max_components_list = max_components else: raise ValueError(str_error) @@ -109,39 +110,60 @@ def check_max_components(max_components, n_levels, var_name): # TODO: document me! -def check_max_iters(max_iters, n_levels): +def check_max_iters(max_iters, n_scales): if type(max_iters) is int: - max_iters = [np.round(max_iters/n_levels) - for _ in range(n_levels)] - elif len(max_iters) == 1 and n_levels > 1: - max_iters = [np.round(max_iters[0]/n_levels) - for _ in range(n_levels)] - elif len(max_iters) != n_levels: + max_iters = [np.round(max_iters/n_scales) + for _ in range(n_scales)] + elif len(max_iters) == 1 and n_scales > 1: + max_iters = [np.round(max_iters[0]/n_scales) + for _ in range(n_scales)] + elif len(max_iters) != n_scales: raise ValueError('max_iters can be integer, integer list ' 'containing 1 or {} elements or ' - 'None'.format(n_levels)) + 'None'.format(n_scales)) return np.require(max_iters, dtype=np.int) # TODO: document me! -def check_sampling(sampling, n_levels): +def check_sampling(sampling, n_scales): if (isinstance(sampling, (list, tuple)) and np.alltrue([isinstance(s, (np.ndarray, np.int)) or sampling is None for s in sampling])): if len(sampling) == 1: - return sampling * n_levels - elif len(sampling) == n_levels: + return sampling * n_scales + elif len(sampling) == n_scales: return sampling else: raise ValueError('A sampling list can only ' 'contain 1 element or {} ' - 'elements'.format(n_levels)) + 'elements'.format(n_scales)) elif isinstance(sampling, (np.ndarray, np.int)) or sampling is None: - return [sampling] * n_levels + return [sampling] * n_scales else: raise ValueError('sampling can be an integer or ndarray, ' 'a integer or ndarray list ' 'containing 1 or {} elements or ' - 'None'.format(n_levels)) + 'None'.format(n_scales)) +def check_algorithm_cls(algorithm_cls, n_scales, base_algorithm_cls): + r""" + """ + if (isinstance(algorithm_cls, partial) and + base_algorithm_cls in algorithm_cls.func.mro()): + return [algorithm_cls] * n_scales + elif (isinstance(algorithm_cls, type) and + base_algorithm_cls in algorithm_cls.mro()): + return [algorithm_cls] * n_scales + elif len(algorithm_cls) == 1: + return check_algorithm_cls(algorithm_cls[0], n_scales, + base_algorithm_cls) + elif len(algorithm_cls) == n_scales: + return [check_algorithm_cls(a, 1, base_algorithm_cls)[0] + for a in algorithm_cls] + else: + raise ValueError("algorithm_cls must be a subclass of {} or a " + "list/tuple of {} subclasses with the same length " + "as the number of scales {}" + .format(base_algorithm_cls, base_algorithm_cls, + n_scales)) diff --git a/menpofit/clm/algorithm/sd.py b/menpofit/clm/algorithm/sd.py index a77b245..29d9bbc 100644 --- a/menpofit/clm/algorithm/sd.py +++ b/menpofit/clm/algorithm/sd.py @@ -5,4 +5,5 @@ class SupervisedDescentCLMAlgorithm(object): r""" """ - raise NotImplementedError + def __init__(self): + raise NotImplementedError diff --git a/menpofit/clm/builder.py b/menpofit/clm/builder.py deleted file mode 100644 index 4f05930..0000000 --- a/menpofit/clm/builder.py +++ /dev/null @@ -1,361 +0,0 @@ -from __future__ import division, print_function -import numpy as np -from menpo.feature import sparse_hog -from menpo.visualize import print_dynamic, progress_bar_str - -from menpofit import checks -from menpofit.base import create_pyramid, build_sampling_grid -from menpofit.builder import (DeformableModelBuilder, build_shape_model, - normalization_wrt_reference_shape) -from .classifier import linear_svm_lr - - -class CLMBuilder(DeformableModelBuilder): - r""" - Class that builds Multilevel Constrained Local Models. - - Parameters - ---------- - classifier_trainers : ``callable -> callable`` or ``[callable -> callable]`` - - Each ``classifier_trainers`` is a callable that will be invoked as: - - classifer = classifier_trainer(X, t) - - where X is a matrix of samples and t is a matrix of classifications - for each sample. `classifier` is then itself a callable, - which will be used to classify novel instance by the CLM. - - If list of length ``n_levels``, then a classifier_trainer callable is - defined per level. The first element of the list specifies the - classifier_trainer to be used at the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified - classifier_trainer will be used for all levels. - - - Examples of such classifier trainers can be found in - `menpo.fitmultilevel.clm.classifier` - - patch_shape : tuple of `int` - The shape of the patches used by the classifier trainers. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - - normalization_diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the ``normalization_diagonal`` - value. - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of each - pyramid level, so the shape models will be scaled. - - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - - max_shape_components : ``None`` or `int` > ``0`` or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - - Returns - ------- - clm : :map:`CLMBuilder` - The CLM Builder object - """ - def __init__(self, classifier_trainers=linear_svm_lr, patch_shape=(5, 5), - features=sparse_hog, normalization_diagonal=None, - n_levels=3, downscale=1.1, scaled_shape_models=True, - max_shape_components=None, boundary=3): - - # general deformable model checks - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - features = checks.check_features(features, n_levels) - - # CLM specific checks - classifier_trainers = check_classifier_trainers(classifier_trainers, n_levels) - patch_shape = check_patch_shape(patch_shape) - - # store parameters - self.classifier_trainers = classifier_trainers - self.patch_shape = patch_shape - self.features = features - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models - self.max_shape_components = max_shape_components - self.boundary = boundary - - def build(self, images, group=None, label=None, verbose=False): - r""" - Builds a Multilevel Constrained Local Model from a list of - landmarked images. - - Parameters - ---------- - images : list of :map:`Image` - The set of landmarked images from which to build the AAM. - group : string, Optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label : `string`, optional - The label of of the landmark manager that you wish to use. If - ``None``, the convex hull of all landmarks is used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - clm : :map:`CLM` - The CLM object - """ - # compute reference_shape and normalize images size - self.reference_shape, normalized_images = \ - normalization_wrt_reference_shape( - images, group, label, self.normalization_diagonal, - verbose=verbose) - - # create pyramid - generators = create_pyramid(normalized_images, self.n_levels, - self.downscale, self.features, - verbose=verbose) - - # build the model at each pyramid level - if verbose: - if self.n_levels > 1: - print_dynamic('- Building model for each of the {} pyramid ' - 'levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building model\n') - - shape_models = [] - classifiers = [] - # for each pyramid level (high --> low) - for j in range(self.n_levels): - # since models are built from highest to lowest level, the - # parameters of type list need to use a reversed index - rj = self.n_levels - j - 1 - - if verbose: - level_str = ' - ' - if self.n_levels > 1: - level_str = ' - Level {}: '.format(j + 1) - - # get images of current level - feature_images = [] - for c, g in enumerate(generators): - if verbose: - print_dynamic( - '{}Computing feature space/rescaling - {}'.format( - level_str, - progress_bar_str((c + 1.) / len(generators), - show_bar=False))) - feature_images.append(next(g)) - - # extract potentially rescaled shapes - shapes = [i.landmarks[group][label] for i in feature_images] - - # define shapes that will be used for training - if j == 0: - original_shapes = shapes - train_shapes = shapes - else: - if self.scaled_shape_models: - train_shapes = shapes - else: - train_shapes = original_shapes - - # train shape model and find reference frame - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = build_shape_model( - train_shapes, self.max_shape_components[rj]) - - # add shape model to the list - shape_models.append(shape_model) - - # build classifiers - sampling_grid = build_sampling_grid(self.patch_shape) - n_points = shapes[0].n_points - level_classifiers = [] - for k in range(n_points): - if verbose: - print_dynamic('{}Building classifiers - {}'.format( - level_str, - progress_bar_str((k + 1.) / n_points, - show_bar=False))) - - positive_labels = [] - negative_labels = [] - positive_samples = [] - negative_samples = [] - - for i, s in zip(feature_images, shapes): - - max_x = i.shape[0] - 1 - max_y = i.shape[1] - 1 - - point = (np.round(s.points[k, :])).astype(int) - patch_grid = sampling_grid + point[None, None, ...] - positive, negative = get_pos_neg_grid_positions( - patch_grid, positive_grid_size=(1, 1)) - - x = positive[:, 0] - y = positive[:, 1] - x[x > max_x] = max_x - y[y > max_y] = max_y - x[x < 0] = 0 - y[y < 0] = 0 - - positive_sample = i.pixels[:, x, y].T - positive_samples.append(positive_sample) - positive_labels.append(np.ones(positive_sample.shape[0])) - - x = negative[:, 0] - y = negative[:, 1] - x[x > max_x] = max_x - y[y > max_y] = max_y - x[x < 0] = 0 - y[y < 0] = 0 - - negative_sample = i.pixels[:, x, y].T - negative_samples.append(negative_sample) - negative_labels.append(-np.ones(negative_sample.shape[0])) - - positive_samples = np.asanyarray(positive_samples) - positive_samples = np.reshape(positive_samples, - (-1, positive_samples.shape[-1])) - positive_labels = np.asanyarray(positive_labels).flatten() - - negative_samples = np.asanyarray(negative_samples) - negative_samples = np.reshape(negative_samples, - (-1, negative_samples.shape[-1])) - negative_labels = np.asanyarray(negative_labels).flatten() - - X = np.vstack((positive_samples, negative_samples)) - t = np.hstack((positive_labels, negative_labels)) - - clf = self.classifier_trainers[rj](X, t) - level_classifiers.append(clf) - - # add level classifiers to the list - classifiers.append(level_classifiers) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - classifiers.reverse() - n_training_images = len(images) - - from .base import CLM - return CLM(shape_models, classifiers, n_training_images, - self.patch_shape, self.features, self.reference_shape, - self.downscale, self.scaled_shape_models) - - -def get_pos_neg_grid_positions(sampling_grid, positive_grid_size=(1, 1)): - r""" - Divides a sampling grid in positive and negative pixel positions. By - default only the centre of the grid is considered to be positive. - """ - positive_grid_size = np.array(positive_grid_size) - mask = np.zeros(sampling_grid.shape[:-1], dtype=np.bool) - centre = np.round(np.array(mask.shape) / 2).astype(int) - positive_grid_size -= [1, 1] - start = centre - positive_grid_size - end = centre + positive_grid_size + 1 - mask[start[0]:end[0], start[1]:end[1]] = True - positive = sampling_grid[mask] - negative = sampling_grid[~mask] - return positive, negative - - -def check_classifier_trainers(classifier_trainers, n_levels): - r""" - Checks the classifier_trainers. Must be a ``callable`` -> - ``callable`` or - or a list containing 1 or {n_levels} callables each of which returns a - callable. - """ - str_error = ("classifier must be a callable " - "of a list containing 1 or {} callables").format(n_levels) - if not isinstance(classifier_trainers, list): - classifier_list = [classifier_trainers] * n_levels - elif len(classifier_trainers) == 1: - classifier_list = [classifier_trainers[0]] * n_levels - elif len(classifier_trainers) == n_levels: - classifier_list = classifier_trainers - else: - raise ValueError(str_error) - for classifier in classifier_list: - if not callable(classifier): - raise ValueError(str_error) - return classifier_list - - -def check_patch_shape(patch_shape): - r""" - Checks the patch shape. It must be a tuple with `int` > ``1``. - """ - str_error = "patch_size mast be a tuple with two integers" - if not isinstance(patch_shape, tuple) or len(patch_shape) != 2: - raise ValueError(str_error) - for sh in patch_shape: - if not isinstance(sh, int) or sh < 2: - raise ValueError(str_error) - return patch_shape diff --git a/menpofit/clm/classifier.py b/menpofit/clm/classifier.py deleted file mode 100644 index 377b4cd..0000000 --- a/menpofit/clm/classifier.py +++ /dev/null @@ -1,19 +0,0 @@ -from sklearn import svm -from sklearn import linear_model - - -class linear_svm_lr(object): - r""" - Binary classifier that combines Linear Support Vector Machines and - Logistic Regression. - """ - def __init__(self, X, t): - self.clf1 = svm.LinearSVC(class_weight='auto') - self.clf1.fit(X, t) - t1 = self.clf1.decision_function(X) - self.clf2 = linear_model.LogisticRegression(class_weight='auto') - self.clf2.fit(t1[..., None], t) - - def __call__(self, x): - t1_pred = self.clf1.decision_function(x) - return self.clf2.predict_proba(t1_pred[..., None])[:, 1] diff --git a/menpofit/clm/fitter.py b/menpofit/clm/fitter.py index c83c223..26fe326 100644 --- a/menpofit/clm/fitter.py +++ b/menpofit/clm/fitter.py @@ -45,4 +45,5 @@ def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, class SupervisedDescentCLMFitter(CLMFitter): r""" """ - raise NotImplementedError + def __init__(self): + raise NotImplementedError diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 72fd8fd..781f7c2 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -13,15 +13,15 @@ class MultiFitter(object): r""" """ @property - def n_levels(self): + def n_scales(self): r""" - The number of pyramidal levels used during alignment. + The number of scales used during alignment. :type: `int` """ return len(self.scales) - def fit(self, image, initial_shape, max_iters=50, gt_shape=None, + def fit(self, image, initial_shape, max_iters=20, gt_shape=None, crop_image=0.5, **kwargs): r""" Fits the multilevel fitter to an image. @@ -68,11 +68,6 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, images, initial_shapes, gt_shapes = self._prepare_image( image, initial_shape, gt_shape=gt_shape, crop_image=crop_image) - # detach added landmarks from image - del image.landmarks['initial_shape'] - if gt_shape: - del image.landmarks['gt_shape'] - # work out the affine transform between the initial shape of the # highest pyramidal level and the initial shape of the original image affine_correction = AlignmentAffine(initial_shapes[-1], initial_shape) @@ -131,50 +126,58 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ - - # attach landmarks to the image - image.landmarks['initial_shape'] = initial_shape + # Attach landmarks to the image + image.landmarks['__initial_shape'] = initial_shape if gt_shape: - image.landmarks['gt_shape'] = gt_shape + image.landmarks['__gt_shape'] = gt_shape - # if specified, crop the image if crop_image: + # If specified, crop the image image = image.crop_to_landmarks_proportion(crop_image, - group='initial_shape') + group='__initial_shape') - # rescale image wrt the scale factor between reference_shape and + # Rescale image wrt the scale factor between reference_shape and # initial_shape - image = image.rescale_to_reference_shape(self.reference_shape, - group='initial_shape') + image = image.rescale_to_pointcloud(self.reference_shape, + group='__initial_shape') - # obtain image representation + # Compute image representation images = [] - for j, s in enumerate(self.scales[::-1]): - if j == 0: - # compute features at highest level - feature_image = self.features[j](image) - elif self.scale_features: - # scale features at other levels - feature_image = images[0].rescale(s) + for i in range(self.n_scales): + # Handle features + if i == 0 or self.features[i] is not self.features[i-1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_image = self.features[i](image) + + # Handle scales + if self.scales[i] != 1: + # Scale feature images only if scale is different than 1 + scaled_image = feature_image.rescale(self.scales[i]) else: - # scale image and compute features at other levels - scaled_image = image.rescale(s) - feature_image = self.features[j](scaled_image) - images.append(feature_image) - images.reverse() + scaled_image = feature_image + + # Add scaled image to list + images.append(scaled_image) - # get initial shapes per level - initial_shapes = [i.landmarks['initial_shape'].lms for i in images] + # Get initial shapes per level + initial_shapes = [i.landmarks['__initial_shape'].lms for i in images] - # get ground truth shapes per level + # Get ground truth shapes per level if gt_shape: - gt_shapes = [i.landmarks['gt_shape'].lms for i in images] + gt_shapes = [i.landmarks['__gt_shape'].lms for i in images] else: gt_shapes = None + # detach added landmarks from image + del image.landmarks['__initial_shape'] + if gt_shape: + del image.landmarks['__gt_shape'] + return images, initial_shapes, gt_shapes - def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, + def _fit(self, images, initial_shape, gt_shapes=None, max_iters=20, **kwargs): r""" Fits the fitter to the multilevel pyramidal images. @@ -188,8 +191,6 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, gt_shapes: :class:`menpo.shape.PointCloud` list, optional The original ground truth shapes associated to the multilevel images. - - Default: None max_iters: int or list, optional The maximum number of iterations. If int, then this will be the overall maximum number of iterations @@ -197,32 +198,41 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, If list, then a maximum number of iterations is specified for each pyramidal level. - Default: 50 - Returns ------- algorithm_results: :class:`menpo.fg2015.fittingresult.FittingResult` list The fitting object containing the state of the whole fitting procedure. """ - max_iters = checks.check_max_iters(max_iters, self.n_levels) + # Perform check + max_iters = checks.check_max_iters(max_iters, self.n_scales) + + # Set initial and ground truth shapes shape = initial_shape gt_shape = None - algorithm_results = [] - for j, (i, alg, it, s) in enumerate(zip(images, self.algorithms, - max_iters, self.scales)): - if gt_shapes: - gt_shape = gt_shapes[j] - algorithm_result = alg.run(i, shape, gt_shape=gt_shape, - max_iters=it, **kwargs) + # Initialize list of algorithm results + algorithm_results = [] + for i in range(self.n_scales): + # Handle ground truth shape + if gt_shapes is not None: + gt_shape = gt_shapes[i] + + # Run algorithm + algorithm_result = self.algorithms[i].run(images[i], shape, + gt_shape=gt_shape, + max_iters=max_iters[i], + **kwargs) + # Add algorithm result to the list algorithm_results.append(algorithm_result) + # Prepare this scale's final shape for the next scale shape = algorithm_result.final_shape - if s != self.scales[-1]: - shape = Scale(self.scales[j+1]/s, + if self.scales[i] != self.scales[-1]: + shape = Scale(self.scales[i+1] / self.scales[i], n_dims=shape.n_dims).apply(shape) + # Return list of algorithm results return algorithm_results @@ -239,10 +249,6 @@ def reference_shape(self): """ return self._model.reference_shape - @property - def reference_bounding_box(self): - return self.reference_shape.bounding_box() - @property def features(self): r""" @@ -257,44 +263,33 @@ def features(self): def scales(self): return self._model.scales - @property - def scale_features(self): - r""" - Flag that defined the nature of Gaussian pyramid used to build the - AAM. - If ``True``, the feature space is computed once at the highest scale - and the Gaussian pyramid is applied to the feature images. - If ``False``, the Gaussian pyramid is applied to the original images - and features are extracted at each level. - - :type: `boolean` - """ - return self._model.scale_features - def _check_n_shape(self, n_shape): if n_shape is not None: if type(n_shape) is int or type(n_shape) is float: for sm in self._model.shape_models: sm.n_active_components = n_shape - elif len(n_shape) == 1 and self._model.n_levels > 1: + elif len(n_shape) == 1 and self._model.n_scales > 1: for sm in self._model.shape_models: sm.n_active_components = n_shape[0] - elif len(n_shape) == self._model.n_levels: + elif len(n_shape) == self._model.n_scales: for sm, n in zip(self._model.shape_models, n_shape): sm.n_active_components = n else: raise ValueError('n_shape can be an integer or a float or None' 'or a list containing 1 or {} of ' - 'those'.format(self._model.n_levels)) - - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): - transform = noisy_alignment_similarity_transform( - self.reference_bounding_box, bounding_box, noise_std=noise_std) - return transform.apply(self.reference_shape) - - def noisy_shape_from_shape(self, shape, noise_std=0.05): - return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std) + 'those'.format(self._model.n_scales)) + + def noisy_shape_from_bounding_box(self, bounding_box, noise_type='uniform', + noise_percentage=0.1, rotation=False): + return noisy_shape_from_bounding_box( + self.reference_shape, bounding_box, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) + + def noisy_shape_from_shape(self, shape, noise_type='uniform', + noise_percentage=0.1, rotation=False): + return noisy_shape_from_shape( + self.reference_shape, shape, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) def noisy_alignment_similarity_transform(source, target, noise_type='uniform', diff --git a/menpofit/fittingresult.py b/menpofit/fittingresult.py deleted file mode 100644 index 7fdedef..0000000 --- a/menpofit/fittingresult.py +++ /dev/null @@ -1,1258 +0,0 @@ -from __future__ import division - -import abc -from itertools import chain -import numpy as np - -from menpo.shape.pointcloud import PointCloud -from menpo.image import Image -from menpo.transform import Scale -from menpofit.base import name_of_callable - - -class FittingResult(object): - r""" - Object that holds the state of a single fitting object, during and after it - has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` or subclass - The fitted image. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - - def __init__(self, image, gt_shape=None): - self.image = image - self._gt_shape = gt_shape - - @property - def n_iters(self): - return len(self.shapes) - 1 - - @abc.abstractproperty - def shapes(self): - r""" - A list containing the shapes obtained at each fitting iteration. - - :type: `list` of :map:`PointCloud` - """ - - def displacements(self): - r""" - A list containing the displacement between the shape of each iteration - and the shape of the previous one. - - :type: `list` of ndarray - """ - return [np.linalg.norm(s1.points - s2.points, axis=1) - for s1, s2 in zip(self.shapes, self.shapes[1:])] - - def displacements_stats(self, stat_type='mean'): - r""" - A list containing the a statistical metric on the displacement between - the shape of each iteration and the shape of the previous one. - - Parameters - ----------- - stat_type : `str` ``{'mean', 'median', 'min', 'max'}``, optional - Specifies a statistic metric to be extracted from the displacements. - - Returns - ------- - :type: `list` of `float` - The statistical metric on the points displacements for each - iteration. - """ - if stat_type == 'mean': - return [np.mean(d) for d in self.displacements()] - elif stat_type == 'median': - return [np.median(d) for d in self.displacements()] - elif stat_type == 'max': - return [np.max(d) for d in self.displacements()] - elif stat_type == 'min': - return [np.min(d) for d in self.displacements()] - else: - raise ValueError("type must be 'mean', 'median', 'min' or 'max'") - - @abc.abstractproperty - def final_shape(self): - r""" - Returns the final fitted shape. - """ - - @abc.abstractproperty - def initial_shape(self): - r""" - Returns the initial shape from which the fitting started. - """ - - @property - def gt_shape(self): - r""" - Returns the original ground truth shape associated to the image. - """ - return self._gt_shape - - @property - def fitted_image(self): - r""" - Returns a copy of the fitted image with the following landmark - groups attached to it: - - ``initial``, containing the initial fitted shape . - - ``final``, containing the final shape. - - ``ground``, containing the ground truth shape. Only returned if - the ground truth shape was provided. - - :type: :map:`Image` - """ - image = Image(self.image.pixels) - - image.landmarks['initial'] = self.initial_shape - image.landmarks['final'] = self.final_shape - if self.gt_shape is not None: - image.landmarks['ground'] = self.gt_shape - return image - - @property - def iter_image(self): - r""" - Returns a copy of the fitted image with as many landmark groups as - iteration run by fitting procedure: - - ``iter_0``, containing the initial shape. - - ``iter_1``, containing the the fitted shape at the first - iteration. - - ``...`` - - ``iter_n``, containing the final fitted shape. - - :type: :map:`Image` - """ - image = Image(self.image.pixels) - for j, s in enumerate(self.shapes): - key = 'iter_{}'.format(j) - image.landmarks[key] = s - return image - - def errors(self, error_type='me_norm'): - r""" - Returns a list containing the error at each fitting iteration. - - Parameters - ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Returns - ------- - errors : `list` of `float` - The errors at each iteration of the fitting process. - """ - if self.gt_shape is not None: - return [compute_error(t, self.gt_shape, error_type) - for t in self.shapes] - else: - raise ValueError('Ground truth has not been set, errors cannot ' - 'be computed') - - def final_error(self, error_type='me_norm'): - r""" - Returns the final fitting error. - - Parameters - ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Returns - ------- - final_error : `float` - The final error at the end of the fitting procedure. - """ - if self.gt_shape is not None: - return compute_error(self.final_shape, self.gt_shape, error_type) - else: - raise ValueError('Ground truth has not been set, final error ' - 'cannot be computed') - - def initial_error(self, error_type='me_norm'): - r""" - Returns the initial fitting error. - - Parameters - ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Returns - ------- - initial_error : `float` - The initial error at the start of the fitting procedure. - """ - if self.gt_shape is not None: - return compute_error(self.initial_shape, self.gt_shape, error_type) - else: - raise ValueError('Ground truth has not been set, final error ' - 'cannot be computed') - - def view_widget(self, browser_style='buttons', figure_size=(10, 8), - style='coloured'): - r""" - Visualizes the multilevel fitting result object using the - `menpo.visualize.widgets.visualize_fitting_result` widget. - - Parameters - ----------- - browser_style : {``'buttons'``, ``'slider'``}, optional - It defines whether the selector of the fitting results will have the - form of plus/minus buttons or a slider. - figure_size : (`int`, `int`), optional - The initial size of the rendered figure. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. - """ - from menpofit.visualize import visualize_fitting_result - visualize_fitting_result(self, figure_size=figure_size, - browser_style=browser_style, style=style) - - def plot_errors(self, error_type='me_norm', figure_id=None, - new_figure=False, render_lines=True, line_colour='b', - line_style='-', line_width=2, render_markers=True, - marker_style='o', marker_size=4, marker_face_colour='b', - marker_edge_colour='k', marker_edge_width=1., - render_axes=True, axes_font_name='sans-serif', - axes_font_size=10, axes_font_style='normal', - axes_font_weight='normal', figure_size=(10, 6), - render_grid=True, grid_line_style='--', - grid_line_width=0.5): - r""" - Plot of the error evolution at each fitting iteration. - - Parameters - ---------- - error_type : {``me_norm``, ``me``, ``rmse``}, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - figure_id : `object`, optional - The id of the figure to be used. - new_figure : `bool`, optional - If ``True``, a new figure is created. - render_lines : `bool`, optional - If ``True``, the line will be rendered. - line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or - ``(3, )`` `ndarray`, optional - The colour of the lines. - line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the lines. - line_width : `float`, optional - The width of the lines. - render_markers : `bool`, optional - If ``True``, the markers will be rendered. - marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, - ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, - ``1``, ``2``, ``3``, ``4``, ``8``}, optional - The style of the markers. - marker_size : `int`, optional - The size of the markers in points^2. - marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The face (filling) colour of the markers. - marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The edge colour of the markers. - marker_edge_width : `float`, optional - The width of the markers' edge. - render_axes : `bool`, optional - If ``True``, the axes will be rendered. - axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the axes. - axes_font_size : `int`, optional - The font size of the axes. - axes_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the axes. - axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, - ``book``, ``medium``, ``roman``, ``semibold``, - ``demibold``, ``demi``, ``bold``, ``heavy``, - ``extra bold``, ``black``}, optional - The font weight of the axes. - figure_size : (`float`, `float`) or `None`, optional - The size of the figure in inches. - render_grid : `bool`, optional - If ``True``, the grid will be rendered. - grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the grid lines. - grid_line_width : `float`, optional - The width of the grid lines. - - Returns - ------- - viewer : :map:`GraphPlotter` - The viewer object. - """ - from menpo.visualize import GraphPlotter - errors_list = self.errors(error_type=error_type) - return GraphPlotter(figure_id=figure_id, new_figure=new_figure, - x_axis=range(len(errors_list)), - y_axis=[errors_list], - title='Fitting Errors per Iteration', - x_label='Iteration', y_label='Fitting Error', - x_axis_limits=(0, len(errors_list)-1), - y_axis_limits=None).render( - render_lines=render_lines, line_colour=line_colour, - line_style=line_style, line_width=line_width, - render_markers=render_markers, marker_style=marker_style, - marker_size=marker_size, marker_face_colour=marker_face_colour, - marker_edge_colour=marker_edge_colour, - marker_edge_width=marker_edge_width, render_legend=False, - render_axes=render_axes, axes_font_name=axes_font_name, - axes_font_size=axes_font_size, axes_font_style=axes_font_style, - axes_font_weight=axes_font_weight, render_grid=render_grid, - grid_line_style=grid_line_style, grid_line_width=grid_line_width, - figure_size=figure_size) - - def plot_displacements(self, stat_type='mean', figure_id=None, - new_figure=False, render_lines=True, line_colour='b', - line_style='-', line_width=2, render_markers=True, - marker_style='o', marker_size=4, - marker_face_colour='b', marker_edge_colour='k', - marker_edge_width=1., render_axes=True, - axes_font_name='sans-serif', axes_font_size=10, - axes_font_style='normal', axes_font_weight='normal', - figure_size=(10, 6), render_grid=True, - grid_line_style='--', grid_line_width=0.5): - r""" - Plot of a statistical metric of the displacement between the shape of - each iteration and the shape of the previous one. - - Parameters - ---------- - stat_type : {``mean``, ``median``, ``min``, ``max``}, optional - Specifies a statistic metric to be extracted from the displacements - (see also `displacements_stats()` method). - figure_id : `object`, optional - The id of the figure to be used. - new_figure : `bool`, optional - If ``True``, a new figure is created. - render_lines : `bool`, optional - If ``True``, the line will be rendered. - line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or - ``(3, )`` `ndarray`, optional - The colour of the lines. - line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the lines. - line_width : `float`, optional - The width of the lines. - render_markers : `bool`, optional - If ``True``, the markers will be rendered. - marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, - ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, - ``1``, ``2``, ``3``, ``4``, ``8``}, optional - The style of the markers. - marker_size : `int`, optional - The size of the markers in points^2. - marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The face (filling) colour of the markers. - marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The edge colour of the markers. - marker_edge_width : `float`, optional - The width of the markers' edge. - render_axes : `bool`, optional - If ``True``, the axes will be rendered. - axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the axes. - axes_font_size : `int`, optional - The font size of the axes. - axes_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the axes. - axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, - ``book``, ``medium``, ``roman``, ``semibold``, - ``demibold``, ``demi``, ``bold``, ``heavy``, - ``extra bold``, ``black``}, optional - The font weight of the axes. - figure_size : (`float`, `float`) or `None`, optional - The size of the figure in inches. - render_grid : `bool`, optional - If ``True``, the grid will be rendered. - grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the grid lines. - grid_line_width : `float`, optional - The width of the grid lines. - - Returns - ------- - viewer : :map:`GraphPlotter` - The viewer object. - """ - from menpo.visualize import GraphPlotter - # set labels - if stat_type == 'max': - ylabel = 'Maximum Displacement' - title = 'Maximum displacement per Iteration' - elif stat_type == 'min': - ylabel = 'Minimum Displacement' - title = 'Minimum displacement per Iteration' - elif stat_type == 'mean': - ylabel = 'Mean Displacement' - title = 'Mean displacement per Iteration' - elif stat_type == 'median': - ylabel = 'Median Displacement' - title = 'Median displacement per Iteration' - else: - raise ValueError('stat_type must be one of {max, min, mean, ' - 'median}.') - # plot - displacements_list = self.displacements_stats(stat_type=stat_type) - return GraphPlotter(figure_id=figure_id, new_figure=new_figure, - x_axis=range(len(displacements_list)), - y_axis=[displacements_list], - title=title, - x_label='Iteration', y_label=ylabel, - x_axis_limits=(0, len(displacements_list)-1), - y_axis_limits=None).render( - render_lines=render_lines, line_colour=line_colour, - line_style=line_style, line_width=line_width, - render_markers=render_markers, marker_style=marker_style, - marker_size=marker_size, marker_face_colour=marker_face_colour, - marker_edge_colour=marker_edge_colour, - marker_edge_width=marker_edge_width, render_legend=False, - render_axes=render_axes, axes_font_name=axes_font_name, - axes_font_size=axes_font_size, axes_font_style=axes_font_style, - axes_font_weight=axes_font_weight, render_grid=render_grid, - grid_line_style=grid_line_style, grid_line_width=grid_line_width, - figure_size=figure_size) - - def as_serializable(self): - r"""" - Returns a serializable version of the fitting result. This is a much - lighter weight object than the initial fitting result. For example, - it won't contain the original fitting object. - - Returns - ------- - serializable_fitting_result : :map:`SerializableFittingResult` - The lightweight serializable version of this fitting result. - """ - if self.parameters is not None: - parameters = [p.copy() for p in self.parameters] - else: - parameters = [] - gt_shape = self.gt_shape.copy() if self.gt_shape else None - return SerializableFittingResult(self.image.copy(), - parameters, - [s.copy() for s in self.shapes], - gt_shape) - - -class NonParametricFittingResult(FittingResult): - r""" - Object that holds the state of a Non Parametric :map:`Fitter` object - before, during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - fitter : :map:`Fitter` - The Fitter object used to fitter the image. - shapes : `list` of :map:`PointCloud` - The list of fitted shapes per iteration of the fitting procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - - def __init__(self, image, fitter, parameters=None, gt_shape=None): - super(NonParametricFittingResult, self).__init__(image, - gt_shape=gt_shape) - self.fitter = fitter - # The parameters are the shapes for Non-Parametric algorithms - self.parameters = parameters - - @property - def shapes(self): - return self.parameters - - @property - def final_shape(self): - return self.parameters[-1].copy() - - @property - def initial_shape(self): - return self.parameters[0].copy() - - @FittingResult.gt_shape.setter - def gt_shape(self, value): - r""" - Setter for the ground truth shape associated to the image. - """ - if isinstance(value, PointCloud): - self._gt_shape = value - else: - raise ValueError("Accepted values for gt_shape setter are " - "PointClouds.") - - -class SemiParametricFittingResult(FittingResult): - r""" - Object that holds the state of a Semi Parametric :map:`Fitter` object - before, during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - fitter : :map:`Fitter` - The Fitter object used to fitter the image. - parameters : `list` of `ndarray` - The list of optimal transform parameters per iteration of the fitting - procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - - def __init__(self, image, fitter, parameters=None, gt_shape=None): - FittingResult.__init__(self, image, gt_shape=gt_shape) - self.fitter = fitter - self.parameters = parameters - - @property - def transforms(self): - r""" - Generates a list containing the transforms obtained at each fitting - iteration. - """ - return [self.fitter.transform.from_vector(p) for p in self.parameters] - - @property - def final_transform(self): - r""" - Returns the final transform. - """ - return self.fitter.transform.from_vector(self.parameters[-1]) - - @property - def initial_transform(self): - r""" - Returns the initial transform from which the fitting started. - """ - return self.fitter.transform.from_vector(self.parameters[0]) - - @property - def shapes(self): - return [self.fitter.transform.from_vector(p).target - for p in self.parameters] - - @property - def final_shape(self): - return self.final_transform.target - - @property - def initial_shape(self): - return self.initial_transform.target - - @FittingResult.gt_shape.setter - def gt_shape(self, value): - r""" - Setter for the ground truth shape associated to the image. - """ - if type(value) is PointCloud: - self._gt_shape = value - elif type(value) is list and value[0] is float: - transform = self.fitter.transform.from_vector(value) - self._gt_shape = transform.target - else: - raise ValueError("Accepted values for gt_shape setter are " - "PointClouds or float lists " - "specifying transform shapes.") - - -class ParametricFittingResult(SemiParametricFittingResult): - r""" - Object that holds the state of a Fully Parametric :map:`Fitter` object - before, during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - fitter : :map:`Fitter` - The Fitter object used to fitter the image. - parameters : `list` of `ndarray` - The list of optimal transform parameters per iteration of the fitting - procedure. - weights : `list` of `ndarray` - The list of optimal appearance parameters per iteration of the fitting - procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - def __init__(self, image, fitter, parameters=None, weights=None, - gt_shape=None): - SemiParametricFittingResult.__init__(self, image, fitter, parameters, - gt_shape=gt_shape) - self.weights = weights - - @property - def warped_images(self): - r""" - The list containing the warped images obtained at each fitting - iteration. - - :type: `list` of :map:`Image` or subclass - """ - mask = self.fitter.template.mask - transform = self.fitter.transform - return [self.image.warp_to_mask(mask, transform.from_vector(p)) - for p in self.parameters] - - @property - def appearance_reconstructions(self): - r""" - The list containing the appearance reconstruction obtained at - each fitting iteration. - - :type: list` of :map:`Image` or subclass - """ - if self.weights: - return [self.fitter.appearance_model.instance(w) - for w in self.weights] - else: - return [self.fitter.template for _ in self.shapes] - - @property - def error_images(self): - r""" - The list containing the error images obtained at - each fitting iteration. - - :type: list` of :map:`Image` or subclass - """ - template = self.fitter.template - warped_images = self.warped_images - appearances = self.appearance_reconstructions - - error_images = [] - for a, i in zip(appearances, warped_images): - error = a.as_vector() - i.as_vector() - error_image = template.from_vector(error) - error_images.append(error_image) - - return error_images - - -class SerializableFittingResult(FittingResult): - r""" - Designed to allow the fitting results to be easily serializable. In - comparison to the other fitting result objects, the serializable fitting - results contain a much stricter set of data. For example, the major data - components of a serializable fitting result are the fitted shapes, the - parameters and the fitted image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - parameters : `list` of `ndarray` - The list of optimal transform parameters per iteration of the fitting - procedure. - shapes : `list` of :map:`PointCloud` - The list of fitted shapes per iteration of the fitting procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - def __init__(self, image, parameters, shapes, gt_shape): - FittingResult.__init__(self, image, gt_shape=gt_shape) - - self.parameters = parameters - self._shapes = shapes - - @property - def shapes(self): - return self._shapes - - @property - def initial_shape(self): - return self._shapes[0] - - @property - def final_shape(self): - return self._shapes[-1] - - -class MultilevelFittingResult(FittingResult): - r""" - Class that holds the state of a :map:`MultilevelFitter` object before, - during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` or subclass - The fitted image. - multilevel_fitter : :map:`MultilevelFitter` - The multilevel fitter object used to fit the image. - fitting_results : `list` of :map:`FittingResult` - The list of fitting results. - affine_correction : :map:`Affine` - The affine transform between the initial shape of the highest - pyramidal level and the initial shape of the original image - gt_shape : class:`PointCloud`, optional - The ground truth shape associated to the image. - """ - def __init__(self, image, multiple_fitter, fitting_results, - affine_correction, gt_shape=None): - super(MultilevelFittingResult, self).__init__(image, gt_shape=gt_shape) - self.fitter = multiple_fitter - self.fitting_results = fitting_results - self._affine_correction = affine_correction - - @property - def n_levels(self): - r""" - The number of levels of the fitter object. - - :type: `int` - """ - return self.fitter.n_levels - - @property - def downscale(self): - r""" - The downscale factor used by the multiple fitter. - - :type: `float` - """ - return self.fitter.downscale - - @property - def n_iters(self): - r""" - The total number of iterations used to fitter the image. - - :type: `int` - """ - n_iters = 0 - for f in self.fitting_results: - n_iters += f.n_iters - return n_iters - - @property - def shapes(self): - r""" - A list containing the shapes obtained at each fitting iteration. - - :type: `list` of :map:`PointCloud` - """ - return _rescale_shapes_to_reference(self.fitting_results, self.n_levels, - self.downscale, - self._affine_correction) - - @property - def final_shape(self): - r""" - The final fitted shape. - - :type: :map:`PointCloud` - """ - return self._affine_correction.apply( - self.fitting_results[-1].final_shape) - - @property - def initial_shape(self): - r""" - The initial shape from which the fitting started. - - :type: :map:`PointCloud` - """ - n = self.n_levels - 1 - initial_shape = self.fitting_results[0].initial_shape - Scale(self.downscale ** n, initial_shape.n_dims).apply_inplace( - initial_shape) - - return self._affine_correction.apply(initial_shape) - - @FittingResult.gt_shape.setter - def gt_shape(self, value): - r""" - Setter for the ground truth shape associated to the image. - - type: :map:`PointCloud` - """ - self._gt_shape = value - - def __str__(self): - if self.fitter.pyramid_on_features: - feat_str = name_of_callable(self.fitter.features) - else: - feat_str = [] - for j in range(self.n_levels): - if isinstance(self.fitter.features[j], str): - feat_str.append(self.fitter.features[j]) - elif self.fitter.features[j] is None: - feat_str.append("none") - else: - feat_str.append(name_of_callable(self.fitter.features[j])) - out = "Fitting Result\n" \ - " - Initial error: {0:.4f}\n" \ - " - Final error: {1:.4f}\n" \ - " - {2} method with {3} pyramid levels, {4} iterations " \ - "and using {5} features.".format( - self.initial_error(), self.final_error(), self.fitter.algorithm, - self.n_levels, self.n_iters, feat_str) - return out - - def as_serializable(self): - r"""" - Returns a serializable version of the fitting result. This is a much - lighter weight object than the initial fitting result. For example, - it won't contain the original fitting object. - - Returns - ------- - serializable_fitting_result : :map:`SerializableFittingResult` - The lightweight serializable version of this fitting result. - """ - gt_shape = self.gt_shape.copy() if self.gt_shape else None - fr_copies = [fr.as_serializable() for fr in self.fitting_results] - - return SerializableMultilevelFittingResult( - self.image.copy(), fr_copies, - gt_shape, self.n_levels, self.downscale, self.n_iters, - self._affine_correction.copy()) - - -class AMMultilevelFittingResult(MultilevelFittingResult): - r""" - Class that holds the state of an Active Model (either AAM or ATM). - """ - @property - def costs(self): - r""" - Returns a list containing the cost at each fitting iteration. - - :type: `list` of `float` - """ - raise ValueError('costs not implemented yet.') - - @property - def final_cost(self): - r""" - Returns the final fitting cost. - - :type: `float` - """ - raise ValueError('costs not implemented yet.') - - @property - def initial_cost(self): - r""" - Returns the initial fitting cost. - - :type: `float` - """ - raise ValueError('costs not implemented yet.') - - @property - def warped_images(self): - r""" - The list containing the warped images obtained at each fitting - iteration. - - :type: `list` of :map:`Image` or subclass - """ - mask = self.fitting_results[-1].fitter.template.mask - transform = self.fitting_results[-1].fitter.transform - warped_images = [] - for s in self.shapes(): - transform.set_target(s) - image = self.image.warp_to_mask(mask, transform) - warped_images.append(image) - - return warped_images - - @property - def error_images(self): - r""" - The list containing the error images obtained at each fitting - iteration. - - :type: `list` of :map:`Image` or subclass - """ - return list(chain( - *[f.error_images for f in self.fitting_results])) - - -class SerializableMultilevelFittingResult(FittingResult): - r""" - Designed to allow the fitting results to be easily serializable. In - comparison to the other fitting result objects, the serializable fitting - results contain a much stricter set of data. For example, the major data - components of a serializable fitting result are the fitted shapes, the - parameters and the fitted image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - shapes : `list` of :map:`PointCloud` - The list of fitted shapes per iteration of the fitting procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - n_levels : `int` - Number of levels within the multilevel fitter. - downscale : `int` - Scale of downscaling applied to the image. - n_iters : `int` - Number of iterations the fitter performed. - """ - def __init__(self, image, fitting_results, gt_shape, n_levels, - downscale, n_iters, affine_correction): - FittingResult.__init__(self, image, gt_shape=gt_shape) - self.fitting_results = fitting_results - self.n_levels = n_levels - self._n_iters = n_iters - self.downscale = downscale - self.affine_correction = affine_correction - - @property - def n_iters(self): - return self._n_iters - - @property - def final_shape(self): - return self.shapes[-1] - - @property - def initial_shape(self): - return self.shapes[0] - - @property - def shapes(self): - return _rescale_shapes_to_reference(self.fitting_results, self.n_levels, - self.downscale, - self.affine_correction) - - -def _rescale_shapes_to_reference(fitting_results, n_levels, downscale, - affine_correction): - n = n_levels - 1 - shapes = [] - for j, f in enumerate(fitting_results): - transform = Scale(downscale ** (n - j), f.final_shape.n_dims) - for t in f.shapes: - t = transform.apply(t) - shapes.append(affine_correction.apply(t)) - return shapes - - -def compute_error(target, ground_truth, error_type='me_norm'): - r""" - """ - gt_points = ground_truth.points - target_points = target.points - - if error_type == 'me_norm': - return _compute_me_norm(target_points, gt_points) - elif error_type == 'me': - return _compute_me(target_points, gt_points) - elif error_type == 'rmse': - return _compute_rmse(target_points, gt_points) - else: - raise ValueError("Unknown error_type string selected. Valid options " - "are: me_norm, me, rmse'") - - -def _compute_me(target, ground_truth): - r""" - """ - return np.mean(np.sqrt(np.sum((target - ground_truth) ** 2, axis=-1))) - - -def _compute_rmse(target, ground_truth): - r""" - """ - return np.sqrt(np.mean((target.flatten() - ground_truth.flatten()) ** 2)) - - -def _compute_me_norm(target, ground_truth): - r""" - """ - normalizer = np.mean(np.max(ground_truth, axis=0) - - np.min(ground_truth, axis=0)) - return _compute_me(target, ground_truth) / normalizer - - -def compute_cumulative_error(errors, x_axis): - r""" - """ - n_errors = len(errors) - return [np.count_nonzero([errors <= x]) / n_errors for x in x_axis] - - -def plot_cumulative_error_distribution(errors, error_range=None, figure_id=None, - new_figure=False, - title='Cumulative Error Distribution', - x_label='Normalized Point-to-Point Error', - y_label='Images Proportion', - legend_entries=None, render_lines=True, - line_colour=None, line_style='-', - line_width=2, render_markers=True, - marker_style='s', marker_size=10, - marker_face_colour='w', - marker_edge_colour=None, - marker_edge_width=2, render_legend=True, - legend_title=None, - legend_font_name='sans-serif', - legend_font_style='normal', - legend_font_size=10, - legend_font_weight='normal', - legend_marker_scale=1., - legend_location=2, - legend_bbox_to_anchor=(1.05, 1.), - legend_border_axes_pad=1., - legend_n_columns=1, - legend_horizontal_spacing=1., - legend_vertical_spacing=1., - legend_border=True, - legend_border_padding=0.5, - legend_shadow=False, - legend_rounded_corners=False, - render_axes=True, - axes_font_name='sans-serif', - axes_font_size=10, - axes_font_style='normal', - axes_font_weight='normal', - axes_x_limits=None, axes_y_limits=None, - figure_size=(10, 8), render_grid=True, - grid_line_style='--', - grid_line_width=0.5): - r""" - Plot the cumulative error distribution (CED) of the provided fitting errors. - - Parameters - ---------- - errors : `list` of `lists` - A `list` with `lists` of fitting errors. A separate CED curve will be - rendered for each errors `list`. - error_range : `list` of `float` with length 3, optional - Specifies the horizontal axis range, i.e. - - :: - - error_range[0] = min_error - error_range[1] = max_error - error_range[2] = error_step - - If ``None``, then ``'error_range = [0., 0.101, 0.005]'``. - figure_id : `object`, optional - The id of the figure to be used. - new_figure : `bool`, optional - If ``True``, a new figure is created. - title : `str`, optional - The figure's title. - x_label : `str`, optional - The label of the horizontal axis. - y_label : `str`, optional - The label of the vertical axis. - legend_entries : `list of `str` or ``None``, optional - If `list` of `str`, it must have the same length as `errors` `list` and - each `str` will be used to name each curve. If ``None``, the CED curves - will be named as `'Curve %d'`. - render_lines : `bool` or `list` of `bool`, optional - If ``True``, the line will be rendered. If `bool`, this value will be - used for all curves. If `list`, a value must be specified for each - fitting errors curve, thus it must have the same length as `errors`. - line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or - ``(3, )`` `ndarray` or `list` of those or ``None``, optional - The colour of the lines. If not a `list`, this value will be - used for all curves. If `list`, a value must be specified for each - fitting errors curve, thus it must have the same length as `errors`. If - ``None``, the colours will be linearly sampled from jet colormap. - line_style : {``-``, ``--``, ``-.``, ``:``} or `list` of those, optional - The style of the lines. If not a `list`, this value will be used for all - curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - line_width : `float` or `list` of `float`, optional - The width of the lines. If `float`, this value will be used for all - curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - render_markers : `bool` or `list` of `bool`, optional - If ``True``, the markers will be rendered. If `bool`, this value will be - used for all curves. If `list`, a value must be specified for each - fitting errors curve, thus it must have the same length as `errors`. - marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, - ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, - ``1``, ``2``, ``3``, ``4``, ``8``} or `list` of those, optional - The style of the markers. If not a `list`, this value will be used for - all curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - marker_size : `int` or `list` of `int`, optional - The size of the markers in points^2. If `int`, this value will be used - for all curves. If `list`, a value must be specified for each fitting - errors curve, thus it must have the same length as `errors`. - marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray` or `list` of those or ``None``, optional - The face (filling) colour of the markers. If not a `list`, this value - will be used for all curves. If `list`, a value must be specified for - each fitting errors curve, thus it must have the same length as - `errors`. If ``None``, the colours will be linearly sampled from jet - colormap. - marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray` or `list` of those or ``None``, optional - The edge colour of the markers. If not a `list`, this value will be used - for all curves. If `list`, a value must be specified for each fitting - errors curve, thus it must have the same length as `errors`. If - ``None``, the colours will be linearly sampled from jet colormap. - marker_edge_width : `float` or `list` of `float`, optional - The width of the markers' edge. If `float`, this value will be used for - all curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - render_legend : `bool`, optional - If ``True``, the legend will be rendered. - legend_title : `str`, optional - The title of the legend. - legend_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the legend. - legend_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the legend. - legend_font_size : `int`, optional - The font size of the legend. - legend_font_weight : {``ultralight``, ``light``, ``normal``, - ``regular``, ``book``, ``medium``, ``roman``, - ``semibold``, ``demibold``, ``demi``, ``bold``, - ``heavy``, ``extra bold``, ``black``}, optional - The font weight of the legend. - legend_marker_scale : `float`, optional - The relative size of the legend markers with respect to the original - legend_location : `int`, optional - The location of the legend. The predefined values are: - - =============== === - 'best' 0 - 'upper right' 1 - 'upper left' 2 - 'lower left' 3 - 'lower right' 4 - 'right' 5 - 'center left' 6 - 'center right' 7 - 'lower center' 8 - 'upper center' 9 - 'center' 10 - =============== === - - legend_bbox_to_anchor : (`float`, `float`), optional - The bbox that the legend will be anchored. - legend_border_axes_pad : `float`, optional - The pad between the axes and legend border. - legend_n_columns : `int`, optional - The number of the legend's columns. - legend_horizontal_spacing : `float`, optional - The spacing between the columns. - legend_vertical_spacing : `float`, optional - The vertical space between the legend entries. - legend_border : `bool`, optional - If ``True``, a frame will be drawn around the legend. - legend_border_padding : `float`, optional - The fractional whitespace inside the legend border. - legend_shadow : `bool`, optional - If ``True``, a shadow will be drawn behind legend. - legend_rounded_corners : `bool`, optional - If ``True``, the frame's corners will be rounded (fancybox). - render_axes : `bool`, optional - If ``True``, the axes will be rendered. - axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the axes. - axes_font_size : `int`, optional - The font size of the axes. - axes_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the axes. - axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, - ``book``, ``medium``, ``roman``, ``semibold``, - ``demibold``, ``demi``, ``bold``, ``heavy``, - ``extra bold``, ``black``}, optional - The font weight of the axes. - axes_x_limits : (`float`, `float`) or ``None``, optional - The limits of the x axis. If ``None``, it is set to - ``(0., 'errors_max')``. - axes_y_limits : (`float`, `float`) or ``None``, optional - The limits of the y axis. If ``None``, it is set to ``(0., 1.)``. - figure_size : (`float`, `float`) or ``None``, optional - The size of the figure in inches. - render_grid : `bool`, optional - If ``True``, the grid will be rendered. - grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the grid lines. - grid_line_width : `float`, optional - The width of the grid lines. - - Raises - ------ - ValueError - legend_entries list has different length than errors list - - Returns - ------- - viewer : :map:`GraphPlotter` - The viewer object. - """ - from menpo.visualize import GraphPlotter - - # make sure that errors is a list even with one list member - if not isinstance(errors[0], list): - errors = [errors] - - # create x and y axes lists - x_axis = list(np.arange(error_range[0], error_range[1], error_range[2])) - ceds = [compute_cumulative_error(e, x_axis) for e in errors] - - # parse legend_entries, axes_x_limits and axes_y_limits - if legend_entries is None: - legend_entries = ["Curve {}".format(k) for k in range(len(ceds))] - if len(legend_entries) != len(ceds): - raise ValueError('legend_entries list has different length than errors ' - 'list') - if axes_x_limits is None: - axes_x_limits = (0., x_axis[-1]) - if axes_y_limits is None: - axes_y_limits = (0., 1.) - - # render - return GraphPlotter(figure_id=figure_id, new_figure=new_figure, - x_axis=x_axis, y_axis=ceds, title=title, - legend_entries=legend_entries, x_label=x_label, - y_label=y_label, x_axis_limits=axes_x_limits, - y_axis_limits=axes_y_limits).render( - render_lines=render_lines, line_colour=line_colour, - line_style=line_style, line_width=line_width, - render_markers=render_markers, marker_style=marker_style, - marker_size=marker_size, marker_face_colour=marker_face_colour, - marker_edge_colour=marker_edge_colour, - marker_edge_width=marker_edge_width, render_legend=render_legend, - legend_title=legend_title, legend_font_name=legend_font_name, - legend_font_style=legend_font_style, legend_font_size=legend_font_size, - legend_font_weight=legend_font_weight, - legend_marker_scale=legend_marker_scale, - legend_location=legend_location, - legend_bbox_to_anchor=legend_bbox_to_anchor, - legend_border_axes_pad=legend_border_axes_pad, - legend_n_columns=legend_n_columns, - legend_horizontal_spacing=legend_horizontal_spacing, - legend_vertical_spacing=legend_vertical_spacing, - legend_border=legend_border, - legend_border_padding=legend_border_padding, - legend_shadow=legend_shadow, - legend_rounded_corners=legend_rounded_corners, render_axes=render_axes, - axes_font_name=axes_font_name, axes_font_size=axes_font_size, - axes_font_style=axes_font_style, axes_font_weight=axes_font_weight, - figure_size=figure_size, render_grid=render_grid, - grid_line_style=grid_line_style, grid_line_width=grid_line_width) - diff --git a/menpofit/gradientdescent/__init__.py b/menpofit/gradientdescent/__init__.py deleted file mode 100755 index 8d1122e..0000000 --- a/menpofit/gradientdescent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import RLMS diff --git a/menpofit/gradientdescent/base.py b/menpofit/gradientdescent/base.py deleted file mode 100644 index c9546ee..0000000 --- a/menpofit/gradientdescent/base.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import division -import numpy as np -from menpofit.base import build_sampling_grid - -multivariate_normal = None # expensive, from scipy.stats - -from menpofit.fitter import Fitter -from menpofit.fittingresult import SemiParametricFittingResult - - -# TODO: incorporate different residuals -# TODO: generalize transform prior, and map the changes to LK methods -class GradientDescent(Fitter): - r""" - Abstract Interface for defining Gradient Descent based fitting algorithms - for Constrained Local Models [1]_. - - Parameters - ---------- - classifiers : `list` of ``classifier_callable`` - The list containing the classifier that will produce the response - maps for each landmark point. - patch_shape : `tuple` of `int` - The shape of the patches used to train the classifiers. - transform : :map:`GlobalPDM` or subclass - The global point distribution model to be used. - - .. note:: - - Only :map:`GlobalPDM` and its subclasses are supported. - :map:`PDM` is not supported at the moment. - eps : `float`, optional - The convergence value. When calculating the level of convergence, if - the norm of the delta parameter updates is less than ``eps``, the - algorithm is considered to have converged. - - References - ---------- - .. [1] J. Saragih, S. Lucey and J. Cohn, ''Deformable Model Fitting by - Regularized Landmark Mean-Shifts", International Journal of Computer - Vision (IJCV), 2010. - """ - def __init__(self, classifiers, patch_shape, pdm, eps=10**-10): - self.classifiers = classifiers - self.patch_shape = patch_shape - self.transform = pdm - self.eps = eps - # pre-computations - self._set_up() - - def _create_fitting_result(self, image, parameters, gt_shape=None): - return SemiParametricFittingResult( - image, self, parameters=[parameters], gt_shape=gt_shape) - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - self.transform.from_vector_inplace(initial_parameters) - return Fitter.fit(self, image, initial_parameters, gt_shape=gt_shape, - **kwargs) - - def get_parameters(self, shape): - self.transform.set_target(shape) - return self.transform.as_vector() - - -class RLMS(GradientDescent): - r""" - Implementation of the Regularized Landmark Mean-Shifts algorithm for - fitting Constrained Local Models described in [1]_. - - Parameters - ---------- - classifiers : `list` of ``classifier_callable`` - The list containing the classifier that will produce the response - maps for each landmark point. - patch_shape : `tuple` of `int` - The shape of the patches used to train the classifiers. - transform : :map:`GlobalPDM` or subclass - The global point distribution model to be used. - - .. note:: - - Only :map:`GlobalPDM` and its subclasses are supported. - :map:`PDM` is not supported at the moment. - eps : `float`, optional - The convergence value. When calculating the level of convergence, if - the norm of the delta parameter updates is less than ``eps``, the - algorithm is considered to have converged. - scale: `float`, optional - Constant value that will be multiplied to the `noise_variance` of - the pdm in order to compute the covariance of the KDE - approximation. - - References - ---------- - .. [1] J. Saragih, S. Lucey and J. Cohn, ''Deformable Model Fitting by - Regularized Landmark Mean-Shifts", International Journal of Computer - Vision (IJCV), 2010. - """ - def __init__(self, classifiers, patch_shape, pdm, eps=10**-10, scale=10): - self.scale = scale - super(RLMS, self).__init__( - classifiers, patch_shape, pdm, eps=eps) - - @property - def algorithm(self): - return 'RLMS' - - def _set_up(self): - global multivariate_normal - if multivariate_normal is None: - from scipy.stats import multivariate_normal # expensive - # Build the sampling grid associated to the patch shape - self._sampling_grid = build_sampling_grid(self.patch_shape) - # Define the 2-dimensional gaussian distribution - mean = np.zeros(self.transform.n_dims) - covariance = self.scale * self.transform.model.noise_variance() - mvn = multivariate_normal(mean=mean, cov=covariance) - # Compute Gaussian-KDE grid - self._kernel_grid = mvn.pdf(self._sampling_grid) - - # Jacobian - self._J = self.transform.d_dp([]) - - # Prior - sim_prior = np.zeros((4,)) - pdm_prior = 1 / self.transform.model.eigenvalues - self._J_prior = np.hstack((sim_prior, pdm_prior)) - - # Inverse Hessian - H = np.einsum('ijk, ilk -> jl', self._J, self._J) - self._inv_H = np.linalg.inv(np.diag(self._J_prior) + H) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - target = self.transform.target - n_iters = 0 - - max_h = image.shape[-2] - 1 - max_w = image.shape[-1] - 1 - - image_pixels = np.reshape(image.pixels, (image.n_channels, -1)).T - response_image = np.zeros((target.n_points, image.shape[-2], - image.shape[-1])) - - # Compute response maps - for j, clf in enumerate(self.classifiers): - response_image[j, :, :] = np.reshape(clf(image_pixels), - image.shape) - - while n_iters < max_iters and error > self.eps: - - mean_shift_target = np.zeros_like(target.points) - - # Compute mean-shift vectors - for j, point in enumerate(target.points): - - patch_grid = (self._sampling_grid + - np.round(point[None, None, ...]).astype(int)) - - x = patch_grid[:, :, 0] - y = patch_grid[:, :, 1] - - # deal with boundaries - x[x > max_h] = max_h - y[y > max_w] = max_w - x[x < 0] = 0 - y[y < 0] = 0 - - kernel_response = response_image[j, x, y] * self._kernel_grid - normalizer = np.sum(kernel_response) - normalized_kernel_response = kernel_response / normalizer - - mean_shift_target[j, :] = np.sum( - normalized_kernel_response * (x, y), axis=(1, 2)) - - # Compute (shape) error term - error = mean_shift_target - target.points - - # Compute steepest descent parameter updates - sd_delta_p = np.einsum('ijk, ik -> j', self._J, error) - - # TODO: a similar approach could be implemented in LK - # Deal with prior - prior = self._J_prior * self.transform.as_vector() - - # Compute parameter updates - delta_p = -np.dot(self._inv_H, prior - sd_delta_p) - - # Update transform weights - parameters = self.transform.as_vector() + delta_p - fitting_result.parameters.append(parameters) - self.transform.from_vector_inplace(parameters) - target = self.transform.target - - # Test convergence - error = np.abs(np.linalg.norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/gradientdescent/residual.py b/menpofit/gradientdescent/residual.py deleted file mode 100755 index 658e0ca..0000000 --- a/menpofit/gradientdescent/residual.py +++ /dev/null @@ -1,104 +0,0 @@ -import abc - - -class Residual(object): - r""" - """ - __metaclass__ = abc.ABCMeta - - @abc.abstractproperty - def error(self): - pass - - @abc.abstractproperty - def error_derivative(self): - pass - - @abc.abstractproperty - def d_dp(self): - pass - - @abc.abstractproperty - def hessian(self): - pass - - -class SSD(Residual): - - type = 'SSD' - - def error(self): - raise ValueError("Not implemented") - - def error_derivative(self): - raise ValueError("Not implemented") - - def d_dp(self): - raise ValueError("Not implemented") - - def hessian(self): - raise ValueError("Not implemented") - - -class Robust(Residual): - - def __init__(self): - raise ValueError("Not implemented") - - def error(self): - raise ValueError("Not implemented") - - def error_derivative(self): - raise ValueError("Not implemented") - - def d_dp(self): - raise ValueError("Not implemented") - - def hessian(self): - raise ValueError("Not implemented") - - @abc.abstractmethod - def _weights(self): - pass - - -class Fair(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class L1L2(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class GemanMcClure(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Cauchy(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Welsch(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Huber(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Turkey(Robust): - - def _weights(self): - raise ValueError("Not implemented") diff --git a/menpofit/lucaskanade/__init__.py b/menpofit/lucaskanade/__init__.py deleted file mode 100755 index a01f4c8..0000000 --- a/menpofit/lucaskanade/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .appearance import SFA, SFC, SIC, AFA, AFC, AIC, PIC - -from .image import FA, FC, IC diff --git a/menpofit/lucaskanade/appearance/__init__.py b/menpofit/lucaskanade/appearance/__init__.py deleted file mode 100644 index 46657ca..0000000 --- a/menpofit/lucaskanade/appearance/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .simultaneous import SFA, SFC, SIC -from .alternating import AFA, AFC, AIC -from .projectout import PIC diff --git a/menpofit/lucaskanade/appearance/alternating.py b/menpofit/lucaskanade/appearance/alternating.py deleted file mode 100644 index b80955d..0000000 --- a/menpofit/lucaskanade/appearance/alternating.py +++ /dev/null @@ -1,174 +0,0 @@ -from scipy.linalg import norm -import numpy as np - -from .base import AppearanceLucasKanade - - -class AFA(AppearanceLucasKanade): - r""" - Alternating Forward Additive algorithm - """ - @property - def algorithm(self): - return 'Alternating-FA' - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [[0]] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute appearance - weights = self.appearance_model.project(IWxp) - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute warp Jacobian - dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images( - image, dW_dp, forward=(self.template, self.transform)) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - fitting_result.parameters.append(parameters) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class AFC(AppearanceLucasKanade): - r""" - Alternating Forward Compositional algorithm - """ - @property - def algorithm(self): - return 'Alternating-FC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [[0]] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute template by projection - weights = self.appearance_model.project(IWxp) - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images(IWxp, self._dW_dp) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class AIC(AppearanceLucasKanade): - r""" - Alternating Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'Alternating-IC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [[0]] - n_iters = 0 - - # Baker-Matthews, Inverse Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute appearance - weights = self.appearance_model.project(IWxp) - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute steepest descent images, VT_dW_dp - self._J = self.residual.steepest_descent_images(self.template, - self._dW_dp) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, self.template) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/lucaskanade/appearance/base.py b/menpofit/lucaskanade/appearance/base.py deleted file mode 100644 index 6cf28d6..0000000 --- a/menpofit/lucaskanade/appearance/base.py +++ /dev/null @@ -1,21 +0,0 @@ -from menpofit.lucaskanade.residual import SSD -from menpofit.lucaskanade.base import LucasKanade - - -class AppearanceLucasKanade(LucasKanade): - - def __init__(self, model, transform, eps=10**-6): - # Note that the only supported residual for Appearance LK is SSD. - # This is because, in general, we don't know how to take the appropriate - # derivatives for arbitrary residuals with (for instance) a project out - # AAM. - # See https://github.com/menpo/menpo/issues/130 for details. - super(AppearanceLucasKanade, self).__init__(SSD(), - transform, eps=eps) - - # in appearance alignment, target image is aligned to appearance model - self.appearance_model = model - # by default, template is assigned to mean appearance - self.template = model.mean() - # pre-compute - self._set_up() diff --git a/menpofit/lucaskanade/appearance/projectout.py b/menpofit/lucaskanade/appearance/projectout.py deleted file mode 100644 index 0d1cd76..0000000 --- a/menpofit/lucaskanade/appearance/projectout.py +++ /dev/null @@ -1,59 +0,0 @@ -import numpy as np -from scipy.linalg import norm - -from .base import AppearanceLucasKanade - - -class PIC(AppearanceLucasKanade): - r""" - Project-Out Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'ProjectOut-IC' - - def _set_up(self): - # Compute warp Jacobian - dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VT_dW_dp - J = self.residual.steepest_descent_images( - self.template, dW_dp) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Baker-Matthews, Inverse Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, self.template) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/lucaskanade/appearance/simultaneous.py b/menpofit/lucaskanade/appearance/simultaneous.py deleted file mode 100644 index a29d35f..0000000 --- a/menpofit/lucaskanade/appearance/simultaneous.py +++ /dev/null @@ -1,184 +0,0 @@ -from scipy.linalg import norm -import numpy as np - -from .base import AppearanceLucasKanade - - -class SFA(AppearanceLucasKanade): - r""" - Simultaneous Forward Additive algorithm - """ - @property - def algorithm(self): - return 'Simultaneous-FA' - - def _fit(self, fitting_result, max_iters=20, project=True): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute warp Jacobian - dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VI_dW_dp - J_aux = self.residual.steepest_descent_images( - image, dW_dp, forward=(self.template, self.transform)) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J_aux.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - fitting_result.parameters.append(parameters) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class SFC(AppearanceLucasKanade): - r""" - Simultaneous Forward Compositional algorithm - """ - @property - def algorithm(self): - return 'Simultaneous-FC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20, project=True): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute steepest descent images, VI_dW_dp - J_aux = self.residual.steepest_descent_images(IWxp, self._dW_dp) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J_aux.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class SIC(AppearanceLucasKanade): - r""" - Simultaneous Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'Simultaneous-IC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20, project=True): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [] - n_iters = 0 - - mean = self.appearance_model.mean() - - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - if n_iters == 0: - # Project image onto the model bases - weights = self.appearance_model.project(IWxp) - else: - # Compute Gauss-Newton appearance parameters updates - diff = (self.template.as_vector() - mean.as_vector()) - self.template.from_vector_inplace(IWxp.as_vector() - diff - - np.dot(J_aux, delta_p)) - delta_weights = self.appearance_model.project(self.template) - weights += delta_weights - - # Reconstruct appearance - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute steepest descent images, VT_dW_dp - J_aux = self.residual.steepest_descent_images(self.template, - self._dW_dp) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J_aux.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, mean) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/lucaskanade/base.py b/menpofit/lucaskanade/base.py deleted file mode 100644 index 9184805..0000000 --- a/menpofit/lucaskanade/base.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import division -import numpy as np - -from menpofit.fitter import Fitter -from menpofit.fittingresult import ParametricFittingResult - - -class LucasKanade(Fitter): - r""" - An abstract base class for implementations of Lucas-Kanade [1]_ - type algorithms. - - This is to abstract away optimisation specific functionality such as the - calculation of the Hessian (which could be derived using a number of - techniques, currently only Gauss-Newton). - - Parameters - ---------- - image : :map:`Image` - The image to perform the alignment upon. - - .. note:: Only the image is expected within the base class because - different algorithms expect different kinds of template - (image/model) - - residual : :map:`Residual` - The kind of residual to be calculated. This is used to quantify the - error between the input image and the reference object. - - transform : :map:`Alignment` - The transformation type used to warp the image in to the appropriate - reference frame. This is used by the warping function to calculate - sub-pixel coordinates of the input image in the reference frame. - - eps : float, optional - The convergence value. When calculating the level of convergence, if - the norm of the delta parameter updates is less than `eps`, the - algorithm is considered to have converged. - - Default: 1**-10 - - Notes - ----- - The type of optimisation technique chosen will determine properties such - as the convergence rate of the algorithm. The supported optimisation - techniques are detailed below: - - ===== ==================== =============================================== - type full name hessian approximation - ===== ==================== =============================================== - 'GN' Gauss-Newton :math:`\mathbf{J^T J}` - ===== ==================== =============================================== - - Attributes - ---------- - transform - weights - n_iters - - References - ---------- - .. [1] Lucas, Bruce D., and Takeo Kanade. - "An iterative image registration technique with an application to - stereo vision." IJCAI. Vol. 81. 1981. - """ - def __init__(self, residual, transform, eps=10**-10): - # set basic state for all Lucas Kanade algorithms - self.transform = transform - self.residual = residual - self.eps = eps - # setup the optimisation approach - self._calculate_delta_p = self._gauss_newton_update - - def _gauss_newton_update(self, sd_delta_p): - return np.linalg.solve(self._H, sd_delta_p) - - def _set_up(self, **kwargs): - pass - - def _create_fitting_result(self, image, parameters, gt_shape=None): - return ParametricFittingResult(image, self, parameters=[parameters], - gt_shape=gt_shape) - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - self.transform.from_vector_inplace(initial_parameters) - return Fitter.fit(self, image, initial_parameters, gt_shape=gt_shape, - **kwargs) - - def get_parameters(self, shape): - self.transform.set_target(shape) - return self.transform.as_vector() diff --git a/menpofit/lucaskanade/image.py b/menpofit/lucaskanade/image.py deleted file mode 100644 index 76d68e9..0000000 --- a/menpofit/lucaskanade/image.py +++ /dev/null @@ -1,184 +0,0 @@ -from scipy.linalg import norm -import numpy as np - -from .base import LucasKanade - - -class ImageLucasKanade(LucasKanade): - - def __init__(self, template, residual, transform, eps=10 ** -6): - super(ImageLucasKanade, self).__init__(residual, transform, eps=eps) - # in image alignment, we align a template image to the target image - self.template = template - # pre-compute - self._set_up() - - -class FA(ImageLucasKanade): - r""" - Forward Additive algorithm - """ - @property - def algorithm(self): - return 'Image-FA' - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute the Jacobian of the warp - dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - # TODO: rename kwarg "forward" to "forward_additive" - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images( - image, dW_dp, forward=(self.template, self.transform)) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - fitting_result.parameters.append(parameters) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - fitting_result.fitted = True - return fitting_result - - -class FC(ImageLucasKanade): - r""" - Forward Compositional algorithm - """ - @property - def algorithm(self): - return 'Image-FC' - - def _set_up(self): - r""" - The forward compositional algorithm pre-computes the Jacobian of the - warp. This is set as an attribute on the class. - """ - # Compute the Jacobian of the warp - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Forward Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # TODO: add "forward_compositional" kwarg with options - # In the forward compositional algorithm there are two different - # ways of computing the steepest descent images: - # 1. V[I(x)](W(x,p)) * dW/dx * dW/dp - # 2. V[I(W(x,p))] * dW/dp -> this is what is currently used - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images(IWxp, self._dW_dp) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - fitting_result.fitted = True - return fitting_result - - -class IC(ImageLucasKanade): - r""" - Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'Image-IC' - - def _set_up(self): - r""" - The Inverse Compositional algorithm pre-computes the Jacobian of the - warp, the steepest descent images and the Hessian. These are all - stored as attributes on the class. - """ - # Compute the Jacobian of the warp - dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VT_dW_dp - self._J = self.residual.steepest_descent_images( - self.template, dW_dp) - - # TODO: Pre-compute the inverse - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Baker-Matthews, Inverse Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute steepest descent parameter updates. - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, self.template) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - fitting_result.fitted = True - return fitting_result diff --git a/menpofit/lucaskanade/residual.py b/menpofit/lucaskanade/residual.py deleted file mode 100755 index f97d06f..0000000 --- a/menpofit/lucaskanade/residual.py +++ /dev/null @@ -1,573 +0,0 @@ -""" -This module contains a set of similarity measures that was designed for use -within the Lucas-Kanade framework. They therefore expose a number of methods -that make them useful for inverse compositional and forward additive -Lucas-Kanade. - -These similarity measures are designed to be dimension independent where -possible. For this reason, some methods look more complicated than would be -normally the case. For example, calculating the Hessian involves summing -a multi-dimensional array, so we dynamically calculate the list of axes -to sum over. However, the basics of the logic, other than dimension -reduction, should be similar to the original algorithms. - -References ----------- - -.. [1] Lucas, Bruce D., and Takeo Kanade. - "An iterative image registration technique with an application to stereo - vision." - IJCAI. Vol. 81. 1981. -""" -import abc -import numpy as np -from numpy.fft import fftshift, fft2 -import scipy.linalg - -from menpo.math import log_gabor -from menpo.image import MaskedImage -from menpo.feature import gradient - - -class Residual(object): - """ - An abstract base class for calculating the residual between two images - within the Lucas-Kanade algorithm. The classes were designed - specifically to work within the Lucas-Kanade framework and so no - guarantee is made that calling methods on these subclasses will generate - correct results. - """ - __metaclass__ = abc.ABCMeta - - def __init__(self): - pass - - @property - def error(self): - r""" - The RMS of the error image. - - :type: float - - Notes - ----- - Will only generate a result if the - :func:`steepest_descent_update` function has been calculated prior. - - .. math:: - error = \sqrt{\sum_x E(x)^2} - - where :math:`E(x) = T(x) - I(W(x;p))` within the forward additive - framework. - """ - return np.sqrt(np.mean(self._error_img ** 2)) - - @abc.abstractmethod - def steepest_descent_images(self, image, dW_dp, **kwargs): - r""" - Calculates the standard steepest descent images. - - Within the forward additive framework this is defined as - - .. math:: - \nabla I \frac{\partial W}{\partial p} - - The input image is vectorised (`N`-pixels) so that masked images can - be handled. - - Parameters - ---------- - image : :class:`menpo.image.base.Image` - The image to calculate the steepest descent images from, could be - either the template or input image depending on which framework is - used. - dW_dp : ndarray - The Jacobian of the warp. - - Returns - ------- - VT_dW_dp : (N, n_params) ndarray - The steepest descent images - """ - pass - - @abc.abstractmethod - def calculate_hessian(self, VT_dW_dp): - r""" - Calculates the Gauss-Newton approximation to the Hessian. - - This is abstracted because some residuals expect the Hessian to be - pre-processed. The Gauss-Newton approximation to the Hessian is - defined as: - - .. math:: - \mathbf{J J^T} - - Parameters - ---------- - VT_dW_dp : (N, n_params) ndarray - The steepest descent images. - - Returns - ------- - H : (n_params, n_params) ndarray - The approximation to the Hessian - """ - pass - - @abc.abstractmethod - def steepest_descent_update(self, VT_dW_dp, IWxp, template): - r""" - Calculates the steepest descent parameter updates. - - These are defined, for the forward additive algorithm, as: - - .. math:: - \sum_x [ \nabla I \frac{\partial W}{\partial p} ]^T [ T(x) - I(W(x;p)) ] - - Parameters - ---------- - VT_dW_dp : (N, n_params) ndarray - The steepest descent images. - IWxp : :class:`menpo.image.base.Image` - Either the warped image or the template - (depending on the framework) - template : :class:`menpo.image.base.Image` - Either the warped image or the template - (depending on the framework) - - Returns - ------- - sd_delta_p : (n_params,) ndarray - The steepest descent parameter updates. - """ - pass - - def _calculate_gradients(self, image, forward=None): - r""" - Calculates the gradients of the given method. - - If `forward` is provided, then the gradients are warped - (as required in the forward additive algorithm) - - Parameters - ---------- - image : :class:`menpo.image.base.Image` - The image to calculate the gradients for - forward : (:map:`Image`, :map:`AlignableTransform>`), optional - A tuple containing the extra weights required for the function - `warp` (which should be passed as a function handle). - - Default: `None` - """ - if forward: - # Calculate the gradient over the image - # grad: (dims x ch) x H x W - grad = gradient(image) - # Warp gradient for forward additive using the given transform - # grad: (dims x ch) x h x w - template, transform = forward - grad = grad.warp_to_mask(template.mask, transform, - warp_landmarks=False) - else: - # Calculate the gradient over the image and set one pixels along - # the boundary of the image mask to zero (no reliable gradient - # can be computed there!) - # grad: (dims x ch) x h x w - grad = gradient(image) - grad.set_boundary_pixels() - return grad - - -class SSD(Residual): - - type = 'SSD' - - def steepest_descent_images(self, image, dW_dp, forward=None): - # compute gradient - # grad: dims x ch x pixels - grad = self._calculate_gradients(image, forward=forward) - grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) - - # compute steepest descent images - # gradient: dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - a = grad[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi, sdi2=None): - # compute hessian - # sdi.T: params x (ch x pixels) - # sdi: (ch x pixels) x params - # hessian: params x x params - if sdi2 is None: - H = sdi.T.dot(sdi) - else: - H = sdi.T.dot(sdi2) - return H - - def steepest_descent_update(self, sdi, IWxp, template): - self._error_img = IWxp.as_vector() - template.as_vector() - return sdi.T.dot(self._error_img) - - -class GaborFourier(Residual): - - type = 'GaborFourier' - - def __init__(self, image_shape, **kwargs): - super(GaborFourier, self).__init__() - - if 'filter_bank' in kwargs: - self._filter_bank = kwargs.get('filter_bank') - if self._filter_bank.shape != image_shape: - raise ValueError('Filter bank shape must match the shape ' - 'of the image') - else: - gabor = log_gabor(np.ones(image_shape), **kwargs) - # Get filter bank matrix - self._filter_bank = gabor[2] - - # Flatten the filter bank for vectorized calculations - self._filter_bank = self._filter_bank.ravel() - - def steepest_descent_images(self, image, dW_dp, forward=None): - n_dims = image.n_dims - n_channels = image.n_channels - n_params = dW_dp.shape[-1] - - # compute gradient - # grad: dims x ch x pixels - grad_img = self._calculate_gradients(image, forward=forward) - grad = grad_img.as_vector().reshape((n_dims, n_channels, -1)) - - # compute steepest descent images - # gradient: dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - a = grad[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - - # make sdi images - # sdi_img: ch x h x w x params - sdi_mask = np.tile(grad_img.mask.pixels[0, ..., None], - (1, 1, n_params)) - sdi_img = MaskedImage.blank(grad_img.shape + (n_params,), - n_channels=n_channels, - mask=sdi_mask) - sdi_img.from_vector_inplace(sdi.ravel()) - - # compute FFT over each channel, parameter and dimension - # fft_sdi: ch x h x w x params - fft_sdi = fftshift(fft2(sdi_img.pixels, axes=(-3, -2)), axes=(-3, -2)) - # Note that, fft_sdi is rectangular, i.e. is not define in - # terms of the mask pixels, but in terms of the whole image. - # Selecting mask pixels once the fft has been computed makes no - # sense because they have lost their original spatial meaning. - - # reshape steepest descent images - # sdi: (ch x h x w) x params - return fft_sdi.reshape((-1, fft_sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # reshape steepest descent images - # sdi: ch x (h x w) x params - sdi = sdi.reshape((-1, self._filter_bank.shape[0], sdi.shape[-1])) - - # compute filtered steepest descent images - # filter_bank: (h x w) - # sdi: ch x (h x w) x params - # filtered_sdi: ch x (h x w) x params - filtered_sdi = (self._filter_bank[None, ..., None] ** 0.5) * sdi - - # reshape filtered steepest descent images - # filtered_sdi: (ch x h x w) x params - filtered_sdi = filtered_sdi.reshape((-1, sdi.shape[-1])) - - # compute filtered hessian - # filtered_sdi.T: params x (ch x h x w) - # filtered_sdi: (ch x h x w) x params - # hessian: params x x n_param - return np.conjugate(filtered_sdi).T.dot(filtered_sdi) - - def steepest_descent_update(self, sdi, IWxp, template): - # compute error image - # error_img: ch x h x w - error_img = IWxp.pixels - template.pixels - - # compute FFT error image - # fft_error_img: ch x (h x w) - fft_error_img = fftshift(fft2(error_img)) - fft_error_img = fft_error_img.reshape((IWxp.n_channels, -1)) - - # compute filtered steepest descent images - # filter_bank: (h x w) - # fft_error_img: ch x (h x w) - # filtered_error_img: ch x (h x w) - filtered_error_img = self._filter_bank * fft_error_img - - # reshape _error_img - # error_img: (ch x h x w) - self._error_img = filtered_error_img.ravel() - - # compute steepest descent update - # sdi: params x (ch x h x w) - # error_img: (ch x h x w) - # sdu: params - return sdi.T.dot(np.conjugate(self._error_img)) - - -class ECC(Residual): - - type = 'ECC' - - def _normalise_images(self, image): - # TODO: do we need to copy the image? - # TODO: is this supposed to be per channel normalization? - norm_image = image.copy() - norm_image.normalize_norm_inplace() - return norm_image - - def steepest_descent_images(self, image, dW_dp, forward=None): - # normalize image - norm_image = self._normalise_images(image) - - # compute gradient - # gradient: dims x ch x pixels - grad = self._calculate_gradients(norm_image, forward=forward) - grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) - - # compute steepest descent images - # gradient: dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - a = grad[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # compute hessian - # sdi.T: params x (ch x pixels) - # sdi: (ch x pixels) x params - # hessian: params x x params - H = sdi.T.dot(sdi) - self._H_inv = scipy.linalg.inv(H) - return H - - def steepest_descent_update(self, sdi, IWxp, template): - normalised_IWxp = self._normalise_images(IWxp).as_vector() - normalised_template = self._normalise_images(template).as_vector() - - Gt = sdi.T.dot(normalised_template) - Gw = sdi.T.dot(normalised_IWxp) - - # Calculate the numerator - IWxp_norm = scipy.linalg.norm(normalised_IWxp) - num1 = IWxp_norm ** 2 - num2 = np.dot(Gw.T, np.dot(self._H_inv, Gw)) - num = num1 - num2 - - # Calculate the denominator - den1 = np.dot(normalised_template, normalised_IWxp) - den2 = np.dot(Gt.T, np.dot(self._H_inv, Gw)) - den = den1 - den2 - - # Calculate lambda to choose the step size - # Avoid division by zero - if den > 0: - l = num / den - else: - den3 = np.dot(Gt.T, np.dot(self._H_inv, Gt)) - l1 = np.sqrt(num2 / den3) - l2 = - den / den3 - l = np.maximum(l1, l2) - - self._error_img = l * normalised_IWxp - normalised_template - - return sdi.T.dot(self._error_img) - - -class GradientImages(Residual): - - type = 'GradientImages' - - def _regularise_gradients(self, grad): - pixels = grad.pixels - ab = np.sqrt(np.sum(pixels**2, axis=0)) - m_ab = np.median(ab) - ab = ab + m_ab - grad.pixels = pixels / ab - return grad - - def steepest_descent_images(self, image, dW_dp, forward=None): - n_dims = image.n_dims - n_channels = image.n_channels - - # compute gradient - first_grad = self._calculate_gradients(image, forward=forward) - self._template_grad = self._regularise_gradients(first_grad) - - # compute gradient - # second_grad: dims x dims x ch x pixels - second_grad = self._calculate_gradients(self._template_grad) - second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels, -1)) - - # Fix crossed derivatives: dydx = dxdy - second_grad[1, 0, ...] = second_grad[0, 1, ...] - - # compute steepest descent images - # gradient: dims x dims x ch x (h x w) - # dw_dp: dims x x (h x w) x params - # sdi: dims x ch x (h x w) x params - sdi = 0 - a = second_grad[..., None] * dW_dp[:, None, None, ...] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (dims x ch x h x w) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # compute hessian - # sdi.T: params x (dims x ch x pixels) - # sdi: (dims x ch x pixels) x params - # hessian: params x x params - return sdi.T.dot(sdi) - - def steepest_descent_update(self, sdi, IWxp, template): - # compute IWxp regularized gradient - IWxp_grad = self._calculate_gradients(IWxp) - IWxp_grad = self._regularise_gradients(IWxp_grad) - - # compute vectorized error_image - # error_img: (dims x ch x pixels) - self._error_img = (IWxp_grad.as_vector() - - self._template_grad.as_vector()) - - # compute steepest descent update - # sdi.T: params x (dims x ch x pixels) - # error_img: (dims x ch x pixels) - # sdu: params - return sdi.T.dot(self._error_img) - - -class GradientCorrelation(Residual): - - type = 'GradientCorrelation' - - def steepest_descent_images(self, image, dW_dp, forward=None): - n_dims = image.n_dims - n_channels = image.n_channels - - # compute gradient - # grad: dims x ch x pixels - grad = self._calculate_gradients(image, forward=forward) - grad2 = grad.as_vector().reshape((n_dims, n_channels, -1)) - - # compute IGOs (remember axis 0 is y, axis 1 is x) - # grad: dims x ch x pixels - # phi: ch x pixels - # cos_phi: ch x pixels - # sin_phi: ch x pixels - phi = np.angle(grad2[1, ...] + 1j * grad2[0, ...]) - self._cos_phi = np.cos(phi) - self._sin_phi = np.sin(phi) - - # concatenate sin and cos terms so that we can take the second - # derivatives correctly. sin(phi) = y and cos(phi) = x which is the - # correct ordering when multiplying against the warp Jacobian - # cos_phi: ch x pixels - # sin_phi: ch x pixels - # grad: (dims x ch) x pixels - grad.from_vector_inplace( - np.concatenate((self._sin_phi[None, ...], - self._cos_phi[None, ...]), axis=0).ravel()) - - # compute IGOs gradient - # second_grad: dims x dims x ch x pixels - second_grad = self._calculate_gradients(grad) - second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels, -1)) - - # Fix crossed derivatives: dydx = dxdy - second_grad[1, 0, ...] = second_grad[0, 1, ...] - - # complete full IGOs gradient computation - # second_grad: dims x dims x ch x pixels - second_grad[1, ...] = (-self._sin_phi[None, ...] * second_grad[1, ...]) - second_grad[0, ...] = (self._cos_phi[None, ...] * second_grad[0, ...]) - - # compute steepest descent images - # gradient: dims x dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - aux = second_grad[..., None] * dW_dp[None, :, None, ...] - for a in aux.reshape(((-1,) + aux.shape[2:])): - sdi += a - - # compute constant N - # N: 1 - self._N = grad.n_parameters / 2 - - # reshape steepest descent images - # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # compute hessian - # sdi.T: params x (dims x ch x pixels) - # sdi: (dims x ch x pixels) x params - # hessian: params x x params - return sdi.T.dot(sdi) - - def steepest_descent_update(self, sdi, IWxp, template): - n_dims = IWxp.n_dims - n_channels = IWxp.n_channels - - # compute IWxp gradient - IWxp_grad = self._calculate_gradients(IWxp) - IWxp_grad = IWxp_grad.as_vector().reshape( - (n_dims, n_channels, -1)) - - # compute IGOs (remember axis 0 is y, axis 1 is x) - # IWxp_grad: dims x ch x pixels - # phi: ch x pixels - # IWxp_cos_phi: ch x pixels - # IWxp_sin_phi: ch x pixels - phi = np.angle(IWxp_grad[1, ...] + 1j * IWxp_grad[0, ...]) - IWxp_cos_phi = np.cos(phi) - IWxp_sin_phi = np.sin(phi) - - # compute error image - # error_img: (ch x h x w) - self._error_img = (self._cos_phi * IWxp_sin_phi - - self._sin_phi * IWxp_cos_phi).ravel() - - # compute steepest descent update - # sdi: (ch x pixels) x params - # error_img: (ch x pixels) - # sdu: params - sdu = sdi.T.dot(self._error_img) - - # compute step size - qp = np.sum(self._cos_phi * IWxp_cos_phi + - self._sin_phi * IWxp_sin_phi) - l = self._N / qp - return l * sdu diff --git a/menpofit/regression/__init__.py b/menpofit/regression/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/menpofit/regression/base.py b/menpofit/regression/base.py deleted file mode 100644 index edd2dcf..0000000 --- a/menpofit/regression/base.py +++ /dev/null @@ -1,295 +0,0 @@ -import abc - -from menpofit.fitter import Fitter -from menpofit.fittingresult import (NonParametricFittingResult, - SemiParametricFittingResult, - ParametricFittingResult) - - -class Regressor(Fitter): - r""" - An abstract base class for fitting Regressors. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features): - self.regressor = regressor - self.features = features - - def _set_up(self): - r""" - Abstract method that sets up the fitter object. - """ - pass - - def _fit(self, fitting_result, max_iters=1): - r""" - Abstract method to fit an image. - - Parameters - ---------- - fitting_result : `menpo.fit.fittingresult` - The fitting result object. - max_iters : int - The maximum number of iterations. - """ - image = fitting_result.image - initial_shape = fitting_result.initial_shape - n_iters = 0 - - while n_iters < max_iters: - features = self.features(image, initial_shape) - delta_p = self.regressor(features) - - fitted_shape, parameters = self.update(delta_p, initial_shape) - fitting_result.parameters.append(parameters) - n_iters += 1 - - return fitting_result - - @abc.abstractmethod - def update(self, delta_p, initial_shape): - r""" - Abstract method to update the parameters. - """ - pass - - -class NonParametricRegressor(Regressor): - r""" - Fitter of Non-Parametric Regressor. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features): - super(NonParametricRegressor, self).__init__( - regressor, features) - - @property - def algorithm(self): - r""" - Returns the regression type. - """ - return "Non-Parametric" - - def _create_fitting_result(self, image, shapes, gt_shape=None): - r""" - Creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image.. - shape : :map:`PointCloud` - The current shape. - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return NonParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def update(self, delta_shape, initial_shape): - r""" - Updates the shape. - - Parameters - ---------- - delta_shape : :map:`PointCloud` - The shape increment. - initial_shape : :map:`PointCloud` - The current shape. - """ - fitted_shape = initial_shape.from_vector( - initial_shape.as_vector() + delta_shape) - return fitted_shape, fitted_shape - - def get_parameters(self, shape): - r""" - Method that makes sure that the parameter passed to the fit method is - the shape. - - Parameters - ---------- - shape: :map:`PointCloud` - The current shape. - """ - return shape - - -class SemiParametricRegressor(Regressor): - r""" - Fitter of Semi-Parametric Regressor. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features, transform, update='composition'): - super(SemiParametricRegressor, self).__init__( - regressor, features) - self.transform = transform - self._update = self._select_update(update) - - @property - def algorithm(self): - r""" - Returns the regression type. - """ - return "Semi-Parametric" - - def _create_fitting_result(self, image, parameters, gt_shape=None): - r""" - Creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image.. - shape : :map:`PointCloud` - The current shape. - gt_shape : :map:`PointCloud`, optional - The ground truth shape. - """ - self.transform.from_vector_inplace(parameters) - return SemiParametricFittingResult( - image, self, parameters=[self.transform.as_vector()], - gt_shape=gt_shape) - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - self.transform.from_vector_inplace(initial_parameters) - return Fitter.fit(self, image, initial_parameters, gt_shape=gt_shape, - **kwargs) - - def _select_update(self, update): - r""" - Select the way to update the parameters. - - Parameters - ---------- - update : {'compositional', 'additive'} - The update method. - - Returns - ------- - update : `function` - The correct function to apply the update chosen. - """ - if update == 'additive': - return self._additive - elif update == 'compositional': - return self._compositional - else: - raise ValueError('Unknown update string selected. Valid' - 'options are: additive, compositional') - - def _additive(self, delta_p): - r""" - Updates the parameters in the additive way. - - Parameters - ---------- - delta_p : `ndarray` - The parameters increment - """ - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - - def _compositional(self, delta_p): - r""" - Updates the parameters in the compositional way. - - Parameters - ---------- - delta_p : `ndarray` - The parameters increment - """ - self.transform.compose_after_from_vector_inplace(delta_p) - - def update(self, delta_p, initial_shape): - r""" - Updates the parameters of the shape model. - - Parameters - ---------- - delta_p : `ndarray` - The parameters increment. - - initial_shape : :map:`PointCloud` - The current shape. - """ - self._update(delta_p) - return self.transform.target, self.transform.as_vector() - - def get_parameters(self, shape): - r""" - Method that makes sure that the parameter passed to the fit method is - the model parameters. - - Parameters - ---------- - shape : :map:`PointCloud` - The current shape. - """ - self.transform.set_target(shape) - return self.transform.as_vector() - - -class ParametricRegressor(SemiParametricRegressor): - r""" - Fitter of Parametric Regressor. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features, appearance_model, transform, - update='composition'): - super(ParametricRegressor, self).__init__( - regressor, features, transform, update=update) - self.appearance_model = appearance_model - self.template = appearance_model.mean() - - @property - def algorithm(self): - r""" - Returns the regression type. - """ - return "Parametric" - - def _create_fitting_result(self, image, parameters, gt_shape=None): - r""" - Creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image.. - shape : :map:`PointCloud` - The current shape. - gt_shape : :map:`PointCloud`, optional - The ground truth shape. - """ - self.transform.from_vector_inplace(parameters) - return ParametricFittingResult( - image, self, parameters=[self.transform.as_vector()], - gt_shape=gt_shape) diff --git a/menpofit/regression/parametricfeatures.py b/menpofit/regression/parametricfeatures.py deleted file mode 100644 index 0d270c8..0000000 --- a/menpofit/regression/parametricfeatures.py +++ /dev/null @@ -1,122 +0,0 @@ -def extract_parametric_features(appearance_model, warped_image, - rergession_features): - r""" - Extracts a particular parametric feature given an appearance model and - a warped image. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - rergession_features : callable - Defines the function from which the parametric features will be - extracted. - - Non-default regression feature options and new experimental features - can be used by defining a callable. In this case, the callable must - define a constructor that receives as an input an appearance model and - a warped masked image and on calling returns a particular parametric - feature representation. - - Returns - ------- - features : `ndarray` - The resulting parametric features. - """ - if rergession_features is None: - features = weights(appearance_model, warped_image) - elif hasattr(rergession_features, '__call__'): - features = rergession_features(appearance_model, warped_image) - else: - raise ValueError("regression_features can only be: (1) None " - "or (2) a callable defining a non-standard " - "feature computation (see `menpo.fit.regression." - "parametricfeatures`") - return features - - -def weights(appearance_model, warped_image): - r""" - Returns the resulting weights after projecting the warped image to the - appearance PCA model. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return appearance_model.project(warped_image) - - -def whitened_weights(appearance_model, warped_image): - r""" - Returns the sheared weights after projecting the warped image to the - appearance PCA model. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return appearance_model.project_whitened(warped_image) - - -def appearance(appearance_model, warped_image): - r""" - Projects the warped image onto the appearance model and rebuilds from the - weights found. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return appearance_model.reconstruct(warped_image).as_vector() - - -def difference(appearance_model, warped_image): - r""" - Returns the difference between the warped image and the image constructed - by projecting the warped image onto the appearance model and rebuilding it - from the weights found. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return (warped_image.as_vector() - - appearance(appearance_model, warped_image)) - - -def project_out(appearance_model, warped_image): - r""" - Returns a version of the whitened warped image where all the basis of the - model have been projected out and which has been scaled by the inverse of - the appearance model's noise_variance. - - Parameters - ---------- - appearance_model: :class:`menpo.model.pca` - The appearance model based on which the parametric features will be - computed. - warped_image: :class:`menpo.image.masked` - The warped image. - """ - diff = warped_image.as_vector() - appearance_model.mean().as_vector() - return appearance_model.distance_to_subspace_vector(diff).ravel() diff --git a/menpofit/regression/regressors.py b/menpofit/regression/regressors.py deleted file mode 100644 index 1310321..0000000 --- a/menpofit/regression/regressors.py +++ /dev/null @@ -1,151 +0,0 @@ -from __future__ import division -import numpy as np - - -class mlr(object): - r""" - Multivariate Linear Regression - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - """ - def __init__(self, X, T): - XX = np.dot(X.T, X) - XX = (XX + XX.T) / 2 - XT = np.dot(X.T, T) - self.R = np.linalg.solve(XX, XT) - - def __call__(self, x): - return np.dot(x, self.R) - - -class mlr_svd(object): - r""" - Multivariate Linear Regression using SVD decomposition - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - def __init__(self, X, T, variance=None): - self.R, _, _, _ = _svd_regression(X, T, variance=variance) - - def __call__(self, x): - return np.dot(x, self.R) - - -class mlr_pca(object): - r""" - Multivariate Linear Regression using PCA reconstructions - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - def __init__(self, X, T, variance=None): - self.R, _, _, self.V = _svd_regression(X, T, variance=variance) - - def _call__(self, x): - x = np.dot(np.dot(x, self.V.T), self.V) - return np.dot(x, self.R) - - -class mlr_pca_weights(object): - r""" - Multivariate Linear Regression using PCA weights - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - def __init__(self, X, T, variance=None): - _, _, _, self.V = _svd_regression(X, T, variance=variance) - W = np.dot(X, self.V.T) - self.R, _, _, _ = _svd_regression(W, T) - - def __call__(self, x): - w = np.dot(x, self.V.T) - return np.dot(w, self.R) - - -def _svd_regression(X, T, variance=None): - r""" - SVD decomposition for regression. - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - if variance is not None and not (0 < variance <= 1): - raise ValueError("variance must be set to a number between 0 and 1.") - - U, s, V = np.linalg.svd(X) - if variance: - total = sum(s) - acc = 0 - for j, y in enumerate(s): - acc += y - if acc / total >= variance: - r = j+1 - break - else: - tol = np.max(X.shape) * np.spacing(np.max(s)) - r = np.sum(s > tol) - U = U[:, :r] - s = 1 / s[:r] - V = V[:r, :] - R = np.dot(np.dot(V.T * s, U.T), T) - - return R, U, s, V diff --git a/menpofit/regression/trainer.py b/menpofit/regression/trainer.py deleted file mode 100644 index 8a70d64..0000000 --- a/menpofit/regression/trainer.py +++ /dev/null @@ -1,634 +0,0 @@ -from __future__ import division, print_function -import abc -import numpy as np -from menpo.image import Image -from menpo.feature import sparse_hog -from menpo.visualize import print_dynamic, progress_bar_str - -from menpofit.base import noisy_align, build_sampling_grid -from menpofit.fittingresult import (NonParametricFittingResult, - SemiParametricFittingResult, - ParametricFittingResult) -from .base import (NonParametricRegressor, SemiParametricRegressor, - ParametricRegressor) -from .parametricfeatures import extract_parametric_features, weights -from .regressors import mlr - - -class RegressorTrainer(object): - r""" - An abstract base class for training regressors. - - Parameters - ---------- - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : ``None`` or `string` or `function`, optional - The features that are used during the regression. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : boolean, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - """ - __metaclass__ = abc.ABCMeta - - def __init__(self, reference_shape, regression_type=mlr, - regression_features=None, noise_std=0.04, rotation=False, - n_perturbations=10): - self.reference_shape = reference_shape - self.regression_type = regression_type - self.regression_features = regression_features - self.rotation = rotation - self.noise_std = noise_std - self.n_perturbations = n_perturbations - - def _regression_data(self, images, gt_shapes, perturbed_shapes, - verbose=False): - r""" - Method that generates the regression data : features and delta_ps. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - gt_shapes : :map:`PointCloud` list - List of the ground truth shapes that correspond to the images. - - perturbed_shapes : :map:`PointCloud` list - List of the perturbed shapes in order to regress. - - verbose : `boolean`, optional - If ``True``, the progress is printed. - """ - if verbose: - print_dynamic('- Generating regression data') - - n_images = len(images) - features = [] - delta_ps = [] - for j, (i, s, p_shape) in enumerate(zip(images, gt_shapes, - perturbed_shapes)): - if verbose: - print_dynamic('- Generating regression data - {}'.format( - progress_bar_str((j + 1.) / n_images, show_bar=False))) - for ps in p_shape: - features.append(self.features(i, ps)) - delta_ps.append(self.delta_ps(s, ps)) - return np.asarray(features), np.asarray(delta_ps) - - @abc.abstractmethod - def features(self, image, shape): - r""" - Abstract method to generate the features for the regression. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - pass - - @abc.abstractmethod - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Abstract method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - pass - - def train(self, images, shapes, perturbed_shapes=None, verbose=False, - **kwargs): - r""" - Trains a Regressor given a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to train the regressor. - - shapes : :map:`PointCloud` list - List of the shapes that correspond to the images. - - perturbed_shapes : :map:`PointCloud` list, optional - List of the perturbed shapes used for the regressor training. - - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - regressor : :map:`Regressor` - A regressor object. - - Raises - ------ - ValueError - The number of shapes must be equal to the number of images. - ValueError - The number of perturbed shapes must be equal or multiple to - the number of images. - """ - n_images = len(images) - n_shapes = len(shapes) - - # generate regression data - if n_images != n_shapes: - raise ValueError("The number of shapes must be equal to " - "the number of images.") - elif not perturbed_shapes: - perturbed_shapes = self.perturb_shapes(shapes) - features, delta_ps = self._regression_data( - images, shapes, perturbed_shapes, verbose=verbose) - elif n_images == len(perturbed_shapes): - features, delta_ps = self._regression_data( - images, shapes, perturbed_shapes, verbose=verbose) - else: - raise ValueError("The number of perturbed shapes must be " - "equal or multiple to the number of images.") - - # perform regression - if verbose: - print_dynamic('- Performing regression...') - # Expected to be a callable - regressor = self.regression_type(features, delta_ps, **kwargs) - - # compute regressor RMSE - estimated_delta_ps = regressor(features) - error = np.sqrt(np.mean(np.sum((delta_ps - estimated_delta_ps) ** 2, - axis=1))) - if verbose: - print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(error)) - return self._build_regressor(regressor, self.features) - - def perturb_shapes(self, gt_shape): - r""" - Perturbs the given shapes. The number of perturbations is defined by - ``n_perturbations``. - - Parameters - ---------- - gt_shape : :map:`PointCloud` list - List of the shapes that correspond to the images. - will be perturbed. - - Returns - ------- - perturbed_shapes : :map:`PointCloud` list - List of the perturbed shapes. - """ - return [[self._perturb_shape(s) for _ in range(self.n_perturbations)] - for s in gt_shape] - - def _perturb_shape(self, gt_shape): - r""" - Method that performs noisy alignment between the given ground truth - shape and the reference shape. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return noisy_align(self.reference_shape, gt_shape, - noise_std=self.noise_std - ).apply(self.reference_shape) - - @abc.abstractmethod - def _build_regressor(self, regressor, features): - r""" - Abstract method to build a regressor model. - """ - pass - - -class NonParametricRegressorTrainer(RegressorTrainer): - r""" - Class for training a Non-Parametric Regressor. - - Parameters - ---------- - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : `function`, optional - The features that are used during the regression. - - See `menpo.features` for details more details on - Menpo's standard image features and feature options. - See :ref:`feature_functions` for non standard - features definitions. - patch_shape : tuple, optional - The shape of the patches that will be extracted. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - - """ - def __init__(self, reference_shape, regression_type=mlr, - regression_features=sparse_hog, patch_shape=(16, 16), - noise_std=0.04, rotation=False, n_perturbations=10): - super(NonParametricRegressorTrainer, self).__init__( - reference_shape, regression_type=regression_type, - regression_features=regression_features, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.patch_shape = patch_shape - self._set_up() - - def _set_up(self): - # work out feature length per patch - patch_img = Image.init_blank(self.patch_shape, fill=0) - self._feature_patch_length = self.regression_features(patch_img).n_parameters - - @property - def algorithm(self): - r""" - Returns the algorithm name. - """ - return "Non-Parametric" - - def _create_fitting(self, image, shapes, gt_shape=None): - r""" - Method that creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The image object. - - shapes : :map:`PointCloud` list - The shapes. - - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return NonParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def features(self, image, shape): - r""" - Method that extracts the features for the regression, which in this - case are patch based. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - # extract patches - patches = image.extract_patches(shape, patch_size=self.patch_shape) - - features = np.zeros((shape.n_points, self._feature_patch_length)) - for j, patch in enumerate(patches): - # compute features - features[j, ...] = self.regression_features(patch).as_vector() - - return np.hstack((features.ravel(), 1)) - - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - return (gt_shape.as_vector() - - perturbed_shape.as_vector()) - - def _build_regressor(self, regressor, features): - r""" - Method to build the NonParametricRegressor regressor object. - """ - return NonParametricRegressor(regressor, features) - - -class SemiParametricRegressorTrainer(NonParametricRegressorTrainer): - r""" - Class for training a Semi-Parametric Regressor. - - This means that a parametric shape model and a non-parametric appearance - representation are employed. - - Parameters - ---------- - reference_shape : PointCloud - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : `function`, optional - The features that are used during the regression. - - See :ref:`menpo.features` for details more details on - Menpos standard image features and feature options. - patch_shape : tuple, optional - The shape of the patches that will be extracted. - update : 'compositional' or 'additive' - Defines the way to update the warp. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - - """ - def __init__(self, transform, reference_shape, regression_type=mlr, - regression_features=sparse_hog, patch_shape=(16, 16), - update='compositional', noise_std=0.04, rotation=False, - n_perturbations=10): - super(SemiParametricRegressorTrainer, self).__init__( - reference_shape, regression_type=regression_type, - regression_features=regression_features, patch_shape=patch_shape, - noise_std=noise_std, rotation=rotation, - n_perturbations=n_perturbations) - self.transform = transform - self.update = update - - @property - def algorithm(self): - r""" - Returns the algorithm name. - """ - return "Semi-Parametric" - - def _create_fitting(self, image, shapes, gt_shape=None): - r""" - Method that creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The image object. - - shapes : :map:`PointCloud` list - The shapes. - - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return SemiParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - self.transform.set_target(gt_shape) - gt_ps = self.transform.as_vector() - self.transform.set_target(perturbed_shape) - perturbed_ps = self.transform.as_vector() - return gt_ps - perturbed_ps - - def _build_regressor(self, regressor, features): - r""" - Method to build the NonParametricRegressor regressor object. - """ - return SemiParametricRegressor(regressor, features, self.transform, - self.update) - - -class ParametricRegressorTrainer(RegressorTrainer): - r""" - Class for training a Parametric Regressor. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model to be used. - transform : :map:`Affine` - The transform used for warping. - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : ``None`` or `function`, optional - The parametric features that are used during the regression. - - If ``None``, the reconstruction appearance weights will be used as - feature. - - If `string` or `function`, the feature representation will be - computed using one of the function in: - - If `string`, the feature representation will be extracted by - executing a parametric feature function. - - Note that this feature type can only be one of the parametric - feature functions defined :ref:`parametric_features`. - patch_shape : tuple, optional - The shape of the patches that will be extracted. - update : 'compositional' or 'additive' - Defines the way to update the warp. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - - """ - def __init__(self, appearance_model, transform, reference_shape, - regression_type=mlr, regression_features=weights, - update='compositional', noise_std=0.04, rotation=False, - n_perturbations=10): - super(ParametricRegressorTrainer, self).__init__( - reference_shape, regression_type=regression_type, - regression_features=regression_features, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.regression_features = regression_features - self.transform = transform - self.update = update - - @property - def algorithm(self): - r""" - Returns the algorithm name. - """ - return "Parametric" - - def _create_fitting(self, image, shapes, gt_shape=None): - r""" - Method that creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The image object. - - shapes : :map:`PointCloud` list - The shapes. - - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return ParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def features(self, image, shape): - r""" - Method that extracts the features for the regression, which in this - case are patch based. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - self.transform.set_target(shape) - # TODO should the template be a mask or a shape? warp_to_shape here - warped_image = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - features = extract_parametric_features( - self.appearance_model, warped_image, self.regression_features) - return np.hstack((features, 1)) - - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - self.transform.set_target(gt_shape) - gt_ps = self.transform.as_vector() - self.transform.set_target(perturbed_shape) - perturbed_ps = self.transform.as_vector() - return gt_ps - perturbed_ps - - def _build_regressor(self, regressor, features): - r""" - Method to build the NonParametricRegressor regressor object. - """ - return ParametricRegressor( - regressor, features, self.appearance_model, self.transform, - self.update) - - -class SemiParametricClassifierBasedRegressorTrainer( - SemiParametricRegressorTrainer): - r""" - Class for training a Semi-Parametric Classifier-Based Regressor. This means - that the classifiers are used instead of features. - - Parameters - ---------- - classifiers : list of :map:`classifiers` - List of classifiers. - transform : :map:`Affine` - The transform used for warping. - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - patch_shape : tuple, optional - The shape of the patches that will be extracted. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - """ - def __init__(self, classifiers, transform, reference_shape, - regression_type=mlr, patch_shape=(16, 16), - update='compositional', noise_std=0.04, rotation=False, - n_perturbations=10): - super(SemiParametricClassifierBasedRegressorTrainer, self).__init__( - transform, reference_shape, regression_type=regression_type, - patch_shape=patch_shape, update=update, - noise_std=noise_std, rotation=rotation, - n_perturbations=n_perturbations) - self.classifiers = classifiers - - def _set_up(self): - # TODO: CLMs should use slices instead of sampling grid, and the - # need of the _set_up method will probably disappear - # set up sampling grid - self.sampling_grid = build_sampling_grid(self.patch_shape) - - def features(self, image, shape): - r""" - Method that extracts the features for the regression, which in this - case are patch based. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - patches = image.extract_patches(shape, patch_size=self.patch_shape) - features = [clf(patch.as_vector(keep_channels=True)) - for (clf, patch) in zip(self.classifiers, patches)] - return np.hstack((np.asarray(features).ravel(), 1)) diff --git a/menpofit/result.py b/menpofit/result.py index f19c516..a8e7364 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -508,13 +508,13 @@ def __init__(self, image, fitter, algorithm_results, affine_correction, self._gt_shape = gt_shape @property - def n_levels(self): + def n_scales(self): r""" The number of levels of the fitter object. :type: `int` """ - return self.fitter.n_levels + return self.fitter.n_scales @property def scales(self): diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index 0238db5..42c1993 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -5,6 +5,7 @@ from menpo.transform.base import Transform, VComposable, VInvertible from menpofit.differentiable import DP + # TODO: Should MDT implement VComposable and VInvertible? class ModelDrivenTransform(Transform, Targetable, Vectorizable, VComposable, VInvertible, DP): diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 302a5e2..1385ee7 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -31,7 +31,7 @@ def _check_n_parameters(n_params, n_levels, max_n_params): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be ``None`` or `int` or `float` or a `list` - of those containing ``1`` or ``n_levels`` elements. + of those containing ``1`` or ``n_scales`` elements. """ str_error = ("n_params must be None or 1 <= int <= max_n_params or " "a list of those containing 1 or {} elements").format(n_levels) @@ -128,7 +128,7 @@ def visualize_shape_model(shape_model, n_parameters=5, mode='multiple', max_n_params = [sp.n_active_components for sp in shape_model] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_parameters = _check_n_parameters(n_parameters, n_levels, max_n_params) # Initial options dictionaries @@ -487,7 +487,7 @@ def visualize_appearance_model(appearance_model, n_parameters=5, max_n_params = [ap.n_active_components for ap in appearance_model] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_parameters = _check_n_parameters(n_parameters, n_levels, max_n_params) # Find initial groups and labels that will be passed to the landmark options @@ -790,7 +790,7 @@ def visualize_aam(aam, n_shape_parameters=5, n_appearance_parameters=5, print('Initializing...') # Get the number of levels - n_levels = aam.n_levels + n_levels = aam.n_scales # Define the styling options if style == 'coloured': @@ -829,7 +829,7 @@ def visualize_aam(aam, n_shape_parameters=5, n_appearance_parameters=5, max_n_appearance = [ap.n_active_components for ap in aam.appearance_models] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_shape_parameters = _check_n_parameters(n_shape_parameters, n_levels, max_n_shape) n_appearance_parameters = _check_n_parameters(n_appearance_parameters, @@ -972,7 +972,7 @@ def update_info(aam, instance, level, group): if n_levels == 1: tmp_shape_models = '' tmp_pyramid = '' - else: # n_levels > 1 + else: # n_scales > 1 # shape models info if aam.scaled_shape_models: tmp_shape_models = "Each level has a scaled shape model " \ @@ -993,7 +993,7 @@ def update_info(aam, instance, level, group): "> Warp using {} transform".format(aam.transform.__name__), "> {}".format(tmp_pyramid), "> Level {}/{} (downscale={:.1f})".format( - level + 1, aam.n_levels, aam.downscale), + level + 1, aam.n_scales, aam.downscale), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -1216,7 +1216,7 @@ def visualize_atm(atm, n_shape_parameters=5, mode='multiple', print('Initializing...') # Get the number of levels - n_levels = atm.n_levels + n_levels = atm.n_scales # Define the styling options if style == 'coloured': @@ -1252,7 +1252,7 @@ def visualize_atm(atm, n_shape_parameters=5, mode='multiple', max_n_shape = [sp.n_active_components for sp in atm.shape_models] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_shape_parameters = _check_n_parameters(n_shape_parameters, n_levels, max_n_shape) @@ -1388,7 +1388,7 @@ def update_info(atm, instance, level, group): if n_levels == 1: tmp_shape_models = '' tmp_pyramid = '' - else: # n_levels > 1 + else: # n_scales > 1 # shape models info if atm.scaled_shape_models: tmp_shape_models = "Each level has a scaled shape model " \ @@ -1409,7 +1409,7 @@ def update_info(atm, instance, level, group): "> Warp using {} transform".format(atm.transform.__name__), "> {}".format(tmp_pyramid), "> Level {}/{} (downscale={:.1f})".format( - level + 1, atm.n_levels, atm.downscale), + level + 1, atm.n_scales, atm.downscale), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -1590,7 +1590,7 @@ def plot_ced(errors, legend_entries=None, error_range=None, as part of a parent widget. If ``False``, the widget object is not returned, it is just visualized. """ - from menpofit.fittingresult import plot_cumulative_error_distribution + from menpofit.result import plot_cumulative_error_distribution print('Initializing...') # Make sure that errors is a list even with one list member @@ -2466,9 +2466,9 @@ def update_info(name, value): else: text_per_line = [ "> {} iterations".format(fitting_results[im].n_iters)] - if hasattr(fitting_results[im], 'n_levels'): # Multilevel result + if hasattr(fitting_results[im], 'n_scales'): # Multilevel result text_per_line.append("> {} levels with downscale of {:.1f}".format( - fitting_results[im].n_levels, fitting_results[im].downscale)) + fitting_results[im].n_scales, fitting_results[im].downscale)) info_wid.set_widget_state(n_lines=len(text_per_line), text_per_line=text_per_line) From a77c9ed197123263e24e0531f943ff52971ccf63 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:15:52 +0100 Subject: [PATCH 175/423] GaussNewton regression needed transposing Seems to be correct as it doesn't cause an exception - but it doesn't work particularly well. --- menpofit/math/regression.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/menpofit/math/regression.py b/menpofit/math/regression.py index 82dcce0..9b9c65d 100644 --- a/menpofit/math/regression.py +++ b/menpofit/math/regression.py @@ -50,8 +50,9 @@ class IIRLRegression(IRLRegression): r""" Indirect Incremental Regularized Linear Regression """ - def __init__(self, alpha=0, bias=True, alpha2=0): - super(IIRLRegression, self).__init__(alpha=alpha, bias=bias) + def __init__(self, alpha=0, bias=False, alpha2=0): + # TODO: Can we model the bias? May need to slice off of prediction? + super(IIRLRegression, self).__init__(alpha=alpha, bias=False) self.alpha2 = alpha2 def train(self, X, Y): @@ -60,9 +61,10 @@ def train(self, X, Y): J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution - H = J.T.dot(J) + # Note that everything is transposed from the above exchanging of roles + H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J.T) + self.W = np.linalg.solve(H, J).T def increment(self, X, Y): # incremental least squares exchanging the roles of X and Y @@ -70,6 +72,7 @@ def increment(self, X, Y): J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution - H = J.T.dot(J) + # Note that everything is transposed from the above exchanging of roles + H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J.T) + self.W = np.linalg.solve(H, J).T From 018a09605434c853a608c8a4998eaefcef2b4bd7 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:16:45 +0100 Subject: [PATCH 176/423] Remove extra space --- menpofit/modelinstance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/menpofit/modelinstance.py b/menpofit/modelinstance.py index 7810e72..e18c540 100644 --- a/menpofit/modelinstance.py +++ b/menpofit/modelinstance.py @@ -246,7 +246,6 @@ def _weights_for_target(self, target): Weights of the statistical model that generate the closest PointCloud to the requested target """ - self._update_global_transform(target) projected_target = self.global_transform.pseudoinverse().apply(target) # now we have the target in model space, project it to recover the From b1c556f137b81c5043211e42295788bee46cbe2c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:17:13 +0100 Subject: [PATCH 177/423] Remove unused methods and correct spelling from SDM package --- menpofit/sdm/algorithm.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index fe713c7..b8affbb 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -160,7 +160,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, self.eps = eps -# TODO: docment me! +# TODO: document me! def features_per_patch(image, shape, patch_shape, features_callable): """r """ @@ -171,7 +171,7 @@ def features_per_patch(image, shape, patch_shape, features_callable): return np.asarray(patch_features).ravel() -# TODO: docment me! +# TODO: document me! def features_per_shape(image, shapes, patch_shape, features_callable): """r """ @@ -182,7 +182,7 @@ def features_per_shape(image, shapes, patch_shape, features_callable): return np.asarray(patch_features) -# TODO: docment me! +# TODO: document me! def features_per_image(images, shapes, patch_shape, features_callable, level_str='', verbose=False): """r @@ -250,25 +250,3 @@ def compute_features_info(image, shape, features_callable, return (features_patch_shape, features_patch_length, features_shape, features_length) - - -# def initialize_sampling(self, image, group=None, label=None): -# if self._sampling is None: -# sampling = np.ones(self.patch_shape, dtype=np.bool) -# else: -# sampling = self._sampling -# -# # TODO: include offsets support? -# patches = image.extract_patches_around_landmarks( -# group=group, label=label, patch_size=self.patch_shape, -# as_single_array=True) -# -# # TODO: include offsets support? -# features_patch_shape = self.features(patches[0, 0]).shape -# self._features_patch_length = np.prod(features_patch_shape) -# self._features_shape = (patches.shape[0], features_patch_shape) -# self._features_length = np.prod(self._features_shape) -# -# feature_mask = np.tile(sampling[None, None, None, ...], -# self._feature_shape[:3] + (1, 1)) -# self._feature_mask = np.nonzero(feature_mask.flatten())[0] From de685583d49b2017e8636abc6683562eb86166ce Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:20:57 +0100 Subject: [PATCH 178/423] Attempting to refactor SD-AAM This attempts to refactor the Supervised Descent AAM code - including attempting to get PO-CR implemented. It hasn't been exhaustively tested, but I need to make a number of other changes and I wanted a discrete commit for this refactoring work. Changes: The SupervisedDescentAAMFitter is a subclass of the SupervisedDescentFitter. Therefore, it acts more like an SDM than an AAM. This is great because it borrows a lot of code from the SDFitter - which we are pretty happy with. As an extension to this, the SDAAM algorithms have been totally refactored. Now, they act exactly like SupervisedDescentAlgorithm, but parametric. The major change is that the take an interface directly rather than an interface class. Thus, the interface properly encapsulates the AAM state rather than the algorithm. Finally, accepting the refactoring of the train method to match the SDAlgorithm class format (but with parameters), the interfaces now don't have a circular dependancy on the SDAlgorithm. The SDInterfaces now encapsulate warping and the transform. The subclasses of SDAlgorithm, like ProjectOut are thus fairly simple. The finaly thing is that results need to be refactored to now have this dependancy on the whole algorithm. Therefore, I've added a horrible shim that just properly returns the transform to allow the algorithms to keep working for the time being. --- menpofit/aam/algorithm/sd.py | 500 +++++++++++++++++------------------ menpofit/aam/fitter.py | 31 +-- 2 files changed, 251 insertions(+), 280 deletions(-) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index 903f4bf..bb3242e 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -1,8 +1,13 @@ from __future__ import division +from functools import partial import numpy as np from menpo.image import Image from menpo.feature import no_op -from menpo.visualize import print_dynamic, progress_bar_str +from menpo.visualize import print_dynamic +from menpofit.math import IRLRegression, IIRLRegression +from menpofit.result import compute_normalise_point_to_point_error +from menpofit.sdm.algorithm import SupervisedDescentAlgorithm +from menpofit.visualize import print_progress from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult @@ -10,9 +15,14 @@ class SupervisedDescentStandardInterface(object): r""" """ - def __init__(self, cr_aam_algorithm, sampling=None): - self.algorithm = cr_aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None): + self.appearance_model = appearance_model + self.transform = transform + self.template = template + + self._build_sampling_mask(sampling) + def _build_sampling_mask(self, sampling): n_true_pixels = self.template.n_true_pixels() n_channels = self.template.n_channels sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) @@ -29,18 +39,6 @@ def __init__(self, cr_aam_algorithm, sampling=None): def shape_model(self): return self.transform.pdm.model - @property - def appearance_model(self): - return self.algorithm.appearance_model - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - @property def n(self): return self.transform.n_parameters @@ -55,8 +53,11 @@ def warp(self, image): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): + # TODO: Faking an 'algorithm' + algorithm = lambda x: x + algorithm.transform = self.transform return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, algorithm, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -70,8 +71,11 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): + # TODO: Faking an 'algorithm' + algorithm = lambda x: x + algorithm.transform = self.transform return LinearAAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, algorithm, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -79,16 +83,20 @@ def algorithm_result(self, image, shape_parameters, class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): r""" """ - def __init__(self, cr_aam_algorithm, sampling=None, patch_shape=(17, 17), - normalize_parts=no_op): - self.algorithm = cr_aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None, + patch_shape=(17, 17), normalize_parts=no_op): self.patch_shape = patch_shape + # TODO: Refactor to patch_features self.normalize_parts = normalize_parts + super(SupervisedDescentPartsInterface, self).__init__( + appearance_model, transform, template, sampling=sampling) + + def _build_sampling_mask(self, sampling): if sampling is None: sampling = np.ones(self.patch_shape, dtype=np.bool) - image_shape = self.algorithm.template.pixels.shape + image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], image_shape[:3] + (1, 1)) self.i_mask = np.nonzero(image_mask.flatten())[0] @@ -102,137 +110,143 @@ def warp(self, image): patch_size=self.patch_shape, as_single_array=True) parts = self.normalize_parts(parts) - return Image(parts) + return Image(parts, copy=False) + + +def _weights_for_target(transform, target): + transform.set_target(target) + return transform.as_vector() # TODO document me! -class SupervisedDescent(object): +def obtain_parametric_delta_x(gt_shapes, current_shapes, transform): + # initialize current and delta parameters arrays + n_samples = len(gt_shapes) * len(current_shapes[0]) + gt_params = np.empty((n_samples, transform.n_parameters)) + delta_params = np.empty_like(gt_params) + + k = 0 + for gt_s, c_s in zip(gt_shapes, current_shapes): + # Compute and cache ground truth parameters + c_gt_params = _weights_for_target(transform, gt_s) + for s in c_s: + gt_params[k] = c_gt_params + + current_params = _weights_for_target(transform, s) + delta_params[k] = c_gt_params - current_params + + k += 1 + + return delta_params, gt_params + + +class ParametricSupervisedDescentAlgorithm(SupervisedDescentAlgorithm): r""" """ - def __init__(self, aam_interface, appearance_model, transform, max_iters=3, - eps=10**-5, **kwargs): - # set common state for all AAM algorithms - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.transform = transform - self.max_iters = max_iters - # TODO: Make use of eps in self.train? + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5): + super(ParametricSupervisedDescentAlgorithm, self).__init__() + + self.interface = aam_interface + self.n_iterations = n_iterations self.eps = eps - # set interface - self.interface = aam_interface(self, **kwargs) - # perform pre-computations + + self._compute_error = compute_error self._precompute() + @property + def appearance_model(self): + return self.interface.appearance_model + + @property + def transform(self): + return self.interface.transform + def _precompute(self): - # grab appearance model mean + # Grab appearance model mean a_bar = self.appearance_model.mean() - # vectorize it and mask it + # Vectorise it and mask it self.a_bar_m = a_bar.as_vector()[self.interface.i_mask] - def train(self, images, gt_shapes, current_shapes, verbose=False, - **kwargs): - n_images = len(images) - n_samples_image = len(current_shapes[0]) + def _train(self, images, gt_shapes, current_shapes, increment=False, + level_str='', verbose=False): - # set number of iterations and initialize list of regressors - self.regressors = [] + if not increment: + # Reset the regressors + self.regressors = [] - # compute current and delta parameters from current and ground truth - # shapes - delta_params, current_params, gt_params = self._generate_params( - gt_shapes, current_shapes) - # initialize iteration counter - k = 0 + n_perturbations = len(current_shapes[0]) + template_shape = gt_shapes[0] + + # obtain delta_x and gt_x (parameters rather than shapes) + delta_x, gt_x = obtain_parametric_delta_x(gt_shapes, current_shapes, + self.transform) # Cascaded Regression loop - while k < self.max_iters: + for k in range(self.n_iterations): # generate regression data - features = self._generate_features(images, current_params, - verbose=verbose) + features = self._generate_features( + images, current_shapes, + level_str='{}(Iteration {}) - '.format(level_str, k), + verbose=verbose) - # perform regression if verbose: - print_dynamic('- Performing regression...') - regressor = self._perform_regression(features, delta_params, - **kwargs) - # add regressor to list - self.regressors.append(regressor) - - # compute regression rmse - estimated_delta_params = regressor(features) - # TODO: Should print a more informative error here? - rmse = _compute_rmse(delta_params, estimated_delta_params) + print_dynamic('{}(Iteration {}) - Performing regression'.format( + level_str, k)) + + if not increment: + r = self._regressor_cls() + r.train(features, delta_x) + self.regressors.append(r) + else: + self.regressors[k].increment(features, delta_x) + + # Estimate delta_points + estimated_delta_x = self.regressors[k].predict(features) if verbose: - print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(rmse)) - - current_params += estimated_delta_params + self._print_regression_info(template_shape, gt_shapes, + n_perturbations, delta_x, + estimated_delta_x, k, + level_str=level_str) + + j = 0 + for shapes in current_shapes: + for s in shapes: + # Estimate parameters + edx = estimated_delta_x[j] + # Current parameters + cx = _weights_for_target(self.transform, s) + edx + + # Uses less memory to find updated target shape + self.transform.from_vector_inplace(cx) + # Update current shape inplace + s.from_vector_inplace(self.transform.target.as_vector()) + + delta_x[j] = gt_x[j] - cx + j += 1 + + return current_shapes + + def _generate_features(self, images, current_shapes, level_str='', + verbose=False): + # Initialize features array - since current_shapes is a list of lists + # we need to know the total size + n_samples = len(images) * len(current_shapes[0]) + features = np.empty((n_samples,) + self.a_bar_m.shape) + + wrap = partial(print_progress, + prefix='{}Computing features'.format(level_str), + end_with_newline=not level_str, verbose=verbose) - delta_params = gt_params - current_params - # increase iteration counter - k += 1 - - # obtain current shapes from current parameters - current_shapes = [] - for p in current_params: - current_shapes.append(self.transform.from_vector(p).target) - - # convert current shapes into a list of list and return - final_shapes = [] - for j in range(n_images): - k = j * n_samples_image - l = k + n_samples_image - final_shapes.append(current_shapes[k:l]) - return final_shapes - - def _generate_params(self, gt_shapes, current_shapes): - # initialize current and delta parameters arrays - n_samples = len(gt_shapes) * len(current_shapes[0]) - current_params = np.empty((n_samples, self.transform.n_parameters)) - gt_params = np.empty((n_samples, self.transform.n_parameters)) - delta_params = np.empty((n_samples, self.transform.n_parameters)) # initialize sample counter k = 0 - # compute ground truth and current shape parameters - for gt_s, c_s in zip(gt_shapes, current_shapes): - for s in c_s: - # compute current parameters - current_params[k] = self._compute_params(s) - # compute ground truth parameters - gt_params[k] = self._compute_params(gt_s) - # compute delta parameters - delta_params[k] = gt_params[k] - current_params[k] - # increment counter - k += 1 + for img, img_shapes in wrap(zip(images, current_shapes)): + for s in img_shapes: + self.transform.set_target(s) + # Assumes that the transform is correctly set + features[k] = self._compute_features(img) - return delta_params, current_params, gt_params - - def _compute_params(self, shape): - self.transform.set_target(shape) - return self.transform.as_vector() - - def _generate_features(self, images, current_params, verbose=False): - # initialize features array - n_images = len(images) - n_samples = len(current_params) - n_samples_image = int(n_samples / n_images) - features = np.zeros((n_samples,) + self.a_bar_m.shape) - - # initialize sample counter - k = 0 - for i in images: - for _ in range(n_samples_image): - if verbose: - print_dynamic('- Generating regression features - {' - '}'.format( - progress_bar_str((k + 1.) / n_samples, - show_bar=False))) - # set transform - self.transform.from_vector_inplace(current_params[k]) - # compute regression features - f = self._compute_train_features(i) - # add to features array - features[k] = f - # increment counter k += 1 return features @@ -242,47 +256,53 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] - # initialize iteration counter - k = 0 - # Cascaded Regression loop - while k < self.max_iters: - # compute regression features - features = self._compute_test_features(image) + for r in self.regressors: + # Assumes that the transform is correctly set + features = self._compute_features(image) # solve for increments on the shape parameters - dp = self.regressors[k](features) + dx = r.predict(features) - # update warp - self.transform.from_vector_inplace(self.transform.as_vector() + dp) - p_list.append(self.transform.as_vector()) - - # increase iteration counter - k += 1 + # We need to update the transform to set the state for the warping + # of the image above. + new_x = p_list[-1] + dx + self.transform.from_vector_inplace(new_x) + p_list.append(new_x) # return algorithm result return self.interface.algorithm_result( image, p_list, gt_shape=gt_shape) + def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, + delta_x, estimated_delta_x, level_index, + level_str=''): + print_dynamic('{}(Iteration {}) - Calculating errors'.format( + level_str, level_index)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + self.transform.from_vector_inplace(dx) + s1 = self.transform.target + self.transform.from_vector_inplace(edx) + s2 = self.transform.target + + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('{}(Iteration {}) - Training error -> ' + 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. + format(level_str, level_index, mean, std, median)) + # TODO: document me! -class MeanTemplate(SupervisedDescent): +class MeanTemplate(ParametricSupervisedDescentAlgorithm): r""" """ - def _compute_train_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - return i_m - self.a_bar_m - - def _compute_test_features(self, image): - # warp image + def _compute_features(self, image): i = self.interface.warp(image) - # vectorize it and mask it i_m = i.as_vector()[self.interface.i_mask] - # compute masked error return i_m - self.a_bar_m @@ -290,111 +310,103 @@ def _compute_test_features(self, image): class MeanTemplateNewton(MeanTemplate): r""" """ - def _perform_regression(self, features, deltas, gamma=None, - dtype=np.float64): - return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, bias=True): + super(MeanTemplateNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) # TODO: document me! class MeanTemplateGaussNewton(MeanTemplate): r""" """ - def _perform_regression(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - return _supervised_gauss_newton(features, deltas, gamma=gamma, - psi=psi, dtype=dtype) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, alpha2=0, bias=True): + super(MeanTemplateGaussNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IIRLRegression, alpha=alpha, + alpha2=alpha2, bias=bias) # TODO: document me! -class ProjectOut(SupervisedDescent): +class ProjectOut(ParametricSupervisedDescentAlgorithm): r""" """ def _precompute(self): - # call super method super(ProjectOut, self)._precompute() - # grab appearance model components A = self.appearance_model.components - # mask them self.A_m = A.T[self.interface.i_mask, :] - # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) def project_out(self, J): - # project-out appearance bases from a particular vector or matrix + # Project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) - def _compute_train_features(self, image): - # warp image + def _compute_features(self, image): i = self.interface.warp(image) - # vectorize it and mask it i_m = i.as_vector()[self.interface.i_mask] - # compute masked error + # TODO: This project out could actually be cached at test time - + # but we need to think about the best way to implement this and still + # allow incrementing e_m = i_m - self.a_bar_m return self.project_out(e_m) - def _compute_test_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - return i_m - self.a_bar_m - # TODO: document me! class ProjectOutNewton(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None, - dtype=np.float64): - regressor = _supervised_newton(features, deltas, gamma=gamma, - dtype=dtype) - regressor.R = self.project_out(regressor.R) - return regressor + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, bias=True): + super(ProjectOutNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) # TODO: document me! class ProjectOutGaussNewton(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - return _supervised_gauss_newton(features, deltas, gamma=gamma, - psi=psi, dtype=dtype) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, alpha2=0, bias=True): + super(ProjectOutGaussNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + self._regressor_cls = partial(IIRLRegression, alpha=alpha, + alpha2=alpha2, bias=bias) # TODO: document me! -class AppearanceWeights(SupervisedDescent): +class AppearanceWeights(ParametricSupervisedDescentAlgorithm): r""" """ def _precompute(self): - # call super method super(AppearanceWeights, self)._precompute() - # grab appearance model components A = self.appearance_model.components - # mask them A_m = A.T[self.interface.i_mask, :] - # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(A_m) def project(self, J): - # project a particular vector or matrix onto the appearance bases + # Project a particular vector or matrix onto the appearance bases return self.pinv_A_m.dot(J - self.a_bar_m) - def _compute_train_features(self, image): - # warp image + def _compute_features(self, image): i = self.interface.warp(image) - # vectorize it and mask it i_m = i.as_vector()[self.interface.i_mask] - # project it onto the appearance model - return self.project(i_m) - - def _compute_test_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # project it onto the appearance model + # Project image onto the appearance model return self.project(i_m) @@ -402,65 +414,27 @@ def _compute_test_features(self, image): class AppearanceWeightsNewton(AppearanceWeights): r""" """ - def _perform_regression(self, features, deltas, gamma=None, - dtype=np.float64): - return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) - + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, bias=True): + super(AppearanceWeightsNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) -# TODO: document me! -class AppearanceWeightsGaussNewton(AppearanceWeights): - r""" - """ - def _perform_regression(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - return _supervised_gauss_newton(features, deltas, gamma=gamma, - psi=psi, dtype=dtype) + self._regressor_cls = partial(IRLRegression, alpha=alpha, + bias=bias) # TODO: document me! -class _supervised_newton(object): - r""" - """ - def __init__(self, features, deltas, gamma=None, dtype=np.float64): - features = features.astype(dtype) - deltas = deltas.astype(dtype) - XX = features.T.dot(features) - XT = features.T.dot(deltas) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # descent direction - self.R = np.linalg.solve(XX, XT) - - def __call__(self, features): - return np.dot(features, self.R) - - -# TODO: document me! -class _supervised_gauss_newton(object): +class AppearanceWeightsGaussNewton(AppearanceWeights): r""" """ - def __init__(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - features = features.astype(dtype) - # ridge regression - deltas = deltas.astype(dtype) - XX = deltas.T.dot(deltas) - XT = deltas.T.dot(features) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # average Jacobian - self.J = np.linalg.solve(XX, XT) - # average Hessian - self.H = self.J.dot(self.J.T) - if psi: - np.fill_diagonal(self.H, psi + np.diag(self.H)) - # descent direction - self.R = np.linalg.solve(self.H, self.J).T - - def __call__(self, features): - return np.dot(features, self.R) - - -# TODO: document me! -def _compute_rmse(x1, x2): - return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, alpha2=0, bias=True): + super(AppearanceWeightsGaussNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IIRLRegression, alpha=alpha, + alpha2=alpha2, bias=bias) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 58d3404..f0c4cb5 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -125,37 +125,34 @@ def _setup_algorithms(self): for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, self._sampling)): - + template = am.mean() if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, - source=am.mean().landmarks['source'].lms) - # set up algorithm using standard aam interface + source=template.landmarks['source'].lms) + interface = SupervisedDescentStandardInterface( + am, md_transform, template, sampling=s) algorithm = self._sd_algorithm_cls( - SupervisedDescentStandardInterface, am, md_transform, - sampling=s, max_iters=self.n_iterations[j]) - + interface, n_iterations=self.n_iterations[j]) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): - # build linear version of orthogonal model driven transform + # Build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) - # set up algorithm using linear aam interface + interface = SupervisedDescentLinearInterface( + am, md_transform, template, sampling=s) algorithm = self._sd_algorithm_cls( - SupervisedDescentLinearInterface, am, md_transform, - sampling=s, max_iters=self.n_iterations[j]) - + interface, n_iterations=self.n_iterations[j]) elif type(self.aam) is PartsAAM: - # build orthogonal point distribution model + # Build orthogonal point distribution model pdm = OrthoPDM(sm) - # set up algorithm using parts aam interface - algorithm = self._sd_algorithm_cls( - SupervisedDescentPartsInterface, am, pdm, - sampling=s, max_iters=self.n_iterations[j], + interface = SupervisedDescentPartsInterface( + am, pdm, template, sampling=s, patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts) - + algorithm = self._sd_algorithm_cls( + interface, n_iterations=self.n_iterations[j]) else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " From afc4c43a6617292ad68503cca1918f7ff5fe0c91 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 16:23:05 +0100 Subject: [PATCH 179/423] Fix incremental Gauss-Newton --- menpofit/math/regression.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/menpofit/math/regression.py b/menpofit/math/regression.py index 9b9c65d..0ec4ac6 100644 --- a/menpofit/math/regression.py +++ b/menpofit/math/regression.py @@ -64,7 +64,7 @@ def train(self, X, Y): # Note that everything is transposed from the above exchanging of roles H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J).T + self.W = np.linalg.solve(H, J) def increment(self, X, Y): # incremental least squares exchanging the roles of X and Y @@ -75,4 +75,7 @@ def increment(self, X, Y): # Note that everything is transposed from the above exchanging of roles H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J).T + self.W = np.linalg.solve(H, J) + + def predict(self, x): + return self.W.dot(x.T).T From b0fefccdaa969f5bba40c2f97fc8c066f2d13d7f Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 17:33:23 +0100 Subject: [PATCH 180/423] Refactoring AAM to match SD-AAM Interfaces are now concrete rather than passing a class. --- menpofit/aam/algorithm/lk.py | 57 +++++++++++++++++++----------------- menpofit/aam/fitter.py | 38 +++++++++++------------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 3750cba..5955b78 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -9,12 +9,18 @@ class LucasKanadeStandardInterface(object): r""" """ - def __init__(self, aam_algorithm, sampling=None): - self.algorithm = aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None): + self.appearance_model = appearance_model + self.transform = transform + self.template = template + self._build_sampling_mask(sampling) + + def _build_sampling_mask(self, sampling): n_true_pixels = self.template.n_true_pixels() n_channels = self.template.n_channels n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) if sampling is None: @@ -37,18 +43,6 @@ def __init__(self, aam_algorithm, sampling=None): def shape_model(self): return self.transform.pdm.model - @property - def appearance_model(self): - return self.algorithm.appearance_model - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - @property def n(self): return self.transform.n_parameters @@ -151,16 +145,20 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, class LucasKanadePartsInterface(LucasKanadeStandardInterface): r""" """ - def __init__(self, aam_algorithm, sampling=None, patch_shape=(17, 17), - normalize_parts=no_op): - self.algorithm = aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None, + patch_shape=(17, 17), normalize_parts=no_op): self.patch_shape = patch_shape + # TODO: Refactor to patch_features self.normalize_parts = normalize_parts + super(LucasKanadePartsInterface, self).__init__( + appearance_model, transform, template, sampling=sampling) + + def _build_sampling_mask(self, sampling): if sampling is None: sampling = np.ones(self.patch_shape, dtype=np.bool) - image_shape = self.algorithm.template.pixels.shape + image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], image_shape[:3] + (1, 1)) self.i_mask = np.nonzero(image_mask.flatten())[0] @@ -210,18 +208,23 @@ def steepest_descent_images(self, nabla, dw_dp): class LucasKanade(object): r""" """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # set common state for all AAM algorithms - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.transform = transform + def __init__(self, aam_interface, eps=10**-5): self.eps = eps - # set interface - self.interface = aam_interface(self, **kwargs) - # perform pre-computations + self.interface = aam_interface self._precompute() + @property + def appearance_model(self): + return self.interface.appearance_model + + @property + def transform(self): + return self.interface.transform + + @property + def template(self): + return self.interface.template + def _precompute(self): # grab number of shape and appearance parameters self.n = self.transform.n_parameters diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f0c4cb5..2afc4b4 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -40,54 +40,50 @@ class LucasKanadeAAMFitter(AAMFitter): r""" """ def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, - n_shape=None, n_appearance=None, sampling=None, **kwargs): + n_shape=None, n_appearance=None, sampling=None): self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_scales) - self._set_up(lk_algorithm_cls, sampling, **kwargs) + self._sampling = checks.check_sampling(sampling, aam.n_scales) + self._set_up(lk_algorithm_cls) - def _set_up(self, lk_algorithm_cls, sampling, **kwargs): + def _set_up(self, lk_algorithm_cls): self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models, sampling)): + self.aam.shape_models, + self._sampling)): + template = am.mean() if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, source=am.mean().landmarks['source'].lms) - # set up algorithm using standard aam interface - algorithm = lk_algorithm_cls( - LucasKanadeStandardInterface, am, md_transform, sampling=s, - **kwargs) - + interface = LucasKanadeStandardInterface(am, md_transform, + template, sampling=s) + algorithm = lk_algorithm_cls(interface) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) - # set up algorithm using linear aam interface - algorithm = lk_algorithm_cls( - LucasKanadeLinearInterface, am, md_transform, sampling=s, - **kwargs) - + interface = LucasKanadeLinearInterface(am, md_transform, + template, sampling=s) + algorithm = lk_algorithm_cls(interface) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) - # set up algorithm using parts aam interface - algorithm = lk_algorithm_cls( - LucasKanadePartsInterface, am, pdm, sampling=s, + interface = LucasKanadePartsInterface( + am, pdm, template, sampling=s, patch_shape=self.aam.patch_shape[j], - normalize_parts=self.aam.normalize_parts, **kwargs) - + normalize_parts=self.aam.normalize_parts) + algorithm = lk_algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " "{}".format(AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM)) - # append algorithms to list self.algorithms.append(algorithm) From b9a4cd93f9f99e28d9949dbfff7e00592c98aa7b Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 17:33:59 +0100 Subject: [PATCH 181/423] Refactor results to take a transform rather than algorithm This makes the objects much less heavyweight --- menpofit/aam/algorithm/lk.py | 4 ++-- menpofit/aam/algorithm/sd.py | 10 ++-------- menpofit/clm/algorithm/gd.py | 6 ++++-- menpofit/result.py | 15 +++++++-------- menpofit/sdm/algorithm.py | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 5955b78..c983f55 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -120,7 +120,7 @@ def solve_all_ml(self, H, J, e): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self.transform, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -136,7 +136,7 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self.transform, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index 0f2b5b8..012dc74 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -53,11 +53,8 @@ def warp(self, image): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): - # TODO: Faking an 'algorithm' - algorithm = lambda x: x - algorithm.transform = self.transform return AAMAlgorithmResult( - image, algorithm, shape_parameters, + image, self.transform, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -71,11 +68,8 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): - # TODO: Faking an 'algorithm' - algorithm = lambda x: x - algorithm.transform = self.transform return LinearAAMAlgorithmResult( - image, algorithm, shape_parameters, + image, self.transform, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/clm/algorithm/gd.py b/menpofit/clm/algorithm/gd.py index 6a7539b..b6fe3ca 100644 --- a/menpofit/clm/algorithm/gd.py +++ b/menpofit/clm/algorithm/gd.py @@ -125,7 +125,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, k += 1 # Return algorithm result - return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) + return CLMAlgorithmResult(image, self.transform, p_list, + gt_shape=gt_shape) # TODO: Document me! @@ -234,4 +235,5 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, k += 1 # Return algorithm result - return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) \ No newline at end of file + return CLMAlgorithmResult(image, self.transform, p_list, + gt_shape=gt_shape) diff --git a/menpofit/result.py b/menpofit/result.py index a8e7364..beae971 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -420,9 +420,9 @@ def as_serializableresult(self): class ParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, algorithm, shape_parameters, gt_shape=None): + def __init__(self, image, transform, shape_parameters, gt_shape=None): self.image = image - self.algorithm = algorithm + self.transform = transform self.shape_parameters = shape_parameters self._gt_shape = gt_shape @@ -436,7 +436,7 @@ def transforms(self): Generates a list containing the transforms obtained at each fitting iteration. """ - return [self.algorithm.transform.from_vector(p) + return [self.transform.from_vector(p) for p in self.shape_parameters] @property @@ -444,18 +444,18 @@ def final_transform(self): r""" Returns the final transform. """ - return self.algorithm.transform.from_vector(self.shape_parameters[-1]) + return self.transform.from_vector(self.shape_parameters[-1]) @property def initial_transform(self): r""" Returns the initial transform from which the fitting started. """ - return self.algorithm.transform.from_vector(self.shape_parameters[0]) + return self.transform.from_vector(self.shape_parameters[0]) @property def shapes(self): - return [self.algorithm.transform.from_vector(p).target + return [self.transform.from_vector(p).target for p in self.shape_parameters] @property @@ -471,9 +471,8 @@ def initial_shape(self): class NonParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, algorithm, shapes, gt_shape=None): + def __init__(self, image, shapes, gt_shape=None): self.image = image - self.algorithm = algorithm self._shapes = shapes self._gt_shape = gt_shape diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index d1ce75a..88dd0b9 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -101,7 +101,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): shapes.append(current_shape) # return algorithm result - return NonParametricAlgorithmResult(image, self, shapes, + return NonParametricAlgorithmResult(image, shapes, gt_shape=gt_shape) def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, From dc6795506f1e66b996051ac9cb4df32629257a1a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 17:34:28 +0100 Subject: [PATCH 182/423] Fix menpofit widgets No more exceptions from iterations or error types. --- menpofit/result.py | 4 ++-- menpofit/visualize/widgets/base.py | 35 ++++++++++++++++-------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index beae971..ecedd93 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -171,7 +171,7 @@ def errors(self, compute_error=None): raise ValueError('Ground truth has not been set, errors cannot ' 'be computed') - def plot_errors(self, error_type='me_norm', figure_id=None, + def plot_errors(self, error_type=None, figure_id=None, new_figure=False, render_lines=True, line_colour='b', line_style='-', line_width=2, render_markers=True, marker_style='o', marker_size=4, marker_face_colour='b', @@ -245,7 +245,7 @@ def plot_errors(self, error_type='me_norm', figure_id=None, The viewer object. """ from menpo.visualize import GraphPlotter - errors_list = self.errors(error_type=error_type) + errors_list = self.errors(compute_error=error_type) return GraphPlotter(figure_id=figure_id, new_figure=new_figure, x_axis=range(len(errors_list)), y_axis=[errors_list], diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 05739ee..3871615 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -2223,7 +2223,7 @@ def plot_errors_function(name): renderer_options_wid.selected_values[0]['figure']['x_scale'] * 10, renderer_options_wid.selected_values[0]['figure']['y_scale'] * 3) renderer = fitting_results[im].plot_errors( - error_type=error_type_wid.value, + error_type=_error_type_key_to_func(error_type_wid.value), figure_id=save_figure_wid.renderer.figure_id, figure_size=new_figure_size) @@ -2517,18 +2517,8 @@ def plot_ced_fun(name): # widget closes plot_ced_but.visible = False - # Get error type error_type = error_type_wid.value - - from menpofit.result import ( - compute_root_mean_square_error, compute_point_to_point_error, - compute_normalise_point_to_point_error) - if error_type is 'me_norm': - func = compute_normalise_point_to_point_error - elif error_type is 'me': - func = compute_point_to_point_error - elif error_type is 'rmse': - func = compute_root_mean_square_error + func = _error_type_key_to_func(error_type) # Create errors list fit_errors = [f.final_error(compute_error=func) @@ -2646,8 +2636,8 @@ def update_widgets(name, value): # animation. Specifically, if the animation is activated and the user # selects the iterations tab, then the animation stops. def results_tab_fun(name, value): - if value == 1 and image_number_wid.play_toggle.value: - image_number_wid.stop_toggle.value = True + if value == 1 and image_number_wid.play_options_toggle.value: + image_number_wid.stop_options_toggle.value = True result_wid.on_trait_change(results_tab_fun, 'selected_index') # Widget titles @@ -2676,8 +2666,8 @@ def results_tab_fun(name, value): # If animation is activated and the user selects the save figure tab, # then the animation stops. def save_fig_tab_fun(name, value): - if value == 3 and image_number_wid.play_toggle.value: - image_number_wid.stop_toggle.value = True + if value == 3 and image_number_wid.play_options_toggle.value: + image_number_wid.stop_options_toggle.value = True options_box.on_trait_change(save_fig_tab_fun, 'selected_index') # Set widget's style @@ -2691,3 +2681,16 @@ def save_fig_tab_fun(name, value): # Reset value to trigger initial visualization renderer_options_wid.options_widgets[3].render_legend_checkbox.value = True + + +def _error_type_key_to_func(error_type): + from menpofit.result import ( + compute_root_mean_square_error, compute_point_to_point_error, + compute_normalise_point_to_point_error) + if error_type is 'me_norm': + func = compute_normalise_point_to_point_error + elif error_type is 'me': + func = compute_point_to_point_error + elif error_type is 'rmse': + func = compute_root_mean_square_error + return func From 6c25a534dc7d86b5ec9f4d4c5659351b7e45ec1b Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 11:42:52 +0100 Subject: [PATCH 183/423] Nasty bug about truncating shape model Because we just deepcopy the shape model, we were throwing the components away too early. Now we build all the shape models, then truncate them at the end. --- menpofit/aam/base.py | 57 ++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index e7c1dea..21d7efd 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -118,7 +118,7 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): - # check parameters + checks.check_diagonal(diagonal) n_scales = len(scales) scales = checks.check_scales(scales) @@ -127,7 +127,7 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_scales, 'max_appearance_components') - # set parameters + self.features = features self.transform = transform self.diagonal = diagonal @@ -244,17 +244,14 @@ def _train(self, images, group=None, verbose=False, increment=False, if not increment: if j == 0: shape_model = self._build_shape_model( - scale_shapes, self.max_shape_components[j], j) - # Store shape model + scale_shapes, j) self.shape_models.append(shape_model) else: - # Copy shape model self.shape_models.append(deepcopy(shape_model)) else: self._increment_shape_model( scale_shapes, self.shape_models[j], - forgetting_factor=shape_forgetting_factor, - max_components=self.max_shape_components[j]) + forgetting_factor=shape_forgetting_factor) # Obtain warped images - we use a scaled version of the # reference shape, computed here. This is because the mean @@ -292,6 +289,14 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print_dynamic('{}Done\n'.format(scale_prefix)) + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for k, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[k] + if max_sc is not None: + sm.trim_components(max_sc) + def increment(self, images, group=None, verbose=False, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, batch_size=None): @@ -304,18 +309,16 @@ def increment(self, images, group=None, verbose=False, appearance_forgetting_factor=aff, increment=True, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, scale_index): - return build_shape_model(shapes, max_components=max_components) + def _build_shape_model(self, shapes, scale_index): + return build_shape_model(shapes) - def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, - max_components=None): + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): # Compute aligned shapes aligned_shapes = align_shapes(shapes) # Increment shape model shape_model.increment(aligned_shapes, forgetting_factor=forgetting_factor) - if max_components is not None: - shape_model.trim_components(max_components) def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): @@ -664,27 +667,32 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, scale_index): + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Active Appearance Model' + + def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_reference_frame(mean_aligned_shape) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model - shape_model = build_shape_model( - dense_shapes, max_components=max_components) + shape_model = build_shape_model(dense_shapes) return shape_model - def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, - max_components=None): + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): aligned_shapes = align_shapes(shapes) dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, self.transform) # Increment shape model shape_model.increment(dense_shapes, forgetting_factor=forgetting_factor) - if max_components is not None: - shape_model.trim_components(max_components) def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): @@ -767,20 +775,17 @@ def _build_shape_model(self, shapes, max_components, scale_index): dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model - shape_model = build_shape_model(dense_shapes, - max_components=max_components) + shape_model = build_shape_model(dense_shapes) return shape_model - def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, - max_components=None): + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): aligned_shapes = align_shapes(shapes) dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, self.transform) # Increment shape model shape_model.increment(dense_shapes, forgetting_factor=forgetting_factor) - if max_components is not None: - shape_model.trim_components(max_components) def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): From ee9e58b2417643c824837002bf05ed100ebfd57f Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 11:43:32 +0100 Subject: [PATCH 184/423] Fix __str__ for AAMs Also, add HolisticAAM as AAM alias --- menpofit/aam/__init__.py | 2 +- menpofit/aam/base.py | 81 +++++++++++++++++++++++++++++++--------- menpofit/sdm/fitter.py | 24 +++++++----- 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index aca4250..b91a3ae 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,4 +1,4 @@ -from .base import AAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM +from .base import HolisticAAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 21d7efd..a9ac753 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -341,7 +341,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Active Appearance Model' + return 'Holistic Active Appearance Model' def instance(self, shape_weights=None, appearance_weights=None, scale_index=-1): @@ -526,9 +526,8 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=parameters_bounds, figure_size=figure_size, mode=mode) - # TODO: fix me! def __str__(self): - return '' + return _aam_str(self) # TODO: document me! @@ -587,7 +586,7 @@ def _warp_images(self, images, shapes, reference_shape, scale_index, @property def _str_title(self): - return 'Patch-Based Active Appearance Model' + return 'Patch-based Active Appearance Model' def _instance(self, scale_index, shape_instance, appearance_instance): template = self.appearance_models[scale_index].mean @@ -611,14 +610,8 @@ def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=parameters_bounds, figure_size=figure_size, mode=mode) - # TODO: fix me! def __str__(self): - out = super(PatchAAM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + return _aam_str(self) # TODO: document me! @@ -716,9 +709,8 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _aam_str(self) # TODO: document me! @@ -767,7 +759,15 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, scale_index): + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Patch-based Active Appearance Model' + + def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( @@ -809,9 +809,8 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _aam_str(self) # TODO: document me! @@ -863,6 +862,14 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Parts-based Active Appearance Model' + def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): return extract_patches(images, shapes, self.patch_shape[scale_index], @@ -885,6 +892,44 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _aam_str(self) + + +def _aam_str(aam): + if aam.diagonal is not None: + diagonal = aam.diagonal + else: + y, x = aam.reference_shape.range() + diagonal = np.sqrt(x ** 2 + y ** 2) + + # Compute scale info strings + scales_info = [] + lvl_str_tmplt = r""" - Scale {} + - Holistic feature: {} + - {} appearance components + - {} shape components""" + for k, s in enumerate(aam.scales): + scales_info.append(lvl_str_tmplt.format( + s, name_of_callable(aam.features[k]), + aam.appearance_models[k].n_components, + aam.shape_models[k].n_components)) + # Patch based AAM + if hasattr(aam, 'patch_shape'): + for k, s in enumerate(scales_info): + s += '\n - Patch shape: {}'.format(aam.patch_shape[k]) + scales_info = '\n'.join(scales_info) + + cls_str = r"""{class_title} + - Images warped with {transform} transform + - Images scaled to diagonal: {diagonal:.2f} + - Scales: {scales} +{scales_info} +""".format(class_title=aam._str_title, + transform=name_of_callable(aam.transform), + diagonal=diagonal, + scales=aam.scales, + scales_info=scales_info) + return cls_str + +HolisticAAM = AAM diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 0a84f34..ed38cce 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -252,28 +252,32 @@ def __str__(self): scales_info = [] lvl_str_tmplt = r""" - Scale {} - {} iterations - - Patch shape: {}""" + - Patch shape: {} + - Holistic feature: {} + - Patch feature: {}""" for k, s in enumerate(self.scales): - scales_info.append(lvl_str_tmplt.format(s, - self.n_iterations[k], - self._patch_shape[k])) + scales_info.append(lvl_str_tmplt.format( + s, self.n_iterations[k], self._patch_shape[k], + name_of_callable(self.features[k]), + name_of_callable(self._patch_features[k]))) scales_info = '\n'.join(scales_info) cls_str = r"""Supervised Descent Method - Regression performed using the {reg_alg} algorithm - Regression class: {reg_cls} - - Scales: {scales} -{scales_info} - Perturbations generated per shape: {n_perturbations} - Images scaled to diagonal: {diagonal:.2f} - - Custom perturbation scheme used: {is_custom_perturb_func}""".format( + - Custom perturbation scheme used: {is_custom_perturb_func} + - Scales: {scales} +{scales_info} +""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), reg_cls=name_of_callable(regressor_cls), - scales=self.scales, - scales_info=scales_info, n_perturbations=self.n_perturbations, diagonal=diagonal, - is_custom_perturb_func=is_custom_perturb_func) + is_custom_perturb_func=is_custom_perturb_func, + scales=self.scales, + scales_info=scales_info) return cls_str From d8870811145f7e6a64e6e65a64e88b5d5d7d7cd0 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 13:53:32 +0100 Subject: [PATCH 185/423] Change results back to taking algorithm After speaking to @jalabort - he found this more useful. --- menpofit/aam/algorithm/lk.py | 2 +- menpofit/aam/algorithm/sd.py | 4 ++-- menpofit/aam/result.py | 1 - menpofit/atm/algorithm.py | 2 +- menpofit/clm/algorithm/gd.py | 3 +-- menpofit/result.py | 12 ++++++------ 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index c983f55..fe63075 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -120,7 +120,7 @@ def solve_all_ml(self, H, J, e): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index 012dc74..ce0d9c0 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -54,7 +54,7 @@ def warp(self, image): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -69,7 +69,7 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 00a500a..ae478f2 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -107,4 +107,3 @@ def costs(self): for a in self.algorithm_results: costs += a.costs return costs - diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index d92df72..387a9a5 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -101,7 +101,7 @@ def solve_shape_ml(cls, H, J, e): def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): return ATMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, gt_shape=gt_shape) diff --git a/menpofit/clm/algorithm/gd.py b/menpofit/clm/algorithm/gd.py index b6fe3ca..a140f5c 100644 --- a/menpofit/clm/algorithm/gd.py +++ b/menpofit/clm/algorithm/gd.py @@ -235,5 +235,4 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, k += 1 # Return algorithm result - return CLMAlgorithmResult(image, self.transform, p_list, - gt_shape=gt_shape) + return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) diff --git a/menpofit/result.py b/menpofit/result.py index ecedd93..20cd791 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -420,9 +420,9 @@ def as_serializableresult(self): class ParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, transform, shape_parameters, gt_shape=None): + def __init__(self, image, algorithm, shape_parameters, gt_shape=None): self.image = image - self.transform = transform + self.algorithm = algorithm self.shape_parameters = shape_parameters self._gt_shape = gt_shape @@ -436,7 +436,7 @@ def transforms(self): Generates a list containing the transforms obtained at each fitting iteration. """ - return [self.transform.from_vector(p) + return [self.algorithm.transform.from_vector(p) for p in self.shape_parameters] @property @@ -444,18 +444,18 @@ def final_transform(self): r""" Returns the final transform. """ - return self.transform.from_vector(self.shape_parameters[-1]) + return self.algorithm.transform.from_vector(self.shape_parameters[-1]) @property def initial_transform(self): r""" Returns the initial transform from which the fitting started. """ - return self.transform.from_vector(self.shape_parameters[0]) + return self.algorithm.transform.from_vector(self.shape_parameters[0]) @property def shapes(self): - return [self.transform.from_vector(p).target + return [self.algorithm.transform.from_vector(p).target for p in self.shape_parameters] @property From b4fe608a718c4c547f2a51be77162b9881e53271 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 16:23:49 +0100 Subject: [PATCH 186/423] Partial commit - moving home due to tube strike Working on refactoring the ATMs --- menpofit/aam/algorithm/lk.py | 112 +++-- menpofit/atm/algorithm.py | 188 ++------ menpofit/atm/base.py | 827 ++++++++++++++++++----------------- menpofit/atm/fitter.py | 42 +- 4 files changed, 554 insertions(+), 615 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index fe63075..8c33efc 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -5,12 +5,30 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult +def _solve_all_map(H, J, e, Ja_prior, c, Js_prior, p, m, n): + if n is not H.shape[0] - m: + # Bidirectional Compositional case + Js_prior = np.hstack((Js_prior, Js_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + J_prior = np.hstack((Ja_prior, Js_prior)) + H += np.diag(J_prior) + Je = J_prior * np.hstack((c, p)) + J.T.dot(e) + dq = - np.linalg.solve(H, Je) + return dq[:m], dq[m:] + + +def _solve_all_ml(H, J, e, m): + # compute ML solution + dq = - np.linalg.solve(H, J.T.dot(e)) + return dq[:m], dq[m:] + + # TODO document me! -class LucasKanadeStandardInterface(object): +class LucasKanadeBaseInterface(object): r""" """ - def __init__(self, appearance_model, transform, template, sampling=None): - self.appearance_model = appearance_model + def __init__(self, transform, template, sampling=None): self.transform = transform self.template = template @@ -47,10 +65,6 @@ def shape_model(self): def n(self): return self.transform.n_parameters - @property - def m(self): - return self.appearance_model.n_active_components - @property def true_indices(self): return self.template.mask.true_indices() @@ -100,22 +114,24 @@ def solve_shape_ml(cls, H, J, e): # compute and return ML solution return -np.linalg.solve(H, J.T.dot(e)) + +class LucasKanadeStandardInterface(LucasKanadeBaseInterface): + + def __init__(self, appearance_model, transform, template, sampling=None): + super(LucasKanadeStandardInterface, self).__init__(transform, template, + sampling=sampling) + self.appearance_model = appearance_model + + @property + def m(self): + return self.appearance_model.n_active_components + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): - if self.n is not H.shape[0] - self.m: - # Bidirectional Compositional case - Js_prior = np.hstack((Js_prior, Js_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - J_prior = np.hstack((Ja_prior, Js_prior)) - H += np.diag(J_prior) - Je = J_prior * np.hstack((c, p)) + J.T.dot(e) - dq = - np.linalg.solve(H, Je) - return dq[:self.m], dq[self.m:] + return _solve_all_map(H, J, e, Ja_prior, c, Js_prior, p, + self.m, self.n) def solve_all_ml(self, H, J, e): - # compute ML solution - dq = - np.linalg.solve(H, J.T.dot(e)) - return dq[:self.m], dq[self.m:] + return _solve_all_ml(H, J, e, self.m) def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): @@ -136,23 +152,23 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) # TODO document me! -class LucasKanadePartsInterface(LucasKanadeStandardInterface): +class LucasKanadePartsBaseInterface(LucasKanadeBaseInterface): r""" """ - def __init__(self, appearance_model, transform, template, sampling=None, + def __init__(self, transform, template, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): self.patch_shape = patch_shape # TODO: Refactor to patch_features self.normalize_parts = normalize_parts - super(LucasKanadePartsInterface, self).__init__( - appearance_model, transform, template, sampling=sampling) + super(LucasKanadeBaseInterface, self).__init__( + transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): if sampling is None: @@ -175,15 +191,18 @@ def warp_jacobian(self): return np.rollaxis(self.transform.d_dp(None), -1) def warp(self, image): - return Image(image.extract_patches( - self.transform.target, patch_size=self.patch_shape, - as_single_array=True)) + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts, copy=False) def gradient(self, image): - nabla = fast_gradient(image.pixels.reshape((-1,) + self.patch_shape)) + pixels = image.pixels + nabla = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) # remove 1st dimension gradient which corresponds to the gradient # between parts - return nabla.reshape((2,) + image.pixels.shape) + return nabla.reshape((2,) + pixels.shape) def steepest_descent_images(self, nabla, dw_dp): # reshape nabla @@ -204,6 +223,39 @@ def steepest_descent_images(self, nabla, dw_dp): return sdi.reshape((-1, sdi.shape[-1])) +# TODO document me! +class LucasKanadePartsInterface(LucasKanadePartsBaseInterface): + r""" + """ + def __init__(self, appearance_model, transform, template, sampling=None, + patch_shape=(17, 17), normalize_parts=no_op): + self.patch_shape = patch_shape + # TODO: Refactor to patch_features + self.normalize_parts = normalize_parts + self.appearance_model = appearance_model + + super(LucasKanadePartsInterface, self).__init__( + transform, template, sampling=sampling) + + @property + def m(self): + return self.appearance_model.n_active_components + + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): + return _solve_all_map(H, J, e, Ja_prior, c, Js_prior, p, + self.m, self.n) + + def solve_all_ml(self, H, J, e): + return _solve_all_ml(H, J, e, self.m) + + def algorithm_result(self, image, shape_parameters, cost_functions=None, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self, shape_parameters, + cost_functions=cost_functions, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + # TODO document me! class LucasKanade(object): r""" diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 387a9a5..90515cd 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,102 +1,17 @@ from __future__ import division import numpy as np -from menpo.image import Image -from menpo.feature import no_op -from menpo.feature import gradient as fast_gradient +from menpofit.aam.algorithm.lk import (LucasKanadeBaseInterface, + LucasKanadePartsBaseInterface) from .result import ATMAlgorithmResult, LinearATMAlgorithmResult # TODO document me! -class LucasKanadeStandardInterface(object): +class ATMLKStandardInterface(LucasKanadeBaseInterface): r""" """ - def __init__(self, lk_algorithm, sampling=None): - self.algorithm = lk_algorithm - - n_true_pixels = self.template.n_true_pixels() - n_channels = self.template.n_channels - n_parameters = self.transform.n_parameters - sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - - if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 - - self.i_mask = np.nonzero(np.tile( - sampling_mask[None, ...], (n_channels, 1)).flatten())[0] - self.dW_dp_mask = np.nonzero(np.tile( - sampling_mask[None, ..., None], (2, 1, n_parameters))) - self.nabla_mask = np.nonzero(np.tile( - sampling_mask[None, None, ...], (2, n_channels, 1))) - self.nabla2_mask = np.nonzero(np.tile( - sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - - @property - def n(self): - return self.transform.n_parameters - - @property - def true_indices(self): - return self.template.mask.true_indices() - - @property - def shape_model(self): - return self.transform.pdm.model - - def warp_jacobian(self): - dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) - return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, - dW_dp.shape[2])) - - def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) - - def gradient(self, img): - nabla = fast_gradient(img) - nabla.set_boundary_pixels() - return nabla.as_vector().reshape((2, img.n_channels, -1)) - - def steepest_descent_images(self, nabla, dW_dp): - # reshape gradient - # nabla: n_dims x n_channels x n_pixels - nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) - # compute steepest descent images - # nabla: n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # sdi: n_channels x n_pixels x n_params - sdi = 0 - a = nabla[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - # reshape steepest descent images - # sdi: (n_channels x n_pixels) x n_params - return sdi.reshape((-1, sdi.shape[2])) - - @classmethod - def solve_shape_map(cls, H, J, e, J_prior, p): - if p.shape[0] is not H.shape[0]: - # Bidirectional Compositional case - J_prior = np.hstack((J_prior, J_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - H += np.diag(J_prior) - Je = J_prior * p + J.T.dot(e) - return - np.linalg.solve(H, Je) - - @classmethod - def solve_shape_ml(cls, H, J, e): - # compute and return ML solution - return -np.linalg.solve(H, J.T.dot(e)) + def __init__(self, transform, template, sampling=None): + super(ATMLKStandardInterface, self).__init__(transform, template, + sampling=sampling) def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): @@ -106,7 +21,7 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, # TODO document me! -class LucasKanadeLinearInterface(LucasKanadeStandardInterface): +class ATMLKLinearInterface(ATMLKStandardInterface): r""" """ @property @@ -116,87 +31,42 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): return LinearATMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! -class LucasKanadePartsInterface(LucasKanadeStandardInterface): +class ATMLKPartsInterface(LucasKanadePartsBaseInterface): r""" """ - def __init__(self, lk_algorithm, patch_shape=(17, 17), - normalize_parts=no_op, sampling=None): - self.algorithm = lk_algorithm - self.patch_shape = patch_shape - self.normalize_parts = normalize_parts - - if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) - - image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling[None, None, None, ...], - image_shape[:3] + (1, 1)) - self.i_mask = np.nonzero(image_mask.flatten())[0] - self.nabla_mask = np.nonzero(np.tile( - image_mask[None, ...], (2, 1, 1, 1, 1, 1))) - self.nabla2_mask = np.nonzero(np.tile( - image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) - - @property - def shape_model(self): - return self.transform.model - - def warp_jacobian(self): - return np.rollaxis(self.transform.d_dp(None), -1) - - def warp(self, image): - parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, - as_single_array=True) - parts = self.normalize_parts(parts) - return Image(parts) - - def gradient(self, image): - pixels = image.pixels - g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) - # remove 1st dimension gradient which corresponds to the gradient - # between parts - return g.reshape((2,) + pixels.shape) - - def steepest_descent_images(self, nabla, dw_dp): - # reshape nabla - # nabla: dims x parts x off x ch x (h x w) - nabla = nabla[self.nabla_mask].reshape( - nabla.shape[:-2] + (-1,)) - # compute steepest descent images - # nabla: dims x parts x off x ch x (h x w) - # ds_dp: dims x parts x x params - # sdi: parts x off x ch x (h x w) x params - sdi = 0 - a = nabla[..., None] * dw_dp[..., None, None, None, :] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (parts x offsets x ch x w x h) x params - return sdi.reshape((-1, sdi.shape[-1])) + def algorithm_result(self, image, shape_parameters, cost_functions=None, + gt_shape=None): + return LinearATMAlgorithmResult( + image, self, shape_parameters, + cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! class LucasKanade(object): - def __init__(self, lk_atm_interface_cls, template, transform, - eps=10**-5, **kwargs): - # set common state for all ATM algorithms - self.template = template - self.transform = transform + def __init__(self, atm_interface, eps=10**-5): self.eps = eps - # set interface - self.interface = lk_atm_interface_cls(self, **kwargs) - # perform pre-computations + self.interface = atm_interface() self._precompute() - def _precompute(self, **kwargs): + @property + def appearance_model(self): + return self.interface.appearance_model + + @property + def transform(self): + return self.interface.transform + + @property + def template(self): + return self.interface.template + + def _precompute(self): # grab number of shape and appearance parameters self.n = self.transform.n_parameters diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index b903075..95d675d 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -1,104 +1,257 @@ from __future__ import division +from copy import deepcopy +import warnings import numpy as np -from menpo.shape import TriMesh -from menpofit.transform import DifferentiableThinPlateSplines -from menpofit.base import name_of_callable -from menpofit.aam.builder import ( - build_patch_reference_frame, build_reference_frame) +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpo.model import PCAModel +from menpo.transform import Scale +from menpo.shape import mean_pointcloud +from menpofit import checks +from menpofit.transform import (DifferentiableThinPlateSplines, + DifferentiablePiecewiseAffine) +from menpofit.base import name_of_callable, batch +from menpofit.builder import ( + build_reference_frame, build_patch_reference_frame, + compute_features, scale_images, build_shape_model, warp_images, + align_shapes, rescale_images_to_reference_shape, densify_shapes, + extract_patches, MenpoFitBuilderWarning, compute_reference_shape) +# TODO: document me! class ATM(object): r""" Active Template Model class. + """ + def __init__(self, images, group=None, verbose=False, reference_shape=None, + features=no_op, transform=DifferentiablePiecewiseAffine, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + batch_size=None): + + checks.check_diagonal(diagonal) + n_scales = len(scales) + scales = checks.check_scales(scales) + features = checks.check_features(features, n_scales) + max_shape_components = checks.check_max_components( + max_shape_components, n_scales, 'max_shape_components') - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the ATM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. + self.features = features + self.transform = transform + self.diagonal = diagonal + self.scales = scales + self.max_shape_components = max_shape_components + self.reference_shape = reference_shape + self.shape_models = [] + self.warped_templates = [] - scales : `int` or float` or list of those, optional + # Train ATM + self._train(images, group=group, verbose=verbose, increment=False, + batch_size=batch_size) - scale_shapes : `boolean` + def _train(self, images, group=None, verbose=False, increment=False, + shape_forgetting_factor=1.0, batch_size=None): + r""" + Builds an Active Template Model from a list of landmarked images. - scale_features : `boolean` + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. - """ - def __init__(self, shape_models, warped_templates, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest scale + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) + checks.check_trilist(image_batch[0], self.transform, + group=group) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): + if verbose: + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) + else: + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract potentially rescaled shapes + scale_shapes = [i.landmarks[group].lms for i in scaled_images] + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model( + scale_shapes, j) + self.shape_models.append(shape_model) + else: + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(scaled_images, scale_shapes, + scaled_reference_shape, + j, scale_prefix, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format( + scale_prefix)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for k, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[k] + if max_sc is not None: + sm.trim_components(max_sc) + + def increment(self, images, group=None, verbose=False, + shape_forgetting_factor=1.0, batch_size=None): + return self._train(images, group=group, + verbose=verbose, + shape_forgetting_factor=shape_forgetting_factor, + increment=True, batch_size=batch_size) + + def _build_shape_model(self, shapes, scale_index): + return build_shape_model(shapes) + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + # Compute aligned shapes + aligned_shapes = align_shapes(shapes) + # Increment shape model + shape_model.increment(aligned_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + prefix=prefix, verbose=verbose) @property def n_scales(self): """ - The number of scale level of the ATM. + The number of scales of the AAM. :type: `int` """ return len(self.scales) - # TODO: Could we directly use class names instead of this? @property def _str_title(self): r""" Returns a string containing name of the model. - :type: `string` """ - return 'Active Template Model' + return 'Holistic Active Template Model' - def instance(self, shape_weights=None, level=-1): + def instance(self, shape_weights=None, scale_index=-1): r""" - Generates a novel ATM instance given a set of shape weights. If no - weights are provided, the mean shape instance is returned. - - Parameters - ----------- - shape_weights : ``(n_weights,)`` `ndarray` or `float` list - Weights of the shape model that will be used to create - a novel shape instance. If ``None``, the mean shape - ``(shape_weights = [0, 0, ..., 0])`` is used. - - level : `int`, optional - The pyramidal level to be used. - Returns ------- image : :map:`Image` - The novel ATM instance. + The novel AAM instance. """ - sm = self.shape_models[level] + sm = self.shape_models[scale_index] + template = self.warped_templates[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel if shape_weights is None: @@ -107,46 +260,41 @@ def instance(self, shape_weights=None, level=-1): shape_weights *= sm.eigenvalues[:n_shape_weights] ** 0.5 shape_instance = sm.instance(shape_weights) - return self._instance(level, shape_instance) + return self._instance(shape_instance, template) - def random_instance(self, level=-1): + def random_instance(self, scale_index=-1): r""" Generates a novel random instance of the ATM. Parameters ----------- - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` - The novel ATM instance. + The novel AAM instance. """ - sm = self.shape_models[level] + sm = self.shape_models[scale_index] + template = self.warped_templates[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel shape_weights = (np.random.randn(sm.n_active_components) * sm.eigenvalues[:sm.n_active_components]**0.5) shape_instance = sm.instance(shape_weights) - return self._instance(level, shape_instance) + return self._instance(scale_index, shape_instance, template) - def _instance(self, level, shape_instance): - template = self.warped_templates[level] + def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) + reference_frame = build_reference_frame(shape_instance) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - return template.as_unmasked().warp_to_mask( + return template.as_unmasked(copy=False).warp_to_mask( reference_frame.mask, transform, warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, @@ -185,25 +333,32 @@ def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): r""" - Visualizes the ATM object using the - menpo.visualize.widgets.visualize_atm widget. - + Visualizes both the shape and appearance models of the AAM object using + the `menpo.visualize.widgets.visualize_aam` widget. Parameters ----------- n_shape_parameters : `int` or `list` of `int` or None, optional The number of shape principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. + n_appearance_parameters : `int` or `list` of `int` or None, optional + The number of appearance principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per scale is the minimum + between `n_parameters` and the number of active components per + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional If ``'single'``, only a single slider is constructed along with a drop down menu. - If ``'multiple'``, a slider is constructed for each pp window. + If ``'multiple'``, a slider is constructed for each parameter. figure_size : (`int`, `int`), optional The size of the plotted figures. """ @@ -212,165 +367,39 @@ def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=parameters_bounds, figure_size=figure_size, mode=mode) - # TODO: fix me! def __str__(self): - out = "{}\n - {} training shapes.\n".format(self._str_title, - self.n_training_shapes) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_scales): - n_channels.append( - self.warped_templates[j].n_channels) - if j == self.n_scales - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_scales - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_scales): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_scales > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_scales, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_scales, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.warped_templates[0].n_true_pixels() * - n_channels[0], - self.warped_templates[0].n_true_pixels(), - n_channels[0], - self.warped_templates[0]._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_scales - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_scales - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.warped_templates[i].n_true_pixels() * - n_channels[i], - self.warped_templates[i].n_true_pixels(), - n_channels[i], - self.warped_templates[i]._str_shape, - n_channels[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} shape components ({10:.2f}% of " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.warped_templates[0].n_true_pixels() * n_channels[0], - self.warped_templates[0].n_true_pixels(), - n_channels[0], - self.warped_templates[0]._str_shape, - n_channels[0], self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100) - return out + return _atm_str(self) +# TODO: document me! class PatchATM(ATM): r""" - Patch Based Active Template Model class. - - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the ATM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based AAM. - - features : `callable` or ``[callable]`` - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those + Patch based Based Active Appearance Model class. + """ - scale_shapes : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - scale_features : `boolean` + super(PatchATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + batch_size=batch_size) - """ - def __init__(self, shape_models, warped_templates, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = DifferentiableThinPlateSplines - self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[scale_index]) + return warp_images(images, shapes, reference_frame, self.transform, + prefix=prefix, verbose=verbose) @property def _str_title(self): - return 'Patch-Based Active Template Model' + return 'Patch-based Active Template Model' - def _instance(self, level, shape_instance): - template = self.warped_templates[level] + def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( @@ -382,224 +411,220 @@ def _instance(self, level, shape_instance): return template.as_unmasked().warp_to_mask( reference_frame.mask, transform, warp_landmarks=True) - # TODO: fix me! def __str__(self): - out = super(PatchBasedATM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + return _atm_str(self) # TODO: document me! class LinearATM(ATM): r""" Linear Active Template Model class. + """ - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those - - scale_shapes : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, + batch_size=None): - scale_features : `boolean` + super(LinearATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=transform, diagonal=diagonal, scales=scales, + max_shape_components=max_shape_components, batch_size=batch_size) - """ - def __init__(self, shape_models, warped_templates, reference_shape, - transform, features, scales, scale_shapes, scale_features, - n_landmarks): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Active Template Model' + + def _build_shape_model(self, shapes, scale_index): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes) + return shape_model + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, prefix=prefix, + verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance): + def _instance(self, shape_instance, template): raise NotImplemented # TODO: implement me! - def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _atm_str(self) # TODO: document me! class LinearPatchATM(ATM): r""" Linear Patch based Active Template Model class. + """ - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based AAM. - - features : `callable` or ``[callable]`` - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those - - scale_shapes : `boolean` - - scale_features : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - n_landmarks: `int` + super(LinearPatchATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + batch_size=batch_size) - """ - def __init__(self, shape_models, warped_templates, reference_shape, - patch_shape, features, scales, scale_shapes, - scale_features, n_landmarks): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = DifferentiableThinPlateSplines - self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Patch-based Active Template Model' + + def _build_shape_model(self, shapes, scale_index): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes) + return shape_model + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, prefix=prefix, + verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance): + def _instance(self, shape_instance, template): raise NotImplemented # TODO: implement me! - def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _atm_str(self) # TODO: document me! +# TODO: implement offsets support? class PartsATM(ATM): r""" Parts based Active Template Model class. + """ - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based AAM. - - features : `callable` or ``[callable]`` - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - normalize_parts: `callable` - - scales : `int` or float` or list of those + def __init__(self, images, group=None, verbose=False, features=no_op, + normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.normalize_parts = normalize_parts - scale_shapes : `boolean` + super(PartsATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + batch_size=batch_size) - scale_features : `boolean` + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Parts-based Active Template Model' - """ - def __init__(self, shape_models, warped_templates, reference_shape, - patch_shape, features, normalize_parts, scales, - scale_shapes, scale_features): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return extract_patches(images, shapes, self.patch_shape[scale_index], + normalize_function=self.normalize_parts, + prefix=prefix, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance): + def _instance(self, shape_instance, template): raise NotImplemented # TODO: implement me! - def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _atm_str(self) + + +def _atm_str(atm): + if atm.diagonal is not None: + diagonal = atm.diagonal + else: + y, x = atm.reference_shape.range() + diagonal = np.sqrt(x ** 2 + y ** 2) + + # Compute scale info strings + scales_info = [] + lvl_str_tmplt = r""" - Scale {} + - Holistic feature: {} + - {} appearance components + - {} shape components""" + for k, s in enumerate(atm.scales): + scales_info.append(lvl_str_tmplt.format( + s, name_of_callable(atm.features[k]), + atm.appearance_models[k].n_components, + atm.shape_models[k].n_components)) + # Patch based ATM + if hasattr(atm, 'patch_shape'): + for k, s in enumerate(scales_info): + s += '\n - Patch shape: {}'.format(atm.patch_shape[k]) + scales_info = '\n'.join(scales_info) + + cls_str = r"""{class_title} + - Images warped with {transform} transform + - Images scaled to diagonal: {diagonal:.2f} + - Scales: {scales} +{scales_info} +""".format(class_title=atm._str_title, + transform=name_of_callable(atm.transform), + diagonal=diagonal, + scales=atm.scales, + scales_info=scales_info) + return cls_str + +HolisticATM = ATM diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 4a5ce71..0861e11 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -1,11 +1,12 @@ from __future__ import division +from menpofit import checks from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM from .algorithm import ( - LucasKanadeStandardInterface, LucasKanadeLinearInterface, - LucasKanadePartsInterface, InverseCompositional) + ATMLKStandardInterface, ATMLKPartsInterface, ATMLKLinearInterface, + InverseCompositional) from .result import ATMFitterResult @@ -14,56 +15,47 @@ class LucasKanadeATMFitter(ModelFitter): r""" """ def __init__(self, atm, algorithm_cls=InverseCompositional, - n_shape=None, sampling=None, **kwargs): + n_shape=None, sampling=None): self._model = atm - self._check_n_shape(n_shape) - self._set_up(algorithm_cls, sampling, **kwargs) + checks.set_models_components(atm.shape_models, n_shape) + self._sampling = checks.check_sampling(sampling, atm.n_scales) + self._set_up(algorithm_cls) @property def atm(self): return self._model - def _set_up(self, algorithm_cls, sampling, **kwargs): + def _set_up(self, algorithm_cls): self.algorithms = [] - for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, - self.atm.shape_models)): + for j, (wt, sm, s) in enumerate(zip(self.atm.warped_templates, + self.atm.shape_models, + self._sampling)): if type(self.atm) is ATM or type(self.atm) is PatchATM: - # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.atm.transform, source=wt.landmarks['source'].lms) - # set up algorithm using standard aam interface - algorithm = algorithm_cls(LucasKanadeStandardInterface, wt, - md_transform, sampling=sampling, - **kwargs) - + interface = ATMLKStandardInterface(md_transform, wt, sampling=s) + algorithm = algorithm_cls(interface) elif (type(self.atm) is LinearATM or type(self.atm) is LinearPatchATM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.atm.reference_shape) - # set up algorithm using linear aam interface - algorithm = algorithm_cls(LucasKanadeLinearInterface, wt, - md_transform, sampling=sampling, - **kwargs) - + interface = ATMLKLinearInterface(md_transform, wt, sampling=s) + algorithm = algorithm_cls(interface) elif type(self.atm) is PartsATM: - # build orthogonal point distribution model pdm = OrthoPDM(sm) - # set up algorithm using parts aam interface + interface = ATMLKPartsInterface(pdm, wt, sampling=s) algorithm = algorithm_cls( - LucasKanadePartsInterface, wt, pdm, sampling=sampling, + interface, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) - else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " "{}".format(ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM)) - - # append algorithms to list self.algorithms.append(algorithm) def _fitter_result(self, image, algorithm_results, affine_correction, From 537b6df42ad1569176ff5a915e990c49b5b72176 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:35:03 +0100 Subject: [PATCH 187/423] Remove xrange - for Python 3 compat --- menpofit/aam/algorithm/lk.py | 4 +- menpofit/atm/builder.py | 770 ----------------------------------- 2 files changed, 2 insertions(+), 772 deletions(-) delete mode 100644 menpofit/atm/builder.py diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 8c33efc..720be4c 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -42,9 +42,9 @@ def _build_sampling_mask(self, sampling): sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) if sampling is None: - sampling = xrange(0, n_true_pixels, 1) + sampling = range(0, n_true_pixels, 1) elif isinstance(sampling, np.int): - sampling = xrange(0, n_true_pixels, sampling) + sampling = range(0, n_true_pixels, sampling) sampling_mask[sampling] = 1 diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py deleted file mode 100644 index 6954856..0000000 --- a/menpofit/atm/builder.py +++ /dev/null @@ -1,770 +0,0 @@ -from __future__ import division -from copy import deepcopy -from menpo.transform import Scale -from menpofit.transform import ( - DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -from menpo.shape import mean_pointcloud -from menpo.image import Image -from menpo.feature import no_op -from menpo.visualize import print_dynamic -from menpofit import checks -from menpofit.aam.builder import ( - align_shapes, densify_shapes, - build_reference_frame, build_patch_reference_frame) -from menpofit.builder import build_shape_model, compute_reference_shape - - -# TODO: document me! -class ATMBuilder(object): - r""" - Class that builds Active Template Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, 0.5), - scale_shapes=False, scale_features=True, - max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def build(self, shapes, template, group=None, verbose=False): - r""" - Builds a Multilevel Active Template Model given a list of shapes and a - template image. - - Parameters - ---------- - shapes : list of :map:`PointCloud` - The set of shapes from which to build the shape model of the ATM. - template : :map:`Image` or subclass - The image to be used as template. - group : `str`, optional - The key of the landmark set of the template that should be used. If - ``None``, and if there is only one set of landmarks, this set will - be used. - verbose : `bool`, optional - Flag that controls information and progress printing. - - Returns - ------- - atm : :map:`ATM` - The ATM object. Shape and appearance models are stored from lowest - to highest level. - """ - # compute reference_shape - reference_shape = compute_reference_shape(shapes, self.diagonal, - verbose=verbose) - - # normalize the template size using the reference_shape scaling - template = template.rescale_to_pointcloud( - reference_shape, group=group) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - shape_models = [] - warped_templates = [] - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - prefix = ' - Level {}: '.format(j) - else: - prefix = ' - ' - - # obtain shape representation - if j == 0 or self.scale_shapes: - if j == 0: - level_shapes = shapes - level_reference_shape = reference_shape - else: - scale_transform = Scale(scale_factor=s, n_dims=2) - level_shapes = [scale_transform.apply(s) for s in shapes] - level_reference_shape = scale_transform.apply( - reference_shape) - # obtain shape model - if verbose: - print_dynamic('{}Building shape model'.format(prefix)) - shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) - # add shape model to the list - shape_models.append(shape_model) - else: - # copy precious shape model and add it to the list - shape_models.append(deepcopy(shape_model)) - - if verbose: - print_dynamic('{}Building template model'.format(prefix)) - # obtain template representation - if j == 0: - # compute features at highest level - feature_template = self.features[j](template) - level_template = feature_template - elif self.scale_features: - # scale features at other levels - level_template = feature_template.rescale(s) - else: - # scale template and compute features at other levels - scaled_template = template.rescale(s) - level_template = self.features[j](scaled_template) - - # extract potentially rescaled template shape - level_template_shape = level_template.landmarks[group].lms - - # obtain warped template - warped_template = self._warp_template(level_template, - level_template_shape, - level_reference_shape, j) - # add warped template to the list - warped_templates.append(warped_template) - - if verbose: - print_dynamic('{}Done\n'.format(prefix)) - - # reverse the list of shape and warped templates so that they are - # ordered from lower to higher resolution - shape_models.reverse() - warped_templates.reverse() - self.scales.reverse() - - return self._build_atm(shape_models, warped_templates, reference_shape) - - @classmethod - def _build_shape_model(cls, shapes, max_components, level): - return build_shape_model(shapes, max_components=max_components) - - def _warp_template(self, template, template_shape, reference_shape, level): - # build reference frame - reference_frame = build_reference_frame(reference_shape) - # compute transforms - t = self.transform(reference_frame.landmarks['source'].lms, - template_shape) - # warp template - warped_template = template.warp_to_mask(reference_frame.mask, t) - # attach landmarks - warped_template.landmarks['source'] = reference_frame.landmarks[ - 'source'] - return warped_template - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return ATM(shape_models, warped_templates, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features) - - -class PatchATMBuilder(ATMBuilder): - r""" - Class that builds Multilevel Patch-Based Active Template Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=True, - scale_features=True, max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _warp_template(self, template, template_shape, reference_shape, level): - # build reference frame - reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[level]) - # compute transforms - t = self.transform(reference_frame.landmarks['source'].lms, - template_shape) - # warp template - warped_template = template.warp_to_mask(reference_frame.mask, t) - # attach landmarks - warped_template.landmarks['source'] = reference_frame.landmarks[ - 'source'] - return warped_template - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return PatchATM(shape_models, warped_templates, reference_shape, - self.patch_shape, self.features, self.scales, - self.scale_shapes, self.scale_features) - - -# TODO: document me! -class LinearATMBuilder(ATMBuilder): - r""" - Class that builds Linear Active Template Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_reference_frame(mean_aligned_shape) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model( - dense_shapes, max_components=max_components) - return shape_model - - def _warp_template(self, template, template_shape, reference_shape, level): - # compute transforms - t = self.transform(self.reference_frame.landmarks['source'].lms, - template_shape) - # warp template - warped_template = template.warp_to_mask(self.reference_frame.mask, t) - # attach landmarks - warped_template.landmarks['source'] = self.reference_frame.landmarks[ - 'source'] - return warped_template - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return LinearATM(shape_models, warped_templates, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features, - self.n_landmarks) - - -# TODO: document me! -class LinearPatchATMBuilder(LinearATMBuilder): - r""" - Class that builds Linear Patch based Active Template Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=False, - scale_features=True, max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[level]) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model(dense_shapes, - max_components=max_components) - return shape_model - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return LinearPatchATM(shape_models, warped_templates, - reference_shape, self.patch_shape, - self.features, self.scales, self.scale_shapes, - self.scale_features, self.n_landmarks) - - -# TODO: document me! -# TODO: implement offsets support? -class PartsATMBuilder(ATMBuilder): - r""" - Class that builds Parts based Active Template Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - normalize_parts : `callable`, optional - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - normalize_parts=no_op, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _warp_template(self, template, template_shape, reference_shape, level): - parts = template.extract_patches(template_shape, - patch_size=self.patch_shape[level], - as_single_array=True) - if self.normalize_parts: - parts = self.normalize_parts(parts) - - return Image(parts) - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return PartsATM(shape_models, warped_templates, reference_shape, - self.patch_shape, self.features, - self.normalize_parts, self.scales, - self.scale_shapes, self.scale_features) - - -from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM From 15706b6b8c0da068b8c646bb8f95624b70719e76 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:35:32 +0100 Subject: [PATCH 188/423] Incorrect call to super - fixed --- menpofit/aam/algorithm/lk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 720be4c..c21f934 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -167,7 +167,7 @@ def __init__(self, transform, template, sampling=None, # TODO: Refactor to patch_features self.normalize_parts = normalize_parts - super(LucasKanadeBaseInterface, self).__init__( + super(LucasKanadePartsBaseInterface, self).__init__( transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): From ca0f3f7bbe6ca167e6063d6fde0efebf1a45c3a1 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:35:45 +0100 Subject: [PATCH 189/423] Fix noise variance bug This fix is like not modelling the noise at all - its a reasonable fix but we should think carefully about it. --- menpofit/aam/algorithm/lk.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index c21f934..bacf76d 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -298,8 +298,9 @@ def _precompute(self): self.dW_dp = self.interface.warp_jacobian() # compute shape model prior - s2 = (self.appearance_model.noise_variance() / - self.interface.shape_model.noise_variance()) + # TODO: Is this correct? It's like modelling no noise at all + sm_noise_variance = self.interface.shape_model.noise_variance() or 1 + s2 = self.appearance_model.noise_variance() / sm_noise_variance L = self.interface.shape_model.eigenvalues self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) # compute appearance model prior From a95f861fc86c6f285fb301aa2fb3089fb848bc70 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:37:03 +0100 Subject: [PATCH 190/423] Refactor trilist check method Handle images and shapes --- menpofit/aam/base.py | 6 +++--- menpofit/checks.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index a9ac753..409de66 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -116,7 +116,7 @@ class AAM(object): """ def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): checks.check_diagonal(diagonal) @@ -190,8 +190,8 @@ def _train(self, images, group=None, verbose=False, increment=False, 'shape. If the batch mean is not ' 'representative of the true mean, this may ' 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(image_batch[0], self.transform, - group=group) + checks.check_landmark_trilist(image_batch[0], self.transform, + group=group) self.reference_shape = compute_reference_shape( [i.landmarks[group].lms for i in image_batch], self.diagonal, verbose=verbose) diff --git a/menpofit/checks.py b/menpofit/checks.py index 359f441..330cc84 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -15,11 +15,14 @@ def check_diagonal(diagonal): return diagonal -def check_trilist(image, transform, group=None): - trilist = image.landmarks[group].lms +def check_landmark_trilist(image, transform, group=None): + shape = image.landmarks[group].lms + check_trilist(shape, transform) - if not isinstance(trilist, TriMesh) and isinstance(transform, - PiecewiseAffine): + +def check_trilist(shape, transform): + if not isinstance(shape, TriMesh) and isinstance(transform, + PiecewiseAffine): warnings.warn('The given images do not have an explicit triangulation ' 'applied. A Delaunay Triangulation will be computed ' 'and used for warping. This may be suboptimal and cause ' From 03c3216f09890d0f86c4d7ebc7ee262204cfc457 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:37:35 +0100 Subject: [PATCH 191/423] Remove mention of downscale etc from visualize Helps visualize just work at the moment --- menpofit/visualize/widgets/base.py | 60 ++++-------------------------- 1 file changed, 8 insertions(+), 52 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 3871615..2fbd2b9 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -961,39 +961,17 @@ def update_info(aam, instance, level, group): aam_mean = lvl_app_mod.mean() n_channels = aam_mean.n_channels tmplt_inst = lvl_app_mod.template_instance - feat = (aam.features if aam.pyramid_on_features - else aam.features[level]) + feat = aam.features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( name_of_callable(feat), n_channels, 's' * (n_channels > 1)) - # create info str - if n_levels == 1: - tmp_shape_models = '' - tmp_pyramid = '' - else: # n_scales > 1 - # shape models info - if aam.scaled_shape_models: - tmp_shape_models = "Each level has a scaled shape model " \ - "(reference frame)" - else: - tmp_shape_models = "Shape models (reference frames) are " \ - "not scaled" - # pyramid info - if aam.pyramid_on_features: - tmp_pyramid = "Pyramid was applied on feature space" - else: - tmp_pyramid = "Features were extracted at each pyramid level" - # update info widgets text_per_line = [ - "> {} training images".format(aam.n_training_images), - "> {}".format(tmp_shape_models), "> Warp using {} transform".format(aam.transform.__name__), - "> {}".format(tmp_pyramid), - "> Level {}/{} (downscale={:.1f})".format( - level + 1, aam.n_scales, aam.downscale), + "> Level {}/{}".format( + level + 1, aam.n_scales), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -1377,39 +1355,17 @@ def update_info(atm, instance, level, group): lvl_shape_mod = atm.shape_models[level] tmplt_inst = atm.warped_templates[level] n_channels = tmplt_inst.n_channels - feat = (atm.features if atm.pyramid_on_features - else atm.features[level]) + feat = atm.features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( name_of_callable(feat), n_channels, 's' * (n_channels > 1)) - # create info str - if n_levels == 1: - tmp_shape_models = '' - tmp_pyramid = '' - else: # n_scales > 1 - # shape models info - if atm.scaled_shape_models: - tmp_shape_models = "Each level has a scaled shape model " \ - "(reference frame)" - else: - tmp_shape_models = "Shape models (reference frames) are " \ - "not scaled" - # pyramid info - if atm.pyramid_on_features: - tmp_pyramid = "Pyramid was applied on feature space" - else: - tmp_pyramid = "Features were extracted at each pyramid level" - # update info widgets text_per_line = [ - "> {} training shapes".format(atm.n_training_shapes), - "> {}".format(tmp_shape_models), "> Warp using {} transform".format(atm.transform.__name__), - "> {}".format(tmp_pyramid), - "> Level {}/{} (downscale={:.1f})".format( - level + 1, atm.n_scales, atm.downscale), + "> Level {}/{}".format( + level + 1, atm.n_scales), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -2467,8 +2423,8 @@ def update_info(name, value): text_per_line = [ "> {} iterations".format(fitting_results[im].n_iters)] if hasattr(fitting_results[im], 'n_scales'): # Multilevel result - text_per_line.append("> {} levels with downscale of {:.1f}".format( - fitting_results[im].n_scales, fitting_results[im].downscale)) + text_per_line.append("> {} scales".format( + fitting_results[im].n_scales)) info_wid.set_widget_state(n_lines=len(text_per_line), text_per_line=text_per_line) From f5b4217476fb627c7952c7f550efa35b9c18a78e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:38:02 +0100 Subject: [PATCH 192/423] Remove unused methods --- menpofit/base.py | 87 ------------------------------------------------ 1 file changed, 87 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index 18b3f0f..638aca1 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -26,93 +26,6 @@ def batch(iterable, n): yield chunk -def is_pyramid_on_features(features): - r""" - True if feature extraction happens once and then a gaussian pyramid - is taken. False if a gaussian pyramid is taken and then features are - extracted at each level. - """ - return callable(features) - - -def create_pyramid(images, n_levels, downscale, features, verbose=False): - r""" - Function that creates a generator function for Gaussian pyramid. The - pyramid can be created either on the feature space or the original - (intensities) space. - - Parameters - ---------- - images: list of :map:`Image` - The set of landmarked images from which to build the AAM. - - n_scales: int - The number of multi-resolution pyramidal levels to be used. - - downscale: float - The downscale factor that will be used to create the different - pyramidal levels. - - features: ``callable`` ``[callable]`` - If a single callable, then the feature calculation will happen once - followed by a gaussian pyramid. If a list of callables then a - gaussian pyramid is generated with features extracted at each level - (after downsizing and blurring). - - Returns - ------- - list of generators : - The generator function of the Gaussian pyramid. - - """ - will_take_a_while = is_pyramid_on_features(features) - pyramids = [] - for i, img in enumerate(images): - if will_take_a_while and verbose: - print_dynamic( - 'Computing top level feature space - {}'.format( - progress_bar_str((i + 1.) / len(images), - show_bar=False))) - pyramids.append(pyramid_of_feature_images(n_levels, downscale, - features, img)) - return pyramids - - -def pyramid_of_feature_images(n_levels, downscale, features, image): - r""" - Generates a gaussian pyramid of feature images for a single image. - """ - if is_pyramid_on_features(features): - # compute feature image at the top - feature_image = features(image) - # create pyramid on the feature image - return feature_image.gaussian_pyramid(n_levels=n_levels, - downscale=downscale) - else: - # create pyramid on intensities image - # feature will be computed per level - pyramid = image.gaussian_pyramid(n_levels=n_levels, - downscale=downscale) - # add the feature generation here - return feature_images(pyramid, features) - - -# adds feature extraction to a generator of images -def feature_images(images, features): - for feature, level in zip(reversed(features), images): - yield feature(level) - - -class DeformableModel(object): - - def __init__(self, features): - self.features = features - - @property - def pyramid_on_features(self): - return is_pyramid_on_features(self.features) - - def build_grid(shape): r""" """ From d782cf877d98d7672184ea743b635c8e84e27ac0 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:38:16 +0100 Subject: [PATCH 193/423] Fix bugs using wrong index in AAM --- menpofit/aam/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 409de66..70e875d 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -292,8 +292,8 @@ def _train(self, images, group=None, verbose=False, increment=False, # Because we just copy the shape model, we need to wait to trim # it after building each model. This ensures we can have a different # number of components per level - for k, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[k] + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] if max_sc is not None: sm.trim_components(max_sc) @@ -916,8 +916,9 @@ def _aam_str(aam): aam.shape_models[k].n_components)) # Patch based AAM if hasattr(aam, 'patch_shape'): - for k, s in enumerate(scales_info): - s += '\n - Patch shape: {}'.format(aam.patch_shape[k]) + for k in range(len(scales_info)): + scales_info[k] += '\n - Patch shape: {}'.format( + aam.patch_shape[k]) scales_info = '\n'.join(scales_info) cls_str = r"""{class_title} From 5640d604aad2291068ae96158d624531cc3f3f2a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:38:30 +0100 Subject: [PATCH 194/423] Update ATM to work with AAM layout Now ATMs act like AAMs - including the new interface usage so that the algorithms just take a single interface. A lot of the code is similar to the AAM - but not identical! --- menpofit/atm/__init__.py | 4 +- menpofit/atm/algorithm.py | 12 ++- menpofit/atm/base.py | 173 ++++++++++++++++++-------------------- menpofit/atm/fitter.py | 18 ++-- 4 files changed, 94 insertions(+), 113 deletions(-) diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index cea05ea..2ed3a89 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,5 +1,3 @@ -from .builder import ( - ATMBuilder, PatchATMBuilder, LinearATMBuilder, - LinearPatchATMBuilder, PartsATMBuilder) +from .base import HolisticATM, PartsATM, PatchATM, LinearATM, LinearPatchATM from .fitter import LucasKanadeATMFitter from .algorithm import ForwardCompositional, InverseCompositional diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 90515cd..3a020c6 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -41,7 +41,7 @@ class ATMLKPartsInterface(LucasKanadePartsBaseInterface): """ def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): - return LinearATMAlgorithmResult( + return ATMAlgorithmResult( image, self, shape_parameters, cost_functions=cost_functions, gt_shape=gt_shape) @@ -51,13 +51,9 @@ class LucasKanade(object): def __init__(self, atm_interface, eps=10**-5): self.eps = eps - self.interface = atm_interface() + self.interface = atm_interface self._precompute() - @property - def appearance_model(self): - return self.interface.appearance_model - @property def transform(self): return self.interface.transform @@ -77,7 +73,9 @@ def _precompute(self): self.dW_dp = self.interface.warp_jacobian() # compute shape model prior - s2 = 1 / self.interface.shape_model.noise_variance() + # TODO: Is this correct? It's like modelling no noise at all + noise_variance = self.interface.shape_model.noise_variance() or 1 + s2 = 1.0 / noise_variance L = self.interface.shape_model.eigenvalues self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 95d675d..b8f13d4 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -4,7 +4,6 @@ import numpy as np from menpo.feature import no_op from menpo.visualize import print_dynamic -from menpo.model import PCAModel from menpo.transform import Scale from menpo.shape import mean_pointcloud from menpofit import checks @@ -14,7 +13,7 @@ from menpofit.builder import ( build_reference_frame, build_patch_reference_frame, compute_features, scale_images, build_shape_model, warp_images, - align_shapes, rescale_images_to_reference_shape, densify_shapes, + align_shapes, densify_shapes, extract_patches, MenpoFitBuilderWarning, compute_reference_shape) @@ -23,9 +22,10 @@ class ATM(object): r""" Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, reference_shape=None, - features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + def __init__(self, template, shapes, group=None, verbose=False, + reference_shape=None, features=no_op, + transform=DifferentiablePiecewiseAffine, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, batch_size=None): checks.check_diagonal(diagonal) @@ -45,11 +45,11 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, self.warped_templates = [] # Train ATM - self._train(images, group=group, verbose=verbose, increment=False, - batch_size=batch_size) + self._train(template, shapes, group=group, verbose=verbose, + increment=False, batch_size=batch_size) - def _train(self, images, group=None, verbose=False, increment=False, - shape_forgetting_factor=1.0, batch_size=None): + def _train(self, template, shapes, group=None, verbose=False, + increment=False, shape_forgetting_factor=1.0, batch_size=None): r""" Builds an Active Template Model from a list of landmarked images. @@ -74,11 +74,11 @@ def _train(self, images, group=None, verbose=False, increment=False, if batch_size is not None: # Create a generator of fixed sized batches. Will still work even # on an infinite list. - image_batches = batch(images, batch_size) + shape_batches = batch(shapes, batch_size) else: - image_batches = [list(images)] + shape_batches = [list(shapes)] - for k, image_batch in enumerate(image_batches): + for k, shape_batch in enumerate(shape_batches): # After the first batch, we are incrementing the model if k > 0: increment = True @@ -95,16 +95,14 @@ def _train(self, images, group=None, verbose=False, increment=False, 'shape. If the batch mean is not ' 'representative of the true mean, this may ' 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(image_batch[0], self.transform, - group=group) + checks.check_trilist(shape_batch[0], self.transform) self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) + shape_batch, self.diagonal, verbose=verbose) - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) + if k == 0: + # Rescale the template the reference shape + template = template.rescale_to_pointcloud( + self.reference_shape, group=group) # build models at each scale if verbose: @@ -126,7 +124,7 @@ def _train(self, images, group=None, verbose=False, increment=False, # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale - feature_images = compute_features(image_batch, + feature_images = compute_features([template], self.features[j], prefix=scale_prefix, verbose=verbose) @@ -136,11 +134,14 @@ def _train(self, images, group=None, verbose=False, increment=False, scaled_images = scale_images(feature_images, self.scales[j], prefix=scale_prefix, verbose=verbose) + # Extract potentially rescaled shapes + scale_transform = Scale(scale_factor=self.scales[j], + n_dims=2) + scale_shapes = [scale_transform.apply(s) + for s in shape_batch] else: scaled_images = feature_images - - # Extract potentially rescaled shapes - scale_shapes = [i.landmarks[group].lms for i in scaled_images] + scale_shapes = shape_batch # Build the shape model if verbose: @@ -148,8 +149,7 @@ def _train(self, images, group=None, verbose=False, increment=False, if not increment: if j == 0: - shape_model = self._build_shape_model( - scale_shapes, j) + shape_model = self._build_shape_model(scale_shapes, j) self.shape_models.append(shape_model) else: self.shape_models.append(deepcopy(shape_model)) @@ -164,32 +164,10 @@ def _train(self, images, group=None, verbose=False, increment=False, # reference frame. scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( self.reference_shape) - warped_images = self._warp_images(scaled_images, scale_shapes, - scaled_reference_shape, - j, scale_prefix, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format( - scale_prefix)) - - if not increment: - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - self.appearance_models.append(appearance_model) - else: - # increment appearance model - self.appearance_models[j].increment( - warped_images, - forgetting_factor=appearance_forgetting_factor) - # trim appearance model if required - if self.max_appearance_components is not None: - self.appearance_models[j].trim_components( - self.max_appearance_components[j]) + warped_template = self._warp_template(scaled_images[0], group, + scaled_reference_shape, + j, scale_prefix, verbose) + self.warped_templates.append(warped_template[0]) if verbose: print_dynamic('{}Done\n'.format(scale_prefix)) @@ -197,14 +175,14 @@ def _train(self, images, group=None, verbose=False, increment=False, # Because we just copy the shape model, we need to wait to trim # it after building each model. This ensures we can have a different # number of components per level - for k, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[k] + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] if max_sc is not None: sm.trim_components(max_sc) - def increment(self, images, group=None, verbose=False, + def increment(self, template, shapes, group=None, verbose=False, shape_forgetting_factor=1.0, batch_size=None): - return self._train(images, group=group, + return self._train(template, shapes, group=group, verbose=verbose, shape_forgetting_factor=shape_forgetting_factor, increment=True, batch_size=batch_size) @@ -220,10 +198,11 @@ def _increment_shape_model(self, shapes, shape_model, shape_model.increment(aligned_shapes, forgetting_factor=forgetting_factor) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, reference_frame, self.transform, + shape = template.landmarks[group].lms + return warp_images([template], [shape], reference_frame, self.transform, prefix=prefix, verbose=verbose) @property @@ -284,7 +263,7 @@ def random_instance(self, scale_index=-1): sm.eigenvalues[:sm.n_active_components]**0.5) shape_instance = sm.instance(shape_weights) - return self._instance(scale_index, shape_instance, template) + return self._instance(shape_instance, template) def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms @@ -377,22 +356,24 @@ class PatchATM(ATM): Patch based Based Active Appearance Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - max_shape_components=None, batch_size=None): + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(PatchATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): reference_frame = build_patch_reference_frame( reference_shape, patch_shape=self.patch_shape[scale_index]) - return warp_images(images, shapes, reference_frame, self.transform, + shape = template.landmarks[group].lms + return warp_images([template], [shape], reference_frame, self.transform, prefix=prefix, verbose=verbose) @property @@ -421,13 +402,13 @@ class LinearATM(ATM): Linear Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - transform=DifferentiableThinPlateSplines, diagonal=None, - scales=(0.5, 1.0), max_shape_components=None, + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, transform=DifferentiableThinPlateSplines, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, batch_size=None): super(LinearATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=transform, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -458,9 +439,10 @@ def _increment_shape_model(self, shapes, shape_model, shape_model.increment(dense_shapes, forgetting_factor=forgetting_factor) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): - return warp_images(images, shapes, self.reference_frame, + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): + shape = template.landmarks[group].lms + return warp_images([template], [shape], self.reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -484,13 +466,14 @@ class LinearPatchATM(ATM): Linear Patch based Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - max_shape_components=None, batch_size=None): + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(LinearPatchATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -523,9 +506,10 @@ def _increment_shape_model(self, shapes, shape_model, shape_model.increment(dense_shapes, forgetting_factor=forgetting_factor) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): - return warp_images(images, shapes, self.reference_frame, + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): + shape = template.landmarks[group].lms + return warp_images([template], [shape], self.reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -550,15 +534,15 @@ class PartsATM(ATM): Parts based Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, - batch_size=None): + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, normalize_parts=no_op, diagonal=None, + scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts super(PartsATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -571,9 +555,11 @@ def _str_title(self): """ return 'Parts-based Active Template Model' - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): - return extract_patches(images, shapes, self.patch_shape[scale_index], + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): + shape = template.landmarks[group].lms + return extract_patches([template], [shape], + self.patch_shape[scale_index], normalize_function=self.normalize_parts, prefix=prefix, verbose=verbose) @@ -602,17 +588,18 @@ def _atm_str(atm): scales_info = [] lvl_str_tmplt = r""" - Scale {} - Holistic feature: {} - - {} appearance components + - Template shape: {} - {} shape components""" for k, s in enumerate(atm.scales): scales_info.append(lvl_str_tmplt.format( s, name_of_callable(atm.features[k]), - atm.appearance_models[k].n_components, + atm.warped_templates[k].shape, atm.shape_models[k].n_components)) # Patch based ATM if hasattr(atm, 'patch_shape'): - for k, s in enumerate(scales_info): - s += '\n - Patch shape: {}'.format(atm.patch_shape[k]) + for k in range(len(scales_info)): + scales_info[k] += '\n - Patch shape: {}'.format( + atm.patch_shape[k]) scales_info = '\n'.join(scales_info) cls_str = r"""{class_title} diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 0861e11..a7b334e 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -32,25 +32,23 @@ def _set_up(self, algorithm_cls): self._sampling)): if type(self.atm) is ATM or type(self.atm) is PatchATM: - md_transform = OrthoMDTransform( - sm, self.atm.transform, - source=wt.landmarks['source'].lms) + source_lmarks = wt.landmarks['source'].lms + md_transform = OrthoMDTransform(sm, self.atm.transform, + source=source_lmarks) interface = ATMLKStandardInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) elif (type(self.atm) is LinearATM or type(self.atm) is LinearPatchATM): - # build linear version of orthogonal model driven transform - md_transform = LinearOrthoMDTransform( - sm, self.atm.reference_shape) + md_transform = LinearOrthoMDTransform(sm, + self.atm.reference_shape) interface = ATMLKLinearInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) elif type(self.atm) is PartsATM: pdm = OrthoPDM(sm) - interface = ATMLKPartsInterface(pdm, wt, sampling=s) - algorithm = algorithm_cls( - interface, - patch_shape=self.atm.patch_shape[j], + interface = ATMLKPartsInterface( + pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) + algorithm = algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " From 665ef0a4051a69af97481c1cbb02cc12cb696417 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 13:23:34 +0100 Subject: [PATCH 195/423] Refactor LK to match new scheme Just use the correct ordering for scales etc. Also, use _prepare_image rather than _prepare_template to make sure everything is consistent --- menpofit/lk/fitter.py | 87 ++++++++++++++++------------------------- menpofit/lk/residual.py | 1 + 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 11130c2..2319248 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -1,10 +1,11 @@ from __future__ import division from menpo.feature import no_op from menpofit.transform import DifferentiableAlignmentAffine -from menpofit.fitter import MultiFitter, noisy_target_alignment_transform +from menpofit.fitter import (MultiFitter, noisy_shape_from_shape, + noisy_shape_from_bounding_box) from menpofit import checks from .algorithm import InverseCompositional -from .residual import SSD, FourierSSD +from .residual import SSD from .result import LucasKanadeFitterResult @@ -14,76 +15,54 @@ class LucasKanadeFitter(MultiFitter): """ def __init__(self, template, group=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, - scales=(1, .5), scale_features=True, - algorithm_cls=InverseCompositional, residual_cls=SSD, - **kwargs): - # check parameters + scales=(0.5, 1.0), algorithm_cls=InverseCompositional, + residual_cls=SSD): + checks.check_diagonal(diagonal) scales = checks.check_scales(scales) features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - # set parameters + self.features = features self.transform_cls = transform_cls self.diagonal = diagonal self.scales = list(scales) - self.scales.reverse() - self.scale_features = scale_features + # Make template masked for warping + template = template.as_masked(copy=False) - self.templates, self.sources = self._prepare_template( - template, group=group) + if self.diagonal: + template = template.rescale_landmarks_to_diagonal_range( + self.diagonal, group=group) + self.reference_shape = template.landmarks[group].lms - self.reference_shape = self.sources[0] + self.templates, self.sources = self._prepare_template(template, + group=group) + self._set_up(algorithm_cls, residual_cls) + def _set_up(self, algorithm_cls, residual_cls): self.algorithms = [] for j, (t, s) in enumerate(zip(self.templates, self.sources)): transform = self.transform_cls(s, s) - if ('kernel_func' in kwargs and - (residual_cls is SSD or - residual_cls is FourierSSD)): - kernel_func = kwargs.pop('kernel_func') - kernel = kernel_func(t.shape) - residual = residual_cls(kernel=kernel) - else: - residual = residual_cls() - algorithm = algorithm_cls(t, transform, residual, **kwargs) + residual = residual_cls() + algorithm = algorithm_cls(t, transform, residual) self.algorithms.append(algorithm) def _prepare_template(self, template, group=None): - template = template.crop_to_landmarks(group=group) - template = template.as_masked() - - # rescale template to diagonal range - if self.diagonal: - template = template.rescale_landmarks_to_diagonal_range( - self.diagonal, group=group) - - # obtain image representation - templates = [] - for j, s in enumerate(self.scales[::-1]): - if j == 0: - # compute features at highest level - feature_template = self.features[j](template) - elif self.scale_features: - # scale features at other levels - feature_template = templates[0].rescale(s) - else: - # scale image and compute features at other levels - scaled_template = template.rescale(s) - feature_template = self.features[j](scaled_template) - templates.append(feature_template) - templates.reverse() - - # get sources per level - sources = [i.landmarks[group].lms for i in templates] - + gt_shape = template.landmarks[group].lms + templates, _, sources = self._prepare_image(template, gt_shape, + gt_shape=gt_shape) return templates, sources - def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): - transform = noisy_target_alignment_transform( - self.reference_shape, gt_shape, - alignment_transform_cls=self.transform_cls, noise_std=noise_std) - return transform.apply(self.reference_shape) + def noisy_shape_from_bounding_box(self, bounding_box, noise_type='uniform', + noise_percentage=0.1, rotation=False): + return noisy_shape_from_bounding_box( + self.reference_shape, bounding_box, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) + + def noisy_shape_from_shape(self, shape, noise_type='uniform', + noise_percentage=0.1, rotation=False): + return noisy_shape_from_shape( + self.reference_shape, shape, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index d406c0f..9a44f20 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -5,6 +5,7 @@ import scipy.linalg from menpo.feature import gradient + # TODO: Do we want residuals to support masked templates? class Residual(object): """ From 268410056f283922fd1fd363b0b9a3dd755b657e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 13:27:30 +0100 Subject: [PATCH 196/423] Rename Patch{ATM,AAM} to Masked and Parts to Patch Since we use Patch all the time in SDM etc, we should probably use it for the 'parts' AAM which is more similar to SDM. Since 'Patch' was being used merely for an AAM where the appearance model has a disjoint mask applied - we change the name to 'Masked' --- menpofit/aam/__init__.py | 2 +- menpofit/aam/base.py | 24 ++++++++++++------------ menpofit/aam/fitter.py | 22 +++++++++++----------- menpofit/atm/__init__.py | 2 +- menpofit/atm/base.py | 24 ++++++++++++------------ menpofit/atm/fitter.py | 12 ++++++------ 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index b91a3ae..cbc979c 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,4 +1,4 @@ -from .base import HolisticAAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM +from .base import HolisticAAM, LinearAAM, LinearMaskedAAM, PatchAAM, MaskedAAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 70e875d..8ec78c9 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -531,9 +531,9 @@ def __str__(self): # TODO: document me! -class PatchAAM(AAM): +class MaskedAAM(AAM): r""" - Patch based Based Active Appearance Model class. + Masked Active Appearance Model class. Parameters ----------- @@ -570,7 +570,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(PatchAAM, self).__init__( + super(MaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -586,7 +586,7 @@ def _warp_images(self, images, shapes, reference_shape, scale_index, @property def _str_title(self): - return 'Patch-based Active Appearance Model' + return 'Masked Active Appearance Model' def _instance(self, scale_index, shape_instance, appearance_instance): template = self.appearance_models[scale_index].mean @@ -714,9 +714,9 @@ def __str__(self): # TODO: document me! -class LinearPatchAAM(AAM): +class LinearMaskedAAM(AAM): r""" - Linear Patch based Active Appearance Model class. + Linear Masked Active Appearance Model class. Parameters ----------- @@ -752,7 +752,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(LinearPatchAAM, self).__init__( + super(LinearMaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -765,7 +765,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Linear Patch-based Active Appearance Model' + return 'Linear Masked Active Appearance Model' def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) @@ -815,9 +815,9 @@ def __str__(self): # TODO: document me! # TODO: implement offsets support? -class PartsAAM(AAM): +class PatchAAM(AAM): r""" - Parts based Active Appearance Model class. + Patch-based Active Appearance Model class. Parameters ----------- @@ -855,7 +855,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts - super(PartsAAM, self).__init__( + super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -868,7 +868,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Parts-based Active Appearance Model' + return 'Patch-based Active Appearance Model' def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 2afc4b4..ff42ea6 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -8,7 +8,7 @@ from menpofit.sdm import SupervisedDescentFitter from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform import menpofit.checks as checks -from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM +from .base import AAM, MaskedAAM, LinearAAM, LinearMaskedAAM, PatchAAM from .algorithm.lk import ( LucasKanadeStandardInterface, LucasKanadeLinearInterface, LucasKanadePartsInterface, WibergInverseCompositional) @@ -54,7 +54,7 @@ def _set_up(self, lk_algorithm_cls): self._sampling)): template = am.mean() - if type(self.aam) is AAM or type(self.aam) is PatchAAM: + if type(self.aam) is AAM or type(self.aam) is MaskedAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, @@ -63,14 +63,14 @@ def _set_up(self, lk_algorithm_cls): template, sampling=s) algorithm = lk_algorithm_cls(interface) elif (type(self.aam) is LinearAAM or - type(self.aam) is LinearPatchAAM): + type(self.aam) is LinearMaskedAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) interface = LucasKanadeLinearInterface(am, md_transform, template, sampling=s) algorithm = lk_algorithm_cls(interface) - elif type(self.aam) is PartsAAM: + elif type(self.aam) is PatchAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) interface = LucasKanadePartsInterface( @@ -81,8 +81,8 @@ def _set_up(self, lk_algorithm_cls): else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " - "{}".format(AAM, PatchAAM, LinearAAM, - LinearPatchAAM, PartsAAM)) + "{}".format(AAM, MaskedAAM, LinearAAM, + LinearMaskedAAM, PatchAAM)) self.algorithms.append(algorithm) @@ -122,7 +122,7 @@ def _setup_algorithms(self): self.aam.shape_models, self._sampling)): template = am.mean() - if type(self.aam) is AAM or type(self.aam) is PatchAAM: + if type(self.aam) is AAM or type(self.aam) is MaskedAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, @@ -132,7 +132,7 @@ def _setup_algorithms(self): algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) elif (type(self.aam) is LinearAAM or - type(self.aam) is LinearPatchAAM): + type(self.aam) is LinearMaskedAAM): # Build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) @@ -140,7 +140,7 @@ def _setup_algorithms(self): am, md_transform, template, sampling=s) algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) - elif type(self.aam) is PartsAAM: + elif type(self.aam) is PatchAAM: # Build orthogonal point distribution model pdm = OrthoPDM(sm) interface = SupervisedDescentPartsInterface( @@ -152,8 +152,8 @@ def _setup_algorithms(self): else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " - "{}".format(AAM, PatchAAM, LinearAAM, - LinearPatchAAM, PartsAAM)) + "{}".format(AAM, MaskedAAM, LinearAAM, + LinearMaskedAAM, PatchAAM)) # append algorithms to list self.algorithms.append(algorithm) diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index 2ed3a89..6705775 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,3 +1,3 @@ -from .base import HolisticATM, PartsATM, PatchATM, LinearATM, LinearPatchATM +from .base import HolisticATM, PatchATM, MaskedATM, LinearATM, LinearMaskedATM from .fitter import LucasKanadeATMFitter from .algorithm import ForwardCompositional, InverseCompositional diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index b8f13d4..57d961f 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -351,9 +351,9 @@ def __str__(self): # TODO: document me! -class PatchATM(ATM): +class MaskedATM(ATM): r""" - Patch based Based Active Appearance Model class. + Masked Based Active Appearance Model class. """ def __init__(self, template, shapes, group=None, verbose=False, @@ -362,7 +362,7 @@ def __init__(self, template, shapes, group=None, verbose=False, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(PatchATM, self).__init__( + super(MaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -378,7 +378,7 @@ def _warp_template(self, template, group, reference_shape, scale_index, @property def _str_title(self): - return 'Patch-based Active Template Model' + return 'Masked Active Template Model' def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms @@ -461,9 +461,9 @@ def __str__(self): # TODO: document me! -class LinearPatchATM(ATM): +class LinearMaskedATM(ATM): r""" - Linear Patch based Active Template Model class. + Linear Masked Active Template Model class. """ def __init__(self, template, shapes, group=None, verbose=False, @@ -472,7 +472,7 @@ def __init__(self, template, shapes, group=None, verbose=False, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(LinearPatchATM, self).__init__( + super(LinearMaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -484,7 +484,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Linear Patch-based Active Template Model' + return 'Linear Masked Active Template Model' def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) @@ -529,9 +529,9 @@ def __str__(self): # TODO: document me! # TODO: implement offsets support? -class PartsATM(ATM): +class PatchATM(ATM): r""" - Parts based Active Template Model class. + Patch-based Active Template Model class. """ def __init__(self, template, shapes, group=None, verbose=False, @@ -541,7 +541,7 @@ def __init__(self, template, shapes, group=None, verbose=False, self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts - super(PartsATM, self).__init__( + super(PatchATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -553,7 +553,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Parts-based Active Template Model' + return 'Patch-based Active Template Model' def _warp_template(self, template, group, reference_shape, scale_index, prefix, verbose): diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index a7b334e..dc126e4 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -3,7 +3,7 @@ from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform -from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM +from .base import ATM, MaskedATM, LinearATM, LinearMaskedATM, PatchATM from .algorithm import ( ATMLKStandardInterface, ATMLKPartsInterface, ATMLKLinearInterface, InverseCompositional) @@ -31,19 +31,19 @@ def _set_up(self, algorithm_cls): self.atm.shape_models, self._sampling)): - if type(self.atm) is ATM or type(self.atm) is PatchATM: + if type(self.atm) is ATM or type(self.atm) is MaskedATM: source_lmarks = wt.landmarks['source'].lms md_transform = OrthoMDTransform(sm, self.atm.transform, source=source_lmarks) interface = ATMLKStandardInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) elif (type(self.atm) is LinearATM or - type(self.atm) is LinearPatchATM): + type(self.atm) is LinearMaskedATM): md_transform = LinearOrthoMDTransform(sm, self.atm.reference_shape) interface = ATMLKLinearInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) - elif type(self.atm) is PartsATM: + elif type(self.atm) is PatchATM: pdm = OrthoPDM(sm) interface = ATMLKPartsInterface( pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], @@ -52,8 +52,8 @@ def _set_up(self, algorithm_cls): else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " - "{}".format(ATM, PatchATM, LinearATM, - LinearPatchATM, PartsATM)) + "{}".format(ATM, MaskedATM, LinearATM, + LinearMaskedATM, PatchATM)) self.algorithms.append(algorithm) def _fitter_result(self, image, algorithm_results, affine_correction, From b548310a9eb371635087ba17b512f8df13381658 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 7 Aug 2015 10:52:20 +0100 Subject: [PATCH 197/423] Update travis and appveyor --- .travis.yml | 14 +++++--------- appveyor.yml | 24 +++++++++--------------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 05ec7f4..cae360d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ +sudo: false + os: - linux -- osx git: depth: 200 @@ -12,17 +13,12 @@ env: - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.4 -matrix: - allow_failures: - - env: PYTHON_VERSION=3.4 - install: -- wget https://raw.githubusercontent.com/menpo/condaci/v0.2.4/condaci.py -O condaci.py -- python condaci.py setup $PYTHON_VERSION --channel $BINSTAR_USER -- export PATH=$HOME/miniconda/bin:$PATH +- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py -O condaci.py +- python condaci.py setup script: -- python condaci.py auto ./conda --binstaruser $BINSTAR_USER --binstarkey $BINSTAR_KEY +- ~/miniconda/bin/python condaci.py build ./conda notifications: slack: diff --git a/appveyor.yml b/appveyor.yml index 3711a56..0e70524 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,30 +5,23 @@ environment: BINSTAR_USER: menpo BINSTAR_KEY: secure: mw1Fz5a5C0lT4CXzsOCADoo/Xa9YymZI3yjVZNR8f5GwYrVAOC2YXxyEG6NaSWZY - PYTHON_VERSION: 2.7 -# matrix: -# - PYTHON_VERSION: 2.7 -# - PYTHON_VERSION: 3.4 -# -# + matrix: + - PYTHON_VERSION: 2.7 + - PYTHON_VERSION: 3.4 + matrix: - fast_finish: true -# allow_failures: -# - platform: x86 -# PYTHON_VERSION: 3.4 -# - platform: x64 -# PYTHON_VERSION: 3.4 + fast_finish: true platform: - x86 - x64 init: -- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.2.4/condaci.py' C:\\condaci.py; echo "Done" -- cmd: python C:\\condaci.py setup %PYTHON_VERSION% --channel %BINSTAR_USER% +- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py' C:\\condaci.py; echo "Done" +- cmd: python C:\\condaci.py setup install: -- cmd: C:\\Miniconda\\python C:\\condaci.py auto ./conda --binstaruser %BINSTAR_USER% --binstarkey %BINSTAR_KEY% +- cmd: C:\\Miniconda\\python C:\\condaci.py build ./conda notifications: - provider: Slack @@ -38,3 +31,4 @@ notifications: on_build_status_changed: true on_build_success: false on_build_failure: false + From 13394e1b24ac31eddf49086929653b7e430bf68c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 7 Aug 2015 11:04:53 +0100 Subject: [PATCH 198/423] Fix a bunch of relative import errors --- menpofit/clm/algorithm/__init__.py | 2 +- menpofit/clm/base.py | 2 +- menpofit/clm/expert/__init__.py | 4 ++-- menpofit/feature/__init__.py | 2 +- menpofit/math/__init__.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/menpofit/clm/algorithm/__init__.py b/menpofit/clm/algorithm/__init__.py index 1df3fab..a4ba38c 100644 --- a/menpofit/clm/algorithm/__init__.py +++ b/menpofit/clm/algorithm/__init__.py @@ -1,3 +1,3 @@ -from gd import ( +from .gd import ( GradientDescentCLMAlgorithm, ActiveShapeModel, RegularisedLandmarkMeanShift) diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 7feabf3..1bc41ae 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -6,7 +6,7 @@ from menpofit.builder import ( normalization_wrt_reference_shape, compute_features, scale_images, build_shape_model, increment_shape_model) -from expert import ExpertEnsemble, CorrelationFilterExpertEnsemble +from .expert import ExpertEnsemble, CorrelationFilterExpertEnsemble # TODO: Document me! diff --git a/menpofit/clm/expert/__init__.py b/menpofit/clm/expert/__init__.py index 673e02b..c0e7ae7 100644 --- a/menpofit/clm/expert/__init__.py +++ b/menpofit/clm/expert/__init__.py @@ -1,2 +1,2 @@ -from ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble -from base import IncrementalCorrelationFilterThinWrapper +from .ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble +from .base import IncrementalCorrelationFilterThinWrapper diff --git a/menpofit/feature/__init__.py b/menpofit/feature/__init__.py index 03a2607..2d00a6a 100644 --- a/menpofit/feature/__init__.py +++ b/menpofit/feature/__init__.py @@ -1,2 +1,2 @@ -from features import ( +from .features import ( centralize, normalize_norm, normalize_std, normalize_var, probability_map) diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py index d916940..fa310c7 100644 --- a/menpofit/math/__init__.py +++ b/menpofit/math/__init__.py @@ -1 +1 @@ -from regression import IRLRegression, IIRLRegression \ No newline at end of file +from .regression import IRLRegression, IIRLRegression From 5a01c0628a063c80aca398a1af5cad3ab6ba3ee5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 10 Aug 2015 13:05:37 +0100 Subject: [PATCH 199/423] Refactor training to be more consistent Now everyone method has essentially the same _train method. Then, this method calls a _train_batch method that is different for each method and actually performs training. --- menpofit/aam/base.py | 280 +++++++++++++++++++------------------ menpofit/atm/base.py | 216 ++++++++++++++-------------- menpofit/clm/base.py | 56 +++++--- menpofit/sdm/fitter.py | 309 +++++++++++++++++++++-------------------- 4 files changed, 449 insertions(+), 412 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 8ec78c9..e5257c5 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -139,33 +139,18 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, self.appearance_models = [] # Train AAM - self._train(images, group=group, verbose=verbose, increment=False, + self._train(images, increment=False, group=group, verbose=verbose, batch_size=batch_size) - def _train(self, images, group=None, verbose=False, increment=False, + def _train(self, images, increment=False, group=None, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, - batch_size=None): + verbose=False, batch_size=None): r""" - Builds an Active Appearance Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest scale """ # If batch_size is not None, then we may have a generator, else we # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. if batch_size is not None: # Create a generator of fixed sized batches. Will still work even # on an infinite list. @@ -174,6 +159,23 @@ def _train(self, images, group=None, verbose=False, increment=False, image_batches = [list(images)] for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + checks.check_landmark_trilist(image_batch[0], + self.transform, group=group) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + # After the first batch, we are incrementing the model if k > 0: increment = True @@ -181,121 +183,135 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print('Computing batch {}'.format(k)) - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - checks.check_landmark_trilist(image_batch[0], self.transform, - group=group) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, + # Train each batch + self._train_batch( + image_batch, increment=increment, group=group, + shape_forgetting_factor=shape_forgetting_factor, + appearance_forgetting_factor=appearance_forgetting_factor, verbose=verbose) - # build models at each scale + def _train_batch(self, image_batch, increment=False, group=None, + verbose=False, shape_forgetting_factor=1.0, + appearance_forgetting_factor=1.0): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest scale + """ + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): if verbose: - print_dynamic('- Building models\n') - - feature_images = [] - # for each scale (low --> high) - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features(image_batch, - self.features[j], - prefix=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - prefix=scale_prefix, - verbose=verbose) - else: - scaled_images = feature_images - - # Extract potentially rescaled shapes - scale_shapes = [i.landmarks[group].lms for i in scaled_images] - - # Build the shape model - if verbose: - print_dynamic('{}Building shape model'.format(scale_prefix)) - - if not increment: - if j == 0: - shape_model = self._build_shape_model( - scale_shapes, j) - self.shape_models.append(shape_model) - else: - self.shape_models.append(deepcopy(shape_model)) - else: - self._increment_shape_model( - scale_shapes, self.shape_models[j], - forgetting_factor=shape_forgetting_factor) - - # Obtain warped images - we use a scaled version of the - # reference shape, computed here. This is because the mean - # moves when we are incrementing, and we need a consistent - # reference frame. - scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( - self.reference_shape) - warped_images = self._warp_images(scaled_images, scale_shapes, - scaled_reference_shape, - j, scale_prefix, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format( - scale_prefix)) - - if not increment: - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - self.appearance_models.append(appearance_model) + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract potentially rescaled shapes + scale_shapes = [i.landmarks[group].lms for i in scaled_images] + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model( + scale_shapes, j) + self.shape_models.append(shape_model) else: - # increment appearance model - self.appearance_models[j].increment( - warped_images, - forgetting_factor=appearance_forgetting_factor) - # trim appearance model if required - if self.max_appearance_components is not None: - self.appearance_models[j].trim_components( - self.max_appearance_components[j]) - - if verbose: - print_dynamic('{}Done\n'.format(scale_prefix)) - - # Because we just copy the shape model, we need to wait to trim - # it after building each model. This ensures we can have a different - # number of components per level - for j, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[j] - if max_sc is not None: - sm.trim_components(max_sc) + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(scaled_images, scale_shapes, + scaled_reference_shape, + j, scale_prefix, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format( + scale_prefix)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] + if max_sc is not None: + sm.trim_components(max_sc) def increment(self, images, group=None, verbose=False, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, @@ -303,11 +319,11 @@ def increment(self, images, group=None, verbose=False, # Literally just to fit under 80 characters, but maintain the sensible # parameter name aff = appearance_forgetting_factor - return self._train(images, group=group, + return self._train(images, increment=True, group=group, verbose=verbose, shape_forgetting_factor=shape_forgetting_factor, appearance_forgetting_factor=aff, - increment=True, batch_size=batch_size) + batch_size=batch_size) def _build_shape_model(self, shapes, scale_index): return build_shape_model(shapes) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 57d961f..f6fa2ff 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -45,32 +45,17 @@ def __init__(self, template, shapes, group=None, verbose=False, self.warped_templates = [] # Train ATM - self._train(template, shapes, group=group, verbose=verbose, - increment=False, batch_size=batch_size) + self._train(template, shapes, increment=False, group=group, + verbose=verbose, batch_size=batch_size) - def _train(self, template, shapes, group=None, verbose=False, - increment=False, shape_forgetting_factor=1.0, batch_size=None): + def _train(self, template, shapes, increment=False, group=None, + shape_forgetting_factor=1.0, verbose=False, batch_size=None): r""" - Builds an Active Template Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest scale """ # If batch_size is not None, then we may have a generator, else we # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. if batch_size is not None: # Create a generator of fixed sized batches. Will still work even # on an infinite list. @@ -79,6 +64,26 @@ def _train(self, template, shapes, group=None, verbose=False, shape_batches = [list(shapes)] for k, shape_batch in enumerate(shape_batches): + if k == 0: + # Rescale the template the reference shape + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + checks.check_trilist(shape_batch[0], self.transform) + self.reference_shape = compute_reference_shape( + shape_batch, self.diagonal, verbose=verbose) + + # Rescale the template the reference shape + template = template.rescale_to_pointcloud( + self.reference_shape, group=group) + # After the first batch, we are incrementing the model if k > 0: increment = True @@ -86,99 +91,92 @@ def _train(self, template, shapes, group=None, verbose=False, if verbose: print('Computing batch {}'.format(k)) - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(shape_batch[0], self.transform) - self.reference_shape = compute_reference_shape( - shape_batch, self.diagonal, verbose=verbose) + # Train each batch + self._train_batch(template, shape_batch, increment=increment, + group=group, + shape_forgetting_factor=shape_forgetting_factor, + verbose=verbose) - if k == 0: - # Rescale the template the reference shape - template = template.rescale_to_pointcloud( - self.reference_shape, group=group) + def _train_batch(self, template, shape_batch, increment=False, group=None, + shape_forgetting_factor=1.0, verbose=False): + r""" + Builds an Active Template Model from a list of landmarked images. + """ + # build models at each scale + if verbose: + print_dynamic('- Building models\n') - # build models at each scale + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): if verbose: - print_dynamic('- Building models\n') - - feature_images = [] - # for each scale (low --> high) - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' - else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features([template], - self.features[j], - prefix=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - prefix=scale_prefix, - verbose=verbose) - # Extract potentially rescaled shapes - scale_transform = Scale(scale_factor=self.scales[j], - n_dims=2) - scale_shapes = [scale_transform.apply(s) - for s in shape_batch] + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) else: - scaled_images = feature_images - scale_shapes = shape_batch - - # Build the shape model - if verbose: - print_dynamic('{}Building shape model'.format(scale_prefix)) - - if not increment: - if j == 0: - shape_model = self._build_shape_model(scale_shapes, j) - self.shape_models.append(shape_model) - else: - self.shape_models.append(deepcopy(shape_model)) + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features([template], + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + # Extract potentially rescaled shapes + scale_transform = Scale(scale_factor=self.scales[j], + n_dims=2) + scale_shapes = [scale_transform.apply(s) + for s in shape_batch] + else: + scaled_images = feature_images + scale_shapes = shape_batch + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model(scale_shapes, j) + self.shape_models.append(shape_model) else: - self._increment_shape_model( - scale_shapes, self.shape_models[j], - forgetting_factor=shape_forgetting_factor) - - # Obtain warped images - we use a scaled version of the - # reference shape, computed here. This is because the mean - # moves when we are incrementing, and we need a consistent - # reference frame. - scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( - self.reference_shape) - warped_template = self._warp_template(scaled_images[0], group, - scaled_reference_shape, - j, scale_prefix, verbose) - self.warped_templates.append(warped_template[0]) - - if verbose: - print_dynamic('{}Done\n'.format(scale_prefix)) - - # Because we just copy the shape model, we need to wait to trim - # it after building each model. This ensures we can have a different - # number of components per level - for j, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[j] - if max_sc is not None: - sm.trim_components(max_sc) + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_template = self._warp_template(scaled_images[0], group, + scaled_reference_shape, + j, scale_prefix, verbose) + self.warped_templates.append(warped_template[0]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] + if max_sc is not None: + sm.trim_components(max_sc) def increment(self, template, shapes, group=None, verbose=False, shape_forgetting_factor=1.0, batch_size=None): diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 1bc41ae..fb64599 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -1,11 +1,13 @@ from __future__ import division +import warnings from menpo.feature import no_op from menpo.visualize import print_dynamic from menpofit import checks from menpofit.base import batch from menpofit.builder import ( normalization_wrt_reference_shape, compute_features, scale_images, - build_shape_model, increment_shape_model) + build_shape_model, increment_shape_model, MenpoFitBuilderWarning, + compute_reference_shape, rescale_images_to_reference_shape) from .expert import ExpertEnsemble, CorrelationFilterExpertEnsemble @@ -28,7 +30,7 @@ def __init__(self, images, group=None, verbose=False, batch_size=None, diagonal=None, scales=(0.5, 1), features=no_op, # shape_model_cls=build_normalised_pca_shape_model, expert_ensemble_cls=CorrelationFilterExpertEnsemble, - max_shape_components=None, + max_shape_components=None, reference_shape=None, shape_forgetting_factor=1.0): self.diagonal = checks.check_diagonal(diagonal) self.scales = checks.check_scales(scales) @@ -41,9 +43,13 @@ def __init__(self, images, group=None, verbose=False, batch_size=None, self.max_shape_components = checks.check_max_components( max_shape_components, self.n_scales, 'max_shape_components') self.shape_forgetting_factor = shape_forgetting_factor + self.reference_shape = reference_shape + self.shape_models = [] + self.expert_ensembles = [] # Train CLM - self.train(images, group=group, verbose=verbose, batch_size=batch_size) + self._train(images, increment=False, group=group, verbose=verbose, + batch_size=batch_size) @property def n_scales(self): @@ -54,18 +60,13 @@ def n_scales(self): """ return len(self.scales) - def _train_batch(self, image_batch, increment, group=None, verbose=False): + def _train_batch(self, image_batch, increment=False, group=None, + verbose=False): r""" """ - # If increment is False, we need to initialise/reset both shape models - # and ensembles of experts - if not increment: - self.shape_models = [] - self.expert_ensembles = [] - - # normalize images and compute reference shape - self.reference_shape, image_batch = normalization_wrt_reference_shape( - image_batch, group, self.diagonal, verbose=verbose) + # normalize images + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, verbose=verbose) # build models at each scale if verbose: @@ -139,7 +140,7 @@ def _train_batch(self, image_batch, increment, group=None, verbose=False): if verbose: print_dynamic('{}Done\n'.format(prefix)) - def _train(self, images, increment, group=None, verbose=False, + def _train(self, images, increment=False, group=None, verbose=False, batch_size=None): r""" """ @@ -155,27 +156,36 @@ def _train(self, images, increment, group=None, verbose=False, image_batches = [list(images)] for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + # After the first batch, we are incrementing the model if k > 0: increment = True if verbose: - print('Batch {}'.format(k)) + print('Computing batch {}'.format(k)) # Train each batch - self._train_batch(image_batch, increment, group=group, + self._train_batch(image_batch, increment=increment, group=group, verbose=verbose) - def train(self, images, group=None, verbose=False, batch_size=None): - r""" - """ - return self._train(images, False, group=group, verbose=verbose, - batch_size=batch_size) - def increment(self, images, group=None, verbose=False, batch_size=None): r""" """ - return self._train(images, True, group=group, verbose=verbose, + return self._train(images, increment=True, group=group, verbose=verbose, batch_size=batch_size) def view_shape_models_widget(self, n_parameters=5, diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index ed38cce..93d4e70 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -50,8 +50,9 @@ def __init__(self, images, group=None, bounding_box_group=None, self._setup_algorithms() # Now, train the model! - self._train(images, group=group, bounding_box_group=bounding_box_group, - verbose=verbose, increment=False, batch_size=batch_size) + self._train(images,increment=False, group=group, + bounding_box_group=bounding_box_group, verbose=verbose, + batch_size=batch_size) def _setup_algorithms(self): for j in range(self.n_scales): @@ -60,13 +61,12 @@ def _setup_algorithms(self): patch_shape=self._patch_shape[j], n_iterations=self.n_iterations[j])) - def perturb_from_bounding_box(self, bounding_box): - return self._perturb_from_bounding_box(self.reference_shape, - bounding_box) - - def _train(self, images, group=None, bounding_box_group=None, - verbose=False, increment=False, batch_size=None): - + def _train(self, images, increment=False, group=None, + bounding_box_group=None, verbose=False, batch_size=None): + r""" + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. # If batch_size is not None, then we may have a generator, else we # assume we have a list. if batch_size is not None: @@ -77,154 +77,163 @@ def _train(self, images, group=None, bounding_box_group=None, image_batches = [list(images)] for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + # We set landmarks on the images to archive the perturbations, so + # when the default 'None' is used, we need to grab the actual + # label to sort out the ambiguity + if group is None: + group = image_batch[0].landmarks.group_labels[0] + # After the first batch, we are incrementing the model if k > 0: increment = True if verbose: - print('Computing batch {} - ({})'.format(k, len(image_batch))) - - # In the case where group is None, we need to get the only key so - # that we can attach landmarks below and not get a complaint about - # using None - if group is None: - group = image_batch[0].landmarks.group_labels[0] + print('Computing batch {}'.format(k)) - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, + # Train each batch + self._train_batch( + image_batch, increment=increment, group=group, + bounding_box_group=bounding_box_group, verbose=verbose) - # No bounding box is given, so we will use the ground truth box - if bounding_box_group is None: - # It's important to use bb_group for batching, so that we - # generate ground truth bounding boxes for each batch, every - # time - bb_group = '__gt_bb_' - for i in image_batch: - gt_s = i.landmarks[group].lms - perturb_bbox_group = bb_group + '0' - i.landmarks[perturb_bbox_group] = gt_s.bounding_box() - else: - bb_group = bounding_box_group - - # Find all bounding boxes on the images with the given bounding - # box key - all_bb_keys = list(image_batch[0].landmarks.keys_matching( - '*{}*'.format(bb_group))) - n_perturbations = len(all_bb_keys) - - # If there is only one example bounding box, then we will generate - # more perturbations based on the bounding box. - if n_perturbations == 1: - msg = '- Generating {} new initial bounding boxes ' \ - 'per image'.format(self.n_perturbations) - wrap = partial(print_progress, prefix=msg, verbose=verbose) - - for i in wrap(image_batch): - # We assume that the first bounding box is a valid - # perturbation thus create n_perturbations - 1 new bounding - # boxes - for j in range(1, self.n_perturbations): - gt_s = i.landmarks[group].lms.bounding_box() - bb = i.landmarks[all_bb_keys[0]].lms - - # This is customizable by passing in the correct method - p_s = self._perturb_from_bounding_box(gt_s, bb) - perturb_bbox_group = '{}_{}'.format(bb_group, j) - i.landmarks[perturb_bbox_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations), - MenpoFitBuilderWarning) - self.n_perturbations = n_perturbations - - # Re-grab all the bounding box keys for iterating over when - # calculating perturbations - all_bb_keys = list(image_batch[0].landmarks.keys_matching( - '*{}*'.format(bb_group))) - - # for each scale (low --> high) - current_shapes = [] - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' - else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features(image_batch, - self.features[j], - prefix=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - prefix=scale_prefix, - verbose=verbose) - else: - scaled_images = feature_images - - # Extract scaled ground truth shapes for current scale - scaled_shapes = [i.landmarks[group].lms for i in scaled_images] - - if j == 0: - msg = '{}Generating {} perturbations per image'.format( - scale_prefix, self.n_perturbations) - wrap = partial(print_progress, prefix=msg, - end_with_newline=False, verbose=verbose) - - # Extract perturbations at the very bottom level - for i in wrap(scaled_images): - c_shapes = [] - for perturb_bbox_group in all_bb_keys: - bbox = i.landmarks[perturb_bbox_group].lms - c_s = align_shape_with_bounding_box( - self.reference_shape, bbox) - c_shapes.append(c_s) - current_shapes.append(c_shapes) - - # train supervised descent algorithm - if not increment: - current_shapes = self.algorithms[j].train( - scaled_images, scaled_shapes, current_shapes, - prefix=scale_prefix, verbose=verbose) + def _train_batch(self, image_batch, increment=False, group=None, + bounding_box_group=None, verbose=False): + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # No bounding box is given, so we will use the ground truth box + if bounding_box_group is None: + # It's important to use bb_group for batching, so that we + # generate ground truth bounding boxes for each batch, every + # time + bb_group = '__gt_bb_' + for i in image_batch: + gt_s = i.landmarks[group].lms + perturb_bbox_group = bb_group + '0' + i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + else: + bb_group = bounding_box_group + + # Find all bounding boxes on the images with the given bounding + # box key + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) + n_perturbations = len(all_bb_keys) + + # If there is only one example bounding box, then we will generate + # more perturbations based on the bounding box. + if n_perturbations == 1: + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + + for i in wrap(image_batch): + # We assume that the first bounding box is a valid + # perturbation thus create n_perturbations - 1 new bounding + # boxes + for j in range(1, self.n_perturbations): + gt_s = i.landmarks[group].lms.bounding_box() + bb = i.landmarks[all_bb_keys[0]].lms + + # This is customizable by passing in the correct method + p_s = self._perturb_from_bounding_box(gt_s, bb) + perturb_bbox_group = '{}_{}'.format(bb_group, j) + i.landmarks[perturb_bbox_group] = p_s + elif n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided bounding_box_group.'. + format(self.n_perturbations, n_perturbations), + MenpoFitBuilderWarning) + self.n_perturbations = n_perturbations + + # Re-grab all the bounding box keys for iterating over when + # calculating perturbations + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) + + # for each scale (low --> high) + current_shapes = [] + for j in range(self.n_scales): + if verbose: + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) else: - current_shapes = self.algorithms[j].increment( - scaled_images, scaled_shapes, current_shapes, - prefix=scale_prefix, verbose=verbose) - - # Scale current shapes to next resolution, don't bother - # scaling final level - if j != (self.n_scales - 1): - transform = Scale(self.scales[j + 1] / self.scales[j], - n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract scaled ground truth shapes for current scale + scaled_shapes = [i.landmarks[group].lms for i in scaled_images] + + if j == 0: + msg = '{}Generating {} perturbations per image'.format( + scale_prefix, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False, verbose=verbose) + + # Extract perturbations at the very bottom level + for i in wrap(scaled_images): + c_shapes = [] + for perturb_bbox_group in all_bb_keys: + bbox = i.landmarks[perturb_bbox_group].lms + c_s = align_shape_with_bounding_box( + self.reference_shape, bbox) + c_shapes.append(c_s) + current_shapes.append(c_shapes) + + # train supervised descent algorithm + if not increment: + current_shapes = self.algorithms[j].train( + scaled_images, scaled_shapes, current_shapes, + prefix=scale_prefix, verbose=verbose) + else: + current_shapes = self.algorithms[j].increment( + scaled_images, scaled_shapes, current_shapes, + prefix=scale_prefix, verbose=verbose) + + # Scale current shapes to next resolution, don't bother + # scaling final level + if j != (self.n_scales - 1): + transform = Scale(self.scales[j + 1] / self.scales[j], + n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) def increment(self, images, group=None, bounding_box_group=None, verbose=False, batch_size=None): @@ -233,6 +242,10 @@ def increment(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=True, batch_size=batch_size) + def perturb_from_bounding_box(self, bounding_box): + return self._perturb_from_bounding_box(self.reference_shape, + bounding_box) + def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): return MultiFitterResult(image, self, algorithm_results, From 5ab5f670a0d34d24820438e6aea392ef2572ba48 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 10 Aug 2015 13:07:18 +0100 Subject: [PATCH 200/423] Move _train to top in CLM - consistent with AAM etc --- menpofit/clm/base.py | 90 ++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index fb64599..4d8159d 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -5,9 +5,9 @@ from menpofit import checks from menpofit.base import batch from menpofit.builder import ( - normalization_wrt_reference_shape, compute_features, scale_images, - build_shape_model, increment_shape_model, MenpoFitBuilderWarning, - compute_reference_shape, rescale_images_to_reference_shape) + compute_features, scale_images, build_shape_model, increment_shape_model, + MenpoFitBuilderWarning, compute_reference_shape, + rescale_images_to_reference_shape) from .expert import ExpertEnsemble, CorrelationFilterExpertEnsemble @@ -60,6 +60,48 @@ def n_scales(self): """ return len(self.scales) + def _train(self, images, increment=False, group=None, verbose=False, + batch_size=None): + r""" + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + # Train each batch + self._train_batch(image_batch, increment=increment, group=group, + verbose=verbose) + def _train_batch(self, image_batch, increment=False, group=None, verbose=False): r""" @@ -140,48 +182,6 @@ def _train_batch(self, image_batch, increment=False, group=None, if verbose: print_dynamic('{}Done\n'.format(prefix)) - def _train(self, images, increment=False, group=None, verbose=False, - batch_size=None): - r""" - """ - # If batch_size is not None, then we may have a generator, else we - # assume we have a list. - # If batch_size is not None, then we may have a generator, else we - # assume we have a list. - if batch_size is not None: - # Create a generator of fixed sized batches. Will still work even - # on an infinite list. - image_batches = batch(images, batch_size) - else: - image_batches = [list(images)] - - for k, image_batch in enumerate(image_batches): - if k == 0: - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The ' - 'mean of the first batch will be the ' - 'reference shape. If the batch mean is ' - 'not representative of the true mean, ' - 'this may cause issues.', - MenpoFitBuilderWarning) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # After the first batch, we are incrementing the model - if k > 0: - increment = True - - if verbose: - print('Computing batch {}'.format(k)) - - # Train each batch - self._train_batch(image_batch, increment=increment, group=group, - verbose=verbose) - def increment(self, images, group=None, verbose=False, batch_size=None): r""" """ From d642ab191a356f0886fc18c4bce209634b04aa8d Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:24:42 +0100 Subject: [PATCH 201/423] Fix silly bug in PatchAAM Wasn't passing parameters up to super class properly. --- menpofit/aam/algorithm/lk.py | 5 ++--- menpofit/aam/base.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index bacf76d..181c155 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -229,13 +229,12 @@ class LucasKanadePartsInterface(LucasKanadePartsBaseInterface): """ def __init__(self, appearance_model, transform, template, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): - self.patch_shape = patch_shape # TODO: Refactor to patch_features - self.normalize_parts = normalize_parts self.appearance_model = appearance_model super(LucasKanadePartsInterface, self).__init__( - transform, template, sampling=sampling) + transform, template, patch_shape=patch_shape, + normalize_parts=normalize_parts, sampling=sampling) @property def m(self): diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index e5257c5..a448cb3 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -873,7 +873,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, - transform=DifferentiableThinPlateSplines, diagonal=diagonal, + transform=None, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -937,13 +937,18 @@ def _aam_str(aam): aam.patch_shape[k]) scales_info = '\n'.join(scales_info) + if aam.transform is not None: + transform_str = 'Images warped with {transform} transform' + else: + transform_str = 'No image warping performed' + cls_str = r"""{class_title} - - Images warped with {transform} transform - Images scaled to diagonal: {diagonal:.2f} + - {transform} - Scales: {scales} {scales_info} """.format(class_title=aam._str_title, - transform=name_of_callable(aam.transform), + transform=transform_str, diagonal=diagonal, scales=aam.scales, scales_info=scales_info) From 95c965d6f47a132420a06e7a5606ec1ba3a10130 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:25:18 +0100 Subject: [PATCH 202/423] Small inhancement for no_op When holistic feature is no_op, don't copy images or even bother running through the loop, just skip it. --- menpofit/aam/base.py | 7 +++++-- menpofit/clm/base.py | 7 +++++-- menpofit/sdm/fitter.py | 7 +++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index a448cb3..5e6d52b 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -232,8 +232,11 @@ def _train_batch(self, image_batch, increment=False, group=None, else: scale_prefix = None - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: + # Handle holistic features + if j == 0 and self.features[j] == no_op: + # Saves a lot of memory + feature_images = image_batch + elif j == 0 or self.features[j] is not self.features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 4d8159d..42f3614 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -122,8 +122,11 @@ def _train_batch(self, image_batch, increment=False, group=None, else: prefix = ' - ' - # handle features - if i == 0 or self.features[i] is not self.features[i-1]: + # Handle holistic features + if i == 0 and self.features[i] == no_op: + # Saves a lot of memory + feature_images = image_batch + elif i == 0 or self.features[i] is not self.features[i - 1]: # compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 93d4e70..010dcb8 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -179,8 +179,11 @@ def _train_batch(self, image_batch, increment=False, group=None, else: scale_prefix = None - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: + # Handle holistic features + if j == 0 and self.features[j] == no_op: + # Saves a lot of memory + feature_images = image_batch + elif j == 0 or self.features[j] is not self.features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale From 554ccd1c677cbcd1b0ca2636a941f3b240275794 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:25:54 +0100 Subject: [PATCH 203/423] Fix printing a bit Includes non-verbose bug for CLMs --- menpofit/builder.py | 2 +- menpofit/clm/base.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 50cfdc4..553df59 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -162,7 +162,7 @@ def warp_images(images, shapes, reference_frame, transform, prefix='', def extract_patches(images, shapes, patch_shape, normalize_function=no_op, prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Warping images'.format(prefix), + prefix='{}Extracting patches'.format(prefix), end_with_newline=not prefix, verbose=verbose) parts_images = [] diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 42f3614..e335957 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -118,9 +118,11 @@ def _train_batch(self, image_batch, increment=False, group=None, for i in range(self.n_scales): if verbose: if self.n_scales > 1: - prefix = ' - Scale {}: '.format(i) + prefix = ' - Scale {}: '.format(j) else: - prefix = ' - ' + prefix = ' - ' + else: + prefix = None # Handle holistic features if i == 0 and self.features[i] == no_op: From 343190e644588ff903c800e2e76f2f8b75bba339 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:26:31 +0100 Subject: [PATCH 204/423] Change fitting methods to fit_from_shape/bb This renames the fitting methods and adds a new method that can fit from bounding boxes. --- menpofit/builder.py | 2 +- menpofit/fitter.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 553df59..a174780 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -332,4 +332,4 @@ class MenpoFitBuilderWarning(Warning): r""" A warning that some part of building the model may cause issues. """ - pass \ No newline at end of file + pass diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 91029a9..dcc647e 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -21,8 +21,8 @@ def n_scales(self): """ return len(self.scales) - def fit(self, image, initial_shape, max_iters=20, gt_shape=None, - crop_image=0.5, **kwargs): + def fit_from_shape(self, image, initial_shape, max_iters=20, gt_shape=None, + crop_image=None, **kwargs): r""" Fits the multilevel fitter to an image. @@ -78,6 +78,14 @@ def fit(self, image, initial_shape, max_iters=20, gt_shape=None, return fitter_result + def fit_from_bb(self, image, bounding_box, max_iters=20, gt_shape=None, + crop_image=None, **kwargs): + initial_shape = align_shape_with_bounding_box(self.reference_shape, + bounding_box) + return self.fit_from_shape(image, initial_shape, max_iters=max_iters, + gt_shape=gt_shape, crop_image=crop_image, + **kwargs) + def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): r""" From 497e5279548fb83417ada46ea721b213ef2fa83e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:36:03 +0100 Subject: [PATCH 205/423] Rename Parts interfaces to patch This is consistent with the previous renaming of parts aam to patch aam --- menpofit/aam/algorithm/lk.py | 8 ++++---- menpofit/aam/fitter.py | 4 ++-- menpofit/atm/algorithm.py | 4 ++-- menpofit/atm/fitter.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 181c155..5cf0806 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -158,7 +158,7 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, # TODO document me! -class LucasKanadePartsBaseInterface(LucasKanadeBaseInterface): +class LucasKanadePatchBaseInterface(LucasKanadeBaseInterface): r""" """ def __init__(self, transform, template, sampling=None, @@ -167,7 +167,7 @@ def __init__(self, transform, template, sampling=None, # TODO: Refactor to patch_features self.normalize_parts = normalize_parts - super(LucasKanadePartsBaseInterface, self).__init__( + super(LucasKanadePatchBaseInterface, self).__init__( transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): @@ -224,7 +224,7 @@ def steepest_descent_images(self, nabla, dw_dp): # TODO document me! -class LucasKanadePartsInterface(LucasKanadePartsBaseInterface): +class LucasKanadePatchInterface(LucasKanadePatchBaseInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, @@ -232,7 +232,7 @@ def __init__(self, appearance_model, transform, template, sampling=None, # TODO: Refactor to patch_features self.appearance_model = appearance_model - super(LucasKanadePartsInterface, self).__init__( + super(LucasKanadePatchInterface, self).__init__( transform, template, patch_shape=patch_shape, normalize_parts=normalize_parts, sampling=sampling) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index ff42ea6..57da7f2 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -11,7 +11,7 @@ from .base import AAM, MaskedAAM, LinearAAM, LinearMaskedAAM, PatchAAM from .algorithm.lk import ( LucasKanadeStandardInterface, LucasKanadeLinearInterface, - LucasKanadePartsInterface, WibergInverseCompositional) + LucasKanadePatchInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, SupervisedDescentPartsInterface, ProjectOutNewton) @@ -73,7 +73,7 @@ def _set_up(self, lk_algorithm_cls): elif type(self.aam) is PatchAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) - interface = LucasKanadePartsInterface( + interface = LucasKanadePatchInterface( am, pdm, template, sampling=s, patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 3a020c6..0e5d5db 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,7 +1,7 @@ from __future__ import division import numpy as np from menpofit.aam.algorithm.lk import (LucasKanadeBaseInterface, - LucasKanadePartsBaseInterface) + LucasKanadePatchBaseInterface) from .result import ATMAlgorithmResult, LinearATMAlgorithmResult @@ -36,7 +36,7 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, # TODO document me! -class ATMLKPartsInterface(LucasKanadePartsBaseInterface): +class ATMLKPatchInterface(LucasKanadePatchBaseInterface): r""" """ def algorithm_result(self, image, shape_parameters, cost_functions=None, diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index dc126e4..5370996 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -5,7 +5,7 @@ from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import ATM, MaskedATM, LinearATM, LinearMaskedATM, PatchATM from .algorithm import ( - ATMLKStandardInterface, ATMLKPartsInterface, ATMLKLinearInterface, + ATMLKStandardInterface, ATMLKPatchInterface, ATMLKLinearInterface, InverseCompositional) from .result import ATMFitterResult @@ -45,7 +45,7 @@ def _set_up(self, algorithm_cls): algorithm = algorithm_cls(interface) elif type(self.atm) is PatchATM: pdm = OrthoPDM(sm) - interface = ATMLKPartsInterface( + interface = ATMLKPatchInterface( pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) algorithm = algorithm_cls(interface) From 781c293c885b3b4371abea047c8351d4ae081fcc Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 16:23:02 +0100 Subject: [PATCH 206/423] Universally rename patch_shape to patch_size This is consistent with Menpo's extract_patches method and with CLM --- menpofit/aam/algorithm/lk.py | 14 ++++++------ menpofit/aam/algorithm/sd.py | 8 +++---- menpofit/aam/base.py | 32 +++++++++++++-------------- menpofit/aam/fitter.py | 6 +++--- menpofit/atm/base.py | 26 +++++++++++----------- menpofit/atm/fitter.py | 2 +- menpofit/builder.py | 12 +++++------ menpofit/checks.py | 18 ++++++++-------- menpofit/clm/expert/ensemble.py | 10 ++++----- menpofit/sdm/algorithm.py | 38 ++++++++++++++++----------------- menpofit/sdm/fitter.py | 16 +++++++------- 11 files changed, 89 insertions(+), 93 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 5cf0806..58ce0bf 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -162,8 +162,8 @@ class LucasKanadePatchBaseInterface(LucasKanadeBaseInterface): r""" """ def __init__(self, transform, template, sampling=None, - patch_shape=(17, 17), normalize_parts=no_op): - self.patch_shape = patch_shape + patch_size=(17, 17), normalize_parts=no_op): + self.patch_size = patch_size # TODO: Refactor to patch_features self.normalize_parts = normalize_parts @@ -172,7 +172,7 @@ def __init__(self, transform, template, sampling=None, def _build_sampling_mask(self, sampling): if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) + sampling = np.ones(self.patch_size, dtype=np.bool) image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], @@ -192,14 +192,14 @@ def warp_jacobian(self): def warp(self, image): parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, + patch_size=self.patch_size, as_single_array=True) parts = self.normalize_parts(parts) return Image(parts, copy=False) def gradient(self, image): pixels = image.pixels - nabla = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) + nabla = fast_gradient(pixels.reshape((-1,) + self.patch_size)) # remove 1st dimension gradient which corresponds to the gradient # between parts return nabla.reshape((2,) + pixels.shape) @@ -228,12 +228,12 @@ class LucasKanadePatchInterface(LucasKanadePatchBaseInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_shape=(17, 17), normalize_parts=no_op): + patch_size=(17, 17), normalize_parts=no_op): # TODO: Refactor to patch_features self.appearance_model = appearance_model super(LucasKanadePatchInterface, self).__init__( - transform, template, patch_shape=patch_shape, + transform, template, patch_size=patch_size, normalize_parts=normalize_parts, sampling=sampling) @property diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index ce0d9c0..e2d0415 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -78,8 +78,8 @@ class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_shape=(17, 17), normalize_parts=no_op): - self.patch_shape = patch_shape + patch_size=(17, 17), normalize_parts=no_op): + self.patch_size = patch_size # TODO: Refactor to patch_features self.normalize_parts = normalize_parts @@ -88,7 +88,7 @@ def __init__(self, appearance_model, transform, template, sampling=None, def _build_sampling_mask(self, sampling): if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) + sampling = np.ones(self.patch_size, dtype=np.bool) image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], @@ -101,7 +101,7 @@ def shape_model(self): def warp(self, image): parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, + patch_size=self.patch_size, as_single_array=True) parts = self.normalize_parts(parts) return Image(parts, copy=False) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 5e6d52b..cc608cb 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -563,7 +563,7 @@ class MaskedAAM(AAM): reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` + patch_size : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at @@ -584,10 +584,10 @@ class MaskedAAM(AAM): """ def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -599,7 +599,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[scale_index]) + reference_shape, patch_size=self.patch_size[scale_index]) return warp_images(images, shapes, reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -612,7 +612,7 @@ def _instance(self, scale_index, shape_instance, appearance_instance): landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( - shape_instance, patch_shape=self.patch_shape) + shape_instance, patch_size=self.patch_size) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) @@ -746,7 +746,7 @@ class LinearMaskedAAM(AAM): reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` + patch_size : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at @@ -766,10 +766,10 @@ class LinearMaskedAAM(AAM): """ def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -790,7 +790,7 @@ def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + mean_aligned_shape, patch_size=self.patch_size[scale_index]) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model @@ -847,7 +847,7 @@ class PatchAAM(AAM): reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` + patch_size : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at @@ -869,9 +869,9 @@ class PatchAAM(AAM): def __init__(self, images, group=None, verbose=False, features=no_op, normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, + patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) self.normalize_parts = normalize_parts super(PatchAAM, self).__init__( @@ -891,7 +891,7 @@ def _str_title(self): def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): - return extract_patches(images, shapes, self.patch_shape[scale_index], + return extract_patches(images, shapes, self.patch_size[scale_index], normalize_function=self.normalize_parts, prefix=prefix, verbose=verbose) @@ -934,10 +934,10 @@ def _aam_str(aam): aam.appearance_models[k].n_components, aam.shape_models[k].n_components)) # Patch based AAM - if hasattr(aam, 'patch_shape'): + if hasattr(aam, 'patch_size'): for k in range(len(scales_info)): - scales_info[k] += '\n - Patch shape: {}'.format( - aam.patch_shape[k]) + scales_info[k] += '\n - Patch size: {}'.format( + aam.patch_size[k]) scales_info = '\n'.join(scales_info) if aam.transform is not None: diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 57da7f2..5ab2e82 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -75,7 +75,7 @@ def _set_up(self, lk_algorithm_cls): pdm = OrthoPDM(sm) interface = LucasKanadePatchInterface( am, pdm, template, sampling=s, - patch_shape=self.aam.patch_shape[j], + patch_size=self.aam.patch_size[j], normalize_parts=self.aam.normalize_parts) algorithm = lk_algorithm_cls(interface) else: @@ -102,7 +102,7 @@ def __init__(self, images, aam, group=None, bounding_box_group=None, checks.set_models_components(aam.shape_models, n_shape) self._sampling = checks.check_sampling(sampling, aam.n_scales) - # patch_feature and patch_shape are not actually + # patch_feature and patch_size are not actually # used because they are fully defined by the AAM already. Therefore, # we just leave them as their 'defaults' because they won't be used. super(SupervisedDescentAAMFitter, self).__init__( @@ -145,7 +145,7 @@ def _setup_algorithms(self): pdm = OrthoPDM(sm) interface = SupervisedDescentPartsInterface( am, pdm, template, sampling=s, - patch_shape=self.aam.patch_shape[j], + patch_size=self.aam.patch_size[j], normalize_parts=self.aam.normalize_parts) algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index f6fa2ff..7e8c83f 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -356,9 +356,9 @@ class MaskedATM(ATM): def __init__(self, template, shapes, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, + patch_size=(17, 17), max_shape_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, @@ -369,7 +369,7 @@ def __init__(self, template, shapes, group=None, verbose=False, def _warp_template(self, template, group, reference_shape, scale_index, prefix, verbose): reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[scale_index]) + reference_shape, patch_size=self.patch_size[scale_index]) shape = template.landmarks[group].lms return warp_images([template], [shape], reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -382,7 +382,7 @@ def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( - shape_instance, patch_shape=self.patch_shape) + shape_instance, patch_size=self.patch_size) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) @@ -466,9 +466,9 @@ class LinearMaskedATM(ATM): def __init__(self, template, shapes, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, + patch_size=(17, 17), max_shape_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, @@ -488,7 +488,7 @@ def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + mean_aligned_shape, patch_size=self.patch_size[scale_index]) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model @@ -534,9 +534,9 @@ class PatchATM(ATM): def __init__(self, template, shapes, group=None, verbose=False, features=no_op, normalize_parts=no_op, diagonal=None, - scales=(0.5, 1.0), patch_shape=(17, 17), + scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) self.normalize_parts = normalize_parts super(PatchATM, self).__init__( @@ -557,7 +557,7 @@ def _warp_template(self, template, group, reference_shape, scale_index, prefix, verbose): shape = template.landmarks[group].lms return extract_patches([template], [shape], - self.patch_shape[scale_index], + self.patch_size[scale_index], normalize_function=self.normalize_parts, prefix=prefix, verbose=verbose) @@ -594,10 +594,10 @@ def _atm_str(atm): atm.warped_templates[k].shape, atm.shape_models[k].n_components)) # Patch based ATM - if hasattr(atm, 'patch_shape'): + if hasattr(atm, 'patch_size'): for k in range(len(scales_info)): - scales_info[k] += '\n - Patch shape: {}'.format( - atm.patch_shape[k]) + scales_info[k] += '\n - Patch size: {}'.format( + atm.patch_size[k]) scales_info = '\n'.join(scales_info) cls_str = r"""{class_title} diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 5370996..b4147c3 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -46,7 +46,7 @@ def _set_up(self, algorithm_cls): elif type(self.atm) is PatchATM: pdm = OrthoPDM(sm) interface = ATMLKPatchInterface( - pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], + pdm, wt, sampling=s, patch_size=self.atm.patch_size[j], normalize_parts=self.atm.normalize_parts) algorithm = algorithm_cls(interface) else: diff --git a/menpofit/builder.py b/menpofit/builder.py index a174780..7f952d4 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -159,7 +159,7 @@ def warp_images(images, shapes, reference_frame, transform, prefix='', # TODO: document me! -def extract_patches(images, shapes, patch_shape, normalize_function=no_op, +def extract_patches(images, shapes, patch_size, normalize_function=no_op, prefix='', verbose=False): wrap = partial(print_progress, prefix='{}Extracting patches'.format(prefix), @@ -167,7 +167,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images = [] for i, s in wrap(zip(images, shapes)): - parts = i.extract_patches(s, patch_size=patch_shape, + parts = i.extract_patches(s, patch_size=patch_size, as_single_array=True) parts = normalize_function(parts) parts_images.append(Image(parts)) @@ -207,7 +207,7 @@ def build_reference_frame(landmarks, boundary=3, group='source'): def build_patch_reference_frame(landmarks, boundary=3, group='source', - patch_shape=(17, 17)): + patch_size=(17, 17)): r""" Builds a reference frame from a particular set of landmarks. @@ -225,7 +225,7 @@ def build_patch_reference_frame(landmarks, boundary=3, group='source', Group that will be assigned to the provided set of landmarks on the reference frame. - patch_shape : tuple of ints, optional + patch_size : tuple of ints, optional Tuple specifying the shape of the patches. Returns @@ -233,12 +233,12 @@ def build_patch_reference_frame(landmarks, boundary=3, group='source', patch_based_reference_frame : :map:`Image` The patch based reference frame. """ - boundary = np.max(patch_shape) + boundary + boundary = np.max(patch_size) + boundary reference_frame = _build_reference_frame(landmarks, boundary=boundary, group=group) # mask reference frame - reference_frame.build_mask_around_landmarks(patch_shape, group=group) + reference_frame.build_mask_around_landmarks(patch_size, group=group) return reference_frame diff --git a/menpofit/checks.py b/menpofit/checks.py index 330cc84..34c78b6 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -85,17 +85,17 @@ def check_scale_features(scale_features, features): # TODO: document me! -def check_patch_shape(patch_shape, n_scales): - if len(patch_shape) == 2 and isinstance(patch_shape[0], int): - return [patch_shape] * n_scales - elif len(patch_shape) == 1: - return check_patch_shape(patch_shape[0], 1) - elif len(patch_shape) == n_scales: - l1 = check_patch_shape(patch_shape[0], 1) - l2 = check_patch_shape(patch_shape[1:], n_scales-1) +def check_patch_size(patch_size, n_scales): + if len(patch_size) == 2 and isinstance(patch_size[0], int): + return [patch_size] * n_scales + elif len(patch_size) == 1: + return check_patch_size(patch_size[0], 1) + elif len(patch_size) == n_scales: + l1 = check_patch_size(patch_size[0], 1) + l2 = check_patch_size(patch_size[1:], n_scales-1) return l1 + l2 else: - raise ValueError("patch_shape must be a list/tuple of int or a " + raise ValueError("patch_size must be a list/tuple of int or a " "list/tuple of lit/tuple of int/float with the " "same length as the number of scales") diff --git a/menpofit/clm/expert/ensemble.py b/menpofit/clm/expert/ensemble.py index a120345..fbd663f 100644 --- a/menpofit/clm/expert/ensemble.py +++ b/menpofit/clm/expert/ensemble.py @@ -218,10 +218,10 @@ def _train(self, images, shapes, prefix='', verbose=False, # Increment correlation filter correlation_filter, auto_correlation, cross_correlation = ( self._icf.increment(self.auto_correlations[i], - self.cross_correlations[i], - self.n_images, - patches, - self.response)) + self.cross_correlations[i], + self.n_images, + patches, + self.response)) else: # Train correlation filter correlation_filter, auto_correlation, cross_correlation = ( @@ -258,5 +258,3 @@ def generate_cosine_mask(patch_size): cy = np.hanning(patch_size[0]) cx = np.hanning(patch_size[1]) return cy[..., None].dot(cx[None, ...]) - - diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 88dd0b9..939843a 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -44,7 +44,7 @@ def _train(self, images, gt_shapes, current_shapes, increment=False, for k in range(self.n_iterations): # generate regression data features = features_per_image( - images, current_shapes, self.patch_shape, self.features, + images, current_shapes, self.patch_size, self.features, prefix='{}(Iteration {}) - '.format(prefix, k), verbose=verbose) @@ -90,7 +90,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): for r in self.regressors: # compute regression features features = features_per_patch(image, current_shape, - self.patch_shape, self.features) + self.patch_size, self.features) # solve for increments on the shape vector dx = r.predict(features) @@ -127,15 +127,14 @@ def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, class Newton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, + def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True): super(Newton, self).__init__() self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) - self.patch_shape = patch_shape + self.patch_size = patch_size self.features = features - self.patch_shape = patch_shape self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps @@ -145,26 +144,25 @@ def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, class GaussNewton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, + def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True, alpha2=0): super(GaussNewton, self).__init__() self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, alpha2=alpha2) - self.patch_shape = patch_shape + self.patch_size = patch_size self.features = features - self.patch_shape = patch_shape self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps # TODO: document me! -def features_per_patch(image, shape, patch_shape, features_callable): +def features_per_patch(image, shape, patch_size, features_callable): """r """ - patches = image.extract_patches(shape, patch_size=patch_shape, + patches = image.extract_patches(shape, patch_size=patch_size, as_single_array=True) patch_features = [features_callable(p[0]).ravel() for p in patches] @@ -172,10 +170,10 @@ def features_per_patch(image, shape, patch_shape, features_callable): # TODO: document me! -def features_per_shape(image, shapes, patch_shape, features_callable): +def features_per_shape(image, shapes, patch_size, features_callable): """r """ - patch_features = [features_per_patch(image, s, patch_shape, + patch_features = [features_per_patch(image, s, patch_size, features_callable) for s in shapes] @@ -183,7 +181,7 @@ def features_per_shape(image, shapes, patch_shape, features_callable): # TODO: document me! -def features_per_image(images, shapes, patch_shape, features_callable, +def features_per_image(images, shapes, patch_size, features_callable, prefix='', verbose=False): """r """ @@ -191,7 +189,7 @@ def features_per_image(images, shapes, patch_shape, features_callable, prefix='{}Extracting patches'.format(prefix), end_with_newline=not prefix, verbose=verbose) - patch_features = [features_per_shape(i, shapes[j], patch_shape, + patch_features = [features_per_shape(i, shapes[j], patch_size, features_callable) for j, i in enumerate(wrap(images))] patch_features = np.asarray(patch_features) @@ -237,16 +235,16 @@ def obtain_delta_x(gt_shapes, current_shapes): def compute_features_info(image, shape, features_callable, - patch_shape=(17, 17)): + patch_size=(17, 17)): # TODO: include offsets support? - patches = image.extract_patches(shape, patch_size=patch_shape, + patches = image.extract_patches(shape, patch_size=patch_size, as_single_array=True) # TODO: include offsets support? - features_patch_shape = features_callable(patches[0, 0]).shape - features_patch_length = np.prod(features_patch_shape) - features_shape = patches.shape[:1] + features_patch_shape + features_patch_size = features_callable(patches[0, 0]).shape + features_patch_length = np.prod(features_patch_size) + features_shape = patches.shape[:1] + features_patch_size features_length = np.prod(features_shape) - return (features_patch_shape, features_patch_length, + return (features_patch_size, features_patch_length, features_shape, features_length) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 010dcb8..de7addf 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -23,7 +23,7 @@ class SupervisedDescentFitter(MultiFitter): def __init__(self, images, group=None, bounding_box_group=None, reference_shape=None, sd_algorithm_cls=Newton, holistic_feature=no_op, patch_features=no_op, - patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): @@ -33,14 +33,14 @@ def __init__(self, images, group=None, bounding_box_group=None, scales = checks.check_scales(scales) patch_features = checks.check_features(patch_features, n_scales) holistic_features = checks.check_features(holistic_feature, n_scales) - patch_shape = checks.check_patch_shape(patch_shape, n_scales) + patch_size = checks.check_patch_size(patch_size, n_scales) # set parameters self.algorithms = [] self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls self.features = holistic_features self._patch_features = patch_features - self._patch_shape = patch_shape + self._patch_size = patch_size self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations @@ -58,7 +58,7 @@ def _setup_algorithms(self): for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], - patch_shape=self._patch_shape[j], + patch_size=self._patch_size[j], n_iterations=self.n_iterations[j])) def _train(self, images, increment=False, group=None, @@ -268,12 +268,12 @@ def __str__(self): scales_info = [] lvl_str_tmplt = r""" - Scale {} - {} iterations - - Patch shape: {} + - Patch size: {} - Holistic feature: {} - Patch feature: {}""" for k, s in enumerate(self.scales): scales_info.append(lvl_str_tmplt.format( - s, self.n_iterations[k], self._patch_shape[k], + s, self.n_iterations[k], self._patch_size[k], name_of_callable(self.features[k]), name_of_callable(self._patch_features[k]))) scales_info = '\n'.join(scales_info) @@ -305,7 +305,7 @@ class RegularizedSDM(SupervisedDescentFitter): def __init__(self, images, group=None, bounding_box_group=None, alpha=1.0, reference_shape=None, holistic_feature=no_op, patch_features=no_op, - patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): @@ -314,7 +314,7 @@ def __init__(self, images, group=None, bounding_box_group=None, reference_shape=reference_shape, sd_algorithm_cls=partial(Newton, alpha=alpha), holistic_feature=holistic_feature, patch_features=patch_features, - patch_shape=patch_shape, diagonal=diagonal, scales=scales, + patch_size=patch_size, diagonal=diagonal, scales=scales, n_iterations=n_iterations, n_perturbations=n_perturbations, perturb_from_bounding_box=perturb_from_bounding_box, batch_size=batch_size, verbose=verbose) From efea68a996df6e85c3cb256cfc6d8402e7e1aea2 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 13 Aug 2015 11:52:19 +0100 Subject: [PATCH 207/423] Renaming features parameters features -> holistic_features normalize_parts -> patch_normalisation normalise_callable -> patch_normalisation patch_features -> patch_features At the moment, only SDMs have patch_features due to the lack of distinction on menpo between image and vector features. --- menpofit/aam/algorithm/lk.py | 12 +++--- menpofit/aam/algorithm/sd.py | 7 ++-- menpofit/aam/base.py | 67 +++++++++++++++++------------- menpofit/aam/fitter.py | 6 +-- menpofit/atm/base.py | 44 +++++++++++--------- menpofit/atm/fitter.py | 2 +- menpofit/builder.py | 6 +-- menpofit/clm/base.py | 11 ++--- menpofit/clm/expert/ensemble.py | 10 ++--- menpofit/fitter.py | 8 ++-- menpofit/lk/fitter.py | 7 ++-- menpofit/sdm/algorithm.py | 15 +++---- menpofit/sdm/fitter.py | 30 ++++++------- menpofit/visualize/widgets/base.py | 4 +- 14 files changed, 120 insertions(+), 109 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 58ce0bf..3dd6960 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -162,10 +162,9 @@ class LucasKanadePatchBaseInterface(LucasKanadeBaseInterface): r""" """ def __init__(self, transform, template, sampling=None, - patch_size=(17, 17), normalize_parts=no_op): + patch_size=(17, 17), patch_normalisation=no_op): self.patch_size = patch_size - # TODO: Refactor to patch_features - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(LucasKanadePatchBaseInterface, self).__init__( transform, template, sampling=sampling) @@ -194,7 +193,7 @@ def warp(self, image): parts = image.extract_patches(self.transform.target, patch_size=self.patch_size, as_single_array=True) - parts = self.normalize_parts(parts) + parts = self.patch_normalisation(parts) return Image(parts, copy=False) def gradient(self, image): @@ -228,13 +227,12 @@ class LucasKanadePatchInterface(LucasKanadePatchBaseInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_size=(17, 17), normalize_parts=no_op): - # TODO: Refactor to patch_features + patch_size=(17, 17), patch_normalisation=no_op): self.appearance_model = appearance_model super(LucasKanadePatchInterface, self).__init__( transform, template, patch_size=patch_size, - normalize_parts=normalize_parts, sampling=sampling) + patch_normalisation=patch_normalisation, sampling=sampling) @property def m(self): diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index e2d0415..d6fdb17 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -78,10 +78,9 @@ class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_size=(17, 17), normalize_parts=no_op): + patch_size=(17, 17), patch_normalisation=no_op): self.patch_size = patch_size - # TODO: Refactor to patch_features - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(SupervisedDescentPartsInterface, self).__init__( appearance_model, transform, template, sampling=sampling) @@ -103,7 +102,7 @@ def warp(self, image): parts = image.extract_patches(self.transform.target, patch_size=self.patch_size, as_single_array=True) - parts = self.normalize_parts(parts) + parts = self.patch_normalisation(parts) return Image(parts, copy=False) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index cc608cb..83e634f 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -115,20 +115,21 @@ class AAM(object): ``len(scales)`` elements """ def __init__(self, images, group=None, verbose=False, reference_shape=None, - features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + holistic_features=no_op, + transform=DifferentiablePiecewiseAffine, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): checks.check_diagonal(diagonal) n_scales = len(scales) scales = checks.check_scales(scales) - features = checks.check_features(features, n_scales) + holistic_features = checks.check_features(holistic_features, n_scales) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_scales, 'max_appearance_components') - self.features = features + self.holistic_features = holistic_features self.transform = transform self.diagonal = diagonal self.scales = scales @@ -233,15 +234,15 @@ def _train_batch(self, image_batch, increment=False, group=None, scale_prefix = None # Handle holistic features - if j == 0 and self.features[j] == no_op: + if j == 0 and self.holistic_features[j] == no_op: # Saves a lot of memory feature_images = image_batch - elif j == 0 or self.features[j] is not self.features[j - 1]: + elif j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features(image_batch, - self.features[j], + self.holistic_features[j], prefix=scale_prefix, verbose=verbose) # handle scales @@ -583,14 +584,15 @@ class MaskedAAM(AAM): scale_shapes : `boolean` """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), - max_shape_components=None, max_appearance_components=None, - batch_size=None): + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedAAM, self).__init__( - images, group=group, verbose=verbose, features=features, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, @@ -667,14 +669,16 @@ class LinearAAM(AAM): scales : `int` or float` or list of those """ - def __init__(self, images, group=None, verbose=False, features=no_op, + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, transform=DifferentiableThinPlateSplines, diagonal=None, scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): super(LinearAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, scales=scales, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=transform, + diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -765,14 +769,15 @@ class LinearMaskedAAM(AAM): scales : `int` or float` or list of those """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), - max_shape_components=None, max_appearance_components=None, - batch_size=None): + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedAAM, self).__init__( - images, group=group, verbose=verbose, features=features, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, @@ -867,17 +872,19 @@ class PatchAAM(AAM): scales : `int` or float` or list of those """ - def __init__(self, images, group=None, verbose=False, features=no_op, - normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_size=(17, 17), max_shape_components=None, - max_appearance_components=None, batch_size=None): + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, patch_normalisation=no_op, + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(PatchAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=None, diagonal=diagonal, - scales=scales, max_shape_components=max_shape_components, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=None, + diagonal=diagonal, scales=scales, + max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -892,7 +899,7 @@ def _str_title(self): def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): return extract_patches(images, shapes, self.patch_size[scale_index], - normalize_function=self.normalize_parts, + normalise_function=self.patch_normalisation, prefix=prefix, verbose=verbose) # TODO: implement me! @@ -930,7 +937,7 @@ def _aam_str(aam): - {} shape components""" for k, s in enumerate(aam.scales): scales_info.append(lvl_str_tmplt.format( - s, name_of_callable(aam.features[k]), + s, name_of_callable(aam.holistic_features[k]), aam.appearance_models[k].n_components, aam.shape_models[k].n_components)) # Patch based AAM diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 5ab2e82..3e74017 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -76,7 +76,7 @@ def _set_up(self, lk_algorithm_cls): interface = LucasKanadePatchInterface( am, pdm, template, sampling=s, patch_size=self.aam.patch_size[j], - normalize_parts=self.aam.normalize_parts) + patch_normalisation=self.aam.patch_normalisation) algorithm = lk_algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " @@ -109,7 +109,7 @@ def __init__(self, images, aam, group=None, bounding_box_group=None, images, group=group, bounding_box_group=bounding_box_group, reference_shape=self.aam.reference_shape, sd_algorithm_cls=sd_algorithm_cls, - holistic_feature=self.aam.features, + holistic_feature=self.aam.holistic_features, diagonal=self.aam.diagonal, scales=self.aam.scales, n_iterations=n_iterations, n_perturbations=n_perturbations, @@ -146,7 +146,7 @@ def _setup_algorithms(self): interface = SupervisedDescentPartsInterface( am, pdm, template, sampling=s, patch_size=self.aam.patch_size[j], - normalize_parts=self.aam.normalize_parts) + patch_normalisation=self.aam.patch_normalisation) algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) else: diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 7e8c83f..3b53565 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -23,7 +23,7 @@ class ATM(object): Active Template Model class. """ def __init__(self, template, shapes, group=None, verbose=False, - reference_shape=None, features=no_op, + reference_shape=None, holistic_features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), max_shape_components=None, batch_size=None): @@ -31,11 +31,11 @@ def __init__(self, template, shapes, group=None, verbose=False, checks.check_diagonal(diagonal) n_scales = len(scales) scales = checks.check_scales(scales) - features = checks.check_features(features, n_scales) + holistic_features = checks.check_features(holistic_features, n_scales) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') - self.features = features + self.holistic_features = holistic_features self.transform = transform self.diagonal = diagonal self.scales = scales @@ -118,12 +118,12 @@ def _train_batch(self, template, shape_batch, increment=False, group=None, scale_prefix = None # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: + if j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features([template], - self.features[j], + self.holistic_features[j], prefix=scale_prefix, verbose=verbose) # handle scales @@ -355,13 +355,14 @@ class MaskedATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, diagonal=None, scales=(0.5, 1.0), + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -401,13 +402,14 @@ class LinearATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, transform=DifferentiableThinPlateSplines, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, - batch_size=None): + holistic_features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, batch_size=None): super(LinearATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, scales=scales, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=transform, + diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @property @@ -465,13 +467,14 @@ class LinearMaskedATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, diagonal=None, scales=(0.5, 1.0), + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -533,14 +536,15 @@ class PatchATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, normalize_parts=no_op, diagonal=None, - scales=(0.5, 1.0), patch_size=(17, 17), + holistic_features=no_op, patch_normalisation=no_op, + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(PatchATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -558,7 +562,7 @@ def _warp_template(self, template, group, reference_shape, scale_index, shape = template.landmarks[group].lms return extract_patches([template], [shape], self.patch_size[scale_index], - normalize_function=self.normalize_parts, + normalise_function=self.patch_normalisation, prefix=prefix, verbose=verbose) # TODO: implement me! @@ -590,7 +594,7 @@ def _atm_str(atm): - {} shape components""" for k, s in enumerate(atm.scales): scales_info.append(lvl_str_tmplt.format( - s, name_of_callable(atm.features[k]), + s, name_of_callable(atm.holistic_features[k]), atm.warped_templates[k].shape, atm.shape_models[k].n_components)) # Patch based ATM diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index b4147c3..0450571 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -47,7 +47,7 @@ def _set_up(self, algorithm_cls): pdm = OrthoPDM(sm) interface = ATMLKPatchInterface( pdm, wt, sampling=s, patch_size=self.atm.patch_size[j], - normalize_parts=self.atm.normalize_parts) + patch_normalisation=self.atm.patch_normalisation) algorithm = algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " diff --git a/menpofit/builder.py b/menpofit/builder.py index 7f952d4..6fdeec1 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -159,7 +159,7 @@ def warp_images(images, shapes, reference_frame, transform, prefix='', # TODO: document me! -def extract_patches(images, shapes, patch_size, normalize_function=no_op, +def extract_patches(images, shapes, patch_size, normalise_function=no_op, prefix='', verbose=False): wrap = partial(print_progress, prefix='{}Extracting patches'.format(prefix), @@ -169,8 +169,8 @@ def extract_patches(images, shapes, patch_size, normalize_function=no_op, for i, s in wrap(zip(images, shapes)): parts = i.extract_patches(s, patch_size=patch_size, as_single_array=True) - parts = normalize_function(parts) - parts_images.append(Image(parts)) + parts = normalise_function(parts) + parts_images.append(Image(parts, copy=False)) return parts_images def build_reference_frame(landmarks, boundary=3, group='source'): diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index e335957..780f874 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -27,14 +27,15 @@ class CLM(object): The CLM object """ def __init__(self, images, group=None, verbose=False, batch_size=None, - diagonal=None, scales=(0.5, 1), features=no_op, + diagonal=None, scales=(0.5, 1), holistic_features=no_op, # shape_model_cls=build_normalised_pca_shape_model, expert_ensemble_cls=CorrelationFilterExpertEnsemble, max_shape_components=None, reference_shape=None, shape_forgetting_factor=1.0): self.diagonal = checks.check_diagonal(diagonal) self.scales = checks.check_scales(scales) - self.features = checks.check_features(features, self.n_scales) + self.holistic_features = checks.check_features(holistic_features, + self.n_scales) # self.shape_model_cls = checks.check_algorithm_cls( # shape_model_cls, self.n_scales, ShapeModel) self.expert_ensemble_cls = checks.check_algorithm_cls( @@ -125,15 +126,15 @@ def _train_batch(self, image_batch, increment=False, group=None, prefix = None # Handle holistic features - if i == 0 and self.features[i] == no_op: + if i == 0 and self.holistic_features[i] == no_op: # Saves a lot of memory feature_images = image_batch - elif i == 0 or self.features[i] is not self.features[i - 1]: + elif i == 0 or self.holistic_features[i] is not self.holistic_features[i - 1]: # compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features(image_batch, - self.features[i], + self.holistic_features[i], prefix=prefix, verbose=verbose) # handle scales diff --git a/menpofit/clm/expert/ensemble.py b/menpofit/clm/expert/ensemble.py index fbd663f..1843ee7 100644 --- a/menpofit/clm/expert/ensemble.py +++ b/menpofit/clm/expert/ensemble.py @@ -93,7 +93,7 @@ def _extract_patch(self, image, landmark): # patch: (offsets x ch) x h x w patch = patch.reshape((-1,) + patch.shape[-2:]) # Normalise patch - return self.normalise_callable(patch) + return self.patch_normalisation(patch) def _extract_patches(self, image, shape): r""" @@ -107,7 +107,7 @@ def _extract_patches(self, image, shape): # patches: n_patches x (n_offsets x n_channels) x height x width patches = patches.reshape((patches.shape[0], -1) + patches.shape[-2:]) # Normalise patches - return self.normalise_callable(patches) + return self.patch_normalisation(patches) def predict_response(self, image, shape): r""" @@ -134,7 +134,7 @@ class CorrelationFilterExpertEnsemble(ConvolutionBasedExpertEnsemble): def __init__(self, images, shapes, verbose=False, prefix='', icf_cls=IncrementalCorrelationFilterThinWrapper, patch_size=(17, 17), context_size=(34, 34), - response_covariance=3, normalise_callable=normalize_norm, + response_covariance=3, patch_normalisation=normalize_norm, cosine_mask=True, sample_offsets=None): # TODO: check parameters? # Set parameters @@ -142,7 +142,7 @@ def __init__(self, images, shapes, verbose=False, prefix='', self.patch_size = patch_size self.context_size = context_size self.response_covariance = response_covariance - self.normalise_callable = normalise_callable + self.patch_normalisation = patch_normalisation self.cosine_mask = cosine_mask self.sample_offsets = sample_offsets @@ -168,7 +168,7 @@ def _extract_patch(self, image, landmark): # patch: (offsets x ch) x h x w patch = patch.reshape((-1,) + patch.shape[-2:]) # Normalise patch - patch = self.normalise_callable(patch) + patch = self.patch_normalisation(patch) if self.cosine_mask: # Apply cosine mask if require patch = self._cosine_mask * patch diff --git a/menpofit/fitter.py b/menpofit/fitter.py index dcc647e..899488f 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -141,11 +141,11 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, images = [] for i in range(self.n_scales): # Handle features - if i == 0 or self.features[i] is not self.features[i - 1]: + if i == 0 or self.holistic_features[i] is not self.holistic_features[i - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale - feature_image = self.features[i](image) + feature_image = self.holistic_features[i](image) # Handle scales if self.scales[i] != 1: @@ -246,10 +246,10 @@ def reference_shape(self): return self._model.reference_shape @property - def features(self): + def holistic_features(self): r""" """ - return self._model.features + return self._model.holistic_features @property def scales(self): diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 2319248..b1a639a 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -13,16 +13,17 @@ class LucasKanadeFitter(MultiFitter): r""" """ - def __init__(self, template, group=None, features=no_op, + def __init__(self, template, group=None, holistic_features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, scales=(0.5, 1.0), algorithm_cls=InverseCompositional, residual_cls=SSD): checks.check_diagonal(diagonal) scales = checks.check_scales(scales) - features = checks.check_features(features, len(scales)) + holistic_features = checks.check_features(holistic_features, + len(scales)) - self.features = features + self.holistic_features = holistic_features self.transform_cls = transform_cls self.diagonal = diagonal self.scales = list(scales) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 939843a..b2326be 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -44,7 +44,7 @@ def _train(self, images, gt_shapes, current_shapes, increment=False, for k in range(self.n_iterations): # generate regression data features = features_per_image( - images, current_shapes, self.patch_size, self.features, + images, current_shapes, self.patch_size, self.patch_features, prefix='{}(Iteration {}) - '.format(prefix, k), verbose=verbose) @@ -89,8 +89,8 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # Cascaded Regression loop for r in self.regressors: # compute regression features - features = features_per_patch(image, current_shape, - self.patch_size, self.features) + features = features_per_patch(image, current_shape, self.patch_size, + self.patch_features) # solve for increments on the shape vector dx = r.predict(features) @@ -127,14 +127,15 @@ def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, class Newton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, + def __init__(self, patch_features=no_op, patch_size=(17, 17), + n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True): super(Newton, self).__init__() self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) self.patch_size = patch_size - self.features = features + self.patch_features = patch_features self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps @@ -144,7 +145,7 @@ def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, class GaussNewton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, + def __init__(self, patch_features=no_op, patch_size=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True, alpha2=0): super(GaussNewton, self).__init__() @@ -152,7 +153,7 @@ def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, alpha2=alpha2) self.patch_size = patch_size - self.features = features + self.patch_features = patch_features self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index de7addf..555335d 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -22,7 +22,7 @@ class SupervisedDescentFitter(MultiFitter): """ def __init__(self, images, group=None, bounding_box_group=None, reference_shape=None, sd_algorithm_cls=Newton, - holistic_feature=no_op, patch_features=no_op, + holistic_features=no_op, patch_features=no_op, patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, @@ -32,15 +32,15 @@ def __init__(self, images, group=None, bounding_box_group=None, n_scales = len(scales) scales = checks.check_scales(scales) patch_features = checks.check_features(patch_features, n_scales) - holistic_features = checks.check_features(holistic_feature, n_scales) + holistic_features = checks.check_features(holistic_features, n_scales) patch_size = checks.check_patch_size(patch_size, n_scales) # set parameters self.algorithms = [] self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls - self.features = holistic_features - self._patch_features = patch_features - self._patch_size = patch_size + self.holistic_features = holistic_features + self.patch_features = patch_features + self.patch_size = patch_size self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations @@ -57,8 +57,8 @@ def __init__(self, images, group=None, bounding_box_group=None, def _setup_algorithms(self): for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( - features=self._patch_features[j], - patch_size=self._patch_size[j], + patch_features=self.patch_features[j], + patch_size=self.patch_size[j], n_iterations=self.n_iterations[j])) def _train(self, images, increment=False, group=None, @@ -180,15 +180,15 @@ def _train_batch(self, image_batch, increment=False, group=None, scale_prefix = None # Handle holistic features - if j == 0 and self.features[j] == no_op: + if j == 0 and self.holistic_features[j] == no_op: # Saves a lot of memory feature_images = image_batch - elif j == 0 or self.features[j] is not self.features[j - 1]: + elif j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features(image_batch, - self.features[j], + self.holistic_features[j], prefix=scale_prefix, verbose=verbose) # handle scales @@ -273,9 +273,9 @@ def __str__(self): - Patch feature: {}""" for k, s in enumerate(self.scales): scales_info.append(lvl_str_tmplt.format( - s, self.n_iterations[k], self._patch_size[k], - name_of_callable(self.features[k]), - name_of_callable(self._patch_features[k]))) + s, self.n_iterations[k], self.patch_size[k], + name_of_callable(self.holistic_features[k]), + name_of_callable(self.patch_features[k]))) scales_info = '\n'.join(scales_info) cls_str = r"""Supervised Descent Method @@ -304,7 +304,7 @@ class RegularizedSDM(SupervisedDescentFitter): def __init__(self, images, group=None, bounding_box_group=None, alpha=1.0, reference_shape=None, - holistic_feature=no_op, patch_features=no_op, + holistic_features=no_op, patch_features=no_op, patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, @@ -313,7 +313,7 @@ def __init__(self, images, group=None, bounding_box_group=None, images, group=group, bounding_box_group=bounding_box_group, reference_shape=reference_shape, sd_algorithm_cls=partial(Newton, alpha=alpha), - holistic_feature=holistic_feature, patch_features=patch_features, + holistic_features=holistic_features, patch_features=patch_features, patch_size=patch_size, diagonal=diagonal, scales=scales, n_iterations=n_iterations, n_perturbations=n_perturbations, perturb_from_bounding_box=perturb_from_bounding_box, diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 2fbd2b9..ac881c6 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -961,7 +961,7 @@ def update_info(aam, instance, level, group): aam_mean = lvl_app_mod.mean() n_channels = aam_mean.n_channels tmplt_inst = lvl_app_mod.template_instance - feat = aam.features[level] + feat = aam.holistic_features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( @@ -1355,7 +1355,7 @@ def update_info(atm, instance, level, group): lvl_shape_mod = atm.shape_models[level] tmplt_inst = atm.warped_templates[level] n_channels = tmplt_inst.n_channels - feat = atm.features[level] + feat = atm.holistic_features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( From ad3f7bd0bd06d50bc71366af1c49e262ff5096b8 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 13 Aug 2015 11:54:24 +0100 Subject: [PATCH 208/423] Rename partsinterface to patchinterface Consistent with other renaming --- menpofit/aam/algorithm/sd.py | 4 ++-- menpofit/aam/fitter.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index d6fdb17..762386e 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -74,7 +74,7 @@ def algorithm_result(self, image, shape_parameters, # TODO document me! -class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): +class SupervisedDescentPatchInterface(SupervisedDescentStandardInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, @@ -82,7 +82,7 @@ def __init__(self, appearance_model, transform, template, sampling=None, self.patch_size = patch_size self.patch_normalisation = patch_normalisation - super(SupervisedDescentPartsInterface, self).__init__( + super(SupervisedDescentPatchInterface, self).__init__( appearance_model, transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 3e74017..e7b7d98 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -14,7 +14,7 @@ LucasKanadePatchInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, - SupervisedDescentPartsInterface, ProjectOutNewton) + SupervisedDescentPatchInterface, ProjectOutNewton) from .result import AAMFitterResult @@ -143,7 +143,7 @@ def _setup_algorithms(self): elif type(self.aam) is PatchAAM: # Build orthogonal point distribution model pdm = OrthoPDM(sm) - interface = SupervisedDescentPartsInterface( + interface = SupervisedDescentPatchInterface( am, pdm, template, sampling=s, patch_size=self.aam.patch_size[j], patch_normalisation=self.aam.patch_normalisation) From b753004c20a58f3cb5e619bd25fe4d7f304f354f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 09:51:20 +0100 Subject: [PATCH 209/423] Modify ModelDrivenTransforms - Add new method Jp that computes the parameters' Jacobian. - Add ne LinearOrthoMDTransform --- menpofit/transform/modeldriven.py | 173 ++++++++++++++++++++++++------ 1 file changed, 138 insertions(+), 35 deletions(-) diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index ddf8d00..1db964b 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -1,11 +1,11 @@ import numpy as np - from menpo.base import Targetable, Vectorizable +from menpo.shape import PointCloud from menpofit.modelinstance import PDM, GlobalPDM, OrthoPDM from menpo.transform.base import Transform, VComposable, VInvertible from menpofit.differentiable import DP - +# TODO: Should MDT implement VComposable and VInvertible? class ModelDrivenTransform(Transform, Targetable, Vectorizable, VComposable, VInvertible, DP): r""" @@ -135,22 +135,17 @@ def compose_after_from_vector_inplace(self, delta): r""" Composes two ModelDrivenTransforms together based on the first order approximation proposed by Papandreou and Maragos in [1]. - Parameters ---------- delta : (N,) ndarray Vectorized :class:`ModelDrivenTransform` to be applied **before** self - Returns -------- transform : self self, updated to the result of the composition - - References ---------- - .. [1] G. Papandreou and P. Maragos, "Adaptive and Constrained Algorithms for Inverse Compositional Active Appearance Model Fitting", CVPR08 @@ -176,15 +171,6 @@ def compose_after_from_vector_inplace(self, delta): # (n_points, n_params, n_dims) dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> eilk', dW_dx, dW_dp_0) - #TODO: Can we do this without splitting across the two dimensions? - # dW_dx_x = dW_dx[:, 0, :].flatten()[..., None] - # dW_dx_y = dW_dx[:, 1, :].flatten()[..., None] - # dW_dp_0_mat = np.reshape(dW_dp_0, (n_points * self.n_dims, - # self.n_parameters)) - # dW_dx_dW_dp_0 = dW_dp_0_mat * dW_dx_x + dW_dp_0_mat * dW_dx_y - # dW_dx_dW_dp_0 = np.reshape(dW_dx_dW_dp_0, - # (n_points, self.n_parameters, self.n_dims)) - # (n_params, n_params) J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) # (n_params, n_params) @@ -203,19 +189,14 @@ def _build_pseudoinverse(self): def pseudoinverse_vector(self, vector): r""" The vectorized pseudoinverse of a provided vector instance. - Syntactic sugar for - self.from_vector(vector).pseudoinverse.as_vector() - On ModelDrivenTransform this is especially fast - we just negate the vector provided. - Parameters ---------- vector : (P,) ndarray A vectorized version of self - Returns ------- pseudoinverse_vector : (N,) ndarray @@ -262,7 +243,6 @@ def d_dp(self, points): # dW_dl: n_points x (n_dims) x n_centres x n_dims # dX_dp: (n_points x n_dims) x n_params - # The following is equivalent to # np.einsum('ild, lpd -> ipd', self.dW_dl, dX_dp) @@ -273,6 +253,47 @@ def d_dp(self, points): return dW_dp + def Jp(self): + r""" + Compute parameters' Jacobian. + + References + ---------- + + .. [1] G. Papandreou and P. Maragos, "Adaptive and Constrained + Algorithms for Inverse Compositional Active Appearance Model + Fitting", CVPR08 + """ + # the incremental warp is always evaluated at p=0, ie the mean shape + points = self.pdm.model.mean().points + + # compute: + # - dW/dp when p=0 + # - dW/dp when p!=0 + # - dW/dx when p!=0 evaluated at the source landmarks + + # dW/dp when p=0 and when p!=0 are the same and simply given by + # the Jacobian of the model + # (n_points, n_params, n_dims) + dW_dp_0 = self.pdm.d_dp(points) + # (n_points, n_params, n_dims) + dW_dp = dW_dp_0 + + # (n_points, n_dims, n_dims) + dW_dx = self.transform.d_dx(points) + + # (n_points, n_params, n_dims) + dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> eilk', dW_dx, dW_dp_0) + + # (n_params, n_params) + J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) + # (n_params, n_params) + H = np.einsum('ijk, ilk -> jl', dW_dp, dW_dp) + # (n_params, n_params) + Jp = np.linalg.solve(H, J) + + return Jp + # noinspection PyMissingConstructor class GlobalMDTransform(ModelDrivenTransform): @@ -326,18 +347,76 @@ def compose_after_from_vector_inplace(self, delta): r""" Composes two ModelDrivenTransforms together based on the first order approximation proposed by Papandreou and Maragos in [1]. - Parameters ---------- delta : (N,) ndarray Vectorized :class:`ModelDrivenTransform` to be applied **before** self - Returns -------- transform : self self, updated to the result of the composition + References + ---------- + .. [1] G. Papandreou and P. Maragos, "Adaptive and Constrained + Algorithms for Inverse Compositional Active Appearance Model + Fitting", CVPR08 + """ + # the incremental warp is always evaluated at p=0, ie the mean shape + points = self.pdm.model.mean().points + + # compute: + # - dW/dp when p=0 + # - dW/dp when p!=0 + # - dW/dx when p!=0 evaluated at the source landmarks + + # dW/dq when p=0 and when p!=0 are the same and given by the + # Jacobian of the global transform evaluated at the mean of the + # model + # (n_points, n_global_params, n_dims) + dW_dq = self.pdm._global_transform_d_dp(points) + + # dW/db when p=0, is the Jacobian of the model + # (n_points, n_weights, n_dims) + dW_db_0 = PDM.d_dp(self.pdm, points) + + # dW/dp when p=0, is simply the concatenation of the previous + # two terms + # (n_points, n_params, n_dims) + dW_dp_0 = np.hstack((dW_dq, dW_db_0)) + + # by application of the chain rule dW_db when p!=0, + # is the Jacobian of the global transform wrt the points times + # the Jacobian of the model: dX(S)/db = dX/dS * dS/db + # (n_points, n_dims, n_dims) + dW_dS = self.pdm.global_transform.d_dx(points) + # (n_points, n_weights, n_dims) + dW_db = np.einsum('ilj, idj -> idj', dW_dS, dW_db_0) + + # dW/dp is simply the concatenation of dW_dq with dW_db + # (n_points, n_params, n_dims) + dW_dp = np.hstack((dW_dq, dW_db)) + # dW/dx is the jacobian of the transform evaluated at the source + # landmarks + # (n_points, n_dims, n_dims) + dW_dx = self.transform.d_dx(points) + + # (n_points, n_params, n_dims) + dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> ilk', dW_dx, dW_dp_0) + + # (n_params, n_params) + J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) + # (n_params, n_params) + H = np.einsum('ijk, ilk -> jl', dW_dp, dW_dp) + # (n_params, n_params) + Jp = np.linalg.solve(H, J) + + self.from_vector_inplace(self.as_vector() + np.dot(Jp, delta)) + + def Jp(self): + r""" + Compute parameters Jacobian. References ---------- @@ -389,16 +468,6 @@ def compose_after_from_vector_inplace(self, delta): # (n_points, n_params, n_dims) dW_dx_dW_dp_0 = np.einsum('ijk, ilk -> ilk', dW_dx, dW_dp_0) - #TODO: Can we do this without splitting across the two dimensions? - # dW_dx_x = dW_dx[:, 0, :].flatten()[..., None] - # dW_dx_y = dW_dx[:, 1, :].flatten()[..., None] - # dW_dp_0_mat = np.reshape(dW_dp_0, (n_points * self.n_dims, - # self.n_parameters)) - # dW_dx_dW_dp_0 = dW_dp_0_mat * dW_dx_x + dW_dp_0_mat * dW_dx_y - # # (n_points, n_params, n_dims) - # dW_dx_dW_dp_0 = np.reshape(dW_dx_dW_dp_0, - # (n_points, self.n_parameters, self.n_dims)) - # (n_params, n_params) J = np.einsum('ijk, ilk -> jl', dW_dp, dW_dx_dW_dp_0) # (n_params, n_params) @@ -406,7 +475,7 @@ def compose_after_from_vector_inplace(self, delta): # (n_params, n_params) Jp = np.linalg.solve(H, J) - self.from_vector_inplace(self.as_vector() + np.dot(Jp, delta)) + return Jp # noinspection PyMissingConstructor @@ -455,3 +524,37 @@ def __init__(self, model, transform_cls, global_transform, source=None): self._cached_points = None self.transform = transform_cls(source, self.target) + +# TODO: document me! +class LinearOrthoMDTransform(OrthoPDM, Transform): + r""" + """ + def __init__(self, model, n_landmarks): + super(LinearOrthoMDTransform, self).__init__(model) + self.n_landmarks = n_landmarks + self.W = np.vstack((self.similarity_model.components, + self.model.components)) + V = self.W[:, :self.n_dims*self.n_landmarks] + self.pinv_V = np.linalg.pinv(V) + + @property + def dense_target(self): + return PointCloud(self.target.points[self.n_landmarks:]) + + @property + def sparse_target(self): + return PointCloud(self.target.points[:self.n_landmarks]) + + def set_target(self, target): + if target.n_points == self.n_landmarks: + # densify target + target = np.dot(np.dot(target.as_vector(), self.pinv_V), self.W) + target = PointCloud(np.reshape(target, (-1, self.n_dims))) + OrthoPDM.set_target(self, target) + + def _apply(self, _, **kwargs): + return self.target.points[self.n_landmarks:] + + def d_dp(self, _): + return OrthoPDM.d_dp(self, _)[self.n_landmarks:, ...] + From 720288802fac4bddf580e36f3a2c0589de3022dd Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 10:01:06 +0100 Subject: [PATCH 210/423] Update modelinstance.py --- menpofit/modelinstance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/menpofit/modelinstance.py b/menpofit/modelinstance.py index cf6924a..169612b 100644 --- a/menpofit/modelinstance.py +++ b/menpofit/modelinstance.py @@ -1,5 +1,4 @@ import numpy as np - from menpo.base import Targetable, Vectorizable from menpo.model import MeanInstanceLinearModel from menpofit.differentiable import DP @@ -185,7 +184,7 @@ def d_dp(self, points): return d_dp.swapaxes(0, 1) -# TODO: document me +# TODO: document me! class GlobalPDM(PDM): r""" """ @@ -319,7 +318,7 @@ def _global_transform_d_dp(self, points): return self.global_transform.d_dp(points) -# TODO: document me +# TODO: document me! class OrthoPDM(GlobalPDM): r""" """ From 4f892a9cdfe506622d0eb8f5290fa423ff90fd68 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 16:58:48 +0100 Subject: [PATCH 211/423] Add convenient functions to builder.py - add functions: compute_features, scale_images, warp_images, extract_patches, densify_shapes, align_shapes - move funtions: build_reference_frame, build_patch_reference_frame, _build_reference_frame from aam.builder.py - delete DeformableModelBuilder --- menpofit/builder.py | 239 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 199 insertions(+), 40 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index f9a31fa..29d4ec8 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -1,13 +1,12 @@ from __future__ import division -import abc import numpy as np -from menpo.shape import mean_pointcloud +from menpo.shape import mean_pointcloud, PointCloud, TriMesh +from menpo.image import Image, MaskedImage +from menpo.feature import no_op from menpo.transform import Scale, Translation, GeneralizedProcrustesAnalysis from menpo.model.pca import PCAModel from menpo.visualize import print_dynamic, progress_bar_str -from .base import is_pyramid_on_features - def compute_reference_shape(shapes, normalization_diagonal, verbose=False): r""" @@ -47,8 +46,8 @@ def compute_reference_shape(shapes, normalization_diagonal, verbose=False): return reference_shape -def normalization_wrt_reference_shape(images, group, label, - normalization_diagonal, verbose=False): +def normalization_wrt_reference_shape(images, group, label, diagonal, + verbose=False): r""" Function that normalizes the images sizes with respect to the reference shape (mean shape) scaling. This step is essential before building a @@ -57,7 +56,7 @@ def normalization_wrt_reference_shape(images, group, label, The normalization includes: 1) Computation of the reference shape as the mean shape of the images' landmarks. - 2) Scaling of the reference shape using the normalization_diagonal. + 2) Scaling of the reference shape using the diagonal. 3) Rescaling of all the images so that their shape's scale is in correspondence with the reference shape's scale. @@ -74,10 +73,10 @@ def normalization_wrt_reference_shape(images, group, label, The label of of the landmark manager that you wish to use. If no label is passed, the convex hull of all landmarks is used. - normalization_diagonal: `int` + diagonal: `int` If int, it ensures that the mean shape is scaled so that the diagonal of the bounding box containing it matches the - normalization_diagonal value. + diagonal value. If None, the mean shape is not rescaled. Note that, because the reference frame is computed from the mean @@ -100,7 +99,7 @@ def normalization_wrt_reference_shape(images, group, label, shapes = [i.landmarks[group][label] for i in images] # compute the reference shape and fix its diagonal length - reference_shape = compute_reference_shape(shapes, normalization_diagonal, + reference_shape = compute_reference_shape(shapes, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size @@ -118,7 +117,193 @@ def normalization_wrt_reference_shape(images, group, label, return reference_shape, normalized_images -def build_shape_model(shapes, max_components): +# TODO: document me! +def compute_features(images, features, level_str='', verbose=None): + feature_images = [] + for c, i in enumerate(images): + if verbose: + print_dynamic( + '{}Computing feature space: {}'.format( + level_str, progress_bar_str((c + 1.) / len(images), + show_bar=False))) + i = features(i) + feature_images.append(i) + + return feature_images + + +# TODO: document me! +def scale_images(images, scale, level_str='', verbose=None): + if scale != 1: + scaled_images = [] + for c, i in enumerate(images): + if verbose: + print_dynamic( + '{}Scaling features: {}'.format( + level_str, progress_bar_str((c + 1.) / len(images), + show_bar=False))) + scaled_images.append(i.rescale(scale)) + return scaled_images + else: + return images + + +# TODO: document me! +def warp_images(images, shapes, reference_frame, transform, level_str='', + verbose=None): + warped_images = [] + for c, (i, s) in enumerate(zip(images, shapes)): + if verbose: + print_dynamic('{}Warping images - {}'.format( + level_str, + progress_bar_str(float(c + 1) / len(images), + show_bar=False))) + # compute transforms + t = transform(reference_frame.landmarks['source'].lms, s) + # warp images + warped_i = i.warp_to_mask(reference_frame.mask, t) + # attach reference frame landmarks to images + warped_i.landmarks['source'] = reference_frame.landmarks['source'] + warped_images.append(warped_i) + return warped_images + + +# TODO: document me! +def extract_patches(images, shapes, parts_shape, normalize_function=no_op, + level_str='', verbose=None): + parts_images = [] + for c, (i, s) in enumerate(zip(images, shapes)): + if verbose: + print_dynamic('{}Warping images - {}'.format( + level_str, + progress_bar_str(float(c + 1) / len(images), + show_bar=False))) + parts = i.extract_patches(s, patch_size=parts_shape, + as_single_array=True) + if normalize_function: + parts = normalize_function(parts) + parts_images.append(Image(parts)) + return parts_images + +def build_reference_frame(landmarks, boundary=3, group='source', + trilist=None): + r""" + Builds a reference frame from a particular set of landmarks. + + Parameters + ---------- + landmarks : :map:`PointCloud` + The landmarks that will be used to build the reference frame. + + boundary : `int`, optional + The number of pixels to be left as a safe margin on the boundaries + of the reference frame (has potential effects on the gradient + computation). + + group : `string`, optional + Group that will be assigned to the provided set of landmarks on the + reference frame. + + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. + + If ``None``, defaults to performing Delaunay triangulation on the + points. + + Returns + ------- + reference_frame : :map:`Image` + The reference frame. + """ + reference_frame = _build_reference_frame(landmarks, boundary=boundary, + group=group) + if trilist is not None: + reference_frame.landmarks[group] = TriMesh( + reference_frame.landmarks['source'].lms.points, trilist=trilist) + + # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, + # perhaps the trilist should be directly obtained from the group landmarks + reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) + + return reference_frame + + +def build_patch_reference_frame(landmarks, boundary=3, group='source', + patch_shape=(17, 17)): + r""" + Builds a reference frame from a particular set of landmarks. + + Parameters + ---------- + landmarks : :map:`PointCloud` + The landmarks that will be used to build the reference frame. + + boundary : `int`, optional + The number of pixels to be left as a safe margin on the boundaries + of the reference frame (has potential effects on the gradient + computation). + + group : `string`, optional + Group that will be assigned to the provided set of landmarks on the + reference frame. + + patch_shape : tuple of ints, optional + Tuple specifying the shape of the patches. + + Returns + ------- + patch_based_reference_frame : :map:`Image` + The patch based reference frame. + """ + boundary = np.max(patch_shape) + boundary + reference_frame = _build_reference_frame(landmarks, boundary=boundary, + group=group) + + # mask reference frame + reference_frame.build_mask_around_landmarks(patch_shape, group=group) + + return reference_frame + + +def _build_reference_frame(landmarks, boundary=3, group='source'): + # translate landmarks to the origin + minimum = landmarks.bounds(boundary=boundary)[0] + landmarks = Translation(-minimum).apply(landmarks) + + resolution = landmarks.range(boundary=boundary) + reference_frame = MaskedImage.init_blank(resolution) + reference_frame.landmarks[group] = landmarks + + return reference_frame + + +# TODO: document me! +def densify_shapes(shapes, reference_frame, transform): + # compute non-linear transforms + transforms = [transform(reference_frame.landmarks['source'].lms, s) + for s in shapes] + # build dense shapes + dense_shapes = [] + for (t, s) in zip(transforms, shapes): + warped_points = t.apply(reference_frame.mask.true_indices()) + dense_shape = PointCloud(np.vstack((s.points, warped_points))) + dense_shapes.append(dense_shape) + + return dense_shapes + + +# TODO: document me! +def align_shapes(shapes): + r""" + """ + # centralize shapes + centered_shapes = [Translation(-s.centre()).apply(s) for s in shapes] + # align centralized shape using Procrustes Analysis + gpa = GeneralizedProcrustesAnalysis(centered_shapes) + return [s.aligned_source() for s in gpa.transforms] + + +def build_shape_model(shapes, max_components=None): r""" Builds a shape model given a set of shapes. @@ -137,38 +322,12 @@ def build_shape_model(shapes, max_components): shape_model: :class:`menpo.model.pca` The PCA shape model. """ - # centralize shapes - centered_shapes = [Translation(-s.centre()).apply(s) for s in shapes] - # align centralized shape using Procrustes Analysis - gpa = GeneralizedProcrustesAnalysis(centered_shapes) - aligned_shapes = [s.aligned_source() for s in gpa.transforms] - + # compute aligned shapes + aligned_shapes = align_shapes(shapes) # build shape model shape_model = PCAModel(aligned_shapes) if max_components is not None: # trim shape model if required shape_model.trim_components(max_components) - return shape_model - - -class DeformableModelBuilder(object): - r""" - Abstract class with a set of functions useful to build a Deformable Model. - """ - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def build(self, images, group=None, label=None): - r""" - Builds a Multilevel Deformable Model. - """ - - @property - def pyramid_on_features(self): - r""" - True if feature extraction happens once and then a gaussian pyramid - is taken. False if a gaussian pyramid is taken and then features are - extracted at each level. - """ - return is_pyramid_on_features(self.features) + return shape_model \ No newline at end of file From 7b845adf642bf912f2aad1ba16e01133d2834958 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:07:16 +0100 Subject: [PATCH 212/423] Add check funtions to check.py - Add functions: check_scales, check_patch_shape - Comment functions: check_n_levels, check_downscale, check_boundary --- menpofit/checks.py | 87 +++++++++++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/menpofit/checks.py b/menpofit/checks.py index 27b4e55..5098d03 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -48,38 +48,44 @@ def check_list_callables(callables, n_callables, allow_single=True): return callables -def check_n_levels(n_levels): - r""" - Checks the number of pyramid levels - must be int > 0. - """ - if not isinstance(n_levels, int) or n_levels < 1: - raise ValueError("n_levels must be int > 0") - - -def check_downscale(downscale): - r""" - Checks the downscale factor of the pyramid that must be >= 1. - """ - if downscale < 1: - raise ValueError("downscale must be >= 1") - - -def check_normalization_diagonal(normalization_diagonal): +def check_diagonal(diagonal): r""" Checks the diagonal length used to normalize the images' size that must be >= 20. """ - if normalization_diagonal is not None and normalization_diagonal < 20: - raise ValueError("normalization_diagonal must be >= 20") - - -def check_boundary(boundary): - r""" - Checks the boundary added around the reference shape that must be - int >= 0. - """ - if not isinstance(boundary, int) or boundary < 0: - raise ValueError("boundary must be >= 0") + if diagonal is not None and diagonal < 20: + raise ValueError("diagonal must be >= 20") + + +# TODO: document me! +def check_scales(scales): + if isinstance(scales, (int, float)): + return [scales], 1 + elif len(scales) == 1 and isinstance(scales[0], (int, float)): + return list(scales), 1 + elif len(scales) > 1: + l1, n1 = check_scales(scales[0]) + l2, n2 = check_scales(scales[1:]) + return l1 + l2, n1 + n2 + else: + raise ValueError("scales must be an int/float or a list/tuple of " + "int/float") + + +# TODO: document me! +def check_patch_shape(patch_shape, n_levels): + if len(patch_shape) == 2 and isinstance(patch_shape[0], int): + return [patch_shape] * n_levels + elif len(patch_shape) == 1: + return check_patch_shape(patch_shape[0], 1) + elif len(patch_shape) == n_levels: + l1 = check_patch_shape(patch_shape[0], 1) + l2 = check_patch_shape(patch_shape[1:], n_levels-1) + return l1 + l2 + else: + raise ValueError("patch_shape must be a list/tuple of int or a " + "list/tuple of lit/tuple of int/float with the " + "same length as scales") def check_max_components(max_components, n_levels, var_name): @@ -105,3 +111,28 @@ def check_max_components(max_components, n_levels, var_name): if not isinstance(comp, float): raise ValueError(str_error) return max_components_list + + +# def check_n_levels(n_levels): +# r""" +# Checks the number of pyramid levels - must be int > 0. +# """ +# if not isinstance(n_levels, int) or n_levels < 1: +# raise ValueError("n_levels must be int > 0") +# +# +# def check_downscale(downscale): +# r""" +# Checks the downscale factor of the pyramid that must be >= 1. +# """ +# if downscale < 1: +# raise ValueError("downscale must be >= 1") +# +# +# def check_boundary(boundary): +# r""" +# Checks the boundary added around the reference shape that must be +# int >= 0. +# """ +# if not isinstance(boundary, int) or boundary < 0: +# raise ValueError("boundary must be >= 0") From 9a58a555286e2ce64a97b112932bfca41116ccb6 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:10:47 +0100 Subject: [PATCH 213/423] Update import in transforms.__init__ --- menpofit/transform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/transform/__init__.py b/menpofit/transform/__init__.py index 4c6c12b..2aee5a1 100644 --- a/menpofit/transform/__init__.py +++ b/menpofit/transform/__init__.py @@ -1,4 +1,4 @@ -from .modeldriven import ModelDrivenTransform, OrthoMDTransform +from .modeldriven import OrthoMDTransform, LinearOrthoMDTransform from .homogeneous import (DifferentiableAffine, DifferentiableSimilarity, DifferentiableAlignmentSimilarity, DifferentiableAlignmentAffine) From 9e1fd61efd7d8b161a165b8dc273cf891bd0ebf8 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:19:40 +0100 Subject: [PATCH 214/423] Add new AAM types - LinearGlobalAAM, LinearPatchAAM and PartsAAM - Modified previous GlobalAAM and PatchAAM --- menpofit/aam/base.py | 498 +++++++++++++++++++++++++------------------ 1 file changed, 287 insertions(+), 211 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 68ef24f..8d66070 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,85 +1,25 @@ from __future__ import division - +import abc import numpy as np from menpo.shape import TriMesh - -from menpofit.base import DeformableModel, name_of_callable -from .builder import build_patch_reference_frame, build_reference_frame +from menpofit.transform import DifferentiableThinPlateSplines +from menpofit.base import name_of_callable +from menpofit.builder import ( + build_reference_frame, build_patch_reference_frame) -class AAM(DeformableModel): +class AAM(object): r""" - Active Appearance Model class. - - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - - n_training_images : `int` - The number of training images used to build the AAM. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. - - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. - - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. - + Abstract interface for Active Appearance Model. """ - def __init__(self, shape_models, appearance_models, n_training_images, - transform, features, reference_shape, downscale, - scaled_shape_models): - DeformableModel.__init__(self, features) - self.n_training_images = n_training_images - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.reference_shape = reference_shape - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models - @property def n_levels(self): """ - The number of multi-resolution pyramidal levels of the AAM. + The number of scale levels of the AAM. :type: `int` """ - return len(self.appearance_models) + return len(self.scales) def instance(self, shape_weights=None, appearance_weights=None, level=-1): r""" @@ -151,39 +91,13 @@ def random_instance(self, level=-1): return self._instance(level, shape_instance, appearance_instance) + @abc.abstractmethod def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() - landmarks = template.landmarks['source'].lms - - reference_frame = self._build_reference_frame( - shape_instance, landmarks) + pass - transform = self.transform( - reference_frame.landmarks['source'].lms, landmarks) - - return appearance_instance.as_unmasked(copy=False).warp_to_mask( - reference_frame.mask, transform, warp_landmarks=True) - - def _build_reference_frame(self, reference_shape, landmarks): - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - return build_reference_frame( - reference_shape, trilist=trilist) - - @property - def _str_title(self): - r""" - Returns a string containing name of the model. - - :type: `string` - """ - return 'Active Appearance Model' - - def view_shape_models_widget(self, n_parameters=5, mode='multiple', + def view_shape_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), - figure_size=(10, 8), style='coloured'): + mode='multiple', figure_size=(10, 8)): r""" Visualizes the shape models of the AAM object using the `menpo.visualize.widgets.visualize_shape_model` widget. @@ -191,105 +105,42 @@ def view_shape_models_widget(self, n_parameters=5, mode='multiple', Parameters ----------- n_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the parameters - sliders. If `int`, then the number of sliders per level is the - minimum between `n_parameters` and the number of active components - per level. If `list` of `int`, then a number of sliders is defined - per level. If ``None``, all the active components per level will - have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + popup : `bool`, optional + If ``True``, the widget will appear as a popup window. figure_size : (`int`, `int`), optional The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. """ from menpofit.visualize import visualize_shape_model - visualize_shape_model( - self.shape_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, figure_size=figure_size, - mode=mode, style=style) + visualize_shape_model(self.shape_models, n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode,) - def view_appearance_models_widget(self, n_parameters=5, mode='multiple', + @abc.abstractmethod + def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), - figure_size=(10, 8), style='coloured'): - r""" - Visualizes the appearance models of the AAM object using the - `menpo.visualize.widgets.visualize_appearance_model` widget. - - Parameters - ----------- - n_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the parameters - sliders. If `int`, then the number of sliders per level is the - minimum between `n_parameters` and the number of active components - per level. If `list` of `int`, then a number of sliders is defined - per level. If ``None``, all the active components per level will - have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. - parameters_bounds : (`float`, `float`), optional - The minimum and maximum bounds, in std units, for the sliders. - figure_size : (`int`, `int`), optional - The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. - """ - from menpofit.visualize import visualize_appearance_model - visualize_appearance_model( - self.appearance_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, figure_size=figure_size, - mode=mode, style=style) + mode='multiple', figure_size=(10, 8)): + pass + @abc.abstractmethod def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, - mode='multiple', parameters_bounds=(-3.0, 3.0), - figure_size=(10, 8), style='coloured'): - r""" - Visualizes both the shape and appearance models of the AAM object using - the `menpo.visualize.widgets.visualize_aam` widget. - - Parameters - ----------- - n_shape_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the shape - parameters sliders. If `int`, then the number of sliders per level - is the minimum between `n_parameters` and the number of active - components per level. If `list` of `int`, then a number of sliders - is defined per level. If ``None``, all the active components per - level will have a slider. - n_appearance_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the appearance - parameters sliders. If `int`, then the number of sliders per level - is the minimum between `n_parameters` and the number of active - components per level. If `list` of `int`, then a number of sliders - is defined per level. If ``None``, all the active components per - level will have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. - parameters_bounds : (`float`, `float`), optional - The minimum and maximum bounds, in std units, for the sliders. - figure_size : (`int`, `int`), optional - The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. - """ - from menpofit.visualize import visualize_aam - visualize_aam(self, n_shape_parameters=n_shape_parameters, - n_appearance_parameters=n_appearance_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, style=style) + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + pass + # TODO: fix me! def __str__(self): out = "{}\n - {} training images.\n".format(self._str_title, self.n_training_images) @@ -393,7 +244,111 @@ def __str__(self): return out -class PatchBasedAAM(AAM): +class GlobalAAM(AAM): + r""" + Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + n_training_images : `int` + The number of training images used to build the AAM. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + downscale : `float` + The downscale factor that was used to create the different pyramidal + levels. + + scaled_shape_models : `boolean`, optional + If ``True``, the reference frames are the mean shapes of each pyramid + level, so the shape models are scaled. + + If ``False``, the reference frames of all levels are the mean shape of + the highest level, so the shape models are not scaled; they have the + same size. + + Note that from our experience, if scaled_shape_models is ``False``, AAMs + tend to have slightly better performance. + + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + def _instance(self, level, shape_instance, appearance_instance): + template = self.appearance_models[level].mean() + landmarks = template.landmarks['source'].lms + + if type(landmarks) == TriMesh: + trilist = landmarks.trilist + else: + trilist = None + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) + + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + instance = appearance_instance.warp_to_mask( + reference_frame.mask, transform) + instance.landmarks = reference_frame.landmarks + + return instance + + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + from menpofit.visualize import visualize_appearance_model + visualize_appearance_model(self.appearance_models, + n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + # TODO: fix me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + from menpofit.visualize import visualize_aam + visualize_aam(self, n_shape_parameters=n_shape_parameters, + n_appearance_parameters=n_appearance_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + +class PatchAAM(AAM): r""" Patch Based Active Appearance Model class. @@ -449,31 +404,152 @@ class PatchBasedAAM(AAM): AAMs tend to have slightly better performance. """ - def __init__(self, shape_models, appearance_models, n_training_images, - patch_shape, transform, features, reference_shape, - downscale, scaled_shape_models): - super(PatchBasedAAM, self).__init__( - shape_models, appearance_models, n_training_images, transform, - features, reference_shape, downscale, scaled_shape_models) + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = DifferentiableThinPlateSplines + self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + def _instance(self, level, shape_instance, appearance_instance): + template = self.appearance_models[level].mean + landmarks = template.landmarks['source'].lms + + reference_frame = build_patch_reference_frame( + shape_instance, patch_shape=self.patch_shape) + + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + instance = appearance_instance.warp_to_mask(reference_frame.mask, + transform) + instance.landmarks = reference_frame.landmarks + + return instance + + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + from menpofit.visualize import visualize_appearance_model + visualize_appearance_model(self.appearance_models, + n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + # TODO: fix me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + from menpofit.visualize import visualize_aam + visualize_aam(self, n_shape_parameters=n_shape_parameters, + n_appearance_parameters=n_appearance_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + +# TODO: document me! +class LinearGlobalAAM(AAM): + r""" + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features, + n_landmarks): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks + + # TODO: implement me! + def _instance(self, level, shape_instance, appearance_instance): + raise NotImplemented + + # TODO: implement me! + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + +# TODO: document me! +class LinearPatchAAM(AAM): + r""" + """ + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, scales, scale_shapes, + scale_features, n_landmarks): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = DifferentiableThinPlateSplines self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks - def _build_reference_frame(self, reference_shape, landmarks): - return build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape) + # TODO: implement me! + def _instance(self, level, shape_instance, appearance_instance): + raise NotImplemented - @property - def _str_title(self): - r""" - Returns a string containing name of the model. + # TODO: implement me! + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + raise NotImplemented - :type: `string` - """ - return 'Patch-Based Active Appearance Model' + # TODO: implement me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented - def __str__(self): - out = super(PatchBasedAAM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + +# TODO: document me! +class PartsAAM(AAM): + r""" + """ + def __init__(self, shape_models, appearance_models, reference_shape, + parts_shape, features, normalize_parts, scales, + scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.parts_shape = parts_shape + self.features = features + self.normalize_parts = normalize_parts + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + # TODO: implement me! + def _instance(self, level, shape_instance, appearance_instance): + raise NotImplemented + + # TODO: implement me! + def view_appearance_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented \ No newline at end of file From b8fc1ad998dbe27eb0d3acc6c04b609f5381df3f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:20:52 +0100 Subject: [PATCH 215/423] Add builders for all AAM classes - PatchAAMs, LinearPatchAAMs and PartsAAMs can now have different patch sizes. --- menpofit/aam/builder.py | 1025 +++++++++++++++++++++++---------------- 1 file changed, 618 insertions(+), 407 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 4179d19..886f73c 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,24 +1,146 @@ from __future__ import division -import numpy as np - -from menpo.shape import TriMesh -from menpo.image import MaskedImage -from menpo.transform import Translation -from menpo.feature import igo +import abc +from copy import deepcopy from menpo.model import PCAModel -from menpo.visualize import print_dynamic, print_progress - +from menpo.shape import mean_pointcloud +from menpo.feature import no_op +from menpo.visualize import print_dynamic from menpofit import checks -from menpofit.base import create_pyramid -from menpofit.builder import (DeformableModelBuilder, build_shape_model, - normalization_wrt_reference_shape) -from menpofit.transform import (DifferentiablePiecewiseAffine, - DifferentiableThinPlateSplines) +from menpofit.builder import ( + normalization_wrt_reference_shape, compute_features, scale_images, + warp_images, extract_patches, build_shape_model, align_shapes, + build_reference_frame, build_patch_reference_frame, densify_shapes) +from menpofit.transform import ( + DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) + + +class AAMBuilder(object): + r""" + Abstract interface for Active Appearance Model Builder. + """ + def build(self, images, group=None, label=None, verbose=False): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + + label : `string`, optional + The label of of the landmark manager that you wish to use. If no + label is passed, the convex hull of all landmarks is used. + + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest level + """ + # normalize images and compute reference shape + reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + shape_models = [] + appearance_models = [] + # for each pyramid level (high --> low) + for j, s in enumerate(self.scales): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if j == 0: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, self.features, + level_str=level_str, + verbose=verbose) + + # extract potentially rescaled shapes + level_shapes = [i.landmarks[group][label] + for i in level_images] + + # obtain shape representation + if j == 0 or self.scale_shapes: + # obtain shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # add shape model to the list + shape_models.append(shape_model) + else: + # copy precious shape model and add it to the list + shape_models.append(deepcopy(shape_model)) + + # obtain warped images + warped_images = self._warp_images(level_images, level_shapes, + shape_model.mean(), j, + level_str, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format(level_str)) + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + appearance_models.append(appearance_model) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + # reverse the list of shape and appearance models so that they are + # ordered from lower to higher resolution + shape_models.reverse() + appearance_models.reverse() + self.scales.reverse() + + aam = self._build_aam(shape_models, appearance_models, reference_shape) + + return aam + + @classmethod + def _build_shape_model(cls, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) -class AAMBuilder(DeformableModelBuilder): + @abc.abstractmethod + def _build_aam(self, shape_models, appearance_models, reference_shape): + pass + + +# TODO: implement checker for conflict between features and scale_features +# TODO: document me! +class GlobalAAMBuilder(AAMBuilder): r""" - Class that builds Multilevel Active Appearance Models. + Class that builds a Global Active Appearance Model. Parameters ---------- @@ -44,13 +166,12 @@ class AAMBuilder(DeformableModelBuilder): Triangle list that will be used to build the reference frame. If ``None``, defaults to performing Delaunay triangulation on the points. - normalization_diagonal : `int` >= ``20``, optional + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the normalization_diagonal - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. @@ -59,25 +180,11 @@ class AAMBuilder(DeformableModelBuilder): reference frame (provided that features computation does not change the image size). - n_levels : `int` > 0, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(``n_levels``) + scales : `int` or float` or list of those, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of - each pyramid level, so the shape models will be scaled. + scale_shapes : `boolean`, optional - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -113,11 +220,6 @@ class AAMBuilder(DeformableModelBuilder): If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- aam : :map:`AAMBuilder` @@ -126,247 +228,202 @@ class AAMBuilder(DeformableModelBuilder): Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError ``max_shape_components`` must be ``None`` or an `int` > 0 or a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``n_levels`` elements + ``len(scales)`` elements ValueError ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``n_levels`` elements - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements + ``len(scales)`` elements """ - def __init__(self, features=igo, transform=DifferentiablePiecewiseAffine, - trilist=None, normalization_diagonal=None, n_levels=3, - downscale=2, scaled_shape_models=True, - max_shape_components=None, max_appearance_components=None, - boundary=3): + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, 0.5), + scale_shapes=False, scale_features=True, + max_shape_components=None, max_appearance_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') - features = checks.check_features(features, n_levels) - # store parameters + # set parameters self.features = features self.transform = transform self.trilist = trilist - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - self.boundary = boundary - def build(self, images, group=None, label=None, verbose=False): - r""" - Builds a Multilevel Active Appearance Model from a list of - landmarked images. + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + self.reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. + def _build_aam(self, shape_models, appearance_models, reference_shape): + return GlobalAAM(shape_models, appearance_models, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features) - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. +# TODO: document me! +class PatchAAMBuilder(AAMBuilder): + r""" + Class that builds a Patch Active Appearance Model. - verbose : `boolean`, optional - Flag that controls information and progress printing. + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from lowest - to highest level - """ - # compute reference_shape and normalize images size - self.reference_shape, normalized_images = \ - normalization_wrt_reference_shape(images, group, label, - self.normalization_diagonal, - verbose=verbose) - - # create pyramid - generators = create_pyramid(normalized_images, self.n_levels, - self.downscale, self.features, - verbose=verbose) - - # build the model at each pyramid level - if verbose: - if self.n_levels > 1: - print('- Building model for each of the {} ' - 'pyramid levels'.format(self.n_levels)) - else: - print('- Building model') + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - shape_models = [] - appearance_models = [] - # for each pyramid level (high --> low) - for j in range(self.n_levels): - # since models are built from highest to lowest level, the - # parameters in form of list need to use a reversed index - rj = self.n_levels - j - 1 + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - if verbose: - level_str = ' - ' - if self.n_levels > 1: - level_str = ' - Level {}: '.format(j + 1) + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - # get feature images of current level - feature_images = [] + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - if verbose: - generators_with_print = print_progress( - generators, show_bar=False, show_eta=False, - end_with_newline=False, - prefix='{}Computing feature space/rescaling'.format(level_str)) - else: - generators_with_print = generators + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - for g in generators_with_print: - feature_images.append(next(g)) + If ``None``, the mean shape is not rescaled. - # extract potentially rescaled shapes - shapes = [i.landmarks[group][label] for i in feature_images] + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - # define shapes that will be used for training - if j == 0: - original_shapes = shapes - train_shapes = shapes - else: - if self.scaled_shape_models: - train_shapes = shapes - else: - train_shapes = original_shapes + scales : `int` or float` or list of those, optional - # train shape model and find reference frame - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = build_shape_model( - train_shapes, self.max_shape_components[rj]) - reference_frame = self._build_reference_frame(shape_model.mean()) + scale_shapes : `boolean`, optional - # add shape model to the list - shape_models.append(shape_model) + scale_features : `boolean`, optional - # compute transforms - if verbose: - print_dynamic('{}Computing transforms'.format(level_str)) + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. - # Create a dummy initial transform - s_to_t_transform = self.transform( - reference_frame.landmarks['source'].lms, - reference_frame.landmarks['source'].lms) + Per level: + If `int`, it specifies the exact number of components to be + retained. - if verbose: - feature_images_with_print = print_progress( - feature_images, show_bar=False, show_eta=False, - end_with_newline=False, - prefix='{}Warping images'.format(level_str)) - else: - feature_images_with_print = feature_images - - # warp images to reference frame - warped_images = [] - for i in feature_images_with_print: - # Setting the target can be significantly faster for transforms - # such as CachedPiecewiseAffine - s_to_t_transform.set_target(i.landmarks[group][label]) - warped_images.append(i.warp_to_mask(reference_frame.mask, - s_to_t_transform)) - - # attach reference_frame to images' source shape - for i in warped_images: - i.landmarks['source'] = reference_frame.landmarks['source'] - - # build appearance model - if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components[rj] is not None: - appearance_model.trim_components( - self.max_appearance_components[rj]) + If `float`, it specifies the percentage of variance to be retained. - # add appearance model to the list - appearance_models.append(appearance_model) + If ``None``, all the available components are kept + (100% of variance). - if verbose: - print('{}Done'.format(level_str).ljust(80)) + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - appearance_models.reverse() - n_training_images = len(images) + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. - return self._build_aam(shape_models, appearance_models, - n_training_images) + Per level: + If `int`, it specifies the exact number of components to be + retained. - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + If `float`, it specifies the percentage of variance to be retained. - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + If ``None``, all the available components are kept + (100% of variance). - Returns - ------- - reference_frame : :map:`MaskedImage` - The reference frame. - """ - return build_reference_frame(mean_shape, boundary=self.boundary, - trilist=self.trilist) + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object - def _build_aam(self, shape_models, appearance_models, n_training_images): - r""" - Returns an AAM object. + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=True, + scale_features=True, max_shape_components=None, + max_appearance_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components - Parameters - ---------- - shape_models : :map:`PCAModel` - The trained multilevel shape models. - - appearance_models : :map:`PCAModel` - The trained multilevel appearance models. - - n_training_images : `int` - The number of training images. + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + self.reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[level]) + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) - Returns - ------- - aam : :map:`AAM` - The trained AAM object. - """ - from .base import AAM - return AAM(shape_models, appearance_models, n_training_images, - self.transform, self.features, self.reference_shape, - self.downscale, self.scaled_shape_models) + def _build_aam(self, shape_models, appearance_models, reference_shape): + return PatchAAM(shape_models, appearance_models, reference_shape, + self.patch_shape, self.features, self.scales, + self.scale_shapes, self.scale_features) -class PatchBasedAAMBuilder(AAMBuilder): +# TODO: document me! +class LinearGlobalAAMBuilder(AAMBuilder): r""" - Class that builds Multilevel Patch-Based Active Appearance Models. + Class that builds a Linear Global Active Appearance Model. Parameters ---------- @@ -384,45 +441,33 @@ class PatchBasedAAMBuilder(AAMBuilder): once and then creating a pyramid on top tends to lead to better performing AAMs. - patch_shape : tuple of `int`, optional - The appearance model of the Patch-Based AAM will be obtained by - sampling appearance patches with the specified shape around each - landmark. + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. + + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. - normalization_diagonal : `int` >= ``20``, optional + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the ``normalization_diagonal`` - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. - .. note:: - - Because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - downscale : `float` >= 1, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: + scales : `int` or float` or list of those, optional - (downscale ** k) for k in range(``n_levels``) + scale_shapes : `boolean`, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of each - pyramid level, so the shape models will be scaled. - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -452,213 +497,379 @@ class PatchBasedAAMBuilder(AAMBuilder): Per level: If `int`, it specifies the exact number of components to be retained. + If `float`, it specifies the percentage of variance to be retained. + If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- - aam : ::map:`PatchBasedAAMBuilder` - The Patch-Based AAM Builder object + aam : :map:`AAMBuilder` + The AAM Builder object Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` - ValueError - ``max_shape_components must be ``None`` or an `int` > ``0`` or - a ``0`` <= `float` <= ``1`` or a list of those containing ``1`` - or ``n_levels`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > 0 or a - ``0`` <= `float` <= ``1`` or a list of those containing ``1`` - or ``n_levels`` elements + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError - ``features`` must be a `string` or a `function` or a list of those - containing 1 or ``n_levels`` elements + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements ValueError - ``pyramid_on_features`` is enabled so ``features`` must be a - `string` or a `function` or a list containing one of those + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, features=igo, patch_shape=(16, 16), - normalization_diagonal=None, n_levels=3, downscale=2, - scaled_shape_models=True, max_shape_components=None, - max_appearance_components=None, boundary=3): + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None, max_appearance_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, len(scales), 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') - features = checks.check_features(features, n_levels) - - # store parameters + # set parameters self.features = features - self.patch_shape = patch_shape - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.transform = transform + self.trilist = trilist + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - self.boundary = boundary - # patch-based AAMs can only work with TPS transform - self.transform = DifferentiableThinPlateSplines + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model( + dense_shapes, max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) + + def _build_aam(self, shape_models, appearance_models, reference_shape): + return LinearGlobalAAM(shape_models, appearance_models, + reference_shape, self.transform, + self.features, self.scales, + self.scale_shapes, self.scale_features, + self.n_landmarks) + + +# TODO: document me! +class LinearPatchAAMBuilder(AAMBuilder): + r""" + Class that builds a Linear Patch Active Appearance Model. - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - Returns - ------- - reference_frame : :map:`MaskedImage` - The patch-based reference frame. - """ - return build_patch_reference_frame(mean_shape, boundary=self.boundary, - patch_shape=self.patch_shape) + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - def _mask_image(self, image): - r""" - Creates the patch-based mask of the given image. + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - Parameters - ---------- - image : :map:`MaskedImage` - The image to be masked. - """ - image.build_mask_around_landmarks(self.patch_shape, group='source') + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - def _build_aam(self, shape_models, appearance_models, n_training_images): - r""" - Returns a Patch-Based AAM object. + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - Parameters - ---------- - shape_models : :map:`PCAModel` - The trained multilevel shape models. + If ``None``, the mean shape is not rescaled. - appearance_models : :map:`PCAModel` - The trained multilevel appearance models. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - n_training_images : `int` - The number of training images. + scales : `int` or float` or list of those, optional - Returns - ------- - aam : :map:`PatchBasedAAM` - The trained Patched-Based AAM object. - """ - from .base import PatchBasedAAM - return PatchBasedAAM(shape_models, appearance_models, - n_training_images, self.patch_shape, - self.transform, self.features, - self.reference_shape, self.downscale, - self.scaled_shape_models) + scale_shapes : `boolean`, optional + scale_features : `boolean`, optional -def build_reference_frame(landmarks, boundary=3, group='source', - trilist=None): - r""" - Builds a reference frame from a particular set of landmarks. + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - Parameters - ---------- - landmarks : :map:`PointCloud` - The landmarks that will be used to build the reference frame. + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. - boundary : `int`, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). + Per level: + If `int`, it specifies the exact number of components to be + retained. - group : `string`, optional - Group that will be assigned to the provided set of landmarks on the - reference frame. + If `float`, it specifies the percentage of variance to be retained. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. + If ``None``, all the available components are kept + (100% of variance). - If ``None``, defaults to performing Delaunay triangulation on the - points. + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - Returns - ------- - reference_frame : :map:`Image` - The reference frame. - """ - reference_frame = _build_reference_frame(landmarks, boundary=boundary, - group=group) - if trilist is not None: - reference_frame.landmarks[group] = TriMesh( - reference_frame.landmarks['source'].lms.points, trilist=trilist) + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. - reference_frame.constrain_mask_to_landmarks(group=group) + If `float`, it specifies the percentage of variance to be retained. - return reference_frame + If ``None``, all the available components are kept + (100% of variance). + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=False, + scale_features=True, max_shape_components=None, + max_appearance_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components -def build_patch_reference_frame(landmarks, boundary=3, group='source', - patch_shape=(16, 16)): + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[level]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes, + max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) + + def _build_aam(self, shape_models, appearance_models, reference_shape): + return LinearPatchAAM(shape_models, appearance_models, + reference_shape, self.patch_shape, + self.features, self.scales, self.scale_shapes, + self.scale_features, self.n_landmarks) + + +# TODO: document me! +# TODO: implement offsets support? +class PartsAAMBuilder(AAMBuilder): r""" - Builds a reference frame from a particular set of landmarks. + Class that builds a Parts Active Appearance Model. Parameters ---------- - landmarks : :map:`PointCloud` - The landmarks that will be used to build the reference frame. + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - boundary : `int`, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - group : `string`, optional - Group that will be assigned to the provided set of landmarks on the - reference frame. + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - patch_shape : tuple of ints, optional - Tuple specifying the shape of the patches. + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + normalize_parts : `callable`, optional + + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). + + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). Returns ------- - patch_based_reference_frame : :map:`Image` - The patch based reference frame. + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - boundary = np.max(patch_shape) + boundary - reference_frame = _build_reference_frame(landmarks, boundary=boundary, - group=group) + def __init__(self, patch_shape=(17, 17), features=no_op, + normalize_parts=no_op, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None, max_appearance_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components - # mask reference frame - reference_frame.build_mask_around_landmarks(patch_shape, group=group) + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return extract_patches(images, shapes, self.patch_shape[level], + normalize_function=self.normalize_parts, + level_str=level_str, verbose=verbose) - return reference_frame + def _build_aam(self, shape_models, appearance_models, reference_shape): + return PartsAAM(shape_models, appearance_models, reference_shape, + self.patch_shape, self.features, + self.normalize_parts, self.scales, + self.scale_shapes, self.scale_features) -def _build_reference_frame(landmarks, boundary=3, group='source'): - # translate landmarks to the origin - minimum = landmarks.bounds(boundary=boundary)[0] - landmarks = Translation(-minimum).apply(landmarks) +from .base import ( + GlobalAAM, PatchAAM, LinearGlobalAAM, LinearPatchAAM, PartsAAM) - resolution = landmarks.range(boundary=boundary) - reference_frame = MaskedImage.init_blank(resolution) - reference_frame.landmarks[group] = landmarks - return reference_frame From e4c24a56088b99a12ce00d7fc0450c13d65c9f04 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 17:59:38 +0100 Subject: [PATCH 216/423] Update documentation in aam.base.py --- menpofit/aam/base.py | 172 ++++++++++++++++++++++++++++++++----------- 1 file changed, 130 insertions(+), 42 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 8d66070..2c39d1e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -256,14 +256,15 @@ class GlobalAAM(AAM): appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - n_training_images : `int` - The number of training images used to build the AAM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional + features : `callable` or ``[callable]``, If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -277,24 +278,11 @@ class GlobalAAM(AAM): once and then creating a pyramid on top tends to lead to better performing AAMs. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. + scales : `int` or float` or list of those, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + scale_shapes : `boolean` - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. - - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + scale_features : `boolean` """ def __init__(self, shape_models, appearance_models, reference_shape, @@ -360,17 +348,14 @@ class PatchAAM(AAM): appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - n_training_images : `int` - The number of training images used to build the AAM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, optional + features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -384,24 +369,11 @@ class PatchAAM(AAM): once and then creating a pyramid on top tends to lead to better performing AAMs. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + scales : `int` or float` or list of those - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. + scale_shapes : `boolean` - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean` """ def __init__(self, shape_models, appearance_models, reference_shape, @@ -455,6 +427,44 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearGlobalAAM(AAM): r""" + Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + """ def __init__(self, shape_models, appearance_models, reference_shape, transform, features, scales, scale_shapes, scale_features, @@ -489,6 +499,45 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearPatchAAM(AAM): r""" + Linear Patch Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + n_landmarks: `int` + """ def __init__(self, shape_models, appearance_models, reference_shape, patch_shape, features, scales, scale_shapes, @@ -524,6 +573,45 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class PartsAAM(AAM): r""" + Parts Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + normalize_parts: `callable` + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + """ def __init__(self, shape_models, appearance_models, reference_shape, parts_shape, features, normalize_parts, scales, From e7c93f4e25d0435dcea08ed024ccb0e6177069e7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 20 May 2015 18:37:41 +0100 Subject: [PATCH 217/423] Minor documnetation changes in aam.base and aam.builder --- menpofit/aam/base.py | 8 ++++---- menpofit/aam/builder.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 2c39d1e..af9915e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -338,7 +338,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, class PatchAAM(AAM): r""" - Patch Based Active Appearance Model class. + Patch based Based Active Appearance Model class. Parameters ----------- @@ -427,7 +427,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearGlobalAAM(AAM): r""" - Active Appearance Model class. + Linear Active Appearance Model class. Parameters ----------- @@ -499,7 +499,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class LinearPatchAAM(AAM): r""" - Linear Patch Active Appearance Model class. + Linear Patch based Active Appearance Model class. Parameters ----------- @@ -573,7 +573,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! class PartsAAM(AAM): r""" - Parts Active Appearance Model class. + Parts based Active Appearance Model class. Parameters ----------- diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 886f73c..3a8543a 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -140,7 +140,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class GlobalAAMBuilder(AAMBuilder): r""" - Class that builds a Global Active Appearance Model. + Class that builds Active Appearance Models. Parameters ---------- @@ -282,7 +282,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class PatchAAMBuilder(AAMBuilder): r""" - Class that builds a Patch Active Appearance Model. + Class that builds Patch based Active Appearance Models. Parameters ---------- @@ -423,7 +423,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class LinearGlobalAAMBuilder(AAMBuilder): r""" - Class that builds a Linear Global Active Appearance Model. + Class that builds Linear Active Appearance Models. Parameters ---------- @@ -577,7 +577,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! class LinearPatchAAMBuilder(AAMBuilder): r""" - Class that builds a Linear Patch Active Appearance Model. + Class that builds Linear Patch based Active Appearance Models. Parameters ---------- @@ -730,7 +730,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: implement offsets support? class PartsAAMBuilder(AAMBuilder): r""" - Class that builds a Parts Active Appearance Model. + Class that builds Parts based Active Appearance Models. Parameters ---------- From a21ada4418dea7c08b8707b7acb77f93d221a264 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:20:16 +0100 Subject: [PATCH 218/423] Restructure aam.base - Remove pure AAM interface --- menpofit/aam/base.py | 212 ++++++++++++++++--------------------------- 1 file changed, 79 insertions(+), 133 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index af9915e..4266ae7 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -10,8 +10,56 @@ class AAM(object): r""" - Abstract interface for Active Appearance Model. + Active Appearance Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean` + + scale_features : `boolean` + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + @property def n_levels(self): """ @@ -91,54 +139,44 @@ def random_instance(self, level=-1): return self._instance(level, shape_instance, appearance_instance) - @abc.abstractmethod def _instance(self, level, shape_instance, appearance_instance): - pass + template = self.appearance_models[level].mean() + landmarks = template.landmarks['source'].lms - def view_shape_models_widget(self, n_parameters=5, - parameters_bounds=(-3.0, 3.0), - mode='multiple', figure_size=(10, 8)): - r""" - Visualizes the shape models of the AAM object using the - `menpo.visualize.widgets.visualize_shape_model` widget. + if type(landmarks) == TriMesh: + trilist = landmarks.trilist + else: + trilist = None + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) - Parameters - ----------- - n_parameters : `int` or `list` of `int` or ``None``, optional - The number of shape principal components to be used for the - parameters sliders. - If `int`, then the number of sliders per level is the minimum - between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. - parameters_bounds : (`float`, `float`), optional - The minimum and maximum bounds, in std units, for the sliders. - mode : {``single``, ``multiple``}, optional - If ``'single'``, only a single slider is constructed along with a - drop down menu. - If ``'multiple'``, a slider is constructed for each parameter. - popup : `bool`, optional - If ``True``, the widget will appear as a popup window. - figure_size : (`int`, `int`), optional - The size of the plotted figures. - """ - from menpofit.visualize import visualize_shape_model - visualize_shape_model(self.shape_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode,) + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + instance = appearance_instance.warp_to_mask( + reference_frame.mask, transform) + instance.landmarks = reference_frame.landmarks + + return instance - @abc.abstractmethod def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): - pass + from menpofit.visualize import visualize_appearance_model + visualize_appearance_model(self.appearance_models, + n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) - @abc.abstractmethod + # TODO: fix me! def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): - pass + from menpofit.visualize import visualize_aam + visualize_aam(self, n_shape_parameters=n_shape_parameters, + n_appearance_parameters=n_appearance_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) # TODO: fix me! def __str__(self): @@ -244,98 +282,6 @@ def __str__(self): return out -class GlobalAAM(AAM): - r""" - Active Appearance Model class. - - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean` - - scale_features : `boolean` - - """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() - landmarks = template.landmarks['source'].lms - - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) - - transform = self.transform( - reference_frame.landmarks['source'].lms, landmarks) - - instance = appearance_instance.warp_to_mask( - reference_frame.mask, transform) - instance.landmarks = reference_frame.landmarks - - return instance - - def view_appearance_models_widget(self, n_parameters=5, - parameters_bounds=(-3.0, 3.0), - mode='multiple', figure_size=(10, 8)): - from menpofit.visualize import visualize_appearance_model - visualize_appearance_model(self.appearance_models, - n_parameters=n_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode) - - # TODO: fix me! - def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, - parameters_bounds=(-3.0, 3.0), mode='multiple', - figure_size=(10, 8)): - from menpofit.visualize import visualize_aam - visualize_aam(self, n_shape_parameters=n_shape_parameters, - n_appearance_parameters=n_appearance_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode) - - class PatchAAM(AAM): r""" Patch based Based Active Appearance Model class. @@ -425,7 +371,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: document me! -class LinearGlobalAAM(AAM): +class LinearAAM(AAM): r""" Linear Active Appearance Model class. @@ -614,11 +560,11 @@ class PartsAAM(AAM): """ def __init__(self, shape_models, appearance_models, reference_shape, - parts_shape, features, normalize_parts, scales, + patch_shape, features, normalize_parts, scales, scale_shapes, scale_features): self.shape_models = shape_models self.appearance_models = appearance_models - self.parts_shape = parts_shape + self.patch_shape = patch_shape self.features = features self.normalize_parts = normalize_parts self.reference_shape = reference_shape From 0685876c7e91918ab0be48f88fd438403415b360 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:21:41 +0100 Subject: [PATCH 219/423] Restructure aam.builder - Remove pure AAMBuilder interface --- menpofit/aam/builder.py | 258 +++++++++++++++++++--------------------- 1 file changed, 124 insertions(+), 134 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 3a8543a..6ad44f2 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -14,131 +14,9 @@ DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -class AAMBuilder(object): - r""" - Abstract interface for Active Appearance Model Builder. - """ - def build(self, images, group=None, label=None, verbose=False): - r""" - Builds an Active Appearance Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest level - """ - # normalize images and compute reference shape - reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - shape_models = [] - appearance_models = [] - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if j == 0: - # compute features at highest level - feature_images = compute_features(images, self.features, - level_str=level_str, - verbose=verbose) - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, self.features, - level_str=level_str, - verbose=verbose) - - # extract potentially rescaled shapes - level_shapes = [i.landmarks[group][label] - for i in level_images] - - # obtain shape representation - if j == 0 or self.scale_shapes: - # obtain shape model - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) - # add shape model to the list - shape_models.append(shape_model) - else: - # copy precious shape model and add it to the list - shape_models.append(deepcopy(shape_model)) - - # obtain warped images - warped_images = self._warp_images(level_images, level_shapes, - shape_model.mean(), j, - level_str, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - appearance_models.append(appearance_model) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - appearance_models.reverse() - self.scales.reverse() - - aam = self._build_aam(shape_models, appearance_models, reference_shape) - - return aam - - @classmethod - def _build_shape_model(cls, shapes, max_components, level): - return build_shape_model(shapes, max_components=max_components) - - @abc.abstractmethod - def _build_aam(self, shape_models, appearance_models, reference_shape): - pass - - # TODO: implement checker for conflict between features and scale_features # TODO: document me! -class GlobalAAMBuilder(AAMBuilder): +class AAMBuilder(object): r""" Class that builds Active Appearance Models. @@ -266,6 +144,119 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components + def build(self, images, group=None, label=None, verbose=False): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + + label : `string`, optional + The label of of the landmark manager that you wish to use. If no + label is passed, the convex hull of all landmarks is used. + + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest level + """ + # normalize images and compute reference shape + reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + shape_models = [] + appearance_models = [] + # for each pyramid level (high --> low) + for j, s in enumerate(self.scales): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if j == 0: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, self.features, + level_str=level_str, + verbose=verbose) + + # extract potentially rescaled shapes + level_shapes = [i.landmarks[group][label] + for i in level_images] + + # obtain shape representation + if j == 0 or self.scale_shapes: + # obtain shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # add shape model to the list + shape_models.append(shape_model) + else: + # copy precious shape model and add it to the list + shape_models.append(deepcopy(shape_model)) + + # obtain warped images + warped_images = self._warp_images(level_images, level_shapes, + shape_model.mean(), j, + level_str, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format(level_str)) + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + appearance_models.append(appearance_model) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + # reverse the list of shape and appearance models so that they are + # ordered from lower to higher resolution + shape_models.reverse() + appearance_models.reverse() + self.scales.reverse() + + aam = self._build_aam(shape_models, appearance_models, reference_shape) + + return aam + + @classmethod + def _build_shape_model(cls, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): self.reference_frame = build_reference_frame(reference_shape) @@ -274,9 +265,9 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): - return GlobalAAM(shape_models, appearance_models, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features) + return AAM(shape_models, appearance_models, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features) # TODO: document me! @@ -421,7 +412,7 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): # TODO: document me! -class LinearGlobalAAMBuilder(AAMBuilder): +class LinearAAMBuilder(AAMBuilder): r""" Class that builds Linear Active Appearance Models. @@ -567,11 +558,11 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): - return LinearGlobalAAM(shape_models, appearance_models, - reference_shape, self.transform, - self.features, self.scales, - self.scale_shapes, self.scale_features, - self.n_landmarks) + return LinearAAM(shape_models, appearance_models, + reference_shape, self.transform, + self.features, self.scales, + self.scale_shapes, self.scale_features, + self.n_landmarks) # TODO: document me! @@ -869,7 +860,6 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): self.scale_shapes, self.scale_features) -from .base import ( - GlobalAAM, PatchAAM, LinearGlobalAAM, LinearPatchAAM, PartsAAM) +from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM From 2ce01e0c36a2afef6c2255729ae32dcab11cf637 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:24:23 +0100 Subject: [PATCH 220/423] Restructure fitter.py - Add new MultiFitter, ModelFitter - Move noisy_align from base.py to fitter.py --- menpofit/base.py | 34 ---- menpofit/fitter.py | 395 ++++++++++++++++++++++----------------------- 2 files changed, 190 insertions(+), 239 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index 36b8a08..175054e 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -96,40 +96,6 @@ def pyramid_on_features(self): return is_pyramid_on_features(self.features) -# TODO: Should this be a method on Similarity? AlignableTransforms? -def noisy_align(source, target, noise_std=0.04, rotation=False): - r""" - Constructs and perturbs the optimal similarity transform between source - to the target by adding white noise to its weights. - - Parameters - ---------- - source: :class:`menpo.shape.PointCloud` - The source pointcloud instance used in the alignment - target: :class:`menpo.shape.PointCloud` - The target pointcloud instance used in the alignment - noise_std: float - The standard deviation of the white noise - - Default: 0.04 - rotation: boolean - If False the second parameter of the Similarity, - which captures captures inplane rotations, is set to 0. - - Default:False - - Returns - ------- - noisy_transform : :class: `menpo.transform.Similarity` - The noisy Similarity Transform - """ - transform = AlignmentSimilarity(source, target, rotation=rotation) - parameters = transform.as_vector() - parameter_range = np.hstack((parameters[:2], target.range())) - noise = (parameter_range * noise_std * - np.random.randn(transform.n_parameters)) - return Similarity.init_identity(source.n_dims).from_vector(parameters + noise) - def build_sampling_grid(patch_shape): r""" diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 3d93d55..64f7df8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,112 +1,21 @@ from __future__ import division -import abc -from menpo.transform import AlignmentAffine, Scale, AlignmentSimilarity import numpy as np from menpo.shape import PointCloud -from menpofit.base import is_pyramid_on_features, pyramid_of_feature_images, \ - noisy_align -from menpofit.fittingresult import MultilevelFittingResult +from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity -class Fitter(object): +# TODO: document me! +class MultiFitter(object): r""" - Abstract interface that all :map:`Fitter` objects must implement. """ - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def _set_up(self, **kwargs): - r""" - Abstract method that sets up the fitter object. - """ - pass - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - r""" - Fits the fitter to an image. - - Parameters - ----------- - image: :map:`Image` or subclass - The image to be fitted. - initial_parameters: list - The initial parameters of the model. - gt_shape: :map:`PointCloud` - The ground truth shape associated to the image. - - Returns - ------- - fitting_result: :map:`FittingResult` - The fitting result containing the result of fitting procedure. - """ - fitting_result = self._create_fitting_result( - image, initial_parameters, gt_shape=gt_shape) - return self._fit(fitting_result, **kwargs) - - @abc.abstractmethod - def _create_fitting_result(self, **kwargs): - r""" - Abstract method that defines the fitting result object associated to - the fitter object. - """ - pass - - @abc.abstractmethod - def _fit(self, **kwargs): - r""" - Abstract method implements a particular alignment algorithm. - """ - pass - - def get_parameters(self, shape): - r""" - Abstract method that gets the parameters. - """ - pass - - -class MultilevelFitter(Fitter): - r""" - Abstract interface that all :map:`MultilevelFitter` must implement. - """ - - @abc.abstractproperty - def reference_shape(self): - r""" - The reference shape of the multilevel fitter. - """ - pass - - @abc.abstractproperty - def features(self): - r""" - Returns the feature computation functions applied at each pyramidal - level. - """ - pass - - @abc.abstractproperty + @property def n_levels(self): r""" - The number of pyramidal levels. - """ - pass - - @abc.abstractproperty - def downscale(self): - r""" - The downscale factor used by the multiple fitter. - """ - pass + The number of pyramidal levels used during alignment. - @property - def pyramid_on_features(self): - r""" - Returns True if the pyramid is computed on the feature image and False - if it is computed on the original (intensities) image and features are - extracted at each level. + :type: `int` """ - return is_pyramid_on_features(self.features) + return len(self.scales) def fit(self, image, initial_shape, max_iters=50, gt_shape=None, crop_image=0.5, **kwargs): @@ -165,64 +74,16 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, affine_correction = AlignmentAffine(initial_shapes[-1], initial_shape) # run multilevel fitting - fitting_results = self._fit(images, initial_shapes[0], - max_iters=max_iters, - gt_shapes=gt_shapes, **kwargs) + algorithm_results = self._fit(images, initial_shapes[0], + max_iters=max_iters, + gt_shapes=gt_shapes, **kwargs) # build multilevel fitting result - multi_fitting_result = self._create_fitting_result( - image, fitting_results, affine_correction, gt_shape=gt_shape) - - return multi_fitting_result - - def perturb_shape(self, gt_shape, noise_std=0.04, rotation=False): - r""" - Generates an initial shape by adding gaussian noise to the perfect - similarity alignment between the ground truth and reference_shape. - - Parameters - ----------- - gt_shape: :class:`menpo.shape.PointCloud` - The ground truth shape. - noise_std: float, optional - The standard deviation of the gaussian noise used to produce the - initial shape. - - Default: 0.04 - rotation: boolean, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the initial shape. - - Default: False - - Returns - ------- - initial_shape: :class:`menpo.shape.PointCloud` - The initial shape. - """ - reference_shape = self.reference_shape - return noisy_align(reference_shape, gt_shape, noise_std=noise_std, - rotation=rotation).apply(reference_shape) - - def obtain_shape_from_bb(self, bounding_box): - r""" - Generates an initial shape given a bounding box detection. + fitter_result = self._fitter_result( + image, self, algorithm_results, affine_correction, + gt_shape=gt_shape) - Parameters - ----------- - bounding_box: (2, 2) ndarray - The bounding box specified as: - - np.array([[x_min, y_min], [x_max, y_max]]) - - Returns - ------- - initial_shape: :class:`menpo.shape.PointCloud` - The initial shape. - """ - reference_shape = self.reference_shape - return align_shape_with_bb(reference_shape, - bounding_box).apply(reference_shape) + return fitter_result def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): @@ -267,6 +128,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ + # attach landmarks to the image image.landmarks['initial_shape'] = initial_shape if gt_shape: @@ -283,8 +145,24 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, image = image.rescale_to_reference_shape(self.reference_shape, group='initial_shape') - images = list(reversed(list(pyramid_of_feature_images( - self.n_levels, self.downscale, self.features, image)))) + # obtain image representation + from copy import deepcopy + scales = deepcopy(self.scales) + scales.reverse() + images = [] + for j, s in enumerate(scales): + if j == 0: + # compute features at highest level + feature_image = self.features(image) + elif self.scale_features: + # scale features at other levels + feature_image = images[0].rescale(s) + else: + # scale image and compute features at other levels + scaled_image = image.rescale(s) + feature_image = self.features(scaled_image) + images.append(feature_image) + images.reverse() # get initial shapes per level initial_shapes = [i.landmarks['initial_shape'].lms for i in images] @@ -292,46 +170,11 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # get ground truth shapes per level if gt_shape: gt_shapes = [i.landmarks['gt_shape'].lms for i in images] - del image.landmarks['gt_shape'] else: gt_shapes = None return images, initial_shapes, gt_shapes - def _create_fitting_result(self, image, fitting_results, affine_correction, - gt_shape=None): - r""" - Creates the :class: `menpo.aam.fitting.MultipleFitting` object - associated with a particular Fitter object. - - Parameters - ----------- - image: :class:`menpo.image.masked.MaskedImage` - The original image to be fitted. - fitting_results: :class:`menpo.fit.fittingresult.FittingResultList` - A list of basic fitting objects containing the state of the - different fitting levels. - affine_correction: :class: `menpo.transforms.affine.Affine` - An affine transform that maps the result of the top resolution - fitting level to the space scale of the original image. - gt_shape: class:`menpo.shape.PointCloud`, optional - The ground truth shape associated to the image. - - Default: None - error_type: 'me_norm', 'me' or 'rmse', optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Default: 'me_norm' - - Returns - ------- - fitting: :class:`menpo.fitmultilevel.fittingresult.MultilevelFittingResult` - The fitting object that will hold the state of the fitter. - """ - return MultilevelFittingResult(image, self, fitting_results, - affine_correction, gt_shape=gt_shape) - def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, **kwargs): r""" @@ -359,14 +202,32 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, Returns ------- - fitting_results: :class:`menpo.fit.fittingresult.FittingResult` list + algorithm_results: :class:`menpo.fg2015.fittingresult.FittingResult` list The fitting object containing the state of the whole fitting procedure. """ + max_iters = self._prepare_max_iters(max_iters) shape = initial_shape gt_shape = None - n_levels = self.n_levels + algorithm_results = [] + for j, (i, alg, it, s) in enumerate(zip(images, self.algorithms, + max_iters, self.scales)): + if gt_shapes: + gt_shape = gt_shapes[j] + + algorithm_result = alg.run(i, shape, gt_shape=gt_shape, + max_iters=it, **kwargs) + algorithm_results.append(algorithm_result) + + shape = algorithm_result.final_shape + if s != self.scales[-1]: + Scale(self.scales[j+1]/s, + n_dims=shape.n_dims).apply_inplace(shape) + + return algorithm_results + def _prepare_max_iters(self, max_iters): + n_levels = self.n_levels # check max_iters parameter if type(max_iters) is int: max_iters = [np.round(max_iters/n_levels) @@ -378,22 +239,111 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, raise ValueError('max_iters can be integer, integer list ' 'containing 1 or {} elements or ' 'None'.format(self.n_levels)) + return np.require(max_iters, dtype=np.int) - # fit images - fitting_results = [] - for j, (i, f, it) in enumerate(zip(images, self._fitters, max_iters)): - if gt_shapes is not None: - gt_shape = gt_shapes[j] - parameters = f.get_parameters(shape) - fitting_result = f.fit(i, parameters, gt_shape=gt_shape, - max_iters=it, **kwargs) - fitting_results.append(fitting_result) +# TODO: document me! +class ModelFitter(MultiFitter): + r""" + """ + @property + def reference_shape(self): + r""" + The reference shape of the AAM. + + :type: :map:`PointCloud` + """ + return self._model.reference_shape + + @property + def features(self): + r""" + The feature extracted at each pyramidal level during AAM building. + Stored in ascending pyramidal order. + + :type: `list` + """ + return self._model.features + + @property + def n_levels(self): + r""" + The number of pyramidal levels used during AAM building. + + :type: `int` + """ + return self._model.n_levels + + @property + def scales(self): + return self._model.scales + + @property + def scale_features(self): + r""" + Flag that defined the nature of Gaussian pyramid used to build the + AAM. + If ``True``, the feature space is computed once at the highest scale + and the Gaussian pyramid is applied to the feature images. + If ``False``, the Gaussian pyramid is applied to the original images + and features are extracted at each level. + + :type: `boolean` + """ + return self._model.scale_features + + def _check_n_shape(self, n_shape): + if n_shape is not None: + if type(n_shape) is int or type(n_shape) is float: + for sm in self._model.shape_models: + sm.n_active_components = n_shape + elif len(n_shape) == 1 and self._model.n_levels > 1: + for sm in self._model.shape_models: + sm.n_active_components = n_shape[0] + elif len(n_shape) == self._model.n_levels: + for sm, n in zip(self._model.shape_models, n_shape): + sm.n_active_components = n + else: + raise ValueError('n_shape can be an integer or a float or None' + 'or a list containing 1 or {} of ' + 'those'.format(self._model.n_levels)) + + def perturb_shape(self, gt_shape, noise_std=0.04, rotation=False): + transform = noisy_align(AlignmentSimilarity, self.reference_shape, + gt_shape, noise_std=noise_std, + rotation=rotation) + return transform.apply(self.reference_shape) + + def obtain_shape_from_bb(self, bounding_box): + r""" + Generates an initial shape given a bounding box detection. - shape = fitting_result.final_shape - Scale(self.downscale, n_dims=shape.n_dims).apply_inplace(shape) + Parameters + ----------- + bounding_box: (2, 2) ndarray + The bounding box specified as: + + np.array([[x_min, y_min], [x_max, y_max]]) - return fitting_results + Returns + ------- + initial_shape: :class:`menpo.shape.PointCloud` + The initial shape. + """ + + reference_shape = self.reference_shape + return align_shape_with_bb(reference_shape, + bounding_box).apply(reference_shape) + + +# TODO: document me! +def noisy_align(alignment_transform_cls, source, target, noise_std=0.04, + rotation=True): + r""" + """ + noise = noise_std * np.random.randn(target.n_points, target.n_dims) + noisy_target = PointCloud(target.points + noise) + return alignment_transform_cls(source, noisy_target, rotation=rotation) def align_shape_with_bb(shape, bounding_box): @@ -417,4 +367,39 @@ def align_shape_with_bb(shape, bounding_box): """ shape_box = PointCloud(shape.bounds()) bounding_box = PointCloud(bounding_box) - return AlignmentSimilarity(shape_box, bounding_box, rotation=False) \ No newline at end of file + return AlignmentSimilarity(shape_box, bounding_box, rotation=False) + + +# TODO: implement as a method on Similarity? AlignableTransforms? +# def noisy_align(source, target, noise_std=0.04, rotation=False): +# r""" +# Constructs and perturbs the optimal similarity transform between source +# to the target by adding white noise to its weights. +# +# Parameters +# ---------- +# source: :class:`menpo.shape.PointCloud` +# The source pointcloud instance used in the alignment +# target: :class:`menpo.shape.PointCloud` +# The target pointcloud instance used in the alignment +# noise_std: float +# The standard deviation of the white noise +# +# Default: 0.04 +# rotation: boolean +# If False the second parameter of the Similarity, +# which captures captures inplane rotations, is set to 0. +# +# Default:False +# +# Returns +# ------- +# noisy_transform : :class: `menpo.transform.Similarity` +# The noisy Similarity Transform +# """ +# transform = AlignmentSimilarity(source, target, rotation=rotation) +# parameters = transform.as_vector() +# parameter_range = np.hstack((parameters[:2], target.range())) +# noise = (parameter_range * noise_std * +# np.random.randn(transform.n_parameters)) +# return Similarity.init_identity(source.n_dims).from_vector(parameters + noise) From 3c57634c160e42f924087e48b71ac21de182c589 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:26:39 +0100 Subject: [PATCH 221/423] Force OrthoMDTransform and OrthoMDPDM to use only similarity transforms --- menpofit/modelinstance.py | 7 +++++-- menpofit/transform/modeldriven.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/menpofit/modelinstance.py b/menpofit/modelinstance.py index 169612b..7810e72 100644 --- a/menpofit/modelinstance.py +++ b/menpofit/modelinstance.py @@ -322,7 +322,7 @@ def _global_transform_d_dp(self, points): class OrthoPDM(GlobalPDM): r""" """ - def __init__(self, model, global_transform_cls): + def __init__(self, model): # 1. Construct similarity model from the mean of the model self.similarity_model = similarity_2d_instance_model(model.mean()) # 2. Orthonormalize model and similarity model @@ -330,7 +330,9 @@ def __init__(self, model, global_transform_cls): model_cpy.orthonormalize_against_inplace(self.similarity_model) self.similarity_weights = self.similarity_model.project( model_cpy.mean()) - super(OrthoPDM, self).__init__(model_cpy, global_transform_cls) + from menpofit.transform import DifferentiableAlignmentSimilarity + super(OrthoPDM, self).__init__(model_cpy, + DifferentiableAlignmentSimilarity) @property def global_parameters(self): @@ -353,3 +355,4 @@ def _update_global_weights(self, global_weights): def _global_transform_d_dp(self, points): return self.similarity_model.components.reshape( self.n_global_parameters, -1, self.n_dims).swapaxes(0, 1) + diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index 1db964b..fc5fd8a 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -519,8 +519,8 @@ class OrthoMDTransform(GlobalMDTransform): The source landmarks of the transform. If no `source` is provided the mean of the model is used. """ - def __init__(self, model, transform_cls, global_transform, source=None): - self.pdm = OrthoPDM(model, global_transform) + def __init__(self, model, transform_cls, source=None): + self.pdm = OrthoPDM(model) self._cached_points = None self.transform = transform_cls(source, self.target) From c90639a3f7d667241a67c42bf6f5c1a40ea6eb37 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:29:16 +0100 Subject: [PATCH 222/423] Add LKAAMFitter - All AAM objects are used with a single fitter (this is different to the original implementation in my repo but seems more user friendly) --- menpofit/aam/fitter.py | 460 ++++++----------------------------------- 1 file changed, 61 insertions(+), 399 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 045f388..957bead 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,243 +1,70 @@ from __future__ import division -from itertools import chain +from menpofit.fitter import ModelFitter +from menpofit.modelinstance import OrthoPDM +from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform +from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM +from .algorithm import ( + StandardAAMInterface, LinearAAMInterface, PartsAAMInterface, AIC) +from .result import AAMFitterResult -from menpofit.base import name_of_callable -from menpofit.fitter import MultilevelFitter -from menpofit.fittingresult import AMMultilevelFittingResult -from menpofit.transform import (ModelDrivenTransform, OrthoMDTransform, - DifferentiableAlignmentSimilarity) -from menpofit.lucaskanade.appearance import SIC - -class AAMFitter(MultilevelFitter): +# TODO: document me! +class LKAAMFitter(ModelFitter): r""" - Abstract Interface for defining Active Appearance Models Fitters. - - Parameters - ----------- - aam : :map:`AAM` - The Active Appearance Model to be used. """ - def __init__(self, aam): - self.aam = aam - - @property - def reference_shape(self): - r""" - The reference shape of the AAM. - - :type: :map:`PointCloud` - """ - return self.aam.reference_shape - - @property - def features(self): - r""" - The feature extracted at each pyramidal level during AAM building. - Stored in ascending pyramidal order. - - :type: `list` - """ - return self.aam.features - - @property - def n_levels(self): - r""" - The number of pyramidal levels used during AAM building. - - :type: `int` - """ - return self.aam.n_levels - - @property - def downscale(self): - r""" - The downscale used to generate the final scale factor applied at - each pyramidal level during AAM building. - The scale factor is computed as: - - ``(downscale ** k) for k in range(n_levels)`` - - :type: `float` - """ - return self.aam.downscale - - def _create_fitting_result(self, image, fitting_results, affine_correction, - gt_shape=None): - r""" - Creates a :map:`AAMMultilevelFittingResult` associated to a - particular fitting of the AAM fitter. - - Parameters - ----------- - image : :map:`Image` or subclass - The image to be fitted. - - fitting_results : `list` of :map:`FittingResult` - A list of fitting result objects containing the state of the - the fitting for each pyramidal level. - - affine_correction : :map:`Affine` - An affine transform that maps the result of the top resolution - level to the scale space of the original image. - - gt_shape : :map:`PointCloud`, optional - The ground truth shape associated to the image. - - error_type : 'me_norm', 'me' or 'rmse', optional - Specifies how the error between the fitted and ground truth - shapes must be computed. - - Returns - ------- - fitting : :map:`AAMMultilevelFittingResult` - A fitting result object that will hold the state of the AAM - fitter for a particular fitting. - """ - return AAMMultilevelFittingResult( - image, self, fitting_results, affine_correction, gt_shape=gt_shape) - - -class LucasKanadeAAMFitter(AAMFitter): - r""" - Lucas-Kanade based :map:`Fitter` for Active Appearance Models. - - Parameters - ----------- - aam : :map:`AAM` - The Active Appearance Model to be used. - algorithm : subclass of :map:`AppearanceLucasKanade`, optional - The Appearance Lucas-Kanade class to be used. - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - n_appearance : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of appearance components or amount of appearance variance - to be used per pyramidal level. - - If `None`, all available appearance components - ``(n_active_components)`` will be used. - If `int` ``> 1``, the specified number of appearance components will - be used. - If ``0. <=`` `float` ``<= 1.``, the number of appearance components - capturing the specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - """ - def __init__(self, aam, algorithm=SIC, - md_transform=OrthoMDTransform, n_shape=None, + def __init__(self, aam, algorithm_cls=AIC, n_shape=None, n_appearance=None, **kwargs): - super(LucasKanadeAAMFitter, self).__init__(aam) - self._set_up(algorithm=algorithm, md_transform=md_transform, - n_shape=n_shape, n_appearance=n_appearance, **kwargs) - - @property - def algorithm(self): - r""" - Returns a string containing the name of fitting algorithm. - - :type: `str` - """ - return 'LK-AAM-' + self._fitters[0].algorithm - - def _set_up(self, algorithm=SIC, - md_transform=OrthoMDTransform, - global_transform=DifferentiableAlignmentSimilarity, - n_shape=None, n_appearance=None, **kwargs): - r""" - Sets up the Lucas-Kanade fitter object. - - Parameters - ----------- - algorithm : subclass of :map:`AppearanceLucasKanade`, optional - The Appearance Lucas-Kanade class to be used. - - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - - n_appearance : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of appearance components or amount of appearance variance - to be used per pyramidal level. + super(LKAAMFitter, self).__init__() + self._model = aam + self._algorithms = [] + self._check_n_shape(n_shape) + self._check_n_appearance(n_appearance) + self._set_up(algorithm_cls, **kwargs) + + def _set_up(self, algorithm_cls, **kwargs): + for j, (am, sm) in enumerate(zip(self._model.appearance_models, + self._model.shape_models)): + + if type(self.aam) is AAM or type(self.aam) is PatchAAM: + # build orthonormal model driven transform + md_transform = OrthoMDTransform( + sm, self._model.transform, + source=am.mean().landmarks['source'].lms) + # set up algorithm using standard aam interface + algorithm = algorithm_cls(StandardAAMInterface, am, + md_transform, **kwargs) + + elif (type(self.aam) is LinearAAM or + type(self.aam) is LinearPatchAAM): + # build linear version of orthogonal model driven transform + md_transform = LinearOrthoMDTransform( + sm, self._model.n_landmarks) + # set up algorithm using linear aam interface + algorithm = algorithm_cls(LinearAAMInterface, am, + md_transform, **kwargs) + + elif type(self.aam) is PartsAAM: + # build orthogonal point distribution model + pdm = OrthoPDM(sm) + # set up algorithm using parts aam interface + am.patch_shape = self._model.patch_shape[j] + am.normalize_parts = self._model.normalize_parts + algorithm = algorithm_cls(PartsAAMInterface, am, pdm, **kwargs) - If `None`, all available appearance components - ``(n_active_components)`` will be used. - If `int` ``> 1``, the specified number of appearance components will - be used. - If ``0. <=`` `float` ``<= 1.``, the number of appearance components - capturing the specified variance ratio will be computed and used. + else: + raise ValueError("AAM object must be of one of the " + "following classes: {}, {}, {}, {}, " + "{}".format(AAM, PatchAAM, LinearAAM, + LinearPatchAAM, PartsAAM)) - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. + # append algorithms to list + self._algorithms.append(algorithm) - Raises - ------- - ValueError - ``n_shape`` can be an `int`, `float`, ``None`` or a `list` - containing ``1`` or ``n_levels`` of those. - ValueError - ``n_appearance`` can be an `int`, `float`, `None` or a `list` - containing ``1`` or ``n_levels`` of those. - """ - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.aam.n_levels > 1: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.aam.n_levels: - for sm, n in zip(self.aam.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None ' - 'or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + @property + def aam(self): + return self._model - # check n_appearance parameter + def _check_n_appearance(self, n_appearance): if n_appearance is not None: if type(n_appearance) is int or type(n_appearance) is float: for am in self.aam.appearance_models: @@ -253,173 +80,8 @@ def _set_up(self, algorithm=SIC, 'or None or a list containing 1 or {} of ' 'those'.format(self.aam.n_levels)) - self._fitters = [] - for j, (am, sm) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models)): - - if md_transform is not ModelDrivenTransform: - md_trans = md_transform( - sm, self.aam.transform, global_transform, - source=am.mean().landmarks['source'].lms) - else: - md_trans = md_transform( - sm, self.aam.transform, - source=am.mean().landmarks['source'].lms) - self._fitters.append( - algorithm(am, md_trans, **kwargs)) + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return AAMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) - def __str__(self): - out = "{0} Fitter\n" \ - " - Lucas-Kanade {1}\n" \ - " - Transform is {2} and residual is {3}.\n" \ - " - {4} training images.\n".format( - self.aam._str_title, self._fitters[0].algorithm, - self._fitters[0].transform.__class__.__name__, - self._fitters[0].residual.type, self.aam.n_training_images) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_levels): - n_channels.append( - self._fitters[j].appearance_model.template_instance.n_channels) - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format(name_of_callable( - self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - self.features[j].__name__)) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.aam.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.aam.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self._fitters[0].appearance_model.n_features, - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.aam.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self._fitters[i].appearance_model.n_features, - self._fitters[i].template.n_true_pixels(), - n_channels[i], self._fitters[i].template._str_shape, - n_channels[i]) - out = "{0} - {1} motion components\n - {2} active " \ - "appearance components ({3:.2f}% of original " \ - "variance)\n".format( - out, self._fitters[i].transform.n_parameters, - self._fitters[i].appearance_model.n_active_components, - self._fitters[i].appearance_model.variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} motion parameters\n" \ - " - {10} appearance components ({11:.2f}% of original " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self._fitters[0].appearance_model.n_features, - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0], self._fitters[0].transform.n_parameters, - self._fitters[0].appearance_model.n_active_components, - self._fitters[0].appearance_model.variance_ratio() * 100) - return out - - -class AAMMultilevelFittingResult(AMMultilevelFittingResult): - r""" - Class that holds the state of a :map:`AAMFitter` object before, - during and after it has fitted a particular image. - """ - @property - def appearance_reconstructions(self): - r""" - The list containing the appearance reconstruction obtained at - each fitting iteration. - - :type: `list` of :map:`Image` or subclass - """ - return list(chain( - *[f.appearance_reconstructions for f in self.fitting_results])) - - @property - def aam_reconstructions(self): - r""" - The list containing the aam reconstruction (i.e. the appearance - reconstruction warped on the shape instance reconstruction) obtained at - each fitting iteration. - - Note that this reconstruction is only tested to work for the - :map:`OrthoMDTransform` - - :type: list` of :map:`Image` or subclass - """ - aam_reconstructions = [] - for level, f in enumerate(self.fitting_results): - if f.weights: - for shape_w, aw in zip(f.parameters, f.weights): - shape_w = shape_w[4:] - sm_level = self.fitter.aam.shape_models[level] - am_level = self.fitter.aam.appearance_models[level] - swt = shape_w / sm_level.eigenvalues[:len(shape_w)] ** 0.5 - awt = aw / am_level.eigenvalues[:len(aw)] ** 0.5 - aam_reconstructions.append(self.fitter.aam.instance( - shape_weights=swt, appearance_weights=awt, level=level)) - else: - for shape_w in f.parameters: - shape_w = shape_w[4:] - sm_level = self.fitter.aam.shape_models[level] - swt = shape_w / sm_level.eigenvalues[:len(shape_w)] ** 0.5 - aam_reconstructions.append(self.fitter.aam.instance( - shape_weights=swt, appearance_weights=None, - level=level)) - return aam_reconstructions From d3057ae3fab9263923f34959efb38dd41cea4448 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:32:39 +0100 Subject: [PATCH 223/423] Add results.py - Results will substitute FittingResults --- menpofit/result.py | 903 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 903 insertions(+) create mode 100644 menpofit/result.py diff --git a/menpofit/result.py b/menpofit/result.py new file mode 100644 index 0000000..536145e --- /dev/null +++ b/menpofit/result.py @@ -0,0 +1,903 @@ +from __future__ import division +import abc +import numpy as np +from menpo.transform import Scale +from menpo.image import Image + + +# TODO: document me! +class Result(object): + r""" + """ + @abc.abstractproperty + def final_shape(self): + r""" + Returns the final fitted shape. + """ + + @abc.abstractproperty + def initial_shape(self): + r""" + Returns the initial shape from which the fitting started. + """ + + @property + def gt_shape(self): + r""" + Returns the original ground truth shape associated to the image. + """ + return self._gt_shape + + @property + def fitted_image(self): + r""" + Returns a copy of the fitted image with the following landmark + groups attached to it: + - ``initial``, containing the initial fitted shape . + - ``final``, containing the final shape. + - ``ground``, containing the ground truth shape. Only returned if + the ground truth shape was provided. + + :type: :map:`Image` + """ + image = Image(self.image.pixels) + + image.landmarks['initial'] = self.initial_shape + image.landmarks['final'] = self.final_shape + if self.gt_shape is not None: + image.landmarks['ground'] = self.gt_shape + return image + + def final_error(self, error_type='me_norm'): + r""" + Returns the final fitting error. + + Parameters + ----------- + error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + + Returns + ------- + final_error : `float` + The final error at the end of the fitting procedure. + """ + if self.gt_shape is not None: + return compute_error(self.final_shape, self.gt_shape, error_type) + else: + raise ValueError('Ground truth has not been set, final error ' + 'cannot be computed') + + def initial_error(self, error_type='me_norm'): + r""" + Returns the initial fitting error. + + Parameters + ----------- + error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + + Returns + ------- + initial_error : `float` + The initial error at the start of the fitting procedure. + """ + if self.gt_shape is not None: + return compute_error(self.initial_shape, self.gt_shape, error_type) + else: + raise ValueError('Ground truth has not been set, final error ' + 'cannot be computed') + + def as_serializableresult(self): + return SerializableIterativeResult( + self.image, self.initial_shape, self.final_shape, + gt_shape=self.gt_shape) + + def __str__(self): + out = "Initial error: {0:.4f}\nFinal error: {1:.4f}".format( + self.initial_error(), self.final_error()) + return out + + +# TODO: document me! +class IterativeResult(Result): + r""" + """ + @abc.abstractproperty + def n_iters(self): + r""" + Returns the number of iterations. + """ + + @abc.abstractproperty + def shapes(self, as_points=False): + r""" + Generates a list containing the shapes obtained at each fitting + iteration. + + Parameters + ----------- + as_points : boolean, optional + Whether the results is returned as a list of :map:`PointCloud`s or + ndarrays. + + Default: `False` + + Returns + ------- + shapes : :map:`PointCloud`s or ndarray list + A list containing the shapes obtained at each fitting iteration. + """ + + @property + def iter_image(self): + r""" + Returns a copy of the fitted image with a as many landmark groups as + iteration run by fitting procedure: + - ``iter_0``, containing the initial shape. + - ``iter_1``, containing the the fitted shape at the first + iteration. + - ``...`` + - ``iter_n``, containing the final fitted shape. + + :type: :map:`Image` + """ + image = Image(self.image.pixels) + for j, s in enumerate(self.shapes()): + image.landmarks['iter_'+str(j)] = s + return image + + def errors(self, error_type='me_norm'): + r""" + Returns a list containing the error at each fitting iteration. + + Parameters + ----------- + error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + + Returns + ------- + errors : `list` of `float` + The errors at each iteration of the fitting process. + """ + if self.gt_shape is not None: + return [compute_error(t, self.gt_shape, error_type) + for t in self.shapes()] + else: + raise ValueError('Ground truth has not been set, errors cannot ' + 'be computed') + + def plot_errors(self, error_type='me_norm', figure_id=None, + new_figure=False, render_lines=True, line_colour='b', + line_style='-', line_width=2, render_markers=True, + marker_style='o', marker_size=4, marker_face_colour='b', + marker_edge_colour='k', marker_edge_width=1., + render_axes=True, axes_font_name='sans-serif', + axes_font_size=10, axes_font_style='normal', + axes_font_weight='normal', figure_size=(10, 6), + render_grid=True, grid_line_style='--', + grid_line_width=0.5): + r""" + Plot of the error evolution at each fitting iteration. + Parameters + ---------- + error_type : {``me_norm``, ``me``, ``rmse``}, optional + Specifies the way in which the error between the fitted and + ground truth shapes is to be computed. + figure_id : `object`, optional + The id of the figure to be used. + new_figure : `bool`, optional + If ``True``, a new figure is created. + render_lines : `bool`, optional + If ``True``, the line will be rendered. + line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or + ``(3, )`` `ndarray`, optional + The colour of the lines. + line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the lines. + line_width : `float`, optional + The width of the lines. + render_markers : `bool`, optional + If ``True``, the markers will be rendered. + marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, + ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, + ``1``, ``2``, ``3``, ``4``, ``8``}, optional + The style of the markers. + marker_size : `int`, optional + The size of the markers in points^2. + marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The face (filling) colour of the markers. + marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The edge colour of the markers. + marker_edge_width : `float`, optional + The width of the markers' edge. + render_axes : `bool`, optional + If ``True``, the axes will be rendered. + axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the axes. + axes_font_size : `int`, optional + The font size of the axes. + axes_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the axes. + axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, + ``book``, ``medium``, ``roman``, ``semibold``, + ``demibold``, ``demi``, ``bold``, ``heavy``, + ``extra bold``, ``black``}, optional + The font weight of the axes. + figure_size : (`float`, `float`) or `None`, optional + The size of the figure in inches. + render_grid : `bool`, optional + If ``True``, the grid will be rendered. + grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the grid lines. + grid_line_width : `float`, optional + The width of the grid lines. + Returns + ------- + viewer : :map:`GraphPlotter` + The viewer object. + """ + from menpo.visualize import GraphPlotter + errors_list = self.errors(error_type=error_type) + return GraphPlotter(figure_id=figure_id, new_figure=new_figure, + x_axis=range(len(errors_list)), + y_axis=[errors_list], + title='Fitting Errors per Iteration', + x_label='Iteration', y_label='Fitting Error', + x_axis_limits=(0, len(errors_list)-1), + y_axis_limits=None).render( + render_lines=render_lines, line_colour=line_colour, + line_style=line_style, line_width=line_width, + render_markers=render_markers, marker_style=marker_style, + marker_size=marker_size, marker_face_colour=marker_face_colour, + marker_edge_colour=marker_edge_colour, + marker_edge_width=marker_edge_width, render_legend=False, + render_axes=render_axes, axes_font_name=axes_font_name, + axes_font_size=axes_font_size, axes_font_style=axes_font_style, + axes_font_weight=axes_font_weight, render_grid=render_grid, + grid_line_style=grid_line_style, grid_line_width=grid_line_width, + figure_size=figure_size) + + def displacements(self): + r""" + A list containing the displacement between the shape of each iteration + and the shape of the previous one. + :type: `list` of ndarray + """ + return [np.linalg.norm(s1.points - s2.points, axis=1) + for s1, s2 in zip(self.shapes, self.shapes[1:])] + + def displacements_stats(self, stat_type='mean'): + r""" + A list containing the a statistical metric on the displacement between + the shape of each iteration and the shape of the previous one. + Parameters + ----------- + stat_type : `str` ``{'mean', 'median', 'min', 'max'}``, optional + Specifies a statistic metric to be extracted from the displacements. + Returns + ------- + :type: `list` of `float` + The statistical metric on the points displacements for each + iteration. + """ + if stat_type == 'mean': + return [np.mean(d) for d in self.displacements()] + elif stat_type == 'median': + return [np.median(d) for d in self.displacements()] + elif stat_type == 'max': + return [np.max(d) for d in self.displacements()] + elif stat_type == 'min': + return [np.min(d) for d in self.displacements()] + else: + raise ValueError("type must be 'mean', 'median', 'min' or 'max'") + + def plot_displacements(self, stat_type='mean', figure_id=None, + new_figure=False, render_lines=True, line_colour='b', + line_style='-', line_width=2, render_markers=True, + marker_style='o', marker_size=4, + marker_face_colour='b', marker_edge_colour='k', + marker_edge_width=1., render_axes=True, + axes_font_name='sans-serif', axes_font_size=10, + axes_font_style='normal', axes_font_weight='normal', + figure_size=(10, 6), render_grid=True, + grid_line_style='--', grid_line_width=0.5): + r""" + Plot of a statistical metric of the displacement between the shape of + each iteration and the shape of the previous one. + Parameters + ---------- + stat_type : {``mean``, ``median``, ``min``, ``max``}, optional + Specifies a statistic metric to be extracted from the displacements + (see also `displacements_stats()` method). + figure_id : `object`, optional + The id of the figure to be used. + new_figure : `bool`, optional + If ``True``, a new figure is created. + render_lines : `bool`, optional + If ``True``, the line will be rendered. + line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or + ``(3, )`` `ndarray`, optional + The colour of the lines. + line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the lines. + line_width : `float`, optional + The width of the lines. + render_markers : `bool`, optional + If ``True``, the markers will be rendered. + marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, + ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, + ``1``, ``2``, ``3``, ``4``, ``8``}, optional + The style of the markers. + marker_size : `int`, optional + The size of the markers in points^2. + marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The face (filling) colour of the markers. + marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray`, optional + The edge colour of the markers. + marker_edge_width : `float`, optional + The width of the markers' edge. + render_axes : `bool`, optional + If ``True``, the axes will be rendered. + axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the axes. + axes_font_size : `int`, optional + The font size of the axes. + axes_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the axes. + axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, + ``book``, ``medium``, ``roman``, ``semibold``, + ``demibold``, ``demi``, ``bold``, ``heavy``, + ``extra bold``, ``black``}, optional + The font weight of the axes. + figure_size : (`float`, `float`) or `None`, optional + The size of the figure in inches. + render_grid : `bool`, optional + If ``True``, the grid will be rendered. + grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the grid lines. + grid_line_width : `float`, optional + The width of the grid lines. + Returns + ------- + viewer : :map:`GraphPlotter` + The viewer object. + """ + from menpo.visualize import GraphPlotter + # set labels + if stat_type == 'max': + ylabel = 'Maximum Displacement' + title = 'Maximum displacement per Iteration' + elif stat_type == 'min': + ylabel = 'Minimum Displacement' + title = 'Minimum displacement per Iteration' + elif stat_type == 'mean': + ylabel = 'Mean Displacement' + title = 'Mean displacement per Iteration' + elif stat_type == 'median': + ylabel = 'Median Displacement' + title = 'Median displacement per Iteration' + else: + raise ValueError('stat_type must be one of {max, min, mean, ' + 'median}.') + # plot + displacements_list = self.displacements_stats(stat_type=stat_type) + return GraphPlotter(figure_id=figure_id, new_figure=new_figure, + x_axis=range(len(displacements_list)), + y_axis=[displacements_list], + title=title, + x_label='Iteration', y_label=ylabel, + x_axis_limits=(0, len(displacements_list)-1), + y_axis_limits=None).render( + render_lines=render_lines, line_colour=line_colour, + line_style=line_style, line_width=line_width, + render_markers=render_markers, marker_style=marker_style, + marker_size=marker_size, marker_face_colour=marker_face_colour, + marker_edge_colour=marker_edge_colour, + marker_edge_width=marker_edge_width, render_legend=False, + render_axes=render_axes, axes_font_name=axes_font_name, + axes_font_size=axes_font_size, axes_font_style=axes_font_style, + axes_font_weight=axes_font_weight, render_grid=render_grid, + grid_line_style=grid_line_style, grid_line_width=grid_line_width, + figure_size=figure_size) + + def as_serializableresult(self): + return SerializableIterativeResult( + self.image, self.shapes, self.n_iters, gt_shape=self.gt_shape) + + +# TODO: document me! +class ParametricAlgorithmResult(IterativeResult): + r""" + """ + def __init__(self, image, fitter, shape_parameters, gt_shape=None): + self.image = image + self.fitter = fitter + self.shape_parameters = shape_parameters + self._gt_shape = gt_shape + + @property + def n_iters(self): + return len(self.shapes()) - 1 + + @property + def transforms(self): + r""" + Generates a list containing the transforms obtained at each fitting + iteration. + """ + return [self.fitter.transform.from_vector(p) + for p in self.shape_parameters] + + @property + def final_transform(self): + r""" + Returns the final transform. + """ + return self.fitter.transform.from_vector(self.shape_parameters[-1]) + + @property + def initial_transform(self): + r""" + Returns the initial transform from which the fitting started. + """ + return self.fitter.transform.from_vector(self.shape_parameters[0]) + + def shapes(self, as_points=False): + if as_points: + return [self.fitter.transform.from_vector(p).target.points + for p in self.shape_parameters] + + else: + return [self.fitter.transform.from_vector(p).target + for p in self.shape_parameters] + + @property + def final_shape(self): + return self.final_transform.target + + @property + def initial_shape(self): + return self.initial_transform.target + + +# TODO: document me! +class MultiFitterResult(IterativeResult): + r""" + """ + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(MultiFitterResult, self).__init__() + self.image = image + self.fitter = fitter + self.algorithm_results = algorithm_results + self._affine_correction = affine_correction + self._gt_shape = gt_shape + + @property + def n_levels(self): + r""" + The number of levels of the fitter object. + + :type: `int` + """ + return self.fitter.n_levels + + @property + def scales(self): + return self.fitter.scales + + @property + def n_iters(self): + r""" + The total number of iterations used to fitter the image. + + :type: `int` + """ + n_iters = 0 + for f in self.algorithm_results: + n_iters += f.n_iters + return n_iters + + def shapes(self, as_points=False): + r""" + Generates a list containing the shapes obtained at each fitting + iteration. + + Parameters + ----------- + as_points : `boolean`, optional + Whether the result is returned as a `list` of :map:`PointCloud` or + a `list` of `ndarrays`. + + Returns + ------- + shapes : `list` of :map:`PointCoulds` or `list` of `ndarray` + A list containing the fitted shapes at each iteration of + the fitting procedure. + """ + return _rescale_shapes_to_reference( + self.algorithm_results, self.scales, self._affine_correction) + + @property + def final_shape(self): + r""" + The final fitted shape. + + :type: :map:`PointCloud` + """ + final_shape = self.algorithm_results[-1].final_shape + return self._affine_correction.apply(final_shape) + + @property + def initial_shape(self): + initial_shape = self.algorithm_results[0].initial_shape + Scale(self.scales[-1]/self.scales[0], + initial_shape.n_dims).apply_inplace(initial_shape) + return self._affine_correction.apply(initial_shape) + + +# TODO: document me! +class SerializableIterativeResult(IterativeResult): + r""" + """ + def __init__(self, image, shapes, n_iters, gt_shape=None): + self.image = image + self._gt_shape = gt_shape + self._shapes = shapes + self._n_iters = n_iters + + @property + def n_iters(self): + return self._n_iters + + def shapes(self, as_points=False): + if as_points: + return [s.points for s in self._shapes] + else: + return self._shapes + + @property + def initial_shape(self): + return self._shapes[0] + + @property + def final_shape(self): + return self._shapes[-1] + + +# TODO: Document me! +def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): + r""" + """ + shapes = [] + for j, (alg, s) in enumerate(zip(algorithm_results, scales)): + transform = Scale(scales[-1]/s, alg.final_shape.n_dims) + for t in alg.shapes: + t = transform.apply(t) + shapes.append(affine_correction.apply(t)) + return shapes + + +# TODO: Document me! +def compute_error(target, ground_truth, error_type='me_norm'): + r""" + """ + gt_points = ground_truth.points + target_points = target.points + + if error_type == 'me_norm': + return _compute_me_norm(target_points, gt_points) + elif error_type == 'me': + return _compute_me(target_points, gt_points) + elif error_type == 'rmse': + return _compute_rmse(target_points, gt_points) + else: + raise ValueError("Unknown error_type string selected. Valid options " + "are: me_norm, me, rmse'") + + +# TODO: Document me! +# TODO: rename to more descriptive name +def _compute_me(target, ground_truth): + r""" + """ + return np.mean(np.sqrt(np.sum((target - ground_truth) ** 2, axis=-1))) + + +# TODO: Document me! +# TODO: rename to more descriptive name +def _compute_rmse(target, ground_truth): + r""" + """ + return np.sqrt(np.mean((target.flatten() - ground_truth.flatten()) ** 2)) + + +# TODO: Document me! +# TODO: rename to more descriptive name +def _compute_me_norm(target, ground_truth): + r""" + """ + normalizer = np.mean(np.max(ground_truth, axis=0) - + np.min(ground_truth, axis=0)) + return _compute_me(target, ground_truth) / normalizer + + +# TODO: Document me! +def compute_cumulative_error(errors, x_axis): + r""" + """ + n_errors = len(errors) + return [np.count_nonzero([errors <= x]) / n_errors for x in x_axis] + + +def plot_cumulative_error_distribution(errors, error_range=None, figure_id=None, + new_figure=False, + title='Cumulative Error Distribution', + x_label='Normalized Point-to-Point Error', + y_label='Images Proportion', + legend_entries=None, render_lines=True, + line_colour=None, line_style='-', + line_width=2, render_markers=True, + marker_style='s', marker_size=10, + marker_face_colour='w', + marker_edge_colour=None, + marker_edge_width=2, render_legend=True, + legend_title=None, + legend_font_name='sans-serif', + legend_font_style='normal', + legend_font_size=10, + legend_font_weight='normal', + legend_marker_scale=1., + legend_location=2, + legend_bbox_to_anchor=(1.05, 1.), + legend_border_axes_pad=1., + legend_n_columns=1, + legend_horizontal_spacing=1., + legend_vertical_spacing=1., + legend_border=True, + legend_border_padding=0.5, + legend_shadow=False, + legend_rounded_corners=False, + render_axes=True, + axes_font_name='sans-serif', + axes_font_size=10, + axes_font_style='normal', + axes_font_weight='normal', + axes_x_limits=None, axes_y_limits=None, + figure_size=(10, 8), render_grid=True, + grid_line_style='--', + grid_line_width=0.5): + r""" + Plot the cumulative error distribution (CED) of the provided fitting errors. + + Parameters + ---------- + errors : `list` of `lists` + A `list` with `lists` of fitting errors. A separate CED curve will be + rendered for each errors `list`. + error_range : `list` of `float` with length 3, optional + Specifies the horizontal axis range, i.e. + + :: + + error_range[0] = min_error + error_range[1] = max_error + error_range[2] = error_step + + If ``None``, then ``'error_range = [0., 0.101, 0.005]'``. + figure_id : `object`, optional + The id of the figure to be used. + new_figure : `bool`, optional + If ``True``, a new figure is created. + title : `str`, optional + The figure's title. + x_label : `str`, optional + The label of the horizontal axis. + y_label : `str`, optional + The label of the vertical axis. + legend_entries : `list of `str` or ``None``, optional + If `list` of `str`, it must have the same length as `errors` `list` and + each `str` will be used to name each curve. If ``None``, the CED curves + will be named as `'Curve %d'`. + render_lines : `bool` or `list` of `bool`, optional + If ``True``, the line will be rendered. If `bool`, this value will be + used for all curves. If `list`, a value must be specified for each + fitting errors curve, thus it must have the same length as `errors`. + line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or + ``(3, )`` `ndarray` or `list` of those or ``None``, optional + The colour of the lines. If not a `list`, this value will be + used for all curves. If `list`, a value must be specified for each + fitting errors curve, thus it must have the same length as `errors`. If + ``None``, the colours will be linearly sampled from jet colormap. + line_style : {``-``, ``--``, ``-.``, ``:``} or `list` of those, optional + The style of the lines. If not a `list`, this value will be used for all + curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + line_width : `float` or `list` of `float`, optional + The width of the lines. If `float`, this value will be used for all + curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + render_markers : `bool` or `list` of `bool`, optional + If ``True``, the markers will be rendered. If `bool`, this value will be + used for all curves. If `list`, a value must be specified for each + fitting errors curve, thus it must have the same length as `errors`. + marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, + ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, + ``1``, ``2``, ``3``, ``4``, ``8``} or `list` of those, optional + The style of the markers. If not a `list`, this value will be used for + all curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + marker_size : `int` or `list` of `int`, optional + The size of the markers in points^2. If `int`, this value will be used + for all curves. If `list`, a value must be specified for each fitting + errors curve, thus it must have the same length as `errors`. + marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray` or `list` of those or ``None``, optional + The face (filling) colour of the markers. If not a `list`, this value + will be used for all curves. If `list`, a value must be specified for + each fitting errors curve, thus it must have the same length as + `errors`. If ``None``, the colours will be linearly sampled from jet + colormap. + marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} + or ``(3, )`` `ndarray` or `list` of those or ``None``, optional + The edge colour of the markers. If not a `list`, this value will be used + for all curves. If `list`, a value must be specified for each fitting + errors curve, thus it must have the same length as `errors`. If + ``None``, the colours will be linearly sampled from jet colormap. + marker_edge_width : `float` or `list` of `float`, optional + The width of the markers' edge. If `float`, this value will be used for + all curves. If `list`, a value must be specified for each fitting errors + curve, thus it must have the same length as `errors`. + render_legend : `bool`, optional + If ``True``, the legend will be rendered. + legend_title : `str`, optional + The title of the legend. + legend_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the legend. + legend_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the legend. + legend_font_size : `int`, optional + The font size of the legend. + legend_font_weight : {``ultralight``, ``light``, ``normal``, + ``regular``, ``book``, ``medium``, ``roman``, + ``semibold``, ``demibold``, ``demi``, ``bold``, + ``heavy``, ``extra bold``, ``black``}, optional + The font weight of the legend. + legend_marker_scale : `float`, optional + The relative size of the legend markers with respect to the original + legend_location : `int`, optional + The location of the legend. The predefined values are: + + =============== === + 'best' 0 + 'upper right' 1 + 'upper left' 2 + 'lower left' 3 + 'lower right' 4 + 'right' 5 + 'center left' 6 + 'center right' 7 + 'lower center' 8 + 'upper center' 9 + 'center' 10 + =============== === + + legend_bbox_to_anchor : (`float`, `float`), optional + The bbox that the legend will be anchored. + legend_border_axes_pad : `float`, optional + The pad between the axes and legend border. + legend_n_columns : `int`, optional + The number of the legend's columns. + legend_horizontal_spacing : `float`, optional + The spacing between the columns. + legend_vertical_spacing : `float`, optional + The vertical space between the legend entries. + legend_border : `bool`, optional + If ``True``, a frame will be drawn around the legend. + legend_border_padding : `float`, optional + The fractional whitespace inside the legend border. + legend_shadow : `bool`, optional + If ``True``, a shadow will be drawn behind legend. + legend_rounded_corners : `bool`, optional + If ``True``, the frame's corners will be rounded (fancybox). + render_axes : `bool`, optional + If ``True``, the axes will be rendered. + axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, + ``monospace``}, optional + The font of the axes. + axes_font_size : `int`, optional + The font size of the axes. + axes_font_style : {``normal``, ``italic``, ``oblique``}, optional + The font style of the axes. + axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, + ``book``, ``medium``, ``roman``, ``semibold``, + ``demibold``, ``demi``, ``bold``, ``heavy``, + ``extra bold``, ``black``}, optional + The font weight of the axes. + axes_x_limits : (`float`, `float`) or ``None``, optional + The limits of the x axis. If ``None``, it is set to + ``(0., 'errors_max')``. + axes_y_limits : (`float`, `float`) or ``None``, optional + The limits of the y axis. If ``None``, it is set to ``(0., 1.)``. + figure_size : (`float`, `float`) or ``None``, optional + The size of the figure in inches. + render_grid : `bool`, optional + If ``True``, the grid will be rendered. + grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional + The style of the grid lines. + grid_line_width : `float`, optional + The width of the grid lines. + + Raises + ------ + ValueError + legend_entries list has different length than errors list + + Returns + ------- + viewer : :map:`GraphPlotter` + The viewer object. + """ + from menpo.visualize import GraphPlotter + + # make sure that errors is a list even with one list member + if not isinstance(errors[0], list): + errors = [errors] + + # create x and y axes lists + x_axis = list(np.arange(error_range[0], error_range[1], error_range[2])) + ceds = [compute_cumulative_error(e, x_axis) for e in errors] + + # parse legend_entries, axes_x_limits and axes_y_limits + if legend_entries is None: + legend_entries = ["Curve {}".format(k) for k in range(len(ceds))] + if len(legend_entries) != len(ceds): + raise ValueError('legend_entries list has different length than errors ' + 'list') + if axes_x_limits is None: + axes_x_limits = (0., x_axis[-1]) + if axes_y_limits is None: + axes_y_limits = (0., 1.) + + # render + return GraphPlotter(figure_id=figure_id, new_figure=new_figure, + x_axis=x_axis, y_axis=ceds, title=title, + legend_entries=legend_entries, x_label=x_label, + y_label=y_label, x_axis_limits=axes_x_limits, + y_axis_limits=axes_y_limits).render( + render_lines=render_lines, line_colour=line_colour, + line_style=line_style, line_width=line_width, + render_markers=render_markers, marker_style=marker_style, + marker_size=marker_size, marker_face_colour=marker_face_colour, + marker_edge_colour=marker_edge_colour, + marker_edge_width=marker_edge_width, render_legend=render_legend, + legend_title=legend_title, legend_font_name=legend_font_name, + legend_font_style=legend_font_style, legend_font_size=legend_font_size, + legend_font_weight=legend_font_weight, + legend_marker_scale=legend_marker_scale, + legend_location=legend_location, + legend_bbox_to_anchor=legend_bbox_to_anchor, + legend_border_axes_pad=legend_border_axes_pad, + legend_n_columns=legend_n_columns, + legend_horizontal_spacing=legend_horizontal_spacing, + legend_vertical_spacing=legend_vertical_spacing, + legend_border=legend_border, + legend_border_padding=legend_border_padding, + legend_shadow=legend_shadow, + legend_rounded_corners=legend_rounded_corners, render_axes=render_axes, + axes_font_name=axes_font_name, axes_font_size=axes_font_size, + axes_font_style=axes_font_style, axes_font_weight=axes_font_weight, + figure_size=figure_size, render_grid=render_grid, + grid_line_style=grid_line_style, grid_line_width=grid_line_width) From a12b5af0e868c05a02365ea1981f5512b340f82b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 14:34:23 +0100 Subject: [PATCH 224/423] Add results for AAMs --- menpofit/aam/result.py | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 menpofit/aam/result.py diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py new file mode 100644 index 0000000..5824945 --- /dev/null +++ b/menpofit/aam/result.py @@ -0,0 +1,53 @@ +from __future__ import division +from menpofit.result import ( + ParametricAlgorithmResult, MultiFitterResult, SerializableIterativeResult) + + +# TODO: document me! +# TODO: handle costs +class AAMAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + def __init__(self, image, fitter, shape_parameters, + appearance_parameters=None, gt_shape=None): + super(AAMAlgorithmResult, self).__init__( + image, fitter, shape_parameters, gt_shape=gt_shape) + self.appearance_parameters = appearance_parameters + + +# TODO: document me! +class LinearAAMAlgorithmResult(AAMAlgorithmResult): + r""" + """ + def shapes(self, as_points=False): + if as_points: + return [self.fitter.transform.from_vector(p).sparse_target.points + for p in self.shape_parameters] + + else: + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] + + @property + def final_shape(self): + return self.final_transform.sparse_target + + @property + def initial_shape(self): + return self.initial_transform.sparse_target + + +# TODO: document me! +# TODO: handle costs +class AAMFitterResult(MultiFitterResult): + r""" + """ + pass + + +# TODO: document me! +# TODO: handle costs +class SerializableAAMFitterResult(SerializableIterativeResult): + r""" + """ + pass From d81760e9811608f1f2b994c2e3568ad200dda765 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 15:04:34 +0100 Subject: [PATCH 225/423] Add small fix in aam.fitter --- menpofit/aam/fitter.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 957bead..9bee842 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -3,8 +3,7 @@ from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM -from .algorithm import ( - StandardAAMInterface, LinearAAMInterface, PartsAAMInterface, AIC) +from .algorithm import AAMInterface, LinearAAMInterface, PartsAAMInterface, AIC from .result import AAMFitterResult @@ -15,8 +14,8 @@ class LKAAMFitter(ModelFitter): def __init__(self, aam, algorithm_cls=AIC, n_shape=None, n_appearance=None, **kwargs): super(LKAAMFitter, self).__init__() + self.algorithms = [] self._model = aam - self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) self._set_up(algorithm_cls, **kwargs) @@ -31,8 +30,8 @@ def _set_up(self, algorithm_cls, **kwargs): sm, self._model.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(StandardAAMInterface, am, - md_transform, **kwargs) + algorithm = algorithm_cls(AAMInterface, am, md_transform, + **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -58,7 +57,7 @@ def _set_up(self, algorithm_cls, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) @property def aam(self): From 4671bd79fe0d4c1899430bc2cbeb5ab588fc2311 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 15:19:38 +0100 Subject: [PATCH 226/423] Small corrections to fitters --- menpofit/aam/fitter.py | 10 ++++++---- menpofit/fitter.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 9bee842..4a6d010 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -13,9 +13,8 @@ class LKAAMFitter(ModelFitter): """ def __init__(self, aam, algorithm_cls=AIC, n_shape=None, n_appearance=None, **kwargs): - super(LKAAMFitter, self).__init__() - self.algorithms = [] - self._model = aam + super(LKAAMFitter, self).__init__(aam) + self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) self._set_up(algorithm_cls, **kwargs) @@ -57,12 +56,15 @@ def _set_up(self, algorithm_cls, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self.algorithms.append(algorithm) + self._algorithms.append(algorithm) @property def aam(self): return self._model + def algorithms(self): + return self._algorithms + def _check_n_appearance(self, n_appearance): if n_appearance is not None: if type(n_appearance) is int or type(n_appearance) is float: diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 64f7df8..24252c1 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,4 +1,5 @@ from __future__ import division +import abc import numpy as np from menpo.shape import PointCloud from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity @@ -17,6 +18,26 @@ def n_levels(self): """ return len(self.scales) + @abc.abstractproperty + def algorithms(self): + pass + + @abc.abstractproperty + def reference_shape(self): + pass + + @abc.abstractproperty + def scales(self): + pass + + @abc.abstractproperty + def features(self): + pass + + @abc.abstractproperty + def scale_features(self): + pass + def fit(self, image, initial_shape, max_iters=50, gt_shape=None, crop_image=0.5, **kwargs): r""" @@ -241,11 +262,19 @@ def _prepare_max_iters(self, max_iters): 'None'.format(self.n_levels)) return np.require(max_iters, dtype=np.int) + @abc.abstractmethod + def _fitter_result(self): + pass + + # TODO: document me! class ModelFitter(MultiFitter): r""" """ + def __init__(self, model): + self._model = model + @property def reference_shape(self): r""" @@ -330,7 +359,6 @@ def obtain_shape_from_bb(self, bounding_box): initial_shape: :class:`menpo.shape.PointCloud` The initial shape. """ - reference_shape = self.reference_shape return align_shape_with_bb(reference_shape, bounding_box).apply(reference_shape) From 6570e0408864e66e8ac0118101a7c6f72e98d749 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 15:26:20 +0100 Subject: [PATCH 227/423] More corrections to fitters --- menpofit/aam/fitter.py | 1 + menpofit/fitter.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 4a6d010..594a985 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -62,6 +62,7 @@ def _set_up(self, algorithm_cls, **kwargs): def aam(self): return self._model + @property def algorithms(self): return self._algorithms diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 24252c1..f88fbcb 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -101,8 +101,7 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, # build multilevel fitting result fitter_result = self._fitter_result( - image, self, algorithm_results, affine_correction, - gt_shape=gt_shape) + image, algorithm_results, affine_correction, gt_shape=gt_shape) return fitter_result @@ -263,7 +262,8 @@ def _prepare_max_iters(self, max_iters): return np.require(max_iters, dtype=np.int) @abc.abstractmethod - def _fitter_result(self): + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): pass From f8e3918c6d344cf594c423059b2200572aec6f30 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:14:42 +0100 Subject: [PATCH 228/423] Add basic AAM algorithms - Add PFC, PIC, SFC, SIC, AFC, AIC, MAFC, MAIC, WFC, WIC --- menpofit/aam/algorithm.py | 886 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 886 insertions(+) create mode 100644 menpofit/aam/algorithm.py diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py new file mode 100644 index 0000000..44d4cc5 --- /dev/null +++ b/menpofit/aam/algorithm.py @@ -0,0 +1,886 @@ +from __future__ import division +import abc +import numpy as np +from menpo.image import Image +from menpo.feature import gradient as fast_gradient +from .result import AAMAlgorithmResult, LinearAAMAlgorithmResult + + +class AAMInterface(object): + + def __init__(self, aam_algorithm, sampling_step=None): + self.algorithm = aam_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling_step is None: + sampling_step = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling_step) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def shape_model(self): + return self.transform.pdm.model + + @property + def appearance_model(self): + return self.algorithm.appearance_model + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def m(self): + return self.appearance_model.n_active_components + + @property + def true_indices(self): + return self.template.mask.true_indices() + + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + + def partial_newton_hessian(self, nabla2, dw_dp): + # reshape gradient + # gradient: n_dims x n_dims x n_channels x n_pixels + nabla2 = nabla2[self.nabla2_mask].reshape( + (2,) + nabla2.shape[:2] + (-1,)) + + # compute partial hessian + # gradient: n_dims x n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # h: n_dims x n_channels x n_pixels x n_params + h1 = 0 + aux = nabla2[..., None] * dw_dp[:, None, None, ...] + for d in aux: + h1 += d + # compute partial hessian + # h: n_dims x n_channels x n_pixels x n_params + # warp_jacobian: n_dims x x n_pixels x x n_params + # h: + h2 = 0 + aux = h1[..., None] * dw_dp[..., None, :, None, :] + for d in aux: + h2 += d + + # reshape hessian + # 2: (n_channels x n_pixels) x n_params x n_params + return h2.reshape((-1, h2.shape[3] * h2.shape[4])) + + @classmethod + def solve_shape_map(cls, H, J, e, J_prior, p): + if p.shape[0] is not H.shape[0]: + # Bidirectional Compositional case + J_prior = np.hstack((J_prior, J_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + H += np.diag(J_prior) + Je = J_prior * p + J.T.dot(e) + return - np.linalg.solve(H, Je) + + @classmethod + def solve_shape_ml(cls, H, J, e): + # compute and return ML solution + return -np.linalg.solve(H, J.T.dot(e)) + + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): + if self.n is not H.shape[0] - self.m: + # Bidirectional Compositional case + Js_prior = np.hstack((Js_prior, Js_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + J_prior = np.hstack((Ja_prior, Js_prior)) + H += np.diag(J_prior) + Je = J_prior * np.hstack((c, p)) + J.T.dot(e) + dq = - np.linalg.solve(H, Je) + return dq[:self.m], dq[self.m:] + + def solve_all_ml(self, H, J, e): + # compute ML solution + dq = - np.linalg.solve(H, J.T.dot(e)) + return dq[:self.m], dq[self.m:] + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class LinearAAMInterface(AAMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return LinearAAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class PartsAAMInterface(AAMInterface): + + def __init__(self, aam_algorithm, sampling_mask=None): + self.algorithm = aam_algorithm + + if sampling_mask is None: + sampling_mask = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling_mask[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.gradient_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.gradient2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) + + @property + def shape_model(self): + return self.transform.model + + @property + def patch_shape(self): + return self.appearance_model.patch_shape + + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + def warp(self, image): + return Image(image.extract_patches( + self.transform.target, patch_size=self.patch_shape, + as_single_array=True)) + + def gradient(self, image): + pixels = image.pixels + patch_shape = self.algorithm.appearance_model.patch_shape + g = fast_gradient(pixels.reshape((-1,) + patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return g.reshape((2,) + pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.gradient_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) + + def partial_newton_hessian(self, nabla2, dw_dp): + # reshape gradient + # gradient: dims x dims x parts x off x ch x (h x w) + nabla2 = nabla2[self.gradient2_mask].reshape( + nabla2.shape[:-2] + (-1,)) + + # compute partial hessian + # gradient: dims x dims x parts x off x ch x (h x w) + # dw_dp: dims x x parts x x params + # h: dims x parts x off x ch x (h x w) x params + h1 = 0 + aux = nabla2[..., None] * dw_dp[:, None, :, None, None, None, ...] + for d in aux: + h1 += d + # compute partial hessian + # h: dims x parts x off x ch x (h x w) x params + # dw_dp: dims x parts x x params + # h: + h2 = 0 + aux = h1[..., None] * dw_dp[..., None, None, None, None, :] + for d in aux: + h2 += d + + # reshape hessian + # 2: (parts x off x ch x w x h) x params x params + return h2.reshape((-1, h2.shape[-2] * h2.shape[-1])) + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class AAMAlgorithm(object): + + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # set common state for all AAM algorithms + self.appearance_model = appearance_model + self.template = appearance_model.mean() + self.transform = transform + self.eps = eps + # set interface + self.interface = aam_interface(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self, **kwargs): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + self.m = self.appearance_model.n_active_components + + # grab appearance model components + self.A = self.appearance_model.components + # mask them + self.A_m = self.A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + + # grab appearance model mean + self.a_bar = self.appearance_model.mean() + # vectorize it and mask it + self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] + + # compute warp jacobian + self.dW_dp = self.interface.warp_jacobian() + + # compute shape model prior + s2 = (self.appearance_model.noise_variance() / + self.interface.shape_model.noise_variance()) + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + # compute appearance model prior + S = self.appearance_model.eigenvalues + self.s2_inv_S = s2 / S + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + pass + + +class ProjectOut(AAMAlgorithm): + r""" + Abstract Interface for Project-out AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(ProjectOut, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # solve for increments on the shape parameters + self.dp = self.solve(map_inference) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + @abc.abstractmethod + def solve(self, map_inference): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class PFC(ProjectOut): + r""" + Project-out Forward Compositional (PFC) Gauss-Newton algorithm + """ + def solve(self, map_inference): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # compute masked forward Jacobian + J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) + # project out appearance model from it + QJ_m = self.project_out(J_m) + # compute masked forward Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + JQJ_m, QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class PIC(ProjectOut): + r""" + Project-out Inverse Compositional (PIC) Gauss-Newton algorithm + """ + def precompute(self): + # call super method + super(PIC, self).precompute() + # compute appearance model mean gradient + nabla_a = self.interface.gradient(self.a_bar) + # compute masked inverse Jacobian + J_m = self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + # project out appearance model from it + self.QJ_m = self.project_out(J_m) + # compute masked inverse Hessian + self.JQJ_m = self.QJ_m.T.dot(J_m) + # compute masked Jacobian pseudo-inverse + self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) + + def solve(self, map_inference): + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + self.JQJ_m, self.QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return -self.pinv_QJ_m.dot(self.e_m) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Simultaneous(AAMAlgorithm): + r""" + Abstract Interface for Simultaneous AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(Simultaneous, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [self.c] + + # compute masked error + self.e_m = i_m - a_m + + # solve for increments on the appearance and shape parameters + # simultaneously + dc, self.dp = self.solve(map_inference) + + # update appearance parameters + self.c += dc + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(self.c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + def solve(self, map_inference): + # compute masked Jacobian + J_m = self.compute_jacobian() + # assemble masked simultaneous Jacobian + J_sim_m = np.hstack((-self.A_m, J_m)) + # compute masked Hessian + H_sim_m = J_sim_m.T.dot(J_sim_m) + # solve for increments on the appearance and shape parameters + # simultaneously + if map_inference: + return self.interface.solve_all_map( + H_sim_m, J_sim_m, self.e_m, self.s2_inv_S, self.c, + self.s2_inv_L, self.transform.as_vector()) + else: + return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) + + @abc.abstractmethod + def update_warp(self): + pass + + +class SFC(Simultaneous): + r""" + Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class SIC(Simultaneous): + r""" + Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Alternating(AAMAlgorithm): + r""" + Abstract Interface for Alternating AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(Alternating, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def precompute(self, **kwargs): + # call super method + super(Alternating, self).precompute() + # compute MAP appearance Hessian + self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + Jdp = 0 + else: + Jdp = J_m.dot(self.dp) + + # compute masked error + e_m = i_m - a_m + + # solve for increment on the appearance parameters + if map_inference: + Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) + dc = np.linalg.solve(self.AA_m_map, Ae_m_map) + else: + dc = self.pinv_A_m.dot(e_m + Jdp) + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m - self.A_m.T.dot(dc), self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, + e_m - self.A_m.dot(dc)) + + # update appearance parameters + c += dc + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class AFC(Alternating): + r""" + Alternating Forward Compositional (AFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class AIC(Alternating): + r""" + Alternating Inverse Compositional (AIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class ModifiedAlternating(Alternating): + r""" + Abstract Interface for Modified Alternating AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(ModifiedAlternating, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + a_m = self.a_bar_m + c_list = [] + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + c = self.pinv_A_m.dot(i_m - a_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m, self.s2_inv_L, self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class MAFC(ModifiedAlternating): + r""" + Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class MAIC(ModifiedAlternating): + r""" + Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Wiberg(AAMAlgorithm): + r""" + Abstract Interface for Wiberg AAM algorithms + """ + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # call super constructor + super(Wiberg, self).__init__( + aam_interface, appearance_model, transform, eps, **kwargs) + + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + else: + c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - self.a_bar_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # project out appearance models + QJ_m = self.project_out(J_m) + # compute masked Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + JQJ_m, QJ_m, e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(JQJ_m, QJ_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class WFC(Wiberg): + r""" + Wiberg Forward Compositional (WFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class WIC(Wiberg): + r""" + Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + From b899e41a5c954a5d46753e2e09fb5b2a7188f3b5 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:17:23 +0100 Subject: [PATCH 229/423] Update aam.__init__.py --- menpofit/aam/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 37bd629..119cff0 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,3 +1,11 @@ -from .base import AAM, PatchBasedAAM -from .builder import AAMBuilder, PatchBasedAAMBuilder -from .fitter import LucasKanadeAAMFitter +from .builder import ( + AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, + LinearPatchAAMBuilder, PartsAAMBuilder) +from .fitter import LKAAMFitter +from .algorithm import ( + PFC, PIC, + SFC, SIC, + AFC, AIC, + MAFC, MAIC, + WFC, WIC) +from .result import SerializableAAMFitterResult From 3b155c9741793940de74b8b9aa407118897e323e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:21:48 +0100 Subject: [PATCH 230/423] Slight modification to aam.algorithm --- menpofit/aam/algorithm.py | 54 --------------------------------------- 1 file changed, 54 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 44d4cc5..1e84871 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -88,33 +88,6 @@ def steepest_descent_images(self, nabla, dW_dp): # sdi: (n_channels x n_pixels) x n_params return sdi.reshape((-1, sdi.shape[2])) - def partial_newton_hessian(self, nabla2, dw_dp): - # reshape gradient - # gradient: n_dims x n_dims x n_channels x n_pixels - nabla2 = nabla2[self.nabla2_mask].reshape( - (2,) + nabla2.shape[:2] + (-1,)) - - # compute partial hessian - # gradient: n_dims x n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # h: n_dims x n_channels x n_pixels x n_params - h1 = 0 - aux = nabla2[..., None] * dw_dp[:, None, None, ...] - for d in aux: - h1 += d - # compute partial hessian - # h: n_dims x n_channels x n_pixels x n_params - # warp_jacobian: n_dims x x n_pixels x x n_params - # h: - h2 = 0 - aux = h1[..., None] * dw_dp[..., None, :, None, :] - for d in aux: - h2 += d - - # reshape hessian - # 2: (n_channels x n_pixels) x n_params x n_params - return h2.reshape((-1, h2.shape[3] * h2.shape[4])) - @classmethod def solve_shape_map(cls, H, J, e, J_prior, p): if p.shape[0] is not H.shape[0]: @@ -227,33 +200,6 @@ def steepest_descent_images(self, nabla, dw_dp): # sdi: (parts x offsets x ch x w x h) x params return sdi.reshape((-1, sdi.shape[-1])) - def partial_newton_hessian(self, nabla2, dw_dp): - # reshape gradient - # gradient: dims x dims x parts x off x ch x (h x w) - nabla2 = nabla2[self.gradient2_mask].reshape( - nabla2.shape[:-2] + (-1,)) - - # compute partial hessian - # gradient: dims x dims x parts x off x ch x (h x w) - # dw_dp: dims x x parts x x params - # h: dims x parts x off x ch x (h x w) x params - h1 = 0 - aux = nabla2[..., None] * dw_dp[:, None, :, None, None, None, ...] - for d in aux: - h1 += d - # compute partial hessian - # h: dims x parts x off x ch x (h x w) x params - # dw_dp: dims x parts x x params - # h: - h2 = 0 - aux = h1[..., None] * dw_dp[..., None, None, None, None, :] - for d in aux: - h2 += d - - # reshape hessian - # 2: (parts x off x ch x w x h) x params x params - return h2.reshape((-1, h2.shape[-2] * h2.shape[-1])) - def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( From d1bfa8c13971929dcd0b03f4a830a48e0f4d73d7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:37:32 +0100 Subject: [PATCH 231/423] Unify AAM sampling --- menpofit/aam/algorithm.py | 16 ++++++++-------- menpofit/aam/fitter.py | 14 ++++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 1e84871..58792cf 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -8,7 +8,7 @@ class AAMInterface(object): - def __init__(self, aam_algorithm, sampling_step=None): + def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm n_true_pixels = self.template.n_true_pixels() @@ -16,9 +16,9 @@ def __init__(self, aam_algorithm, sampling_step=None): n_parameters = self.transform.n_parameters sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - if sampling_step is None: - sampling_step = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling_step) + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) sampling_mask[sampling_pattern] = 1 self.i_mask = np.nonzero(np.tile( @@ -143,14 +143,14 @@ def algorithm_result(self, image, shape_parameters, class PartsAAMInterface(AAMInterface): - def __init__(self, aam_algorithm, sampling_mask=None): + def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm - if sampling_mask is None: - sampling_mask = np.ones(self.patch_shape, dtype=np.bool) + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling_mask[None, None, None, ...], + image_mask = np.tile(sampling[None, None, None, ...], image_shape[:3] + (1, 1)) self.i_mask = np.nonzero(image_mask.flatten())[0] self.gradient_mask = np.nonzero(np.tile( diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 594a985..9fe1811 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -12,14 +12,14 @@ class LKAAMFitter(ModelFitter): r""" """ def __init__(self, aam, algorithm_cls=AIC, n_shape=None, - n_appearance=None, **kwargs): + n_appearance=None, sampling=None, **kwargs): super(LKAAMFitter, self).__init__(aam) self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - self._set_up(algorithm_cls, **kwargs) + self._set_up(algorithm_cls, sampling, **kwargs) - def _set_up(self, algorithm_cls, **kwargs): + def _set_up(self, algorithm_cls, sampling, **kwargs): for j, (am, sm) in enumerate(zip(self._model.appearance_models, self._model.shape_models)): @@ -30,7 +30,7 @@ def _set_up(self, algorithm_cls, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = algorithm_cls(AAMInterface, am, md_transform, - **kwargs) + sampling=sampling, **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -39,7 +39,8 @@ def _set_up(self, algorithm_cls, **kwargs): sm, self._model.n_landmarks) # set up algorithm using linear aam interface algorithm = algorithm_cls(LinearAAMInterface, am, - md_transform, **kwargs) + md_transform, sampling=sampling, + **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model @@ -47,7 +48,8 @@ def _set_up(self, algorithm_cls, **kwargs): # set up algorithm using parts aam interface am.patch_shape = self._model.patch_shape[j] am.normalize_parts = self._model.normalize_parts - algorithm = algorithm_cls(PartsAAMInterface, am, pdm, **kwargs) + algorithm = algorithm_cls(PartsAAMInterface, am, pdm, + sampling=sampling, **kwargs) else: raise ValueError("AAM object must be of one of the " From 92aea20facede847ef788c9571fabd8c39e23b22 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:51:36 +0100 Subject: [PATCH 232/423] Small changes to fitter.py --- menpofit/aam/algorithm.py | 1 + menpofit/fitter.py | 13 ++----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 58792cf..aa6882a 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -6,6 +6,7 @@ from .result import AAMAlgorithmResult, LinearAAMAlgorithmResult +# TODO: implement more clever sampling? class AAMInterface(object): def __init__(self, aam_algorithm, sampling=None): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index f88fbcb..5717949 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -27,11 +27,11 @@ def reference_shape(self): pass @abc.abstractproperty - def scales(self): + def features(self): pass @abc.abstractproperty - def features(self): + def scales(self): pass @abc.abstractproperty @@ -294,15 +294,6 @@ def features(self): """ return self._model.features - @property - def n_levels(self): - r""" - The number of pyramidal levels used during AAM building. - - :type: `int` - """ - return self._model.n_levels - @property def scales(self): return self._model.scales From ae959f515f9fa57d22849918e60390a5436bb423 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 16:56:35 +0100 Subject: [PATCH 233/423] Small renaming - AAMAlgorithm -> LKAAMAlgorithm - AAMInterface -> LKAAMInterface --- menpofit/aam/algorithm.py | 18 ++++++++++-------- menpofit/aam/fitter.py | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index aa6882a..9a55ff1 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -7,7 +7,7 @@ # TODO: implement more clever sampling? -class AAMInterface(object): +class LKAAMInterface(object): def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm @@ -129,7 +129,7 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class LinearAAMInterface(AAMInterface): +class LinearLKAAMInterface(LKAAMInterface): @property def shape_model(self): @@ -142,7 +142,7 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class PartsAAMInterface(AAMInterface): +class PartsLKAAMInterface(LKAAMInterface): def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm @@ -208,7 +208,9 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class AAMAlgorithm(object): +# TODO: handle costs for all LKAAMAlgorithms +# TODO document me! +class LKAAMAlgorithm(object): def __init__(self, aam_interface, appearance_model, transform, eps=10**-5, **kwargs): @@ -257,7 +259,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, pass -class ProjectOut(AAMAlgorithm): +class ProjectOut(LKAAMAlgorithm): r""" Abstract Interface for Project-out AAM algorithms """ @@ -378,7 +380,7 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Simultaneous(AAMAlgorithm): +class Simultaneous(LKAAMAlgorithm): r""" Abstract Interface for Simultaneous AAM algorithms """ @@ -498,7 +500,7 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Alternating(AAMAlgorithm): +class Alternating(LKAAMAlgorithm): r""" Abstract Interface for Alternating AAM algorithms """ @@ -723,7 +725,7 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Wiberg(AAMAlgorithm): +class Wiberg(LKAAMAlgorithm): r""" Abstract Interface for Wiberg AAM algorithms """ diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 9fe1811..f693409 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -3,7 +3,8 @@ from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM -from .algorithm import AAMInterface, LinearAAMInterface, PartsAAMInterface, AIC +from .algorithm import ( + LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) from .result import AAMFitterResult @@ -29,7 +30,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): sm, self._model.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(AAMInterface, am, md_transform, + algorithm = algorithm_cls(LKAAMInterface, am, md_transform, sampling=sampling, **kwargs) elif (type(self.aam) is LinearAAM or @@ -38,7 +39,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self._model.n_landmarks) # set up algorithm using linear aam interface - algorithm = algorithm_cls(LinearAAMInterface, am, + algorithm = algorithm_cls(LinearLKAAMInterface, am, md_transform, sampling=sampling, **kwargs) @@ -48,7 +49,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): # set up algorithm using parts aam interface am.patch_shape = self._model.patch_shape[j] am.normalize_parts = self._model.normalize_parts - algorithm = algorithm_cls(PartsAAMInterface, am, pdm, + algorithm = algorithm_cls(PartsLKAAMInterface, am, pdm, sampling=sampling, **kwargs) else: From 75305c37cddadfff1614a2e4bee6806bc32fe073 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 17:56:31 +0100 Subject: [PATCH 234/423] Modify noisy_align in fitter.py --- menpofit/aam/fitter.py | 1 - menpofit/fitter.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f693409..a2080c4 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -89,4 +89,3 @@ def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): return AAMFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 5717949..7094fe2 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -267,7 +267,7 @@ def _fitter_result(self, image, algorithm_results, affine_correction, pass - +# TODO: correctly implement initialization from bounding box # TODO: document me! class ModelFitter(MultiFitter): r""" @@ -356,13 +356,12 @@ def obtain_shape_from_bb(self, bounding_box): # TODO: document me! -def noisy_align(alignment_transform_cls, source, target, noise_std=0.04, - rotation=True): +def noisy_align(alignment_transform_cls, source, target, noise_std=10): r""" """ noise = noise_std * np.random.randn(target.n_points, target.n_dims) noisy_target = PointCloud(target.points + noise) - return alignment_transform_cls(source, noisy_target, rotation=rotation) + return alignment_transform_cls(source, noisy_target) def align_shape_with_bb(shape, bounding_box): From 0e1f951f273484b4b5686294d40f2a9927bc065a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 17:59:10 +0100 Subject: [PATCH 235/423] Add LKFitter --- menpofit/lk/fitter.py | 108 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 menpofit/lk/fitter.py diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py new file mode 100644 index 0000000..5b7d69c --- /dev/null +++ b/menpofit/lk/fitter.py @@ -0,0 +1,108 @@ +from __future__ import division +from menpo.feature import no_op +from menpofit.transform import DifferentiableAlignmentAffine +from menpofit.fitter import MultiFitter, noisy_align +from menpofit.result import MultiFitterResult +from .algorithm import IC +from .residual import SSD, FourierSSD + + +# TODO: document me! +class LKFitter(MultiFitter): + r""" + """ + def __init__(self, template, group=None, label=None, features=no_op, + transform_cls=DifferentiableAlignmentAffine, diagonal=None, + scales=(1, .5), scale_features=True, algorithm_cls=IC, + residual_cls=SSD, **kwargs): + self._features = features + self.transform_cls = transform_cls + self.diagonal = diagonal + self._scales = list(scales) + self._scales.reverse() + self._scale_features = scale_features + + self.templates, self.sources = self._prepare_template( + template, group=group, label=label) + + self._reference_shape = self.sources[0] + + self._algorithms = [] + for j, (t, s) in enumerate(zip(self.templates, self.sources)): + transform = self.transform_cls(s, s) + if ('kernel_func' in kwargs and + (residual_cls is SSD or + residual_cls is FourierSSD)): + kernel_func = kwargs.pop('kernel_func') + kernel = kernel_func(t.shape) + residual = residual_cls(kernel=kernel) + else: + residual = residual_cls() + algorithm = algorithm_cls(t, transform, residual, **kwargs) + self._algorithms.append(algorithm) + + @property + def algorithms(self): + return self._algorithms + + @property + def reference_shape(self): + return self._reference_shape + + @property + def features(self): + return self._features + + @property + def scales(self): + return self._scales + + @property + def scale_features(self): + return self._scale_features + + def _prepare_template(self, template, group=None, label=None): + # copy template + template = template.copy() + + template = template.crop_to_landmarks_inplace(group=group, label=label) + template = template.as_masked() + + # rescale template to diagonal range + if self.diagonal: + template = template.rescale_landmarks_to_diagonal_range( + self.diagonal, group=group, label=label) + + # obtain image representation + from copy import deepcopy + scales = deepcopy(self.scales) + scales.reverse() + templates = [] + for j, s in enumerate(scales): + if j == 0: + # compute features at highest level + feature_template = self.features(template) + elif self.scale_features: + # scale features at other levels + feature_template = templates[0].rescale(s) + else: + # scale image and compute features at other levels + scaled_template = template.rescale(s) + feature_template = self.features(scaled_template) + templates.append(feature_template) + templates.reverse() + + # get sources per level + sources = [i.landmarks[group][label] for i in templates] + + return templates, sources + + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return MultiFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) + + def perturb_shape(self, gt_shape, noise_std=0.04): + transform = noisy_align(self.transform_cls, self.reference_shape, + gt_shape, noise_std=noise_std) + return transform.apply(self.reference_shape) From 86b107f85c25d303612f44797e2b15a1e84006dd Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:04:49 +0100 Subject: [PATCH 236/423] Add results for LK --- menpofit/lk/fitter.py | 12 ++++++------ menpofit/lk/result.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 menpofit/lk/result.py diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 5b7d69c..4633904 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -2,9 +2,9 @@ from menpo.feature import no_op from menpofit.transform import DifferentiableAlignmentAffine from menpofit.fitter import MultiFitter, noisy_align -from menpofit.result import MultiFitterResult from .algorithm import IC from .residual import SSD, FourierSSD +from .result import LKFitterResult # TODO: document me! @@ -97,12 +97,12 @@ def _prepare_template(self, template, group=None, label=None): return templates, sources - def _fitter_result(self, image, algorithm_results, affine_correction, - gt_shape=None): - return MultiFitterResult(image, self, algorithm_results, - affine_correction, gt_shape=gt_shape) - def perturb_shape(self, gt_shape, noise_std=0.04): transform = noisy_align(self.transform_cls, self.reference_shape, gt_shape, noise_std=noise_std) return transform.apply(self.reference_shape) + + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return LKFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) \ No newline at end of file diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py new file mode 100644 index 0000000..1eddf5d --- /dev/null +++ b/menpofit/lk/result.py @@ -0,0 +1,18 @@ +from __future__ import division +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult + + +# TODO: document me! +# TODO: handle costs! +class LKAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + pass + + +# TODO: document me! +# TODO: handle costs +class LKFitterResult(MultiFitterResult): + r""" + """ + pass From 1731a53df8326f8c8a788e8d9834d895f64078b4 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:05:40 +0100 Subject: [PATCH 237/423] Add residuals for LK --- menpofit/lk/residual.py | 500 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 500 insertions(+) create mode 100755 menpofit/lk/residual.py diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py new file mode 100755 index 0000000..401b1e5 --- /dev/null +++ b/menpofit/lk/residual.py @@ -0,0 +1,500 @@ +import abc +import numpy as np +from numpy.fft import fftn, ifftn, fft2 +import scipy.linalg +from menpo.feature import gradient + + +class Residual(object): + """ + An abstract base class for calculating the residual between two images + within the Lucas-Kanade algorithm. The classes were designed + specifically to work within the Lucas-Kanade framework and so no + guarantee is made that calling methods on these subclasses will generate + correct results. + """ + @classmethod + def gradient(cls, image, forward=None): + r""" + Calculates the gradients of the given method. + + If `forward` is provided, then the gradients are warped + (as required in the forward additive algorithm) + + Parameters + ---------- + image : :class:`menpo.image.base.Image` + The image to calculate the gradients for + forward : (:map:`Image`, :map:`AlignableTransform>`), optional + A tuple containing the extra weights required for the function + `warp` (which should be passed as a function handle). + + Default: `None` + """ + if forward: + # Calculate the gradient over the image + # grad: (dims x ch) x H x W + grad = gradient(image) + # Warp gradient for forward additive using the given transform + # grad: (dims x ch) x h x w + template, transform = forward + grad = grad.warp_to_mask(template.mask, transform, + warp_landmarks=False) + else: + # Calculate the gradient over the image and set one pixels along + # the boundary of the image mask to zero (no reliable gradient + # can be computed there!) + # grad: (dims x ch) x h x w + grad = gradient(image) + grad.set_boundary_pixels() + return grad + + @abc.abstractmethod + def steepest_descent_images(self, image, dW_dp, **kwargs): + r""" + Calculates the standard steepest descent images. + + Within the forward additive framework this is defined as + + .. math:: + \nabla I \frac{\partial W}{\partial p} + + The input image is vectorised (`N`-pixels) so that masked images can + be handled. + + Parameters + ---------- + image : :class:`menpo.image.base.Image` + The image to calculate the steepest descent images from, could be + either the template or input image depending on which framework is + used. + dW_dp : ndarray + The Jacobian of the warp. + + Returns + ------- + VT_dW_dp : (N, n_params) ndarray + The steepest descent images + """ + pass + + @abc.abstractmethod + def hessian(self, sdi): + r""" + Calculates the Gauss-Newton approximation to the Hessian. + + This is abstracted because some residuals expect the Hessian to be + pre-processed. The Gauss-Newton approximation to the Hessian is + defined as: + + .. math:: + \mathbf{J J^T} + + Parameters + ---------- + J : (N, n_params) ndarray + The steepest descent images. + + Returns + ------- + H : (n_params, n_params) ndarray + The approximation to the Hessian + """ + pass + + @abc.abstractmethod + def steepest_descent_update(self, sdi, image, template): + r""" + Calculates the steepest descent parameter updates. + + These are defined, for the forward additive algorithm, as: + + .. math:: + \sum_x [ \nabla I \frac{\partial W}{\partial p} ]^T [ T(x) - I(W(x;p)) ] + + Parameters + ---------- + J : (N, n_params) ndarray + The steepest descent images. + image : :class:`menpo.image.base.Image` + Either the warped image or the template + (depending on the framework) + template : :class:`menpo.image.base.Image` + Either the warped image or the template + (depending on the framework) + + Returns + ------- + sd_delta_p : (n_params,) ndarray + The steepest descent parameter updates. + """ + pass + + +class SSD(Residual): + + def __init__(self, kernel=None): + self.kernel = kernel + + def steepest_descent_images(self, image, dW_dp, forward=None): + # compute gradient + # grad: dims x ch x h x w + nabla = self.gradient(image, forward=forward) + nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + + image.shape) + + # compute steepest descent images + # gradient: dims x ch x h x w + # dw_dp: dims x x h x w x params + # sdi: ch x h x w x params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + + if self.kernel is not None: + # if required, filter steepest descent images + # fft_sdi: ch x h x w x params + filtered_sdi = ifftn(self.kernel[..., None] * + fftn(sdi, axes=(-3, -2)), + axes=(-3, -2)) + # reshape steepest descent images + # sdi: (ch x h x w) x params + # filtered_sdi: (ch x h x w) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) + filtered_sdi = filtered_sdi.reshape(sdi.shape) + else: + # reshape steepest descent images + # sdi: (ch x h x w) x params + # filtered_sdi: (ch x h x w) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) + filtered_sdi = sdi + + return filtered_sdi, sdi + + def hessian(self, sdi, sdi2=None): + # compute hessian + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params + # hessian: params x x params + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) + return H + + def steepest_descent_update(self, sdi, image, template): + error_img = image.as_vector() - template.as_vector() + return sdi.T.dot(error_img) + + +class FourierSSD(Residual): + + def __init__(self, kernel=None): + self.kernel = kernel + + def steepest_descent_images(self, image, dW_dp, forward=None): + # compute gradient + # grad: dims x ch x h x w + nabla = self.gradient(image, forward=forward) + nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + + image.shape) + + # compute steepest descent images + # gradient: dims x ch x h x w + # dw_dp: dims x x h x w x params + # sdi: ch x h x w x params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + + # compute steepest descent images fft + # fft_sdi: ch x h x w x params + fft_sdi = fftn(sdi, axes=(-3, -2)) + + if self.kernel is not None: + # if required, filter steepest descent images + filtered_fft_sdi = self.kernel[..., None] * fft_sdi + # reshape steepest descent images + # fft_sdi: (ch x h x w) x params + # filtered_fft_sdi: (ch x h x w) x params + fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) + filtered_fft_sdi = filtered_fft_sdi.reshape(fft_sdi.shape) + else: + # reshape steepest descent images + # fft_sdi: (ch x h x w) x params + # filtered_fft_sdi: (ch x h x w) x params + fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) + filtered_fft_sdi = fft_sdi + + return filtered_fft_sdi, fft_sdi + + def hessian(self, sdi, sdi2=None): + if sdi2 is None: + H = sdi.conjugate().T.dot(sdi) + else: + H = sdi.conjugate().T.dot(sdi2) + return H + + def steepest_descent_update(self, sdi, image, template): + # compute error image + # error_img: ch x h x w + error_img = image.pixels - template.pixels + + # compute error image fft + # fft_error_img: ch x (h x w) + fft_error_img = fft2(error_img) + + # compute steepest descent update + # fft_sdi: params x (ch x h x w) + # fft_error_img: (ch x h x w) + # fft_sdu: params + return sdi.conjugate().T.dot(fft_error_img.ravel()) + + +class ECC(Residual): + + def _normalise_images(self, image): + # TODO: do we need to copy the image? + # TODO: is this supposed to be per channel normalization? + norm_image = image.copy() + norm_image.normalize_norm_inplace() + return norm_image + + def steepest_descent_images(self, image, dW_dp, forward=None): + # normalize image + norm_image = self._normalise_images(image) + + # compute gradient + # gradient: dims x ch x pixels + grad = self.gradient(norm_image, forward=forward) + grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) + + # compute steepest descent images + # gradient: dims x ch x pixels + # dw_dp: dims x x pixels x params + # sdi: ch x pixels x params + sdi = 0 + a = grad[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (ch x pixels) x params + return sdi.reshape((-1, sdi.shape[-1])) + + def hessian(self, sdi): + # compute hessian + # sdi.T: params x (ch x pixels) + # sdi: (ch x pixels) x params + # hessian: params x x params + H = sdi.T.dot(sdi) + self._H_inv = scipy.linalg.inv(H) + return H + + def steepest_descent_update(self, sdi, image, template): + normalised_IWxp = self._normalise_images(image).as_vector() + normalised_template = self._normalise_images(template).as_vector() + + Gt = sdi.T.dot(normalised_template) + Gw = sdi.T.dot(normalised_IWxp) + + # Calculate the numerator + IWxp_norm = scipy.linalg.norm(normalised_IWxp) + num1 = IWxp_norm ** 2 + num2 = np.dot(Gw.T, np.dot(self._H_inv, Gw)) + num = num1 - num2 + + # Calculate the denominator + den1 = np.dot(normalised_template, normalised_IWxp) + den2 = np.dot(Gt.T, np.dot(self._H_inv, Gw)) + den = den1 - den2 + + # Calculate lambda to choose the step size + # Avoid division by zero + if den > 0: + l = num / den + else: + den3 = np.dot(Gt.T, np.dot(self._H_inv, Gt)) + l1 = np.sqrt(num2 / den3) + l2 = - den / den3 + l = np.maximum(l1, l2) + + self._error_img = l * normalised_IWxp - normalised_template + + return sdi.T.dot(self._error_img) + + +class GradientImages(Residual): + + def _regularise_gradients(self, grad): + pixels = grad.pixels + ab = np.sqrt(np.sum(pixels**2, axis=0)) + m_ab = np.median(ab) + ab = ab + m_ab + grad.pixels = pixels / ab + return grad + + def steepest_descent_images(self, image, dW_dp, forward=None): + n_dims = image.n_dims + n_channels = image.n_channels + + # compute gradient + first_grad = self.gradient(image, forward=forward) + self._template_grad = self._regularise_gradients(first_grad) + + # compute gradient + # second_grad: dims x dims x ch x pixels + second_grad = self.gradient(self._template_grad) + second_grad = second_grad.masked_pixels().flatten().reshape( + (n_dims, n_dims, n_channels, -1)) + + # Fix crossed derivatives: dydx = dxdy + second_grad[1, 0, ...] = second_grad[0, 1, ...] + + # compute steepest descent images + # gradient: dims x dims x ch x (h x w) + # dw_dp: dims x x (h x w) x params + # sdi: dims x ch x (h x w) x params + sdi = 0 + a = second_grad[..., None] * dW_dp[:, None, None, ...] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (dims x ch x h x w) x params + return sdi.reshape((-1, sdi.shape[-1])) + + def hessian(self, sdi): + # compute hessian + # sdi.T: params x (dims x ch x pixels) + # sdi: (dims x ch x pixels) x params + # hessian: params x x params + return sdi.T.dot(sdi) + + def steepest_descent_update(self, sdi, image, template): + # compute image regularized gradient + IWxp_grad = self.gradient(image) + IWxp_grad = self._regularise_gradients(IWxp_grad) + + # compute vectorized error_image + # error_img: (dims x ch x pixels) + self._error_img = (IWxp_grad.as_vector() - + self._template_grad.as_vector()) + + # compute steepest descent update + # sdi.T: params x (dims x ch x pixels) + # error_img: (dims x ch x pixels) + # sdu: params + return sdi.T.dot(self._error_img) + + +class GradientCorrelation(Residual): + + def steepest_descent_images(self, image, dW_dp, forward=None): + n_dims = image.n_dims + n_channels = image.n_channels + + # compute gradient + # grad: dims x ch x pixels + grad = self.gradient(image, forward=forward) + grad2 = grad.as_vector().reshape((n_dims, n_channels) + image.shape) + + # compute IGOs (remember axis 0 is y, axis 1 is x) + # grad: dims x ch x pixels + # phi: ch x pixels + # cos_phi: ch x pixels + # sin_phi: ch x pixels + phi = np.angle(grad2[1, ...] + 1j * grad2[0, ...]) + self._cos_phi = np.cos(phi) + self._sin_phi = np.sin(phi) + + # concatenate sin and cos terms so that we can take the second + # derivatives correctly. sin(phi) = y and cos(phi) = x which is the + # correct ordering when multiplying against the warp Jacobian + # cos_phi: ch x pixels + # sin_phi: ch x pixels + # grad: (dims x ch) x pixels + grad.from_vector_inplace( + np.concatenate((self._sin_phi[None, ...], + self._cos_phi[None, ...]), axis=0).ravel()) + + # compute IGOs gradient + # second_grad: dims x dims x ch x pixels + second_grad = self.gradient(grad) + second_grad = second_grad.masked_pixels().flatten().reshape( + (n_dims, n_dims, n_channels) + image.shape) + + # Fix crossed derivatives: dydx = dxdy + second_grad[1, 0, ...] = second_grad[0, 1, ...] + + # complete full IGOs gradient computation + # second_grad: dims x dims x ch x pixels + second_grad[1, ...] = (-self._sin_phi[None, ...] * second_grad[1, ...]) + second_grad[0, ...] = (self._cos_phi[None, ...] * second_grad[0, ...]) + + # compute steepest descent images + # gradient: dims x dims x ch x pixels + # dw_dp: dims x x pixels x params + # sdi: ch x pixels x params + sdi = 0 + aux = second_grad[..., None] * dW_dp[None, :, None, ...] + for a in aux.reshape(((-1,) + aux.shape[2:])): + sdi += a + + # compute constant N + # N: 1 + self._N = grad.n_parameters / 2 + + # reshape steepest descent images + # sdi: (ch x pixels) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) + + return sdi, sdi + + def hessian(self, sdi, sdi2=None): + # compute hessian + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params + # hessian: params x x params + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) + return H + + def steepest_descent_update(self, sdi, image, template): + n_dims = image.n_dims + n_channels = image.n_channels + + # compute image gradient + IWxp_grad = self.gradient(image) + IWxp_grad = IWxp_grad.as_vector().reshape( + (n_dims, n_channels) + image.shape) + + # compute IGOs (remember axis 0 is y, axis 1 is x) + # IWxp_grad: dims x ch x pixels + # phi: ch x pixels + # IWxp_cos_phi: ch x pixels + # IWxp_sin_phi: ch x pixels + phi = np.angle(IWxp_grad[1, ...] + 1j * IWxp_grad[0, ...]) + IWxp_cos_phi = np.cos(phi) + IWxp_sin_phi = np.sin(phi) + + # compute error image + # error_img: (ch x h x w) + self._error_img = (self._cos_phi * IWxp_sin_phi - + self._sin_phi * IWxp_cos_phi).ravel() + + # compute steepest descent update + # sdi: (ch x pixels) x params + # error_img: (ch x pixels) + # sdu: params + sdu = sdi.T.dot(self._error_img) + + # compute step size + qp = np.sum(self._cos_phi * IWxp_cos_phi + + self._sin_phi * IWxp_sin_phi) + l = self._N / qp + return l * sdu From 0abda37130eda1ad9b9aeae9d3cd60b86182b42b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:06:05 +0100 Subject: [PATCH 238/423] Add algorithms for LK --- menpofit/lk/algorithm.py | 179 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 menpofit/lk/algorithm.py diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py new file mode 100644 index 0000000..49d3a25 --- /dev/null +++ b/menpofit/lk/algorithm.py @@ -0,0 +1,179 @@ +from scipy.linalg import norm +import abc +import numpy as np +from .result import LKAlgorithmResult + + +# TODO: implement Linear, Parts interfaces? Will they play nice with residuals? +# TODO: implement sampling? +# TODO: handle costs for all LKAlgorithms +# TODO: document me! +class LKAlgorithm(object): + r""" + """ + def __init__(self, template, transform, residual, eps=10**-10): + self.template = template + self.transform = transform + self.residual = residual + self.eps = eps + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + pass + + +class FA(LKAlgorithm): + r""" + Forward Additive Lucas-Kanade algorithm + """ + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Forward Compositional Algorithm + while k < max_iters and eps > self.eps: + # warp image + IWxp = image.warp_to_mask(self.template.mask, self.transform) + + # compute warp jacobian + dW_dp = np.rollaxis( + self.transform.d_dp(self.template.indices()), -1) + + # compute steepest descent images + filtered_J, J = self.residual.steepest_descent_images( + image, dW_dp, forward=(self.template, self.transform)) + + # compute hessian + H = self.residual.hessian(filtered_J, sdi2=J) + + # compute steepest descent parameter updates. + sd_dp = self.residual.steepest_descent_update( + filtered_J, IWxp, self.template) + + # compute gradient descent parameter updates + dp = np.real(np.linalg.solve(H, sd_dp)) + + # Update warp weights + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(norm(dp)) + + # increase iteration counter + k += 1 + + return LKAlgorithmResult(image, self, p_list, gt_shape=None) + + +class FC(LKAlgorithm): + r""" + Forward Compositional Lucas-Kanade algorithm + """ + def __init__(self, template, transform, residual, eps=10**-10): + super(FC, self).__init__(template, transform, residual, eps=eps) + self.precompute() + + def precompute(self): + # compute warp jacobian + self.dW_dp = np.rollaxis( + self.transform.d_dp(self.template.indices()), -1) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Forward Compositional Algorithm + while k < max_iters and eps > self.eps: + # warp image + IWxp = image.warp_to_mask(self.template.mask, self.transform) + + # compute steepest descent images + filtered_J, J = self.residual.steepest_descent_images( + IWxp, self.dW_dp) + + # compute hessian + H = self.residual.hessian(filtered_J, sdi2=J) + + # compute steepest descent parameter updates. + sd_dp = self.residual.steepest_descent_update( + filtered_J, IWxp, self.template) + + # compute gradient descent parameter updates + dp = np.real(np.linalg.solve(H, sd_dp)) + + # Update warp weights + self.transform.compose_after_from_vector_inplace(dp) + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(norm(dp)) + + # increase iteration counter + k += 1 + + return LKAlgorithmResult(image, self, p_list, gt_shape=None) + + +class IC(LKAlgorithm): + r""" + Inverse Compositional Lucas-Kanade algorithm + """ + def __init__(self, template, transform, residual, eps=10**-10): + super(IC, self).__init__(template, transform, residual, eps=eps) + self.precompute() + + def precompute(self): + # compute warp jacobian + dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) + dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + + dW_dp.shape[-1:]) + # compute steepest descent images + self.filtered_J, J = self.residual.steepest_descent_images( + self.template, dW_dp) + # compute hessian + self.H = self.residual.hessian(self.filtered_J, sdi2=J) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Baker-Matthews, Inverse Compositional Algorithm + while k < max_iters and eps > self.eps: + # warp image + IWxp = image.warp_to_mask(self.template.mask, self.transform) + + # compute steepest descent parameter updates. + sd_dp = self.residual.steepest_descent_update( + self.filtered_J, IWxp, self.template) + + # compute gradient descent parameter updates + dp = np.real(np.linalg.solve(self.H, sd_dp)) + + # update warp + inv_dp = self.transform.pseudoinverse_vector(dp) + self.transform.compose_after_from_vector_inplace(inv_dp) + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(norm(dp)) + + # increase iteration counter + k += 1 + + return LKAlgorithmResult(image, self, p_list, gt_shape=None) From ff1dc0a4e9848b548bced34c6dfa72dd54f577eb Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 21 May 2015 18:06:57 +0100 Subject: [PATCH 239/423] Update lk.__init__ --- menpofit/lk/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 menpofit/lk/__init__.py diff --git a/menpofit/lk/__init__.py b/menpofit/lk/__init__.py new file mode 100644 index 0000000..b01bf94 --- /dev/null +++ b/menpofit/lk/__init__.py @@ -0,0 +1,3 @@ +from .fitter import LKFitter +from .algorithm import FA, FC, IC +from .residual import SSD, FourierSSD, ECC, GradientImages, GradientCorrelation From 284786473555c20a165d0177904275fcba9233f7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 13:50:03 +0100 Subject: [PATCH 240/423] Add accidentaly deleted view_shape_models_widget to AAM - Update some documentation on AAM --- menpofit/aam/base.py | 83 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 4266ae7..54b962a 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -159,9 +159,62 @@ def _instance(self, level, shape_instance, appearance_instance): return instance + def view_shape_models_widget(self, n_parameters=5, + parameters_bounds=(-3.0, 3.0), + mode='multiple', figure_size=(10, 8)): + r""" + Visualizes the shape models of the AAM object using the + `menpo.visualize.widgets.visualize_shape_model` widget. + + Parameters + ----------- + n_parameters : `int` or `list` of `int` or ``None``, optional + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + parameters_bounds : (`float`, `float`), optional + The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + figure_size : (`int`, `int`), optional + The size of the plotted figures. + """ + from menpofit.visualize import visualize_shape_model + visualize_shape_model(self.shape_models, n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): + r""" + Visualizes the appearance models of the AAM object using the + `menpo.visualize.widgets.visualize_appearance_model` widget. + Parameters + ----------- + n_parameters : `int` or `list` of `int` or ``None``, optional + The number of appearance principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + parameters_bounds : (`float`, `float`), optional + The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + figure_size : (`int`, `int`), optional + The size of the plotted figures. + """ from menpofit.visualize import visualize_appearance_model visualize_appearance_model(self.appearance_models, n_parameters=n_parameters, @@ -172,6 +225,36 @@ def view_appearance_models_widget(self, n_parameters=5, def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): + r""" + Visualizes both the shape and appearance models of the AAM object using + the `menpo.visualize.widgets.visualize_aam` widget. + Parameters + ----------- + n_shape_parameters : `int` or `list` of `int` or None, optional + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + n_appearance_parameters : `int` or `list` of `int` or None, optional + The number of appearance principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. + parameters_bounds : (`float`, `float`), optional + The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + figure_size : (`int`, `int`), optional + The size of the plotted figures. + """ from menpofit.visualize import visualize_aam visualize_aam(self, n_shape_parameters=n_shape_parameters, n_appearance_parameters=n_appearance_parameters, From faacec7e0ac669c6163270ff7086114f71ac8d35 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 14:49:23 +0100 Subject: [PATCH 241/423] More small changes to aam.base --- menpofit/aam/base.py | 45 ++++++++++++++++++++++++++++++++--------- menpofit/aam/builder.py | 2 +- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 54b962a..b2b11c0 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.shape import TriMesh from menpofit.transform import DifferentiableThinPlateSplines @@ -8,6 +7,7 @@ build_reference_frame, build_patch_reference_frame) +# TODO: document me! class AAM(object): r""" Active Appearance Model class. @@ -69,6 +69,15 @@ def n_levels(self): """ return len(self.scales) + # TODO: Could we directly use class names instead of this? + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Active Appearance Model' + def instance(self, shape_weights=None, appearance_weights=None, level=-1): r""" Generates a novel AAM instance given a set of shape and appearance @@ -365,6 +374,7 @@ def __str__(self): return out +# TODO: document me! class PatchAAM(AAM): r""" Patch based Based Active Appearance Model class. @@ -417,6 +427,10 @@ def __init__(self, shape_models, appearance_models, reference_shape, self.scale_shapes = scale_shapes self.scale_features = scale_features + @property + def _str_title(self): + return 'Patch-Based Active Appearance Model' + def _instance(self, level, shape_instance, appearance_instance): template = self.appearance_models[level].mean landmarks = template.landmarks['source'].lms @@ -443,14 +457,13 @@ def view_appearance_models_widget(self, n_parameters=5, figure_size=figure_size, mode=mode) # TODO: fix me! - def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, - parameters_bounds=(-3.0, 3.0), mode='multiple', - figure_size=(10, 8)): - from menpofit.visualize import visualize_aam - visualize_aam(self, n_shape_parameters=n_shape_parameters, - n_appearance_parameters=n_appearance_parameters, - parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode) + def __str__(self): + out = super(PatchAAM, self).__str__() + out_splitted = out.splitlines() + out_splitted[0] = self._str_title + out_splitted.insert(5, " - Patch size is {}W x {}H.".format( + self.patch_shape[1], self.patch_shape[0])) + return '\n'.join(out_splitted) # TODO: document me! @@ -524,6 +537,10 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented + # TODO: implement me! + def __str__(self): + raise NotImplemented + # TODO: document me! class LinearPatchAAM(AAM): @@ -598,6 +615,10 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented + # TODO: implement me! + def __str__(self): + raise NotImplemented + # TODO: document me! class PartsAAM(AAM): @@ -669,4 +690,8 @@ def view_appearance_models_widget(self, n_parameters=5, def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): - raise NotImplemented \ No newline at end of file + raise NotImplemented + + # TODO: implement me! + def __str__(self): + raise NotImplemented diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 6ad44f2..bb98c1a 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,5 +1,4 @@ from __future__ import division -import abc from copy import deepcopy from menpo.model import PCAModel from menpo.shape import mean_pointcloud @@ -14,6 +13,7 @@ DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) +# TODO: fix features checker # TODO: implement checker for conflict between features and scale_features # TODO: document me! class AAMBuilder(object): From 001ba8f0c37512a387286b242d95e1640217aa14 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 17:52:18 +0100 Subject: [PATCH 242/423] Small changes to AAM --- menpofit/aam/algorithm.py | 37 ------------------------------------- menpofit/aam/builder.py | 16 ++++++---------- menpofit/aam/fitter.py | 12 ++++++------ 3 files changed, 12 insertions(+), 53 deletions(-) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py index 9a55ff1..6583f00 100644 --- a/menpofit/aam/algorithm.py +++ b/menpofit/aam/algorithm.py @@ -201,12 +201,6 @@ def steepest_descent_images(self, nabla, dw_dp): # sdi: (parts x offsets x ch x w x h) x params return sdi.reshape((-1, sdi.shape[-1])) - def algorithm_result(self, image, shape_parameters, - appearance_parameters=None, gt_shape=None): - return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, - appearance_parameters=appearance_parameters, gt_shape=gt_shape) - # TODO: handle costs for all LKAAMAlgorithms # TODO document me! @@ -263,12 +257,6 @@ class ProjectOut(LKAAMAlgorithm): r""" Abstract Interface for Project-out AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(ProjectOut, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def project_out(self, J): # project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) @@ -384,12 +372,6 @@ class Simultaneous(LKAAMAlgorithm): r""" Abstract Interface for Simultaneous AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(Simultaneous, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): # initialize transform @@ -504,12 +486,6 @@ class Alternating(LKAAMAlgorithm): r""" Abstract Interface for Alternating AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(Alternating, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def precompute(self, **kwargs): # call super method super(Alternating, self).precompute() @@ -633,12 +609,6 @@ class ModifiedAlternating(Alternating): r""" Abstract Interface for Modified Alternating AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(ModifiedAlternating, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): # initialize transform @@ -729,12 +699,6 @@ class Wiberg(LKAAMAlgorithm): r""" Abstract Interface for Wiberg AAM algorithms """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # call super constructor - super(Wiberg, self).__init__( - aam_interface, appearance_model, transform, eps, **kwargs) - def project_out(self, J): # project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) @@ -832,4 +796,3 @@ def update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) - diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index bb98c1a..ff8a783 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -259,10 +259,9 @@ def _build_shape_model(cls, shapes, max_components, level): def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): - self.reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return AAM(shape_models, appearance_models, reference_shape, @@ -399,11 +398,10 @@ def __init__(self, patch_shape=(17, 17), features=no_op, def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): - self.reference_frame = build_patch_reference_frame( + reference_frame = build_patch_reference_frame( reference_shape, patch_shape=self.patch_shape[level]) - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) def _build_aam(self, shape_models, appearance_models, reference_shape): return PatchAAM(shape_models, appearance_models, reference_shape, @@ -861,5 +859,3 @@ def _build_aam(self, shape_models, appearance_models, reference_shape): from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM - - diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a2080c4..9e3587f 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -21,13 +21,13 @@ def __init__(self, aam, algorithm_cls=AIC, n_shape=None, self._set_up(algorithm_cls, sampling, **kwargs) def _set_up(self, algorithm_cls, sampling, **kwargs): - for j, (am, sm) in enumerate(zip(self._model.appearance_models, - self._model.shape_models)): + for j, (am, sm) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( - sm, self._model.transform, + sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = algorithm_cls(LKAAMInterface, am, md_transform, @@ -37,7 +37,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self._model.n_landmarks) + sm, self.aam.n_landmarks) # set up algorithm using linear aam interface algorithm = algorithm_cls(LinearLKAAMInterface, am, md_transform, sampling=sampling, @@ -47,10 +47,10 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - am.patch_shape = self._model.patch_shape[j] - am.normalize_parts = self._model.normalize_parts algorithm = algorithm_cls(PartsLKAAMInterface, am, pdm, sampling=sampling, **kwargs) + algorithm.patch_shape = self.aam.patch_shape[j] + algorithm.normalize_parts = self.aam.normalize_parts else: raise ValueError("AAM object must be of one of the " From 764111a2379d1e0486e4472085b6149ea5467d3a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 22 May 2015 17:55:02 +0100 Subject: [PATCH 243/423] Add TODO in lk.algorithm - Should we implement Inverse Additive? --- menpofit/lk/algorithm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 49d3a25..3034c08 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -4,6 +4,7 @@ from .result import LKAlgorithmResult +# TODO: implement Inverse Additive Algorithm? # TODO: implement Linear, Parts interfaces? Will they play nice with residuals? # TODO: implement sampling? # TODO: handle costs for all LKAlgorithms From 7db14cf8eadf306271daba9e4c3e5e20118667af Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 08:42:38 +0100 Subject: [PATCH 244/423] Add ATM base classes - Add ATM, PatchATM, LinearATM, LinearPatchATM and PartsATM --- menpofit/atm/base.py | 446 ++++++++++++++++++++++++++++++------------- 1 file changed, 316 insertions(+), 130 deletions(-) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 1ebee48..10c513a 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -1,14 +1,13 @@ from __future__ import division - import numpy as np from menpo.shape import TriMesh - -from menpofit.base import DeformableModel, name_of_callable -from menpofit.aam.builder import (build_patch_reference_frame, - build_reference_frame) +from menpofit.transform import DifferentiableThinPlateSplines +from menpofit.base import name_of_callable +from menpofit.aam.builder import ( + build_patch_reference_frame, build_reference_frame) -class ATM(DeformableModel): +class ATM(object): r""" Active Template Model class. @@ -20,14 +19,15 @@ class ATM(DeformableModel): warped_templates : :map:`MaskedImage` list A list containing the warped templates models of the ATM. - n_training_shapes: `int` - The number of training shapes used to build the ATM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional + features : `callable` or ``[callable]``, If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -41,46 +41,42 @@ class ATM(DeformableModel): once and then creating a pyramid on top tends to lead to better performing AAMs. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. + scales : `int` or float` or list of those, optional - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + scale_shapes : `boolean` - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. - - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + scale_features : `boolean` """ - def __init__(self, shape_models, warped_templates, n_training_shapes, - transform, features, reference_shape, downscale, - scaled_shape_models): - DeformableModel.__init__(self, features) - self.n_training_shapes = n_training_shapes + def __init__(self, shape_models, warped_templates, reference_shape, + transform, features, scales, scale_shapes, scale_features): self.shape_models = shape_models self.warped_templates = warped_templates self.transform = transform + self.features = features self.reference_shape = reference_shape - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features @property def n_levels(self): """ - The number of multi-resolution pyramidal levels of the ATM. + The number of scale level of the ATM. :type: `int` """ - return len(self.warped_templates) + return len(self.scales) + + # TODO: Could we directly use class names instead of this? + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + + :type: `string` + """ + return 'Active Template Model' def instance(self, shape_weights=None, level=-1): r""" @@ -140,99 +136,83 @@ def _instance(self, level, shape_instance): template = self.warped_templates[level] landmarks = template.landmarks['source'].lms - reference_frame = self._build_reference_frame( - shape_instance, landmarks) - - transform = self.transform( - reference_frame.landmarks['source'].lms, landmarks) - - return template.as_unmasked(copy=False).warp_to_mask( - reference_frame.mask, transform, warp_landmarks=True) - - def _build_reference_frame(self, reference_shape, landmarks): if type(landmarks) == TriMesh: trilist = landmarks.trilist else: trilist = None - return build_reference_frame( - reference_shape, trilist=trilist) + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) - @property - def _str_title(self): - r""" - Returns a string containing name of the model. + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) - :type: `string` - """ - return 'Active Template Model' + return template.warp_to_mask(reference_frame.mask, transform, + warp_landmarks=True) - def view_shape_models_widget(self, n_parameters=5, mode='multiple', + def view_shape_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), - figure_size=(10, 8), style='coloured'): + mode='multiple', figure_size=(10, 8)): r""" - Visualizes the shape models of the ATM object using the + Visualizes the shape models of the AAM object using the `menpo.visualize.widgets.visualize_shape_model` widget. Parameters ----------- n_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the parameters - sliders. If `int`, then the number of sliders per level is the - minimum between `n_parameters` and the number of active components - per level. If `list` of `int`, then a number of sliders is defined - per level. If ``None``, all the active components per level will - have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. figure_size : (`int`, `int`), optional The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. """ from menpofit.visualize import visualize_shape_model - visualize_shape_model( - self.shape_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, figure_size=figure_size, - mode=mode, style=style) - - def view_atm_widget(self, n_shape_parameters=5, mode='multiple', - parameters_bounds=(-3.0, 3.0), figure_size=(10, 8), - style='coloured'): + visualize_shape_model(self.shape_models, n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode) + + # TODO: fix me! + def view_atm_widget(self, n_shape_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): r""" Visualizes the ATM object using the menpo.visualize.widgets.visualize_atm widget. Parameters ----------- - n_shape_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the shape - parameters sliders. If `int`, then the number of sliders per level - is the minimum between `n_parameters` and the number of active - components per level. If `list` of `int`, then a number of sliders - is defined per level. If ``None``, all the active components per - level will have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. + n_shape_parameters : `int` or `list` of `int` or None, optional + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each pp window. figure_size : (`int`, `int`), optional The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. """ from menpofit.visualize import visualize_atm visualize_atm(self, n_shape_parameters=n_shape_parameters, parameters_bounds=parameters_bounds, - figure_size=figure_size, mode=mode, style=style) + figure_size=figure_size, mode=mode) + # TODO: fix me! def __str__(self): out = "{}\n - {} training shapes.\n".format(self._str_title, self.n_training_shapes) @@ -333,7 +313,7 @@ def __str__(self): return out -class PatchBasedATM(ATM): +class PatchATM(ATM): r""" Patch Based Active Template Model class. @@ -345,14 +325,92 @@ class PatchBasedATM(ATM): warped_templates : :map:`MaskedImage` list A list containing the warped templates models of the ATM. - n_training_shapes: `int` - The number of training shapes used to build the ATM. + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based ATM. + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, warped_templates, reference_shape, + patch_shape, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.transform = DifferentiableThinPlateSplines + self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + @property + def _str_title(self): + return 'Patch-Based Active Template Model' + + def _instance(self, level, shape_instance): + template = self.warped_templates[level] + landmarks = template.landmarks['source'].lms + + reference_frame = build_patch_reference_frame( + shape_instance, patch_shape=self.patch_shape) + + transform = self.transform( + reference_frame.landmarks['source'].lms, landmarks) + + return template.warp_to_mask(reference_frame.mask, transform, + warp_landmarks=True) + + # TODO: fix me! + def __str__(self): + out = super(PatchBasedATM, self).__str__() + out_splitted = out.splitlines() + out_splitted[0] = self._str_title + out_splitted.insert(5, " - Patch size is {}W x {}H.".format( + self.patch_shape[1], self.patch_shape[0])) + return '\n'.join(out_splitted) + + +# TODO: document me! +class LinearATM(ATM): + r""" + Linear Active Template Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + warped_templates : :map:`MaskedImage` list + A list containing the warped templates models of the ATM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the ATM was + The transform used to warp the images from which the AAM was constructed. features : `callable` or ``[callable]``, optional @@ -369,51 +427,179 @@ class PatchBasedATM(ATM): once and then creating a pyramid on top tends to lead to better performing AAMs. + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, warped_templates, reference_shape, + transform, features, scales, scale_shapes, scale_features, + n_landmarks): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks + + # TODO: implement me! + def _instance(self, level, shape_instance): + raise NotImplemented + + # TODO: implement me! + def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! + def __str__(self): + raise NotImplemented + + +# TODO: document me! +class LinearPatchATM(ATM): + r""" + Linear Patch based Active Template Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + warped_templates : :map:`MaskedImage` list + A list containing the warped templates models of the ATM. + reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + scales : `int` or float` or list of those + + scale_shapes : `boolean` - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. + scale_features : `boolean` - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + n_landmarks: `int` """ - def __init__(self, shape_models, warped_templates, n_training_shapes, - patch_shape, transform, features, reference_shape, - downscale, scaled_shape_models): - super(PatchBasedATM, self).__init__( - shape_models, warped_templates, n_training_shapes, transform, - features, reference_shape, downscale, scaled_shape_models) + def __init__(self, shape_models, warped_templates, reference_shape, + patch_shape, features, scales, scale_shapes, + scale_features, n_landmarks): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.transform = DifferentiableThinPlateSplines self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks - def _build_reference_frame(self, reference_shape, landmarks): - return build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape) + # TODO: implement me! + def _instance(self, level, shape_instance): + raise NotImplemented - @property - def _str_title(self): - r""" - Returns a string containing name of the model. + # TODO: implement me! + def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented - :type: `string` - """ - return 'Patch-Based Active Template Model' + # TODO: implement me! + def __str__(self): + raise NotImplemented + +# TODO: document me! +class PartsATM(ATM): + r""" + Parts based Active Template Model class. + + Parameters + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + warped_templates : :map:`MaskedImage` list + A list containing the warped templates models of the ATM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + patch_shape : tuple of `int` + The shape of the patches used to build the Patch Based AAM. + + features : `callable` or ``[callable]`` + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + normalize_parts: `callable` + + scales : `int` or float` or list of those + + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, warped_templates, reference_shape, + patch_shape, features, normalize_parts, scales, + scale_shapes, scale_features): + self.shape_models = shape_models + self.warped_templates = warped_templates + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + + # TODO: implement me! + def _instance(self, level, shape_instance): + raise NotImplemented + + # TODO: implement me! + def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + parameters_bounds=(-3.0, 3.0), mode='multiple', + figure_size=(10, 8)): + raise NotImplemented + + # TODO: implement me! def __str__(self): - out = super(PatchBasedATM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + raise NotImplemented From e5d0ac31b4bfd281b7ba12bf923a21e0b8ed4a5f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:22:20 +0100 Subject: [PATCH 245/423] Remove SerializableAAMFitterResult --- menpofit/aam/__init__.py | 2 +- menpofit/aam/result.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 119cff0..673cb05 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -8,4 +8,4 @@ AFC, AIC, MAFC, MAIC, WFC, WIC) -from .result import SerializableAAMFitterResult + diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 5824945..a60c59b 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -43,11 +43,3 @@ class AAMFitterResult(MultiFitterResult): r""" """ pass - - -# TODO: document me! -# TODO: handle costs -class SerializableAAMFitterResult(SerializableIterativeResult): - r""" - """ - pass From 9033837df70a9a6fd880532fc66505a53f9cc11b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:23:56 +0100 Subject: [PATCH 246/423] Small changes in menpofit.builder --- menpofit/builder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 29d4ec8..735bca0 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -148,6 +148,7 @@ def scale_images(images, scale, level_str='', verbose=None): return images +# TODO: Can be done more efficiently for PWA defining a dummy transform # TODO: document me! def warp_images(images, shapes, reference_frame, transform, level_str='', verbose=None): @@ -169,7 +170,7 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! -def extract_patches(images, shapes, parts_shape, normalize_function=no_op, +def extract_patches(images, shapes, patch_shape, normalize_function=no_op, level_str='', verbose=None): parts_images = [] for c, (i, s) in enumerate(zip(images, shapes)): @@ -178,7 +179,7 @@ def extract_patches(images, shapes, parts_shape, normalize_function=no_op, level_str, progress_bar_str(float(c + 1) / len(images), show_bar=False))) - parts = i.extract_patches(s, patch_size=parts_shape, + parts = i.extract_patches(s, patch_size=patch_shape, as_single_array=True) if normalize_function: parts = normalize_function(parts) From 05ff85ab9a28a3db8cb8fac08d61e993a56ccb40 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:25:09 +0100 Subject: [PATCH 247/423] Small changes in menpofit.fitter --- menpofit/fitter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 7094fe2..13ea1da 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -328,12 +328,13 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - def perturb_shape(self, gt_shape, noise_std=0.04, rotation=False): + # TODO: Fix me! + def perturb_shape(self, gt_shape, noise_std=10, rotation=False): transform = noisy_align(AlignmentSimilarity, self.reference_shape, - gt_shape, noise_std=noise_std, - rotation=rotation) + gt_shape, noise_std=noise_std) return transform.apply(self.reference_shape) + # TODO: Bounding boxes should be PointGraphs def obtain_shape_from_bb(self, bounding_box): r""" Generates an initial shape given a bounding box detection. From fed1723fd94347b25177c507d24be015890583b4 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:33:07 +0100 Subject: [PATCH 248/423] Add Builders for ATMs - Add ATMBuilder, PatchATMBuilder, LinearAT= ``20``, optional + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the normalization_diagonal - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. @@ -56,25 +57,11 @@ class ATMBuilder(DeformableModelBuilder): reference frame (provided that features computation does not change the image size). - n_levels : `int` > 0, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(``n_levels``) - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of - each pyramid level, so the shape models will be scaled. + scales : `int` or float` or list of those, optional - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. + scale_shapes : `boolean`, optional - Note that from our experience, if ``scaled_shape_models`` is ``False``, - AAMs tend to have slightly better performance. + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -93,11 +80,6 @@ class ATMBuilder(DeformableModelBuilder): If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- atm : :map:`ATMBuilder` @@ -106,41 +88,40 @@ class ATMBuilder(DeformableModelBuilder): Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError ``max_shape_components`` must be ``None`` or an `int` > 0 or a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``n_levels`` elements + ``len(scales)`` elements ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, features=igo, transform=DifferentiablePiecewiseAffine, - trilist=None, normalization_diagonal=None, n_levels=3, - downscale=2, scaled_shape_models=True, - max_shape_components=None, boundary=3): + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, 0.5), + scale_shapes=False, scale_features=True, + max_shape_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) - # store parameters + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters self.features = features self.transform = transform self.trilist = trilist - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components - self.boundary = boundary def build(self, shapes, template, group=None, label=None, verbose=False): r""" @@ -175,133 +156,235 @@ def build(self, shapes, template, group=None, label=None, verbose=False): to highest level. """ # compute reference_shape - self.reference_shape = compute_reference_shape( - shapes, self.normalization_diagonal, verbose=verbose) + reference_shape = compute_reference_shape(shapes, self.diagonal, + verbose=verbose) # normalize the template size using the reference_shape scaling - if verbose: - print_dynamic('- Normalizing template size') - normalized_template = template.rescale_to_reference_shape( - self.reference_shape, group=group, label=label) + template = template.rescale_to_reference_shape( + reference_shape, group=group, label=label) - # create pyramid for template image + # build models at each scale if verbose: - print_dynamic('- Creating template pyramid') - generator = create_pyramid([normalized_template], self.n_levels, - self.downscale, self.features) - - # build the model at each pyramid level - if verbose: - if self.n_levels > 1: - print_dynamic('- Building model for each of the {} pyramid ' - 'levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building model\n') - + print_dynamic('- Building models\n') shape_models = [] warped_templates = [] # for each pyramid level (high --> low) - for j in range(self.n_levels): - # since models are built from highest to lowest level, the - # parameters in form of list need to use a reversed index - rj = self.n_levels - j - 1 - - if verbose: - level_str = ' - ' - if self.n_levels > 1: - level_str = ' - Level {}: '.format(j + 1) - - # rescale shapes if required - if j > 0 and self.scaled_shape_models: - scale_transform = Scale(scale_factor=1.0 / self.downscale, - n_dims=2) - shapes = [scale_transform.apply(s) for s in shapes] - - # train shape model and find reference frame + for j, s in enumerate(self.scales): if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = build_shape_model(shapes, - self.max_shape_components[rj]) - reference_frame = self._build_reference_frame(shape_model.mean()) - - # add shape model to the list - shape_models.append(shape_model) + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain shape representation + if j == 0 or self.scale_shapes: + if j == 0: + level_shapes = shapes + level_reference_shape= reference_shape + else: + scale_transform = Scale(scale_factor=s, n_dims=2) + level_shapes = [scale_transform.apply(s) for s in shapes] + level_reference_shape = scale_transform.apply( + reference_shape) + # obtain shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # add shape model to the list + shape_models.append(shape_model) + else: + # copy precious shape model and add it to the list + shape_models.append(deepcopy(shape_model)) - # get template's feature image of current level if verbose: - print_dynamic('{}Warping template'.format(level_str)) - feature_template = next(generator[0]) - - # compute transform - transform = self.transform(reference_frame.landmarks['source'].lms, - feature_template.landmarks[group][label]) + print_dynamic('{}Building template model'.format(level_str)) + # obtain template representation + if j == 0: + # compute features at highest level + feature_template = self.features(template) + level_template = feature_template + elif self.scale_features: + # scale features at other levels + level_template = feature_template.rescale(s) + else: + # scale template and compute features at other levels + scaled_template = template.rescale(s) + level_template = self.features(scaled_template) - # warp template to reference frame - warped_templates.append( - feature_template.warp_to_mask(reference_frame.mask, transform)) + # extract potentially rescaled template shape + level_template_shape = level_template.landmarks[group][label] - # attach reference_frame to template's source shape - warped_templates[j].landmarks['source'] = \ - reference_frame.landmarks['source'] + # obtain warped template + warped_template = self._warp_template(level_template, + level_template_shape, + level_reference_shape, j) + # add warped template to the list + warped_templates.append(warped_template) if verbose: print_dynamic('{}Done\n'.format(level_str)) - # reverse the list of shape and appearance models so that they are + # reverse the list of shape and warped templates so that they are # ordered from lower to higher resolution shape_models.reverse() warped_templates.reverse() - n_training_shapes = len(shapes) + self.scales.reverse() + + return self._build_atm(shape_models, warped_templates, reference_shape) + + @classmethod + def _build_shape_model(cls, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + + def _warp_template(self, template, template_shape, reference_shape, level): + # build reference frame + reference_frame = build_reference_frame(reference_shape) + # compute transforms + t = self.transform(reference_frame.landmarks['source'].lms, + template_shape) + # warp template + warped_template = template.warp_to_mask(reference_frame.mask, t) + # attach landmarks + warped_template.landmarks['source'] = reference_frame.landmarks[ + 'source'] + return warped_template + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return ATM(shape_models, warped_templates, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features) + + +class PatchATMBuilder(ATMBuilder): + r""" + Class that builds Multilevel Patch-Based Active Template Models. - return self._build_atm(shape_models, warped_templates, - n_training_shapes) + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - Returns - ------- - reference_frame : :map:`MaskedImage` - The reference frame. - """ - return build_reference_frame(mean_shape, boundary=self.boundary, - trilist=self.trilist) + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - def _build_atm(self, shape_models, warped_templates, n_training_shapes): - r""" - Returns an ATM object. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - Parameters - ---------- - shape_models : `list` of :map:`PCAModel` - The trained multilevel shape models. + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - warped_templates : `list` of :map:`MaskedImage` - The warped multilevel templates. + If ``None``, the mean shape is not rescaled. - n_training_shapes : `int` - The number of training shapes. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - Returns - ------- - atm : :map:`ATM` - The trained ATM object. - """ - from .base import ATM - return ATM(shape_models, warped_templates, n_training_shapes, - self.transform, self.features, self.reference_shape, - self.downscale, self.scaled_shape_models) + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + atm : :map:`ATMBuilder` + The ATM Builder object + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=True, + scale_features=True, max_shape_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components -class PatchBasedATMBuilder(ATMBuilder): + def _warp_template(self, template, template_shape, reference_shape, level): + # build reference frame + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[level]) + # compute transforms + t = self.transform(reference_frame.landmarks['source'].lms, + template_shape) + # warp template + warped_template = template.warp_to_mask(reference_frame.mask, t) + # attach landmarks + warped_template.landmarks['source'] = reference_frame.landmarks[ + 'source'] + return warped_template + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return PatchATM(shape_models, warped_templates, reference_shape, + self.patch_shape, self.features, self.scales, + self.scale_shapes, self.scale_features) + + +# TODO: document me! +class LinearATMBuilder(ATMBuilder): r""" - Class that builds Multilevel Patch-Based Active Template Models. + Class that builds Linear Active Template Models. Parameters ---------- @@ -319,45 +402,165 @@ class PatchBasedATMBuilder(ATMBuilder): once and then creating a pyramid on top tends to lead to better performing AAMs. - patch_shape : tuple of `int`, optional - The appearance model of the Patch-Based AAM will be obtained by - sampling appearance patches with the specified shape around each - landmark. + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. - normalization_diagonal : `int` >= ``20``, optional + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. + + diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the ``normalization_diagonal`` - value. + of the bounding box containing it matches the diagonal value. If ``None``, the mean shape is not rescaled. - .. note:: + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). + + scales : `int` or float` or list of those, optional - Because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). + scale_shapes : `boolean`, optional - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. - downscale : `float` >= 1, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. - (downscale ** k) for k in range(``n_levels``) + Per level: + If `int`, it specifies the exact number of components to be + retained. - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of each - pyramid level, so the shape models will be scaled. - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - Note that from our experience, if scaled_shape_models is ``False``, AAMs - tend to have slightly better performance. + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + atm : :map:`ATMBuilder` + The ATM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, + trilist=None, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters + self.features = features + self.transform = transform + self.trilist = trilist + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model( + dense_shapes, max_components=max_components) + return shape_model + + def _warp_template(self, template, template_shape, reference_shape, level): + # compute transforms + t = self.transform(self.reference_frame.landmarks['source'].lms, + template_shape) + # warp template + warped_template = template.warp_to_mask(self.reference_frame.mask, t) + # attach landmarks + warped_template.landmarks['source'] = self.reference_frame.landmarks[ + 'source'] + return warped_template + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return LinearATM(shape_models, warped_templates, reference_shape, + self.transform, self.features, self.scales, + self.scale_shapes, self.scale_features, + self.n_landmarks) + + +# TODO: document me! +class LinearPatchATMBuilder(LinearATMBuilder): + r""" + Class that builds Linear Patch based Active Template Models. + + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) + + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. + + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. + + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. + + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). + + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is @@ -376,111 +579,195 @@ class PatchBasedATMBuilder(ATMBuilder): If ``None``, all the available components are kept (100% of variance). - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - Returns ------- - atm : ::map:`PatchBasedATMBuilder` - The Patch-Based ATM Builder object + atm : :map:`ATMBuilder` + The ATM Builder object Raises ------- ValueError - ``n_levels`` must be `int` > ``0`` + ``diagonal`` must be >= ``20``. ValueError - ``downscale`` must be >= ``1`` + ``scales`` must be `int` or `float` or list of those. ValueError - ``normalization_diagonal`` must be >= ``20`` + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. ValueError - ``max_shape_components must be ``None`` or an `int` > ``0`` or - a ``0`` <= `float` <= ``1`` or a list of those containing ``1`` - or ``n_levels`` elements + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements ValueError - ``features`` must be a `string` or a `function` or a list of those - containing 1 or ``n_levels`` elements + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements ValueError - ``pyramid_on_features`` is enabled so ``features`` must be a - `string` or a `function` or a list containing one of those + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, features=igo, patch_shape=(16, 16), - normalization_diagonal=None, n_levels=3, downscale=2, - scaled_shape_models=True, max_shape_components=None, - boundary=3): + def __init__(self, patch_shape=(17, 17), features=no_op, + diagonal=None, scales=(1, .5), scale_shapes=False, + scale_features=True, max_shape_components=None): # check parameters - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) - - # store parameters - self.features = features + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters self.patch_shape = patch_shape - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + self.features = features + self.transform = DifferentiableThinPlateSplines + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features self.max_shape_components = max_shape_components - self.boundary = boundary - # patch-based AAMs can only work with TPS transform - self.transform = ThinPlateSplines + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[level]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes, + max_components=max_components) + return shape_model + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return LinearPatchATM(shape_models, warped_templates, + reference_shape, self.patch_shape, + self.features, self.scales, self.scale_shapes, + self.scale_features, self.n_landmarks) + + +# TODO: document me! +# TODO: implement offsets support? +class PartsATMBuilder(ATMBuilder): + r""" + Class that builds Parts based Active Template Models. - def _build_reference_frame(self, mean_shape): - r""" - Generates the reference frame given a mean shape. + Parameters + ---------- + patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - Parameters - ---------- - mean_shape : :map:`PointCloud` - The mean shape to use. + features : `callable` or ``[callable]``, optional + If list of length ``n_levels``, feature extraction is performed at + each level after downscaling of the image. + The first element of the list specifies the features to be extracted at + the lowest pyramidal level and so on. - Returns - ------- - reference_frame : :map:`MaskedImage` - The patch-based reference frame. - """ - return build_patch_reference_frame(mean_shape, boundary=self.boundary, - patch_shape=self.patch_shape) + If ``callable`` the specified feature will be applied to the original + image and pyramid generation will be performed on top of the feature + image. Also see the `pyramid_on_features` property. - def _mask_image(self, image): - r""" - Creates the patch-based mask of the given image. + Note that from our experience, this approach of extracting features + once and then creating a pyramid on top tends to lead to better + performing AAMs. - Parameters - ---------- - image : :map:`MaskedImage` - The image to be masked. - """ - image.build_mask_around_landmarks(self.patch_shape, group='source') + normalize_parts : `callable`, optional - def _build_atm(self, shape_models, warped_templates, n_training_shapes): - r""" - Returns a Patch-Based ATM object. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. - Parameters - ---------- - shape_models : :map:`PCAModel` - The trained multilevel shape models. + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. - warped_templates : `list` of :map:`MaskedImage` - The warped multilevel templates. + If ``None``, the mean shape is not rescaled. - n_training_shapes : `int` - The number of training shapes. + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). - Returns - ------- - atm : :map:`PatchBasedATM` - The trained Patched-Based ATM object. - """ - from .base import PatchBasedATM - return PatchBasedATM(shape_models, warped_templates, n_training_shapes, - self.patch_shape, self.transform, self.features, - self.reference_shape, self.downscale, - self.scaled_shape_models) + scales : `int` or float` or list of those, optional + + scale_shapes : `boolean`, optional + + scale_features : `boolean`, optional + + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + atm : :map:`ATMBuilder` + The ATM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) + containing 1 or `len(scales)` elements. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + """ + def __init__(self, patch_shape=(17, 17), features=no_op, + normalize_parts=no_op, diagonal=None, scales=(1, .5), + scale_shapes=False, scale_features=True, + max_shape_components=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + features = checks.check_features(features, n_levels) + max_shape_components = checks.check_max_components( + max_shape_components, len(scales), 'max_shape_components') + # set parameters + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.diagonal = diagonal + self.scales = list(scales) + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.max_shape_components = max_shape_components + + def _warp_template(self, template, template_shape, reference_shape, level): + parts = template.extract_patches(template_shape, + patch_size=self.patch_shape[level], + as_single_array=True) + if self.normalize_parts: + parts = self.normalize_parts(parts) + + return Image(parts) + + def _build_atm(self, shape_models, warped_templates, reference_shape): + return PartsATM(shape_models, warped_templates, reference_shape, + self.patch_shape, self.features, + self.normalize_parts, self.scales, + self.scale_shapes, self.scale_features) + + +from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM diff --git a/menpofit/builder.py b/menpofit/builder.py index 735bca0..571fcf4 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -181,8 +181,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, show_bar=False))) parts = i.extract_patches(s, patch_size=patch_shape, as_single_array=True) - if normalize_function: - parts = normalize_function(parts) + parts = normalize_function(parts) parts_images.append(Image(parts)) return parts_images From 29a9db4b38905c416d7600a080d620b7d01ad878 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:34:58 +0100 Subject: [PATCH 249/423] Add Results for ATMs --- menpofit/atm/result.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 menpofit/atm/result.py diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py new file mode 100644 index 0000000..2b91276 --- /dev/null +++ b/menpofit/atm/result.py @@ -0,0 +1,38 @@ +from __future__ import division +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult + + +# TODO: document me! +# TODO: handle costs +class ATMAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + +# TODO: document me! +class LinearATMAlgorithmResult(ATMAlgorithmResult): + r""" + """ + def shapes(self, as_points=False): + if as_points: + return [self.fitter.transform.from_vector(p).sparse_target.points + for p in self.shape_parameters] + + else: + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] + + @property + def final_shape(self): + return self.final_transform.sparse_target + + @property + def initial_shape(self): + return self.initial_transform.sparse_target + + +# TODO: document me! +# TODO: handle costs +class ATMFitterResult(MultiFitterResult): + r""" + """ + pass From 563d25c9000c29641c2d9e584d12a40a86e5189e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:35:40 +0100 Subject: [PATCH 250/423] Add LKFitters for ATMs --- menpofit/atm/fitter.py | 400 +++++++---------------------------------- 1 file changed, 61 insertions(+), 339 deletions(-) diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index a5bfbe4..53f8820 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -1,352 +1,74 @@ from __future__ import division +from menpofit.fitter import ModelFitter +from menpofit.modelinstance import OrthoPDM +from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform +from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM +from .algorithm import ( + LKATMInterface, LKLinearATMInterface, LKPartsATMInterface, IC) +from .result import ATMFitterResult -from menpofit.fitter import MultilevelFitter -from menpofit.fittingresult import AMMultilevelFittingResult -from menpofit.transform import (ModelDrivenTransform, OrthoMDTransform, - DifferentiableAlignmentSimilarity) -from menpofit.lucaskanade.residual import SSD, GaborFourier -from menpofit.lucaskanade.image import IC -from menpofit.base import name_of_callable - -class ATMFitter(MultilevelFitter): +# TODO: document me! +class LKATMFitter(ModelFitter): r""" - Abstract Interface for defining Active Template Models Fitters. - - Parameters - ----------- - atm : :map:`ATM` - The Active Template Model to be used. """ - def __init__(self, atm): - self.atm = atm - - @property - def reference_shape(self): - r""" - The reference shape of the ATM. - - :type: :map:`PointCloud` - """ - return self.atm.reference_shape - - @property - def features(self): - r""" - The feature extracted at each pyramidal level during ATM building. - Stored in ascending pyramidal order. - - :type: `list` - """ - return self.atm.features - - @property - def n_levels(self): - r""" - The number of pyramidal levels used during ATM building. - - :type: `int` - """ - return self.atm.n_levels - - @property - def downscale(self): - r""" - The downscale used to generate the final scale factor applied at - each pyramidal level during ATM building. - The scale factor is computed as: - - ``(downscale ** k) for k in range(n_levels)`` - - :type: `float` - """ - return self.atm.downscale - - def _create_fitting_result(self, image, fitting_results, affine_correction, - gt_shape=None): - r""" - Creates a :map:`ATMMultilevelFittingResult` associated to a - particular fitting of the ATM fitter. - - Parameters - ----------- - image : :map:`Image` or subclass - The image to be fitted. - - fitting_results : `list` of :map:`FittingResult` - A list of fitting result objects containing the state of the - the fitting for each pyramidal level. - - affine_correction : :map:`Affine` - An affine transform that maps the result of the top resolution - level to the scale space of the original image. - - gt_shape : :map:`PointCloud`, optional - The ground truth shape associated to the image. - - error_type : 'me_norm', 'me' or 'rmse', optional - Specifies how the error between the fitted and ground truth - shapes must be computed. - - Returns - ------- - fitting : :map:`ATMMultilevelFittingResult` - A fitting result object that will hold the state of the ATM - fitter for a particular fitting. - """ - return ATMMultilevelFittingResult( - image, self, fitting_results, affine_correction, gt_shape=gt_shape) - - -class LucasKanadeATMFitter(ATMFitter): - r""" - Lucas-Kanade based :map:`Fitter` for Active Template Models. - - Parameters - ----------- - atm : :map:`ATM` - The Active Template Model to be used. - - algorithm : subclass of :map:`ImageLucasKanade`, optional - The Image Lucas-Kanade class to be used. - - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - """ - def __init__(self, atm, algorithm=IC, residual=SSD, - md_transform=OrthoMDTransform, n_shape=None, **kwargs): - super(LucasKanadeATMFitter, self).__init__(atm) - self._set_up(algorithm=algorithm, residual=residual, - md_transform=md_transform, n_shape=n_shape, **kwargs) - - @property - def algorithm(self): - r""" - Returns a string containing the name of fitting algorithm. - - :type: `str` - """ - return 'LK-ATM-' + self._fitters[0].algorithm - - def _set_up(self, algorithm=IC, - residual=SSD, md_transform=OrthoMDTransform, - global_transform=DifferentiableAlignmentSimilarity, - n_shape=None, **kwargs): - r""" - Sets up the Lucas-Kanade fitter object. - - Parameters - ----------- - algorithm : subclass of :map:`ImageLucasKanade`, optional - The Image Lucas-Kanade class to be used. - - md_transform : :map:`ModelDrivenTransform` or subclass, optional - The model driven transform class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing - the specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number - of components will be used for all levels. - - Raises - ------- - ValueError - ``n_shape`` can be an `int`, `float`, ``None`` or a `list` - containing ``1`` or ``n_levels`` of those. - """ - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.atm.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.atm.n_levels > 1: - for sm in self.atm.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.atm.n_levels: - for sm, n in zip(self.atm.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None ' - 'or a list containing 1 or {} of ' - 'those'.format(self.atm.n_levels)) - - self._fitters = [] - for j, (t, sm) in enumerate(zip(self.atm.warped_templates, - self.atm.shape_models)): - - if md_transform is not ModelDrivenTransform: - md_trans = md_transform( - sm, self.atm.transform, global_transform, - source=t.landmarks['source'].lms) - else: - md_trans = md_transform( + def __init__(self, atm, algorithm_cls=IC, n_shape=None, sampling=None, + **kwargs): + super(LKATMFitter, self).__init__(atm) + self._algorithms = [] + self._check_n_shape(n_shape) + self._set_up(algorithm_cls, sampling, **kwargs) + + def _set_up(self, algorithm_cls, sampling, **kwargs): + for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, + self.atm.shape_models)): + + if type(self.atm) is ATM or type(self.atm) is PatchATM: + # build orthonormal model driven transform + md_transform = OrthoMDTransform( sm, self.atm.transform, - source=t.landmarks['source'].lms) + source=wt.landmarks['source'].lms) + # set up algorithm using standard aam interface + algorithm = algorithm_cls(LKATMInterface, wt, md_transform, + sampling=sampling, **kwargs) + + elif (type(self.atm) is LinearATM or + type(self.atm) is LinearPatchATM): + # build linear version of orthogonal model driven transform + md_transform = LinearOrthoMDTransform( + sm, self.atm.n_landmarks) + # set up algorithm using linear aam interface + algorithm = algorithm_cls(LKLinearATMInterface, wt, + md_transform, sampling=sampling, + **kwargs) + + elif type(self.atm) is PartsATM: + # build orthogonal point distribution model + pdm = OrthoPDM(sm) + # set up algorithm using parts aam interface + algorithm = algorithm_cls( + LKPartsATMInterface, wt, pdm, sampling=sampling, + patch_shape=self.atm.patch_shape[j], + normalize_parts=self.atm.normalize_parts) - if residual is not GaborFourier: - self._fitters.append( - algorithm(t, residual(), md_trans, **kwargs)) else: - self._fitters.append( - algorithm(t, residual(t.shape), md_trans, - **kwargs)) + raise ValueError("AAM object must be of one of the " + "following classes: {}, {}, {}, {}, " + "{}".format(ATM, PatchATM, LinearATM, + LinearPatchATM, PartsATM)) - def __str__(self): - out = "{0} Fitter\n" \ - " - Lucas-Kanade {1}\n" \ - " - Transform is {2} and residual is {3}.\n" \ - " - {4} training images.\n".format( - self.atm._str_title, self._fitters[0].algorithm, - self._fitters[0].transform.__class__.__name__, - self._fitters[0].residual.type, self.atm.n_training_shapes) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_levels): - n_channels.append( - self._fitters[j].template.n_channels) - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format(name_of_callable( - self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - self.features[j].__name__)) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.atm.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, - self.downscale) + # append algorithms to list + self._algorithms.append(algorithm) - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.atm.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self._fitters[0].template.n_true_pixels() * - n_channels[0], - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.atm.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self._fitters[i].template.n_true_pixels() * - n_channels[i], - self._fitters[i].template.n_true_pixels(), - n_channels[i], self._fitters[i].template._str_shape, - n_channels[i]) - out = "{0} - {1} motion components\n\n".format( - out, self._fitters[i].transform.n_parameters) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} motion parameters\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self._fitters[0].template.n_true_pixels() * n_channels[0], - self._fitters[0].template.n_true_pixels(), - n_channels[0], self._fitters[0].template._str_shape, - n_channels[0], self._fitters[0].transform.n_parameters) - return out - - -class ATMMultilevelFittingResult(AMMultilevelFittingResult): - r""" - Class that holds the state of a :map:`ATMFitter` object before, - during and after it has fitted a particular image. - """ @property - def atm_reconstructions(self): - r""" - The list containing the atm reconstruction (i.e. the template warped on - the shape instance reconstruction) obtained at each fitting iteration. + def atm(self): + return self._model - Note that this reconstruction is only tested to work for the - :map:`OrthoMDTransform` + @property + def algorithms(self): + return self._algorithms - :type: list` of :map:`Image` or subclass - """ - atm_reconstructions = [] - for level, f in enumerate(self.fitting_results): - for shape_w in f.parameters: - shape_w = shape_w[4:] - sm_level = self.fitter.aam.shape_models[level] - swt = shape_w / sm_level.eigenvalues[:len(shape_w)] ** 0.5 - atm_reconstructions.append(self.fitter.aam.instance( - shape_weights=swt, level=level)) - return atm_reconstructions + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return ATMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) From 82dad4c95b32d6b7fa60c63a322a71ad451b49fe Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:36:25 +0100 Subject: [PATCH 251/423] Add algorithms for ATMs - Add menpofit.algorithm.py --- menpofit/algorithm.py | 134 ++++++++++++++++++++++++++ menpofit/atm/algorithm.py | 193 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 menpofit/algorithm.py create mode 100644 menpofit/atm/algorithm.py diff --git a/menpofit/algorithm.py b/menpofit/algorithm.py new file mode 100644 index 0000000..b651ba6 --- /dev/null +++ b/menpofit/algorithm.py @@ -0,0 +1,134 @@ +from __future__ import division +import numpy as np +from menpo.image import Image +from menpo.feature import no_op +from menpo.feature import gradient as fast_gradient + + +# TODO: implement more clever sampling? +class LKInterface(object): + + def __init__(self, lk_algorithm, sampling=None): + self.algorithm = lk_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def true_indices(self): + return self.template.mask.true_indices() + + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + + +class LKPartsInterface(LKInterface): + + def __init__(self, lk_algorithm, patch_shape=(17, 17), + normalize_parts=no_op, sampling=None): + self.algorithm = lk_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.nabla_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.nabla2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) + + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + # TODO: add parts normalization + def warp(self, image): + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts) + + def gradient(self, image): + pixels = image.pixels + g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return g.reshape((2,) + pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.nabla_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py new file mode 100644 index 0000000..dce40fb --- /dev/null +++ b/menpofit/atm/algorithm.py @@ -0,0 +1,193 @@ +from __future__ import division +import abc +import numpy as np +from menpofit.algorithm import LKInterface, LKPartsInterface +from .result import ATMAlgorithmResult, LinearATMAlgorithmResult + + +class LKATMInterface(LKInterface): + + @property + def shape_model(self): + return self.transform.pdm.model + + @classmethod + def solve_shape_map(cls, H, J, e, J_prior, p): + if p.shape[0] is not H.shape[0]: + # Bidirectional Compositional case + J_prior = np.hstack((J_prior, J_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + H += np.diag(J_prior) + Je = J_prior * p + J.T.dot(e) + return - np.linalg.solve(H, Je) + + @classmethod + def solve_shape_ml(cls, H, J, e): + # compute and return ML solution + return -np.linalg.solve(H, J.T.dot(e)) + + def algorithm_result(self, image, shape_parameters, gt_shape=None): + return ATMAlgorithmResult( + image, self.algorithm, shape_parameters, gt_shape=gt_shape) + + +class LKLinearATMInterface(LKATMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, gt_shape=None): + return LinearATMAlgorithmResult( + image, self.algorithm, shape_parameters, gt_shape=gt_shape) + + +class LKPartsATMInterface(LKPartsInterface, LKATMInterface): + + @property + def shape_model(self): + return self.transform.model + + +# TODO: handle costs for all LKAAMAlgorithms +# TODO document me! +class LKATMAlgorithm(object): + + def __init__(self, lk_atm_interface_cls, template, transform, + eps=10**-5, **kwargs): + # set common state for all ATM algorithms + self.template = template + self.transform = transform + self.eps = eps + # set interface + self.interface = lk_atm_interface_cls(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self, **kwargs): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + + # vectorize template and mask it + self.t_m = self.template.as_vector()[self.interface.i_mask] + + # compute warp jacobian + self.dW_dp = self.interface.warp_jacobian() + + # compute shape model prior + s2 = 1 / self.interface.shape_model.noise_variance() + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + pass + + +class Compositional(LKATMAlgorithm): + r""" + Abstract Interface for Compositional ATM algorithms + """ + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.t_m + + # solve for increments on the shape parameters + self.dp = self.solve(map_inference) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + @abc.abstractmethod + def solve(self, map_inference): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class FC(Compositional): + r""" + Forward Compositional (FC) Gauss-Newton algorithm + """ + def solve(self, map_inference): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # compute masked forward Jacobian + J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) + # compute masked forward Hessian + JJ_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + JJ_m, J_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return self.interface.solve_shape_ml(JJ_m, J_m, self.e_m) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class IC(Compositional): + r""" + Inverse Compositional (IC) Gauss-Newton algorithm + """ + def precompute(self): + # call super method + super(IC, self).precompute() + # compute appearance model mean gradient + nabla_t = self.interface.gradient(self.template) + # compute masked inverse Jacobian + self.J_m = self.interface.steepest_descent_images(-nabla_t, self.dW_dp) + # compute masked inverse Hessian + self.JJ_m = self.J_m.T.dot(self.J_m) + # compute masked Jacobian pseudo-inverse + self.pinv_J_m = np.linalg.solve(self.JJ_m, self.J_m.T) + + def solve(self, map_inference): + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + self.JJ_m, self.J_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return -self.pinv_J_m.dot(self.e_m) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) From e57dbcd6e03a2937e0ef7b6475df783252ae60e0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 26 May 2015 11:37:24 +0100 Subject: [PATCH 252/423] Add __init__.py for ATMs --- menpofit/atm/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index 304b4d3..12bd84b 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,3 +1,5 @@ -from .base import ATM, PatchBasedATM -from .builder import ATMBuilder, PatchBasedATMBuilder -from .fitter import LucasKanadeATMFitter +from .builder import ( + ATMBuilder, PatchATMBuilder, LinearATMBuilder, + LinearPatchATMBuilder, PartsATMBuilder) +from .fitter import LKATMFitter +from .algorithm import FC, IC \ No newline at end of file From e049ac71cbf54aea395e80394103cf15c89a2607 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 09:11:44 +0100 Subject: [PATCH 253/423] Make shapes a property in results.py --- menpofit/result.py | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 536145e..531f759 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -112,19 +112,11 @@ def n_iters(self): """ @abc.abstractproperty - def shapes(self, as_points=False): + def shapes(self): r""" Generates a list containing the shapes obtained at each fitting iteration. - Parameters - ----------- - as_points : boolean, optional - Whether the results is returned as a list of :map:`PointCloud`s or - ndarrays. - - Default: `False` - Returns ------- shapes : :map:`PointCloud`s or ndarray list @@ -145,7 +137,7 @@ def iter_image(self): :type: :map:`Image` """ image = Image(self.image.pixels) - for j, s in enumerate(self.shapes()): + for j, s in enumerate(self.shapes): image.landmarks['iter_'+str(j)] = s return image @@ -166,7 +158,7 @@ def errors(self, error_type='me_norm'): """ if self.gt_shape is not None: return [compute_error(t, self.gt_shape, error_type) - for t in self.shapes()] + for t in self.shapes] else: raise ValueError('Ground truth has not been set, errors cannot ' 'be computed') @@ -428,7 +420,7 @@ def __init__(self, image, fitter, shape_parameters, gt_shape=None): @property def n_iters(self): - return len(self.shapes()) - 1 + return len(self.shapes) - 1 @property def transforms(self): @@ -453,14 +445,10 @@ def initial_transform(self): """ return self.fitter.transform.from_vector(self.shape_parameters[0]) - def shapes(self, as_points=False): - if as_points: - return [self.fitter.transform.from_vector(p).target.points - for p in self.shape_parameters] - - else: - return [self.fitter.transform.from_vector(p).target - for p in self.shape_parameters] + @property + def shapes(self): + return [self.fitter.transform.from_vector(p).target + for p in self.shape_parameters] @property def final_shape(self): @@ -509,7 +497,8 @@ def n_iters(self): n_iters += f.n_iters return n_iters - def shapes(self, as_points=False): + @property + def shapes(self): r""" Generates a list containing the shapes obtained at each fitting iteration. @@ -561,11 +550,9 @@ def __init__(self, image, shapes, n_iters, gt_shape=None): def n_iters(self): return self._n_iters - def shapes(self, as_points=False): - if as_points: - return [s.points for s in self._shapes] - else: - return self._shapes + @property + def shapes(self): + return self._shapes @property def initial_shape(self): From 651c0df48d5e01b4d35f891ac2684a5e265c93f8 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 09:13:06 +0100 Subject: [PATCH 254/423] Make shape a property for LinearAAMAlgorithmResult --- menpofit/aam/result.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index a60c59b..afd516c 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -19,14 +19,10 @@ def __init__(self, image, fitter, shape_parameters, class LinearAAMAlgorithmResult(AAMAlgorithmResult): r""" """ + @property def shapes(self, as_points=False): - if as_points: - return [self.fitter.transform.from_vector(p).sparse_target.points - for p in self.shape_parameters] - - else: - return [self.fitter.transform.from_vector(p).sparse_target - for p in self.shape_parameters] + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] @property def final_shape(self): From 29ac18ec45c810a74b4ce01c4798db7e080371de Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 09:14:41 +0100 Subject: [PATCH 255/423] Add funtion rescale_images_to_reference_frame to menpofit.builder --- menpofit/builder.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 571fcf4..b4cfa57 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -46,6 +46,26 @@ def compute_reference_shape(shapes, normalization_diagonal, verbose=False): return reference_shape +# TODO: document me! +def rescale_images_to_reference_shape(images, group, label, reference_shape, + verbose=False): + r""" + """ + # normalize the scaling of all images wrt the reference_shape size + normalized_images = [] + for c, i in enumerate(images): + if verbose: + print_dynamic('- Normalizing images size: {}'.format( + progress_bar_str((c + 1.) / len(images), + show_bar=False))) + normalized_images.append(i.rescale_to_reference_shape( + reference_shape, group=group, label=label)) + + if verbose: + print_dynamic('- Normalizing images size: Done\n') + return normalized_images + + def normalization_wrt_reference_shape(images, group, label, diagonal, verbose=False): r""" @@ -103,17 +123,8 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size - normalized_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic('- Normalizing images size: {}'.format( - progress_bar_str((c + 1.) / len(images), - show_bar=False))) - normalized_images.append(i.rescale_to_reference_shape( - reference_shape, group=group, label=label)) - - if verbose: - print_dynamic('- Normalizing images size: Done\n') + normalized_images = rescale_images_to_reference_shape( + images, group, label, reference_shape, verbose=False) return reference_shape, normalized_images From e18272efe3d9e1380ed82d5518e461f6e38a367f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:18:33 +0100 Subject: [PATCH 256/423] Add CR AAM Fitter - This will substitute the previous SD AAM Fitter. --- menpofit/aam/fitter.py | 216 +++++++++++++++++++++++++++++++++-------- 1 file changed, 176 insertions(+), 40 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 9e3587f..f80d677 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,26 +1,70 @@ from __future__ import division +import abc +from menpo.transform import Scale +from menpofit.builder import ( + rescale_images_to_reference_shape, compute_features, scale_images) from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM -from .algorithm import ( +from .algorithm.lk import ( LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) +from .algorithm.cr import ( + CRAAMInterface, CRLinearAAMInterface, CRPartsAAMInterface, PAJ) from .result import AAMFitterResult # TODO: document me! -class LKAAMFitter(ModelFitter): +class AAMFitter(ModelFitter): r""" """ - def __init__(self, aam, algorithm_cls=AIC, n_shape=None, - n_appearance=None, sampling=None, **kwargs): - super(LKAAMFitter, self).__init__(aam) + def __init__(self, aam, n_shape=None, n_appearance=None): + super(AAMFitter, self).__init__(aam) self._algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - self._set_up(algorithm_cls, sampling, **kwargs) - def _set_up(self, algorithm_cls, sampling, **kwargs): + @property + def aam(self): + return self._model + + @property + def algorithms(self): + return self._algorithms + + def _check_n_appearance(self, n_appearance): + if n_appearance is not None: + if type(n_appearance) is int or type(n_appearance) is float: + for am in self.aam.appearance_models: + am.n_active_components = n_appearance + elif len(n_appearance) == 1 and self.aam.n_levels > 1: + for am in self.aam.appearance_models: + am.n_active_components = n_appearance[0] + elif len(n_appearance) == self.aam.n_levels: + for am, n in zip(self.aam.appearance_models, n_appearance): + am.n_active_components = n + else: + raise ValueError('n_appearance can be an integer or a float ' + 'or None or a list containing 1 or {} of ' + 'those'.format(self.aam.n_levels)) + + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return AAMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) + + +# TODO: document me! +class LKAAMFitter(AAMFitter): + r""" + """ + def __init__(self, aam, n_shape=None, n_appearance=None, + lk_algorithm_cls=AIC, sampling=None, **kwargs): + super(LKAAMFitter, self).__init__( + aam, n_shape=n_shape, n_appearance=n_appearance) + self._set_up(lk_algorithm_cls, sampling, **kwargs) + + def _set_up(self, lk_algorithm_cls, sampling, **kwargs): for j, (am, sm) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models)): @@ -30,8 +74,9 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(LKAAMInterface, am, md_transform, - sampling=sampling, **kwargs) + algorithm = lk_algorithm_cls( + LKAAMInterface, am, md_transform, sampling=sampling, + **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -39,18 +84,18 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.aam.n_landmarks) # set up algorithm using linear aam interface - algorithm = algorithm_cls(LinearLKAAMInterface, am, - md_transform, sampling=sampling, - **kwargs) + algorithm = lk_algorithm_cls( + LinearLKAAMInterface, am, md_transform, sampling=sampling, + **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - algorithm = algorithm_cls(PartsLKAAMInterface, am, pdm, - sampling=sampling, **kwargs) - algorithm.patch_shape = self.aam.patch_shape[j] - algorithm.normalize_parts = self.aam.normalize_parts + algorithm = lk_algorithm_cls( + PartsLKAAMInterface, am, pdm, + sampling=sampling, patch_shape=self.aam.patch_shape[j], + normalize_parts=self.aam.normalize_parts, **kwargs) else: raise ValueError("AAM object must be of one of the " @@ -61,31 +106,122 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): # append algorithms to list self._algorithms.append(algorithm) - @property - def aam(self): - return self._model - @property - def algorithms(self): - return self._algorithms +# TODO: document me! +class CRAAMFitter(AAMFitter): + r""" + """ + def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, + n_appearance=None, sampling=None, n_perturbations=10, + max_iters=6, **kwargs): + super(CRAAMFitter, self).__init__( + aam, n_shape=n_shape, n_appearance=n_appearance) + self.n_perturbations = n_perturbations + self.max_iters = self._prepare_max_iters(max_iters) + self._set_up(cr_algorithm_cls, sampling, **kwargs) + + def _set_up(self, cr_algorithm_cls, sampling, **kwargs): + for j, (am, sm) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models)): + + if type(self.aam) is AAM or type(self.aam) is PatchAAM: + # build orthonormal model driven transform + md_transform = OrthoMDTransform( + sm, self.aam.transform, + source=am.mean().landmarks['source'].lms) + # set up algorithm using standard aam interface + algorithm = cr_algorithm_cls( + CRAAMInterface, am, md_transform, sampling=sampling, + max_iters=self.max_iters[j], **kwargs) + + elif (type(self.aam) is LinearAAM or + type(self.aam) is LinearPatchAAM): + # build linear version of orthogonal model driven transform + md_transform = LinearOrthoMDTransform( + sm, self.aam.n_landmarks) + # set up algorithm using linear aam interface + algorithm = cr_algorithm_cls( + CRLinearAAMInterface, am, md_transform, + sampling=sampling, max_iters=self.max_iters[j], **kwargs) + + elif type(self.aam) is PartsAAM: + # build orthogonal point distribution model + pdm = OrthoPDM(sm) + # set up algorithm using parts aam interface + algorithm = cr_algorithm_cls( + CRPartsAAMInterface, am, pdm, + sampling=sampling, max_iters=self.max_iters[j], + patch_shape=self.aam.patch_shape[j], + normalize_parts=self.aam.normalize_parts, **kwargs) - def _check_n_appearance(self, n_appearance): - if n_appearance is not None: - if type(n_appearance) is int or type(n_appearance) is float: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: - for am, n in zip(self.aam.appearance_models, n_appearance): - am.n_active_components = n else: - raise ValueError('n_appearance can be an integer or a float ' - 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + raise ValueError("AAM object must be of one of the " + "following classes: {}, {}, {}, {}, " + "{}".format(AAM, PatchAAM, LinearAAM, + LinearPatchAAM, PartsAAM)) + + # append algorithms to list + self._algorithms.append(algorithm) + + def train(self, images, group=None, label=None, verbose=False, **kwargs): + # normalize images with respect to reference shape of aam + images = rescale_images_to_reference_shape( + images, group, label, self.reference_shape, verbose=verbose) + + # for each pyramid level (low --> high) + for j, s in enumerate(self.scales): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if s == self.scales[-1]: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # compute features at highest level + feature_images = compute_features(images, self.features, + level_str=level_str, + verbose=verbose) + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, self.features, + level_str=level_str, + verbose=verbose) + + # extract ground truth shapes for current level + level_gt_shapes = [i.landmarks[group][label] for i in level_images] + + if j == 0: + # generate perturbed shapes + current_shapes = [] + for gt_s in level_gt_shapes: + perturbed_shapes = [] + for _ in range(self.n_perturbations): + perturbed_shapes.append(self.perturb_shape(gt_s)) + current_shapes.append(perturbed_shapes) + + # train cascaded regression algorithm + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + + # scale current shapes to next level resolution + if s != self.scales[-1]: + transform = Scale(self.scales[j+1]/s, n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + - def _fitter_result(self, image, algorithm_results, affine_correction, - gt_shape=None): - return AAMFitterResult(image, self, algorithm_results, - affine_correction, gt_shape=gt_shape) From 4d79e9a4c141537c1993971f31a5f32835c5a2df Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:21:22 +0100 Subject: [PATCH 257/423] Rename aam.algorithm.py to aam.algorithm.lk.py - Add a new subpackage algorithm to aam --- menpofit/aam/algorithm/lk.py | 796 +++++++++++++++++++++++++++++++++++ 1 file changed, 796 insertions(+) create mode 100644 menpofit/aam/algorithm/lk.py diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py new file mode 100644 index 0000000..1241d88 --- /dev/null +++ b/menpofit/aam/algorithm/lk.py @@ -0,0 +1,796 @@ +from __future__ import division +import abc +import numpy as np +from menpo.image import Image +from menpo.feature import gradient as fast_gradient, no_op +from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult + + +# TODO: needs to use interfaces in menpofit.algorithm.py +# TODO: implement more clever sampling? +class LKAAMInterface(object): + + def __init__(self, aam_algorithm, sampling=None): + self.algorithm = aam_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def shape_model(self): + return self.transform.pdm.model + + @property + def appearance_model(self): + return self.algorithm.appearance_model + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def m(self): + return self.appearance_model.n_active_components + + @property + def true_indices(self): + return self.template.mask.true_indices() + + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + + @classmethod + def solve_shape_map(cls, H, J, e, J_prior, p): + if p.shape[0] is not H.shape[0]: + # Bidirectional Compositional case + J_prior = np.hstack((J_prior, J_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + H += np.diag(J_prior) + Je = J_prior * p + J.T.dot(e) + return - np.linalg.solve(H, Je) + + @classmethod + def solve_shape_ml(cls, H, J, e): + # compute and return ML solution + return -np.linalg.solve(H, J.T.dot(e)) + + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): + if self.n is not H.shape[0] - self.m: + # Bidirectional Compositional case + Js_prior = np.hstack((Js_prior, Js_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + J_prior = np.hstack((Ja_prior, Js_prior)) + H += np.diag(J_prior) + Je = J_prior * np.hstack((c, p)) + J.T.dot(e) + dq = - np.linalg.solve(H, Je) + return dq[:self.m], dq[self.m:] + + def solve_all_ml(self, H, J, e): + # compute ML solution + dq = - np.linalg.solve(H, J.T.dot(e)) + return dq[:self.m], dq[self.m:] + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class LinearLKAAMInterface(LKAAMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return LinearAAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class PartsLKAAMInterface(LKAAMInterface): + + def __init__(self, aam_algorithm, sampling=None, patch_shape=(17, 17), + normalize_parts=no_op): + self.algorithm = aam_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.gradient_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.gradient2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) + + @property + def shape_model(self): + return self.transform.model + + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + def warp(self, image): + return Image(image.extract_patches( + self.transform.target, patch_size=self.patch_shape, + as_single_array=True)) + + def gradient(self, image): + nabla = fast_gradient(image.pixels.reshape((-1,) + self.patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return nabla.reshape((2,) + image.pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.gradient_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) + + +# TODO: handle costs for all LKAAMAlgorithms +# TODO document me! +class LKAAMAlgorithm(object): + + def __init__(self, aam_interface, appearance_model, transform, + eps=10**-5, **kwargs): + # set common state for all AAM algorithms + self.appearance_model = appearance_model + self.template = appearance_model.mean() + self.transform = transform + self.eps = eps + # set interface + self.interface = aam_interface(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self, **kwargs): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + self.m = self.appearance_model.n_active_components + + # grab appearance model components + self.A = self.appearance_model.components + # mask them + self.A_m = self.A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + + # grab appearance model mean + self.a_bar = self.appearance_model.mean() + # vectorize it and mask it + self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] + + # compute warp jacobian + self.dW_dp = self.interface.warp_jacobian() + + # compute shape model prior + s2 = (self.appearance_model.noise_variance() / + self.interface.shape_model.noise_variance()) + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + # compute appearance model prior + S = self.appearance_model.eigenvalues + self.s2_inv_S = s2 / S + + @abc.abstractmethod + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + pass + + +class ProjectOut(LKAAMAlgorithm): + r""" + Abstract Interface for Project-out AAM algorithms + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # solve for increments on the shape parameters + self.dp = self.solve(map_inference) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + @abc.abstractmethod + def solve(self, map_inference): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class PFC(ProjectOut): + r""" + Project-out Forward Compositional (PFC) Gauss-Newton algorithm + """ + def solve(self, map_inference): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # compute masked forward Jacobian + J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) + # project out appearance model from it + QJ_m = self.project_out(J_m) + # compute masked forward Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + JQJ_m, QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class PIC(ProjectOut): + r""" + Project-out Inverse Compositional (PIC) Gauss-Newton algorithm + """ + def precompute(self): + # call super method + super(PIC, self).precompute() + # compute appearance model mean gradient + nabla_a = self.interface.gradient(self.a_bar) + # compute masked inverse Jacobian + J_m = self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + # project out appearance model from it + self.QJ_m = self.project_out(J_m) + # compute masked inverse Hessian + self.JQJ_m = self.QJ_m.T.dot(J_m) + # compute masked Jacobian pseudo-inverse + self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) + + def solve(self, map_inference): + # solve for increments on the shape parameters + if map_inference: + return self.interface.solve_shape_map( + self.JQJ_m, self.QJ_m, self.e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + return -self.pinv_QJ_m.dot(self.e_m) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Simultaneous(LKAAMAlgorithm): + r""" + Abstract Interface for Simultaneous AAM algorithms + """ + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [self.c] + + # compute masked error + self.e_m = i_m - a_m + + # solve for increments on the appearance and shape parameters + # simultaneously + dc, self.dp = self.solve(map_inference) + + # update appearance parameters + self.c += dc + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(self.c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + def solve(self, map_inference): + # compute masked Jacobian + J_m = self.compute_jacobian() + # assemble masked simultaneous Jacobian + J_sim_m = np.hstack((-self.A_m, J_m)) + # compute masked Hessian + H_sim_m = J_sim_m.T.dot(J_sim_m) + # solve for increments on the appearance and shape parameters + # simultaneously + if map_inference: + return self.interface.solve_all_map( + H_sim_m, J_sim_m, self.e_m, self.s2_inv_S, self.c, + self.s2_inv_L, self.transform.as_vector()) + else: + return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) + + @abc.abstractmethod + def update_warp(self): + pass + + +class SFC(Simultaneous): + r""" + Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class SIC(Simultaneous): + r""" + Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Alternating(LKAAMAlgorithm): + r""" + Abstract Interface for Alternating AAM algorithms + """ + def precompute(self, **kwargs): + # call super method + super(Alternating, self).precompute() + # compute MAP appearance Hessian + self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + Jdp = 0 + else: + Jdp = J_m.dot(self.dp) + + # compute masked error + e_m = i_m - a_m + + # solve for increment on the appearance parameters + if map_inference: + Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) + dc = np.linalg.solve(self.AA_m_map, Ae_m_map) + else: + dc = self.pinv_A_m.dot(e_m + Jdp) + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m - self.A_m.T.dot(dc), self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, + e_m - self.A_m.dot(dc)) + + # update appearance parameters + c += dc + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + @abc.abstractmethod + def compute_jacobian(self): + pass + + @abc.abstractmethod + def update_warp(self): + pass + + +class AFC(Alternating): + r""" + Alternating Forward Compositional (AFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class AIC(Alternating): + r""" + Alternating Inverse Compositional (AIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class ModifiedAlternating(Alternating): + r""" + Abstract Interface for Modified Alternating AAM algorithms + """ + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + a_m = self.a_bar_m + c_list = [] + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + c = self.pinv_A_m.dot(i_m - a_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # compute masked Hessian + H_m = J_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + H_m, J_m, e_m, self.s2_inv_L, self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(H_m, J_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class MAFC(ModifiedAlternating): + r""" + Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class MAIC(ModifiedAlternating): + r""" + Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) + + +class Wiberg(LKAAMAlgorithm): + r""" + Abstract Interface for Wiberg AAM algorithms + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def run(self, image, initial_shape, gt_shape=None, max_iters=20, + map_inference=False): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Compositional Gauss-Newton loop + while k < max_iters and eps > self.eps: + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + if k == 0: + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + else: + c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - self.a_bar_m + + # compute masked Jacobian + J_m = self.compute_jacobian() + # project out appearance models + QJ_m = self.project_out(J_m) + # compute masked Hessian + JQJ_m = QJ_m.T.dot(J_m) + # solve for increments on the shape parameters + if map_inference: + self.dp = self.interface.solve_shape_map( + JQJ_m, QJ_m, e_m, self.s2_inv_L, + self.transform.as_vector()) + else: + self.dp = self.interface.solve_shape_ml(JQJ_m, QJ_m, e_m) + + # update warp + s_k = self.transform.target.points + self.update_warp() + p_list.append(self.transform.as_vector()) + + # test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + + +class WFC(Wiberg): + r""" + Wiberg Forward Compositional (WFC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped image gradient + nabla_i = self.interface.gradient(self.i) + # return forward Jacobian + return self.interface.steepest_descent_images(nabla_i, self.dW_dp) + + def update_warp(self): + # update warp based on forward composition + self.transform.from_vector_inplace( + self.transform.as_vector() + self.dp) + + +class WIC(Wiberg): + r""" + Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm + """ + def compute_jacobian(self): + # compute warped appearance model gradient + nabla_a = self.interface.gradient(self.a) + # return inverse Jacobian + return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) + + def update_warp(self): + # update warp based on inverse composition + self.transform.from_vector_inplace( + self.transform.as_vector() - self.dp) From 3668774b57b06f62f9647fe63a11fe8eb935f50e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:23:34 +0100 Subject: [PATCH 258/423] Add first version of CR Algorithms - Contains 2 Project-Out algorithms PJA and PSD. --- menpofit/aam/algorithm/cr.py | 408 +++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 menpofit/aam/algorithm/cr.py diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/cr.py new file mode 100644 index 0000000..b68bb1e --- /dev/null +++ b/menpofit/aam/algorithm/cr.py @@ -0,0 +1,408 @@ +from __future__ import division +import abc +import numpy as np +from menpo.image import Image +from menpo.feature import no_op +from menpo.visualize import print_dynamic, progress_bar_str +from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult + + +# TODO: implement more clever sampling? +class CRAAMInterface(object): + + def __init__(self, cr_aam_algorithm, sampling=None): + self.algorithm = cr_aam_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + + @property + def shape_model(self): + return self.transform.pdm.model + + @property + def appearance_model(self): + return self.algorithm.appearance_model + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def m(self): + return self.appearance_model.n_active_components + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class CRLinearAAMInterface(CRAAMInterface): + + @property + def shape_model(self): + return self.transform.model + + def algorithm_result(self, image, shape_parameters, + appearance_parameters=None, gt_shape=None): + return LinearAAMAlgorithmResult( + image, self.algorithm, shape_parameters, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + +class CRPartsAAMInterface(CRAAMInterface): + + def __init__(self, cr_aam_algorithm, sampling=None, patch_shape=(17, 17), + normalize_parts=no_op): + self.algorithm = cr_aam_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + + @property + def shape_model(self): + return self.transform.model + + def warp(self, image): + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts) + + +# TODO document me! +class CRAAMAlgorithm(object): + + def __init__(self, aam_interface, appearance_model, transform, max_iters=3, + eps=10**-5, **kwargs): + # set common state for all AAM algorithms + self.appearance_model = appearance_model + self.template = appearance_model.mean() + self.transform = transform + self.max_iters = max_iters + self.eps = eps + # set interface + self.interface = aam_interface(self, **kwargs) + # perform pre-computations + self.precompute() + + def precompute(self): + # grab number of shape and appearance parameters + self.n = self.transform.n_parameters + self.m = self.appearance_model.n_active_components + + # grab appearance model components + self.A = self.appearance_model.components + # mask them + self.A_m = self.A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + + # grab appearance model mean + self.a_bar = self.appearance_model.mean() + # vectorize it and mask it + self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] + + # compute shape model prior + s2 = (self.appearance_model.noise_variance() / + self.interface.shape_model.noise_variance()) + L = self.interface.shape_model.eigenvalues + self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) + # compute appearance model prior + S = self.appearance_model.eigenvalues + self.s2_inv_S = s2 / S + + def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): + # check training data + self._check_training_data(images, gt_shapes, current_shapes) + + n_images = len(images) + n_samples_image = len(current_shapes[0]) + + # set number of iterations and initialize list of regressors + self.regressors = [] + + # compute current and delta parameters from current and ground truth + # shapes + delta_params, current_params, gt_params = self._generate_params( + gt_shapes, current_shapes) + # initialize iteration counter + k = 0 + + # Cascaded Regression loop + while k < self.max_iters: + # generate regression data + features = self._generate_features(images, current_params, + verbose=verbose) + + # perform regression + if verbose: + print_dynamic('- Performing regression...') + regressor = self._perform_regression(features, delta_params, + **kwargs) + # add regressor to list + self.regressors.append(regressor) + # compute regression rmse + estimated_delta_params = regressor(features) + rmse = _compute_rmse(delta_params, estimated_delta_params) + if verbose: + print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(rmse)) + + current_params += estimated_delta_params + + delta_params = gt_params - current_params + # increase iteration counter + k += 1 + + # obtain current shapes from current parameters + current_shapes = [] + for p in current_params: + current_shapes.append(self.transform.from_vector(p).target) + + # convert current shapes into a list of list and return + final_shapes = [] + for j in range(n_images): + k = j * n_samples_image + l = k + n_samples_image + final_shapes.append(current_shapes[k:l]) + return final_shapes + + @staticmethod + def _check_training_data(images, gt_shapes, current_shapes): + if len(images) != len(gt_shapes): + raise ValueError("The number of shapes must be equal to " + "the number of images.") + elif len(images) != len(current_shapes): + raise ValueError("The number of current shapes must be " + "equal or multiple to the number of images.") + + def _generate_params(self, gt_shapes, current_shapes): + # initialize current and delta parameters arrays + n_samples = len(gt_shapes) * len(current_shapes[0]) + current_params = np.empty((n_samples, self.transform.n_parameters)) + gt_params = np.empty((n_samples, self.transform.n_parameters)) + delta_params = np.empty((n_samples, self.transform.n_parameters)) + # initialize sample counter + k = 0 + # compute ground truth and current shape parameters + for gt_s, c_s in zip(gt_shapes, current_shapes): + for s in c_s: + # compute current parameters + current_params[k] = self._compute_params(s) + # compute ground truth parameters + gt_params[k] = self._compute_params(gt_s) + # compute delta parameters + delta_params[k] = gt_params[k] - current_params[k] + # increment counter + k += 1 + + return delta_params, current_params, gt_params + + def _compute_params(self, shape): + self.transform.set_target(shape) + return self.transform.as_vector() + + def _generate_features(self, images, current_params, verbose=False): + # initialize features array + n_images = len(images) + n_samples = len(current_params) + n_samples_image = int(n_samples / n_images) + features = np.zeros((n_samples,) + self.a_bar_m.shape) + + # initialize sample counter + k = 0 + for i in images: + for _ in range(n_samples_image): + if verbose: + print_dynamic('- Generating regression features - {' + '}'.format( + progress_bar_str((k + 1.) / n_samples, + show_bar=False))) + # set transform + self.transform.from_vector_inplace(current_params[k]) + # compute regression features + f = self._compute_features(i) + # add to features array + features[k] = f + # increment counter + k += 1 + + return features + + @abc.abstractmethod + def _compute_features(self, image): + pass + + @abc.abstractmethod + def _perform_regression(self, features, deltas, gamma=None): + pass + + def run(self, image, initial_shape, gt_shape=None, **kwargs): + # initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # initialize iteration counter + k = 0 + + # Cascaded Regression loop + while k < self.max_iters: + # compute regression features + features = self._compute_features2(image) + + # solve for increments on the shape parameters + dp = self.regressors[k](features) + + # update warp + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # increase iteration counter + k += 1 + + # return algorithm result + return self.interface.algorithm_result( + image, p_list, gt_shape=gt_shape) + + +# TODO: document me! +class ProjectOut(CRAAMAlgorithm): + r""" + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def _compute_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + e_m = i_m - self.a_bar_m + return self.project_out(e_m) + + def _compute_features2(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + +# TODO: document me! +class ProjectOut2(CRAAMAlgorithm): + r""" + """ + def project_out(self, J): + # project-out appearance bases from a particular vector or matrix + return J - self.A_m.dot(self.pinv_A_m.dot(J)) + + def _compute_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + e_m = i_m - self.a_bar_m + return self.project_out(e_m) + + def _compute_features2(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + +# TODO: document me! +class PSD(ProjectOut): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None): + return _supervised_descent(features, deltas, gamma=gamma) + + +# TODO: document me! +class PAJ(ProjectOut): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None): + return _average_jacobian(features, deltas, gamma=gamma) + + +# TODO: document me! +class _supervised_descent(object): + r""" + """ + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = features.T.dot(features) + XT = features.T.dot(deltas) + if gamma: + XX += gamma * np.eye(features.shape[1]) + # descent direction + self.R = np.linalg.solve(XX, XT) + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +class _average_jacobian(object): + r""" + """ + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = deltas.T.dot(deltas) + XT = deltas.T.dot(features) + if gamma: + XX += gamma * np.eye(deltas.shape[1]) + # average Jacobian + self.J = np.linalg.solve(XX, XT) + # average Hessian + self.H = self.J.dot(self.J.T) + # descent direction + self.R = np.linalg.solve(self.H, self.J).T + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +def _compute_rmse(x1, x2): + return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + From 7e654ac846df7511930e35a242a4c606ac52ef57 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:25:06 +0100 Subject: [PATCH 259/423] Delete old aam.alagorithm.py and update aam.algorithm.__init__.py --- menpofit/aam/__init__.py | 5 +- menpofit/aam/algorithm.py | 798 -------------------------------------- 2 files changed, 3 insertions(+), 800 deletions(-) delete mode 100644 menpofit/aam/algorithm.py diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 673cb05..32a3556 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,11 +1,12 @@ from .builder import ( AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, LinearPatchAAMBuilder, PartsAAMBuilder) -from .fitter import LKAAMFitter +from .fitter import LKAAMFitter, CRAAMFitter from .algorithm import ( PFC, PIC, SFC, SIC, AFC, AIC, MAFC, MAIC, - WFC, WIC) + WFC, WIC, + PSD, PAJ) diff --git a/menpofit/aam/algorithm.py b/menpofit/aam/algorithm.py deleted file mode 100644 index 6583f00..0000000 --- a/menpofit/aam/algorithm.py +++ /dev/null @@ -1,798 +0,0 @@ -from __future__ import division -import abc -import numpy as np -from menpo.image import Image -from menpo.feature import gradient as fast_gradient -from .result import AAMAlgorithmResult, LinearAAMAlgorithmResult - - -# TODO: implement more clever sampling? -class LKAAMInterface(object): - - def __init__(self, aam_algorithm, sampling=None): - self.algorithm = aam_algorithm - - n_true_pixels = self.template.n_true_pixels() - n_channels = self.template.n_channels - n_parameters = self.transform.n_parameters - sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - - if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 - - self.i_mask = np.nonzero(np.tile( - sampling_mask[None, ...], (n_channels, 1)).flatten())[0] - self.dW_dp_mask = np.nonzero(np.tile( - sampling_mask[None, ..., None], (2, 1, n_parameters))) - self.nabla_mask = np.nonzero(np.tile( - sampling_mask[None, None, ...], (2, n_channels, 1))) - self.nabla2_mask = np.nonzero(np.tile( - sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) - - @property - def shape_model(self): - return self.transform.pdm.model - - @property - def appearance_model(self): - return self.algorithm.appearance_model - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - - @property - def n(self): - return self.transform.n_parameters - - @property - def m(self): - return self.appearance_model.n_active_components - - @property - def true_indices(self): - return self.template.mask.true_indices() - - def warp_jacobian(self): - dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) - return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, - dW_dp.shape[2])) - - def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) - - def gradient(self, img): - nabla = fast_gradient(img) - nabla.set_boundary_pixels() - return nabla.as_vector().reshape((2, img.n_channels, -1)) - - def steepest_descent_images(self, nabla, dW_dp): - # reshape gradient - # nabla: n_dims x n_channels x n_pixels - nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) - # compute steepest descent images - # nabla: n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # sdi: n_channels x n_pixels x n_params - sdi = 0 - a = nabla[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - # reshape steepest descent images - # sdi: (n_channels x n_pixels) x n_params - return sdi.reshape((-1, sdi.shape[2])) - - @classmethod - def solve_shape_map(cls, H, J, e, J_prior, p): - if p.shape[0] is not H.shape[0]: - # Bidirectional Compositional case - J_prior = np.hstack((J_prior, J_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - H += np.diag(J_prior) - Je = J_prior * p + J.T.dot(e) - return - np.linalg.solve(H, Je) - - @classmethod - def solve_shape_ml(cls, H, J, e): - # compute and return ML solution - return -np.linalg.solve(H, J.T.dot(e)) - - def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): - if self.n is not H.shape[0] - self.m: - # Bidirectional Compositional case - Js_prior = np.hstack((Js_prior, Js_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - J_prior = np.hstack((Ja_prior, Js_prior)) - H += np.diag(J_prior) - Je = J_prior * np.hstack((c, p)) + J.T.dot(e) - dq = - np.linalg.solve(H, Je) - return dq[:self.m], dq[self.m:] - - def solve_all_ml(self, H, J, e): - # compute ML solution - dq = - np.linalg.solve(H, J.T.dot(e)) - return dq[:self.m], dq[self.m:] - - def algorithm_result(self, image, shape_parameters, - appearance_parameters=None, gt_shape=None): - return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, - appearance_parameters=appearance_parameters, gt_shape=gt_shape) - - -class LinearLKAAMInterface(LKAAMInterface): - - @property - def shape_model(self): - return self.transform.model - - def algorithm_result(self, image, shape_parameters, - appearance_parameters=None, gt_shape=None): - return LinearAAMAlgorithmResult( - image, self.algorithm, shape_parameters, - appearance_parameters=appearance_parameters, gt_shape=gt_shape) - - -class PartsLKAAMInterface(LKAAMInterface): - - def __init__(self, aam_algorithm, sampling=None): - self.algorithm = aam_algorithm - - if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) - - image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling[None, None, None, ...], - image_shape[:3] + (1, 1)) - self.i_mask = np.nonzero(image_mask.flatten())[0] - self.gradient_mask = np.nonzero(np.tile( - image_mask[None, ...], (2, 1, 1, 1, 1, 1))) - self.gradient2_mask = np.nonzero(np.tile( - image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) - - @property - def shape_model(self): - return self.transform.model - - @property - def patch_shape(self): - return self.appearance_model.patch_shape - - def warp_jacobian(self): - return np.rollaxis(self.transform.d_dp(None), -1) - - def warp(self, image): - return Image(image.extract_patches( - self.transform.target, patch_size=self.patch_shape, - as_single_array=True)) - - def gradient(self, image): - pixels = image.pixels - patch_shape = self.algorithm.appearance_model.patch_shape - g = fast_gradient(pixels.reshape((-1,) + patch_shape)) - # remove 1st dimension gradient which corresponds to the gradient - # between parts - return g.reshape((2,) + pixels.shape) - - def steepest_descent_images(self, nabla, dw_dp): - # reshape nabla - # nabla: dims x parts x off x ch x (h x w) - nabla = nabla[self.gradient_mask].reshape( - nabla.shape[:-2] + (-1,)) - # compute steepest descent images - # nabla: dims x parts x off x ch x (h x w) - # ds_dp: dims x parts x x params - # sdi: parts x off x ch x (h x w) x params - sdi = 0 - a = nabla[..., None] * dw_dp[..., None, None, None, :] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (parts x offsets x ch x w x h) x params - return sdi.reshape((-1, sdi.shape[-1])) - - -# TODO: handle costs for all LKAAMAlgorithms -# TODO document me! -class LKAAMAlgorithm(object): - - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # set common state for all AAM algorithms - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.transform = transform - self.eps = eps - # set interface - self.interface = aam_interface(self, **kwargs) - # perform pre-computations - self.precompute() - - def precompute(self, **kwargs): - # grab number of shape and appearance parameters - self.n = self.transform.n_parameters - self.m = self.appearance_model.n_active_components - - # grab appearance model components - self.A = self.appearance_model.components - # mask them - self.A_m = self.A.T[self.interface.i_mask, :] - # compute their pseudoinverse - self.pinv_A_m = np.linalg.pinv(self.A_m) - - # grab appearance model mean - self.a_bar = self.appearance_model.mean() - # vectorize it and mask it - self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] - - # compute warp jacobian - self.dW_dp = self.interface.warp_jacobian() - - # compute shape model prior - s2 = (self.appearance_model.noise_variance() / - self.interface.shape_model.noise_variance()) - L = self.interface.shape_model.eigenvalues - self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) - # compute appearance model prior - S = self.appearance_model.eigenvalues - self.s2_inv_S = s2 / S - - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None, - map_inference=False): - pass - - -class ProjectOut(LKAAMAlgorithm): - r""" - Abstract Interface for Project-out AAM algorithms - """ - def project_out(self, J): - # project-out appearance bases from a particular vector or matrix - return J - self.A_m.dot(self.pinv_A_m.dot(J)) - - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # vectorize it and mask it - i_m = self.i.as_vector()[self.interface.i_mask] - - # compute masked error - self.e_m = i_m - self.a_bar_m - - # solve for increments on the shape parameters - self.dp = self.solve(map_inference) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, gt_shape=gt_shape) - - @abc.abstractmethod - def solve(self, map_inference): - pass - - @abc.abstractmethod - def update_warp(self): - pass - - -class PFC(ProjectOut): - r""" - Project-out Forward Compositional (PFC) Gauss-Newton algorithm - """ - def solve(self, map_inference): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # compute masked forward Jacobian - J_m = self.interface.steepest_descent_images(nabla_i, self.dW_dp) - # project out appearance model from it - QJ_m = self.project_out(J_m) - # compute masked forward Hessian - JQJ_m = QJ_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - return self.interface.solve_shape_map( - JQJ_m, QJ_m, self.e_m, self.s2_inv_L, - self.transform.as_vector()) - else: - return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class PIC(ProjectOut): - r""" - Project-out Inverse Compositional (PIC) Gauss-Newton algorithm - """ - def precompute(self): - # call super method - super(PIC, self).precompute() - # compute appearance model mean gradient - nabla_a = self.interface.gradient(self.a_bar) - # compute masked inverse Jacobian - J_m = self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - # project out appearance model from it - self.QJ_m = self.project_out(J_m) - # compute masked inverse Hessian - self.JQJ_m = self.QJ_m.T.dot(J_m) - # compute masked Jacobian pseudo-inverse - self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) - - def solve(self, map_inference): - # solve for increments on the shape parameters - if map_inference: - return self.interface.solve_shape_map( - self.JQJ_m, self.QJ_m, self.e_m, self.s2_inv_L, - self.transform.as_vector()) - else: - return -self.pinv_QJ_m.dot(self.e_m) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class Simultaneous(LKAAMAlgorithm): - r""" - Abstract Interface for Simultaneous AAM algorithms - """ - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(self.c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [self.c] - - # compute masked error - self.e_m = i_m - a_m - - # solve for increments on the appearance and shape parameters - # simultaneously - dc, self.dp = self.solve(map_inference) - - # update appearance parameters - self.c += dc - self.a = self.appearance_model.instance(self.c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(self.c) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - @abc.abstractmethod - def compute_jacobian(self): - pass - - def solve(self, map_inference): - # compute masked Jacobian - J_m = self.compute_jacobian() - # assemble masked simultaneous Jacobian - J_sim_m = np.hstack((-self.A_m, J_m)) - # compute masked Hessian - H_sim_m = J_sim_m.T.dot(J_sim_m) - # solve for increments on the appearance and shape parameters - # simultaneously - if map_inference: - return self.interface.solve_all_map( - H_sim_m, J_sim_m, self.e_m, self.s2_inv_S, self.c, - self.s2_inv_L, self.transform.as_vector()) - else: - return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) - - @abc.abstractmethod - def update_warp(self): - pass - - -class SFC(Simultaneous): - r""" - Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class SIC(Simultaneous): - r""" - Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class Alternating(LKAAMAlgorithm): - r""" - Abstract Interface for Alternating AAM algorithms - """ - def precompute(self, **kwargs): - # call super method - super(Alternating, self).precompute() - # compute MAP appearance Hessian - self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) - - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - Jdp = 0 - else: - Jdp = J_m.dot(self.dp) - - # compute masked error - e_m = i_m - a_m - - # solve for increment on the appearance parameters - if map_inference: - Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) - dc = np.linalg.solve(self.AA_m_map, Ae_m_map) - else: - dc = self.pinv_A_m.dot(e_m + Jdp) - - # compute masked Jacobian - J_m = self.compute_jacobian() - # compute masked Hessian - H_m = J_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - self.dp = self.interface.solve_shape_map( - H_m, J_m, e_m - self.A_m.T.dot(dc), self.s2_inv_L, - self.transform.as_vector()) - else: - self.dp = self.interface.solve_shape_ml(H_m, J_m, - e_m - self.A_m.dot(dc)) - - # update appearance parameters - c += dc - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - @abc.abstractmethod - def compute_jacobian(self): - pass - - @abc.abstractmethod - def update_warp(self): - pass - - -class AFC(Alternating): - r""" - Alternating Forward Compositional (AFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class AIC(Alternating): - r""" - Alternating Inverse Compositional (AIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class ModifiedAlternating(Alternating): - r""" - Abstract Interface for Modified Alternating AAM algorithms - """ - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - a_m = self.a_bar_m - c_list = [] - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - c = self.pinv_A_m.dot(i_m - a_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) - - # compute masked error - e_m = i_m - a_m - - # compute masked Jacobian - J_m = self.compute_jacobian() - # compute masked Hessian - H_m = J_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - self.dp = self.interface.solve_shape_map( - H_m, J_m, e_m, self.s2_inv_L, self.transform.as_vector()) - else: - self.dp = self.interface.solve_shape_ml(H_m, J_m, e_m) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - -class MAFC(ModifiedAlternating): - r""" - Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class MAIC(ModifiedAlternating): - r""" - Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) - - -class Wiberg(LKAAMAlgorithm): - r""" - Abstract Interface for Wiberg AAM algorithms - """ - def project_out(self, J): - # project-out appearance bases from a particular vector or matrix - return J - self.A_m.dot(self.pinv_A_m.dot(J)) - - def run(self, image, initial_shape, gt_shape=None, max_iters=20, - map_inference=False): - # initialize transform - self.transform.set_target(initial_shape) - p_list = [self.transform.as_vector()] - - # initialize iteration counter and epsilon - k = 0 - eps = np.Inf - - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] - - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - else: - c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) - - # compute masked error - e_m = i_m - self.a_bar_m - - # compute masked Jacobian - J_m = self.compute_jacobian() - # project out appearance models - QJ_m = self.project_out(J_m) - # compute masked Hessian - JQJ_m = QJ_m.T.dot(J_m) - # solve for increments on the shape parameters - if map_inference: - self.dp = self.interface.solve_shape_map( - JQJ_m, QJ_m, e_m, self.s2_inv_L, - self.transform.as_vector()) - else: - self.dp = self.interface.solve_shape_ml(JQJ_m, QJ_m, e_m) - - # update warp - s_k = self.transform.target.points - self.update_warp() - p_list.append(self.transform.as_vector()) - - # test convergence - eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) - - # increase iteration counter - k += 1 - - # return algorithm result - return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - - -class WFC(Wiberg): - r""" - Wiberg Forward Compositional (WFC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped image gradient - nabla_i = self.interface.gradient(self.i) - # return forward Jacobian - return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - - def update_warp(self): - # update warp based on forward composition - self.transform.from_vector_inplace( - self.transform.as_vector() + self.dp) - - -class WIC(Wiberg): - r""" - Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm - """ - def compute_jacobian(self): - # compute warped appearance model gradient - nabla_a = self.interface.gradient(self.a) - # return inverse Jacobian - return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - - def update_warp(self): - # update warp based on inverse composition - self.transform.from_vector_inplace( - self.transform.as_vector() - self.dp) From 7026f9570dd1536a61e69f58bcf4b04ec9c9e79d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 12:34:29 +0100 Subject: [PATCH 260/423] Update aam.alagorithm.__init__.py --- menpofit/aam/algorithm/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 menpofit/aam/algorithm/__init__.py diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py new file mode 100644 index 0000000..3416b8c --- /dev/null +++ b/menpofit/aam/algorithm/__init__.py @@ -0,0 +1,7 @@ +from .lk import ( + PFC, PIC, + SFC, SIC, + AFC, AIC, + MAFC, MAIC, + WFC, WIC) +from .cr import PSD, PAJ From 21d08702f440488c3bb9fe5e4715afa46bed4ed9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 14:23:15 +0100 Subject: [PATCH 261/423] Add new initializations for ModelFitter --- menpofit/fitter.py | 107 +++++++++++---------------------------------- 1 file changed, 25 insertions(+), 82 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 13ea1da..ae871b5 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -284,6 +284,10 @@ def reference_shape(self): """ return self._model.reference_shape + @property + def reference_bounding_box(self): + return self.reference_shape.bounding_box() + @property def features(self): r""" @@ -328,97 +332,36 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - # TODO: Fix me! - def perturb_shape(self, gt_shape, noise_std=10, rotation=False): - transform = noisy_align(AlignmentSimilarity, self.reference_shape, - gt_shape, noise_std=noise_std) - return transform.apply(self.reference_shape) - # TODO: Bounding boxes should be PointGraphs - def obtain_shape_from_bb(self, bounding_box): - r""" - Generates an initial shape given a bounding box detection. - - Parameters - ----------- - bounding_box: (2, 2) ndarray - The bounding box specified as: - - np.array([[x_min, y_min], [x_max, y_max]]) + def get_initial_shape_from_bounding_box(self, bounding_box, noise_std=0.04, + rotation=False): + transform = noisy_align(AlignmentSimilarity, + self.reference_bounding_box, bounding_box, + noise_std=noise_std, rotation=rotation) + return transform.apply(self.reference_shape) - Returns - ------- - initial_shape: :class:`menpo.shape.PointCloud` - The initial shape. - """ - reference_shape = self.reference_shape - return align_shape_with_bb(reference_shape, - bounding_box).apply(reference_shape) + def get_initial_shape_from_shape(self, shape, noise_std=0.04, + rotation=False): + return self.get_initial_shape_from_bounding_box( + shape.bounding_box(), noise_std=noise_std, rotation=rotation) # TODO: document me! -def noisy_align(alignment_transform_cls, source, target, noise_std=10): +def noisy_align(alignment_transform_cls, source, target, noise_std=0.1, + **kwargs): r""" """ - noise = noise_std * np.random.randn(target.n_points, target.n_dims) + noise = noise_std * target.range() * np.random.randn(target.n_points, + target.n_dims) noisy_target = PointCloud(target.points + noise) - return alignment_transform_cls(source, noisy_target) + return alignment_transform_cls(source, noisy_target, **kwargs) -def align_shape_with_bb(shape, bounding_box): +# TODO: document me! +def align_shape_with_bounding_box(alignment_transform_cls, shape, + bounding_box, **kwargs): r""" - Returns the Similarity transform that aligns the provided shape with the - provided bounding box. - - Parameters - ---------- - shape: :class:`menpo.shape.PointCloud` - The shape to be aligned. - bounding_box: (2, 2) ndarray - The bounding box specified as: - - np.array([[x_min, y_min], [x_max, y_max]]) - - Returns - ------- - transform : :class: `menpo.transform.Similarity` - The align transform """ - shape_box = PointCloud(shape.bounds()) - bounding_box = PointCloud(bounding_box) - return AlignmentSimilarity(shape_box, bounding_box, rotation=False) - - -# TODO: implement as a method on Similarity? AlignableTransforms? -# def noisy_align(source, target, noise_std=0.04, rotation=False): -# r""" -# Constructs and perturbs the optimal similarity transform between source -# to the target by adding white noise to its weights. -# -# Parameters -# ---------- -# source: :class:`menpo.shape.PointCloud` -# The source pointcloud instance used in the alignment -# target: :class:`menpo.shape.PointCloud` -# The target pointcloud instance used in the alignment -# noise_std: float -# The standard deviation of the white noise -# -# Default: 0.04 -# rotation: boolean -# If False the second parameter of the Similarity, -# which captures captures inplane rotations, is set to 0. -# -# Default:False -# -# Returns -# ------- -# noisy_transform : :class: `menpo.transform.Similarity` -# The noisy Similarity Transform -# """ -# transform = AlignmentSimilarity(source, target, rotation=rotation) -# parameters = transform.as_vector() -# parameter_range = np.hstack((parameters[:2], target.range())) -# noise = (parameter_range * noise_std * -# np.random.randn(transform.n_parameters)) -# return Similarity.init_identity(source.n_dims).from_vector(parameters + noise) + shape_bb = shape.bounding_box() + return alignment_transform_cls(shape_bb, bounding_box, **kwargs) + From 7db862273e38a4551ffc75408b9556a4b462acc5 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 14:37:58 +0100 Subject: [PATCH 262/423] Change prepare_max_iters in Fitter for a checking function --- menpofit/aam/fitter.py | 4 ++-- menpofit/checks.py | 16 ++++++++++++++++ menpofit/fitter.py | 18 ++---------------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f80d677..93fb5d5 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,11 +1,11 @@ from __future__ import division -import abc from menpo.transform import Scale from menpofit.builder import ( rescale_images_to_reference_shape, compute_features, scale_images) from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform +import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM from .algorithm.lk import ( LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) @@ -117,7 +117,7 @@ def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, super(CRAAMFitter, self).__init__( aam, n_shape=n_shape, n_appearance=n_appearance) self.n_perturbations = n_perturbations - self.max_iters = self._prepare_max_iters(max_iters) + self.max_iters = checks.check_max_iters(max_iters, self.n_levels) self._set_up(cr_algorithm_cls, sampling, **kwargs) def _set_up(self, cr_algorithm_cls, sampling, **kwargs): diff --git a/menpofit/checks.py b/menpofit/checks.py index 5098d03..e48cb42 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,3 +1,4 @@ +import numpy as np from menpofit.base import is_pyramid_on_features @@ -113,6 +114,21 @@ def check_max_components(max_components, n_levels, var_name): return max_components_list +def check_max_iters(max_iters, n_levels): + # check max_iters parameter + if type(max_iters) is int: + max_iters = [np.round(max_iters/n_levels) + for _ in range(n_levels)] + elif len(max_iters) == 1 and n_levels > 1: + max_iters = [np.round(max_iters[0]/n_levels) + for _ in range(n_levels)] + elif len(max_iters) != n_levels: + raise ValueError('max_iters can be integer, integer list ' + 'containing 1 or {} elements or ' + 'None'.format(n_levels)) + return np.require(max_iters, dtype=np.int) + + # def check_n_levels(n_levels): # r""" # Checks the number of pyramid levels - must be int > 0. diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ae871b5..6b92870 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -3,6 +3,7 @@ import numpy as np from menpo.shape import PointCloud from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity +import menpofit.checks as checks # TODO: document me! @@ -226,7 +227,7 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, The fitting object containing the state of the whole fitting procedure. """ - max_iters = self._prepare_max_iters(max_iters) + max_iters = checks.check_max_iters(max_iters, self.n_levels) shape = initial_shape gt_shape = None algorithm_results = [] @@ -246,21 +247,6 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, return algorithm_results - def _prepare_max_iters(self, max_iters): - n_levels = self.n_levels - # check max_iters parameter - if type(max_iters) is int: - max_iters = [np.round(max_iters/n_levels) - for _ in range(n_levels)] - elif len(max_iters) == 1 and n_levels > 1: - max_iters = [np.round(max_iters[0]/n_levels) - for _ in range(n_levels)] - elif len(max_iters) != n_levels: - raise ValueError('max_iters can be integer, integer list ' - 'containing 1 or {} elements or ' - 'None'.format(self.n_levels)) - return np.require(max_iters, dtype=np.int) - @abc.abstractmethod def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): From 67b9d34d543c95e6bb9f628724e403b18ce4e8f3 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 15:34:36 +0100 Subject: [PATCH 263/423] Add support for sampling being a list --- menpofit/aam/fitter.py | 29 ++++++++++++++++------------- menpofit/checks.py | 20 +++++++++++++++++++- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 93fb5d5..f40a450 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -62,11 +62,12 @@ def __init__(self, aam, n_shape=None, n_appearance=None, lk_algorithm_cls=AIC, sampling=None, **kwargs): super(LKAAMFitter, self).__init__( aam, n_shape=n_shape, n_appearance=n_appearance) + sampling = checks.check_sampling(sampling, self.n_levels) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): - for j, (am, sm) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models)): + for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models, sampling)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform @@ -75,7 +76,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = lk_algorithm_cls( - LKAAMInterface, am, md_transform, sampling=sampling, + LKAAMInterface, am, md_transform, sampling=s, **kwargs) elif (type(self.aam) is LinearAAM or @@ -85,7 +86,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): sm, self.aam.n_landmarks) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( - LinearLKAAMInterface, am, md_transform, sampling=sampling, + LinearLKAAMInterface, am, md_transform, sampling=s, **kwargs) elif type(self.aam) is PartsAAM: @@ -93,8 +94,8 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = lk_algorithm_cls( - PartsLKAAMInterface, am, pdm, - sampling=sampling, patch_shape=self.aam.patch_shape[j], + PartsLKAAMInterface, am, pdm, sampling=s, + patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) else: @@ -116,13 +117,14 @@ def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, max_iters=6, **kwargs): super(CRAAMFitter, self).__init__( aam, n_shape=n_shape, n_appearance=n_appearance) + sampling = checks.check_sampling(sampling, self.n_levels) self.n_perturbations = n_perturbations self.max_iters = checks.check_max_iters(max_iters, self.n_levels) self._set_up(cr_algorithm_cls, sampling, **kwargs) def _set_up(self, cr_algorithm_cls, sampling, **kwargs): - for j, (am, sm) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models)): + for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, + self.aam.shape_models, sampling)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform @@ -131,7 +133,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = cr_algorithm_cls( - CRAAMInterface, am, md_transform, sampling=sampling, + CRAAMInterface, am, md_transform, sampling=s, max_iters=self.max_iters[j], **kwargs) elif (type(self.aam) is LinearAAM or @@ -141,8 +143,8 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): sm, self.aam.n_landmarks) # set up algorithm using linear aam interface algorithm = cr_algorithm_cls( - CRLinearAAMInterface, am, md_transform, - sampling=sampling, max_iters=self.max_iters[j], **kwargs) + CRLinearAAMInterface, am, md_transform, sampling=s, + max_iters=self.max_iters[j], **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model @@ -150,7 +152,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): # set up algorithm using parts aam interface algorithm = cr_algorithm_cls( CRPartsAAMInterface, am, pdm, - sampling=sampling, max_iters=self.max_iters[j], + sampling=s, max_iters=self.max_iters[j], patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) @@ -209,7 +211,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - perturbed_shapes.append(self.perturb_shape(gt_s)) + p_s = self.get_initial_shape_from_shape(gt_s) + perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) # train cascaded regression algorithm diff --git a/menpofit/checks.py b/menpofit/checks.py index e48cb42..2ac6e66 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -114,8 +114,8 @@ def check_max_components(max_components, n_levels, var_name): return max_components_list +# TODO: document me! def check_max_iters(max_iters, n_levels): - # check max_iters parameter if type(max_iters) is int: max_iters = [np.round(max_iters/n_levels) for _ in range(n_levels)] @@ -129,6 +129,24 @@ def check_max_iters(max_iters, n_levels): return np.require(max_iters, dtype=np.int) +# TODO: document me! +def check_sampling(sampling, n_levels): + if isinstance(sampling, (list, tuple)): + if len(sampling) == 1: + sampling = sampling * n_levels + elif len(sampling) != n_levels: + raise ValueError('A sampling list can only ' + 'contain 1 element or {} ' + 'elements'.format(n_levels)) + elif isinstance(sampling, np.ndarray): + sampling = [sampling] * n_levels + else: + raise ValueError('sampling can be a ndarray, a ndarray list ' + 'containing 1 or {} elements or ' + 'None'.format(n_levels)) + return sampling + + # def check_n_levels(n_levels): # r""" # Checks the number of pyramid levels - must be int > 0. From c771c0351f64ac942fdf532633ebe6c51d0a6e0e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 29 May 2015 17:46:28 +0100 Subject: [PATCH 264/423] Remove ProjectOut2 from aam.algorithm.cr --- menpofit/aam/algorithm/cr.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/cr.py index b68bb1e..f9c5d7c 100644 --- a/menpofit/aam/algorithm/cr.py +++ b/menpofit/aam/algorithm/cr.py @@ -322,32 +322,6 @@ def _compute_features2(self, image): return i_m - self.a_bar_m -# TODO: document me! -class ProjectOut2(CRAAMAlgorithm): - r""" - """ - def project_out(self, J): - # project-out appearance bases from a particular vector or matrix - return J - self.A_m.dot(self.pinv_A_m.dot(J)) - - def _compute_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - self.a_bar_m - return self.project_out(e_m) - - def _compute_features2(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - return i_m - self.a_bar_m - - # TODO: document me! class PSD(ProjectOut): r""" From 6dc7b869f0fa3b346f8c42ddc687126f03a7f113 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 1 Jun 2015 11:52:29 +0100 Subject: [PATCH 265/423] Small changes to training regression based AAMs - Change previous perturb_shape methods in Fitters to noisy_shape_from_shape and noisy_shape_from_bounding_box --- menpofit/aam/fitter.py | 15 ++++++--------- menpofit/fitter.py | 7 +++---- menpofit/lk/fitter.py | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f40a450..de7bed1 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -170,6 +170,11 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): images = rescale_images_to_reference_shape( images, group, label, self.reference_shape, verbose=verbose) + if self.scale_features: + # compute features at highest level + feature_images = compute_features(images, self.features, + verbose=verbose) + # for each pyramid level (low --> high) for j, s in enumerate(self.scales): if verbose: @@ -180,16 +185,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): # obtain image representation if s == self.scales[-1]: - # compute features at highest level - feature_images = compute_features(images, self.features, - level_str=level_str, - verbose=verbose) level_images = feature_images elif self.scale_features: - # compute features at highest level - feature_images = compute_features(images, self.features, - level_str=level_str, - verbose=verbose) # scale features at other levels level_images = scale_images(feature_images, s, level_str=level_str, @@ -211,7 +208,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - p_s = self.get_initial_shape_from_shape(gt_s) + p_s = self.noisy_shape_from_shape(gt_s) perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 6b92870..9b1f906 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -319,15 +319,14 @@ def _check_n_shape(self, n_shape): 'those'.format(self._model.n_levels)) # TODO: Bounding boxes should be PointGraphs - def get_initial_shape_from_bounding_box(self, bounding_box, noise_std=0.04, - rotation=False): + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, + rotation=False): transform = noisy_align(AlignmentSimilarity, self.reference_bounding_box, bounding_box, noise_std=noise_std, rotation=rotation) return transform.apply(self.reference_shape) - def get_initial_shape_from_shape(self, shape, noise_std=0.04, - rotation=False): + def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): return self.get_initial_shape_from_bounding_box( shape.bounding_box(), noise_std=noise_std, rotation=rotation) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 4633904..73f6eb1 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -97,7 +97,7 @@ def _prepare_template(self, template, group=None, label=None): return templates, sources - def perturb_shape(self, gt_shape, noise_std=0.04): + def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): transform = noisy_align(self.transform_cls, self.reference_shape, gt_shape, noise_std=noise_std) return transform.apply(self.reference_shape) From a100930e53c9cf3edcdf5f313f9864d2cc21113d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 1 Jun 2015 14:32:33 +0100 Subject: [PATCH 266/423] Removed algorithm.py - This decouples the structure of ATM and AAM fitters --- menpofit/algorithm.py | 134 -------------------------------------- menpofit/atm/algorithm.py | 130 +++++++++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 137 deletions(-) delete mode 100644 menpofit/algorithm.py diff --git a/menpofit/algorithm.py b/menpofit/algorithm.py deleted file mode 100644 index b651ba6..0000000 --- a/menpofit/algorithm.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import division -import numpy as np -from menpo.image import Image -from menpo.feature import no_op -from menpo.feature import gradient as fast_gradient - - -# TODO: implement more clever sampling? -class LKInterface(object): - - def __init__(self, lk_algorithm, sampling=None): - self.algorithm = lk_algorithm - - n_true_pixels = self.template.n_true_pixels() - n_channels = self.template.n_channels - n_parameters = self.transform.n_parameters - sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - - if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 - - self.i_mask = np.nonzero(np.tile( - sampling_mask[None, ...], (n_channels, 1)).flatten())[0] - self.dW_dp_mask = np.nonzero(np.tile( - sampling_mask[None, ..., None], (2, 1, n_parameters))) - self.nabla_mask = np.nonzero(np.tile( - sampling_mask[None, None, ...], (2, n_channels, 1))) - self.nabla2_mask = np.nonzero(np.tile( - sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - - @property - def n(self): - return self.transform.n_parameters - - @property - def true_indices(self): - return self.template.mask.true_indices() - - def warp_jacobian(self): - dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) - return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, - dW_dp.shape[2])) - - def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) - - def gradient(self, img): - nabla = fast_gradient(img) - nabla.set_boundary_pixels() - return nabla.as_vector().reshape((2, img.n_channels, -1)) - - def steepest_descent_images(self, nabla, dW_dp): - # reshape gradient - # nabla: n_dims x n_channels x n_pixels - nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) - # compute steepest descent images - # nabla: n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # sdi: n_channels x n_pixels x n_params - sdi = 0 - a = nabla[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - # reshape steepest descent images - # sdi: (n_channels x n_pixels) x n_params - return sdi.reshape((-1, sdi.shape[2])) - - -class LKPartsInterface(LKInterface): - - def __init__(self, lk_algorithm, patch_shape=(17, 17), - normalize_parts=no_op, sampling=None): - self.algorithm = lk_algorithm - self.patch_shape = patch_shape - self.normalize_parts = normalize_parts - - if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) - - image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling[None, None, None, ...], - image_shape[:3] + (1, 1)) - self.i_mask = np.nonzero(image_mask.flatten())[0] - self.nabla_mask = np.nonzero(np.tile( - image_mask[None, ...], (2, 1, 1, 1, 1, 1))) - self.nabla2_mask = np.nonzero(np.tile( - image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) - - def warp_jacobian(self): - return np.rollaxis(self.transform.d_dp(None), -1) - - # TODO: add parts normalization - def warp(self, image): - parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, - as_single_array=True) - parts = self.normalize_parts(parts) - return Image(parts) - - def gradient(self, image): - pixels = image.pixels - g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) - # remove 1st dimension gradient which corresponds to the gradient - # between parts - return g.reshape((2,) + pixels.shape) - - def steepest_descent_images(self, nabla, dw_dp): - # reshape nabla - # nabla: dims x parts x off x ch x (h x w) - nabla = nabla[self.nabla_mask].reshape( - nabla.shape[:-2] + (-1,)) - # compute steepest descent images - # nabla: dims x parts x off x ch x (h x w) - # ds_dp: dims x parts x x params - # sdi: parts x off x ch x (h x w) x params - sdi = 0 - a = nabla[..., None] * dw_dp[..., None, None, None, :] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (parts x offsets x ch x w x h) x params - return sdi.reshape((-1, sdi.shape[-1])) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index dce40fb..2193d70 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,16 +1,87 @@ from __future__ import division import abc import numpy as np -from menpofit.algorithm import LKInterface, LKPartsInterface +from menpo.image import Image +from menpo.feature import no_op +from menpo.feature import gradient as fast_gradient from .result import ATMAlgorithmResult, LinearATMAlgorithmResult -class LKATMInterface(LKInterface): +# TODO: implement more clever sampling? +class LKATMInterface(object): + + def __init__(self, lk_algorithm, sampling=None): + self.algorithm = lk_algorithm + + n_true_pixels = self.template.n_true_pixels() + n_channels = self.template.n_channels + n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) + + if sampling is None: + sampling = 1 + sampling_pattern = xrange(0, n_true_pixels, sampling) + sampling_mask[sampling_pattern] = 1 + + self.i_mask = np.nonzero(np.tile( + sampling_mask[None, ...], (n_channels, 1)).flatten())[0] + self.dW_dp_mask = np.nonzero(np.tile( + sampling_mask[None, ..., None], (2, 1, n_parameters))) + self.nabla_mask = np.nonzero(np.tile( + sampling_mask[None, None, ...], (2, n_channels, 1))) + self.nabla2_mask = np.nonzero(np.tile( + sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) + + @property + def template(self): + return self.algorithm.template + + @property + def transform(self): + return self.algorithm.transform + + @property + def n(self): + return self.transform.n_parameters + + @property + def true_indices(self): + return self.template.mask.true_indices() @property def shape_model(self): return self.transform.pdm.model + def warp_jacobian(self): + dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) + return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, + dW_dp.shape[2])) + + def warp(self, image): + return image.warp_to_mask(self.template.mask, + self.transform) + + def gradient(self, img): + nabla = fast_gradient(img) + nabla.set_boundary_pixels() + return nabla.as_vector().reshape((2, img.n_channels, -1)) + + def steepest_descent_images(self, nabla, dW_dp): + # reshape gradient + # nabla: n_dims x n_channels x n_pixels + nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) + # compute steepest descent images + # nabla: n_dims x n_channels x n_pixels + # warp_jacobian: n_dims x x n_pixels x n_params + # sdi: n_channels x n_pixels x n_params + sdi = 0 + a = nabla[..., None] * dW_dp[:, None, ...] + for d in a: + sdi += d + # reshape steepest descent images + # sdi: (n_channels x n_pixels) x n_params + return sdi.reshape((-1, sdi.shape[2])) + @classmethod def solve_shape_map(cls, H, J, e, J_prior, p): if p.shape[0] is not H.shape[0]: @@ -43,12 +114,65 @@ def algorithm_result(self, image, shape_parameters, gt_shape=None): image, self.algorithm, shape_parameters, gt_shape=gt_shape) -class LKPartsATMInterface(LKPartsInterface, LKATMInterface): +class LKPartsATMInterface(LKATMInterface): + + def __init__(self, lk_algorithm, patch_shape=(17, 17), + normalize_parts=no_op, sampling=None): + self.algorithm = lk_algorithm + self.patch_shape = patch_shape + self.normalize_parts = normalize_parts + + if sampling is None: + sampling = np.ones(self.patch_shape, dtype=np.bool) + + image_shape = self.algorithm.template.pixels.shape + image_mask = np.tile(sampling[None, None, None, ...], + image_shape[:3] + (1, 1)) + self.i_mask = np.nonzero(image_mask.flatten())[0] + self.nabla_mask = np.nonzero(np.tile( + image_mask[None, ...], (2, 1, 1, 1, 1, 1))) + self.nabla2_mask = np.nonzero(np.tile( + image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) @property def shape_model(self): return self.transform.model + def warp_jacobian(self): + return np.rollaxis(self.transform.d_dp(None), -1) + + def warp(self, image): + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts) + + def gradient(self, image): + pixels = image.pixels + g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) + # remove 1st dimension gradient which corresponds to the gradient + # between parts + return g.reshape((2,) + pixels.shape) + + def steepest_descent_images(self, nabla, dw_dp): + # reshape nabla + # nabla: dims x parts x off x ch x (h x w) + nabla = nabla[self.nabla_mask].reshape( + nabla.shape[:-2] + (-1,)) + # compute steepest descent images + # nabla: dims x parts x off x ch x (h x w) + # ds_dp: dims x parts x x params + # sdi: parts x off x ch x (h x w) x params + sdi = 0 + a = nabla[..., None] * dw_dp[..., None, None, None, :] + for d in a: + sdi += d + + # reshape steepest descent images + # sdi: (parts x offsets x ch x w x h) x params + return sdi.reshape((-1, sdi.shape[-1])) + # TODO: handle costs for all LKAAMAlgorithms # TODO document me! From ee3b74ba70e0fd7775ed83f3bd03dd294a11f893 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 1 Jun 2015 14:53:44 +0100 Subject: [PATCH 267/423] Update fitter.py and LinearATMAlgorithmResult --- menpofit/atm/result.py | 12 ++++-------- menpofit/fitter.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 2b91276..4ee4cab 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -12,14 +12,10 @@ class ATMAlgorithmResult(ParametricAlgorithmResult): class LinearATMAlgorithmResult(ATMAlgorithmResult): r""" """ - def shapes(self, as_points=False): - if as_points: - return [self.fitter.transform.from_vector(p).sparse_target.points - for p in self.shape_parameters] - - else: - return [self.fitter.transform.from_vector(p).sparse_target - for p in self.shape_parameters] + @property + def shapes(self): + return [self.fitter.transform.from_vector(p).sparse_target + for p in self.shape_parameters] @property def final_shape(self): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 9b1f906..34fef7a 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -327,7 +327,7 @@ def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, return transform.apply(self.reference_shape) def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): - return self.get_initial_shape_from_bounding_box( + return self.noisy_shape_from_bounding_box( shape.bounding_box(), noise_std=noise_std, rotation=rotation) From 779a052b9951b79d288a4b08818936d42aea31f7 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:20:14 +0100 Subject: [PATCH 268/423] Add small changes to fitter.py --- menpofit/fitter.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 34fef7a..3d8ce11 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -23,21 +23,21 @@ def n_levels(self): def algorithms(self): pass - @abc.abstractproperty - def reference_shape(self): - pass - - @abc.abstractproperty - def features(self): - pass - - @abc.abstractproperty - def scales(self): - pass - - @abc.abstractproperty - def scale_features(self): - pass + # @abc.abstractproperty + # def reference_shape(self): + # pass + # + # @abc.abstractproperty + # def features(self): + # pass + # + # @abc.abstractproperty + # def scales(self): + # pass + # + # @abc.abstractproperty + # def scale_features(self): + # pass def fit(self, image, initial_shape, max_iters=50, gt_shape=None, crop_image=0.5, **kwargs): From d4618d0c6384747ab52e787f92ea3ebff8fbd5cf Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:21:19 +0100 Subject: [PATCH 269/423] Add first version of sdm fitter --- menpofit/sdm/fitter.py | 661 ++++++++++++++++++++++++----------------- 1 file changed, 384 insertions(+), 277 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 3993c75..2230555 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,304 +1,411 @@ -import numpy as np -from menpo.image import Image - -from menpofit.base import name_of_callable -from menpofit.aam.fitter import AAMFitter -from menpofit.clm.fitter import CLMFitter -from menpofit.fitter import MultilevelFitter - - -class SDFitter(MultilevelFitter): +from __future__ import division +from functools import partial +from menpo.transform import Scale, AlignmentSimilarity +from menpo.feature import no_op +from menpofit.builder import normalization_wrt_reference_shape, scale_images +from menpofit.fitter import MultiFitter, noisy_align +from menpofit.result import MultiFitterResult +import menpofit.checks as checks +from .algorithm import SN + + +# TODO: document me! +class CRFitter(MultiFitter): r""" - Abstract Supervised Descent Fitter. """ - def _set_up(self): - r""" - Sets up the SD fitter object. - """ - - def fit(self, image, initial_shape, max_iters=None, gt_shape=None, - **kwargs): - r""" - Fits a single image. - - Parameters - ----------- - image : :map:`MaskedImage` - The image to be fitted. - initial_shape : :map:`PointCloud` - The initial shape estimate from which the fitting procedure - will start. - max_iters : int or `list`, optional - The maximum number of iterations. - - If `int`, then this will be the overall maximum number of iterations - for all the pyramidal levels. - - If `list`, then a maximum number of iterations is specified for each - pyramidal level. - - gt_shape : :map:`PointCloud` - The ground truth shape of the image. - - **kwargs : `dict` - optional arguments to be passed through. - - Returns - ------- - fitting_list : :map:`FittingResultList` - A fitting result object. - """ - if max_iters is None: - max_iters = self.n_levels - return MultilevelFitter.fit(self, image, initial_shape, - max_iters=max_iters, gt_shape=gt_shape, - **kwargs) - - -class SDMFitter(SDFitter): - r""" - Supervised Descent Method. - - Parameters - ----------- - regressors : :map:`RegressorTrainer` - The trained regressors. - - n_training_images : `int` - The number of images that were used to train the SDM fitter. It is - only used for informational reasons. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - - References - ---------- - .. [XiongD13] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 - """ - def __init__(self, regressors, n_training_images, features, - reference_shape, downscale): - self._fitters = regressors - self._features = features - self._reference_shape = reference_shape - self._downscale = downscale - self._n_training_images = n_training_images + def __init__(self, cr_algorithm_cls=SN, features=no_op, + patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), + iterations=6, n_perturbations=10, **kwargs): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + patch_shape = checks.check_patch_shape(patch_shape, n_levels) + # set parameters + self._algorithms = [] + self.diagonal = diagonal + self.scales = list(scales)[::-1] + self.n_perturbations = n_perturbations + self.iterations = checks.check_iterations(iterations, n_levels) + # set up algorithms + self._set_up(cr_algorithm_cls, features, patch_shape, **kwargs) @property - def algorithm(self): - r""" - Returns a string containing the algorithm used from the SDM family. - - : str - """ - return 'SDM-' + self._fitters[0].algorithm + def algorithms(self): + return self._algorithms @property - def reference_shape(self): - r""" - The reference shape used during training. + def reference_bounding_box(self): + return self.reference_shape.bounding_box() - :type: :map:`PointCloud` - """ - return self._reference_shape - - @property - def features(self): - r""" - The feature type per pyramid level. Note that they are stored from - lowest to highest level resolution. - - :type: `list` - """ - return self._features - - @property - def n_levels(self): - r""" - The number of pyramidal levels used during training. + def _set_up(self, cr_algorithm_cls, features, patch_shape, **kwargs): + for j in range(self.n_levels): + algorithm = cr_algorithm_cls( + features=features[j], patch_shape=patch_shape[j], + iterations=self.iterations[j], **kwargs) + self._algorithms.append(algorithm) - : int - """ - return len(self._fitters) + def train(self, images, group=None, label=None, verbose=False, **kwargs): + # normalize images and compute reference shape + self.reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) - @property - def downscale(self): + # for each pyramid level (low --> high) + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # scale images and compute features at other levels + level_images = scale_images(images, self.scales[j], + level_str=level_str, verbose=verbose) + + # extract ground truth shapes for current level + level_gt_shapes = [i.landmarks[group][label] for i in level_images] + + if j == 0: + # generate perturbed shapes + current_shapes = [] + for gt_s in level_gt_shapes: + perturbed_shapes = [] + for _ in range(self.n_perturbations): + p_s = self.noisy_shape_from_shape(gt_s) + perturbed_shapes.append(p_s) + current_shapes.append(perturbed_shapes) + + # train cascaded regression algorithm + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + + # scale current shapes to next level resolution + if self.scales[j] != (1 or self.scales[-1]): + transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + + def _prepare_image(self, image, initial_shape, gt_shape=None, + crop_image=0.5): r""" - The downscale per pyramidal level used during building the AAM. - The scale factor is: (downscale ** k) for k in range(n_levels) + Prepares the image to be fitted. - :type: `float` - """ - return self._downscale + The image is first rescaled wrt the ``reference_landmarks`` and then + a gaussian pyramid is applied. Depending on the + ``pyramid_on_features`` flag, the pyramid is either applied to the + features image computed from the rescaled imaged or applied to the + rescaled image and features extracted at each pyramidal level. - def __str__(self): - out = "Supervised Descent Method\n" \ - " - Non-Parametric '{}' Regressor\n" \ - " - {} training images.\n".format( - name_of_callable(self._fitters[0].regressor), - self._n_training_images) - # small strings about number of channels, channels string and downscale - down_str = [] - for j in range(self.n_levels): - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - temp_img = Image(image_data=np.random.rand(40, 40)) - if self.pyramid_on_features: - temp = self.features(temp_img) - n_channels = [temp.n_channels] * self.n_levels - else: - n_channels = [] - for j in range(self.n_levels): - temp = self.features[j](temp_img) - n_channels.append(temp.n_channels) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - self.features[j].__name__)) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n {}{} {} per " \ - "image.\n".format( - out, self.n_levels - i, down_str[i], feat_str[i], - n_channels[i], ch_str[i]) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( - out, feat_str[0], n_channels[0], ch_str[0]) - return out + Parameters + ---------- + image : :map:`Image` or subclass + The image to be fitted. + initial_shape : :map:`PointCloud` + The initial shape from which the fitting will start. -class SDAAMFitter(AAMFitter, SDFitter): - r""" - Supervised Descent Fitter for AAMs. + gt_shape : class : :map:`PointCloud`, optional + The original ground truth shape associated to the image. - Parameters - ----------- - aam : :map:`AAM` - The Active Appearance Model to be used. + crop_image: `None` or float`, optional + If `float`, it specifies the proportion of the border wrt the + initial shape to which the image will be internally cropped around + the initial shape range. + If `None`, no cropping is performed. - regressors : :map:``RegressorTrainer` - The trained regressors. + This will limit the fitting algorithm search region but is + likely to speed up its running time, specially when the + modeled object occupies a small portion of the image. - n_training_images : `int` - The number of training images used to train the SDM fitter. - """ - def __init__(self, aam, regressors, n_training_images): - super(SDAAMFitter, self).__init__(aam) - self._fitters = regressors - self._n_training_images = n_training_images + Returns + ------- + images : `list` of :map:`Image` or subclass + The list of images that will be fitted by the fitters. - @property - def algorithm(self): - r""" - Returns a string containing the algorithm used from the SDM family. + initial_shapes : `list` of :map:`PointCloud` + The initial shape for each one of the previous images. - :type: `string` + gt_shapes : `list` of :map:`PointCloud` + The ground truth shape for each one of the previous images. """ - return 'SD-AAM-' + self._fitters[0].algorithm + # attach landmarks to the image + image.landmarks['initial_shape'] = initial_shape + if gt_shape: + image.landmarks['gt_shape'] = gt_shape + + # if specified, crop the image + if crop_image: + image = image.copy() + image.crop_to_landmarks_proportion_inplace(crop_image, + group='initial_shape') + + # rescale image wrt the scale factor between reference_shape and + # initial_shape + image = image.rescale_to_reference_shape(self.reference_shape, + group='initial_shape') + + # obtain image representation + images = [] + for s in self.scales: + if s != 1: + # scale image + scaled_image = image.rescale(s) + else: + scaled_image = image + images.append(scaled_image) - def __str__(self): - return "{}Supervised Descent Method for AAMs:\n" \ - " - Parametric '{}' Regressor\n" \ - " - {} training images.\n".format( - self.aam.__str__(), name_of_callable(self._fitters[0].regressor), - self._n_training_images) + # get initial shapes per level + initial_shapes = [i.landmarks['initial_shape'].lms for i in images] + # get ground truth shapes per level + if gt_shape: + gt_shapes = [i.landmarks['gt_shape'].lms for i in images] + else: + gt_shapes = None -class SDCLMFitter(CLMFitter, SDFitter): - r""" - Supervised Descent Fitter for CLMs. - - Parameters - ----------- - clm : :map:`CLM` - The Constrained Local Model to be used. - - regressors : :map:`RegressorTrainer` - The trained regressors. - - n_training_images : `int` - The number of training images used to train the SDM fitter. - - References - ---------- - .. [Asthana13] Robust Discriminative Response Map Fitting with Constrained - Local Models - A. Asthana, S. Zafeiriou, S. Cheng, M. Pantic. - IEEE Conference onComputer Vision and Pattern Recognition. - Portland, Oregon, USA, June 2013. - """ - def __init__(self, clm, regressors, n_training_images): - super(SDCLMFitter, self).__init__(clm) - self._fitters = regressors - self._n_training_images = n_training_images + return images, initial_shapes, gt_shapes - @property - def algorithm(self): - r""" - Returns a string containing the algorithm used from the SDM family. + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return MultiFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) - :type: `string` - """ - return 'SD-CLM-' + self._fitters[0].algorithm + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, + rotation=False): + transform = noisy_align(AlignmentSimilarity, + self.reference_bounding_box, bounding_box, + noise_std=noise_std, rotation=rotation) + return transform.apply(self.reference_shape) + + def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): + return self.noisy_shape_from_bounding_box( + shape.bounding_box(), noise_std=noise_std, rotation=rotation) + # TODO: fix me! def __str__(self): - return "{}Supervised Descent Method for CLMs:\n" \ - " - Parametric '{}' Regressor\n" \ - " - {} training images.\n".format( - self.clm.__str__(), name_of_callable(self._fitters[0].regressor), - self._n_training_images) + pass + # out = "Supervised Descent Method\n" \ + # " - Non-Parametric '{}' Regressor\n" \ + # " - {} training images.\n".format( + # name_of_callable(self._fitters[0].regressor), + # self._n_training_images) + # # small strings about number of channels, channels string and downscale + # down_str = [] + # for j in range(self.n_levels): + # if j == self.n_levels - 1: + # down_str.append('(no downscale)') + # else: + # down_str.append('(downscale by {})'.format( + # self.downscale**(self.n_levels - j - 1))) + # temp_img = Image(image_data=np.random.rand(40, 40)) + # if self.pyramid_on_features: + # temp = self.features(temp_img) + # n_channels = [temp.n_channels] * self.n_levels + # else: + # n_channels = [] + # for j in range(self.n_levels): + # temp = self.features[j](temp_img) + # n_channels.append(temp.n_channels) + # # string about features and channels + # if self.pyramid_on_features: + # feat_str = "- Feature is {} with ".format( + # name_of_callable(self.features)) + # if n_channels[0] == 1: + # ch_str = ["channel"] + # else: + # ch_str = ["channels"] + # else: + # feat_str = [] + # ch_str = [] + # for j in range(self.n_levels): + # if isinstance(self.features[j], str): + # feat_str.append("- Feature is {} with ".format( + # self.features[j])) + # elif self.features[j] is None: + # feat_str.append("- No features extracted. ") + # else: + # feat_str.append("- Feature is {} with ".format( + # self.features[j].__name__)) + # if n_channels[j] == 1: + # ch_str.append("channel") + # else: + # ch_str.append("channels") + # if self.n_levels > 1: + # out = "{} - Gaussian pyramid with {} levels and downscale " \ + # "factor of {}.\n".format(out, self.n_levels, + # self.downscale) + # if self.pyramid_on_features: + # out = "{} - Pyramid was applied on feature space.\n " \ + # "{}{} {} per image.\n".format(out, feat_str, + # n_channels[0], ch_str[0]) + # else: + # out = "{} - Features were extracted at each pyramid " \ + # "level.\n".format(out) + # for i in range(self.n_levels - 1, -1, -1): + # out = "{} - Level {} {}: \n {}{} {} per " \ + # "image.\n".format( + # out, self.n_levels - i, down_str[i], feat_str[i], + # n_channels[i], ch_str[i]) + # else: + # if self.pyramid_on_features: + # feat_str = [feat_str] + # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( + # out, feat_str[0], n_channels[0], ch_str[0]) + # return out + + +# TODO: document me! +SDMFitter = partial(CRFitter, cr_algorithm_cls=SN) + + +# class CRFitter(MultiFitter): +# r""" +# """ +# def __init__(self, cr_algorithm_cls=SN, features=no_op, diagonal=None, +# scales=(1, 0.5), sampling=None, n_perturbations=10, +# iterations=6, **kwargs): +# # check parameters +# checks.check_diagonal(diagonal) +# scales, n_levels = checks.check_scales(scales) +# features = checks.check_features(features, n_levels) +# sampling = checks.check_sampling(sampling, n_levels) +# # set parameters +# self._algorithms = [] +# self.diagonal = diagonal +# self.scales = list(scales) +# self.n_perturbations = n_perturbations +# self.iterations = checks.check_iterations(iterations, n_levels) +# # set up algorithms +# self._set_up(cr_algorithm_cls, features, sampling, **kwargs) +# +# @property +# def algorithms(self): +# return self._algorithms +# +# def _set_up(self, cr_algorithm_cls, features, sampling, **kwargs): +# for j, s in range(self.n_levels): +# algorithm = cr_algorithm_cls( +# features=features[j], sampling=sampling[j], +# max_iters=self.iterations[j], **kwargs) +# self._algorithms.append(algorithm) +# +# def train(self, images, group=None, label=None, verbose=False, **kwargs): +# # normalize images and compute reference shape +# reference_shape, images = normalization_wrt_reference_shape( +# images, group, label, self.diagonal, verbose=verbose) +# +# # for each pyramid level (low --> high) +# for j in range(self.n_levels): +# if verbose: +# if len(self.scales) > 1: +# level_str = ' - Level {}: '.format(j) +# else: +# level_str = ' - ' +# +# # scale images and compute features at other levels +# level_images = scale_images(images, self.scales[j], +# level_str=level_str, verbose=verbose) +# +# # extract ground truth shapes for current level +# level_gt_shapes = [i.landmarks[group][label] for i in level_images] +# +# if j == 0: +# # generate perturbed shapes +# current_shapes = [] +# for gt_s in level_gt_shapes: +# perturbed_shapes = [] +# for _ in range(self.n_perturbations): +# p_s = self.noisy_shape_from_shape(gt_s) +# perturbed_shapes.append(p_s) +# current_shapes.append(perturbed_shapes) +# +# # train cascaded regression algorithm +# current_shapes = self.algorithms[j].train( +# level_images, level_gt_shapes, current_shapes, +# verbose=verbose, **kwargs) +# +# # scale current shapes to next level resolution +# if self.scales[j] != self.scales[-1]: +# transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) +# for image_shapes in current_shapes: +# for shape in image_shapes: +# transform.apply_inplace(shape) +# +# def _fitter_result(self, image, algorithm_results, affine_correction, +# gt_shape=None): +# return MultiFitterResult(image, algorithm_results, affine_correction, +# gt_shape=gt_shape) +# +# # TODO: fix me! +# def __str__(self): +# pass +# # out = "Supervised Descent Method\n" \ +# # " - Non-Parametric '{}' Regressor\n" \ +# # " - {} training images.\n".format( +# # name_of_callable(self._fitters[0].regressor), +# # self._n_training_images) +# # # small strings about number of channels, channels string and downscale +# # down_str = [] +# # for j in range(self.n_levels): +# # if j == self.n_levels - 1: +# # down_str.append('(no downscale)') +# # else: +# # down_str.append('(downscale by {})'.format( +# # self.downscale**(self.n_levels - j - 1))) +# # temp_img = Image(image_data=np.random.rand(40, 40)) +# # if self.pyramid_on_features: +# # temp = self.features(temp_img) +# # n_channels = [temp.n_channels] * self.n_levels +# # else: +# # n_channels = [] +# # for j in range(self.n_levels): +# # temp = self.features[j](temp_img) +# # n_channels.append(temp.n_channels) +# # # string about features and channels +# # if self.pyramid_on_features: +# # feat_str = "- Feature is {} with ".format( +# # name_of_callable(self.features)) +# # if n_channels[0] == 1: +# # ch_str = ["channel"] +# # else: +# # ch_str = ["channels"] +# # else: +# # feat_str = [] +# # ch_str = [] +# # for j in range(self.n_levels): +# # if isinstance(self.features[j], str): +# # feat_str.append("- Feature is {} with ".format( +# # self.features[j])) +# # elif self.features[j] is None: +# # feat_str.append("- No features extracted. ") +# # else: +# # feat_str.append("- Feature is {} with ".format( +# # self.features[j].__name__)) +# # if n_channels[j] == 1: +# # ch_str.append("channel") +# # else: +# # ch_str.append("channels") +# # if self.n_levels > 1: +# # out = "{} - Gaussian pyramid with {} levels and downscale " \ +# # "factor of {}.\n".format(out, self.n_levels, +# # self.downscale) +# # if self.pyramid_on_features: +# # out = "{} - Pyramid was applied on feature space.\n " \ +# # "{}{} {} per image.\n".format(out, feat_str, +# # n_channels[0], ch_str[0]) +# # else: +# # out = "{} - Features were extracted at each pyramid " \ +# # "level.\n".format(out) +# # for i in range(self.n_levels - 1, -1, -1): +# # out = "{} - Level {} {}: \n {}{} {} per " \ +# # "image.\n".format( +# # out, self.n_levels - i, down_str[i], feat_str[i], +# # n_channels[i], ch_str[i]) +# # else: +# # if self.pyramid_on_features: +# # feat_str = [feat_str] +# # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( +# # out, feat_str[0], n_channels[0], ch_str[0]) +# # return out \ No newline at end of file From 395a0b0fe62d7c241c01a104be1747fd7714f9b0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:22:33 +0100 Subject: [PATCH 270/423] Add first version of sdm algorithms --- menpofit/sdm/algorithm.py | 337 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 menpofit/sdm/algorithm.py diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py new file mode 100644 index 0000000..ce921c1 --- /dev/null +++ b/menpofit/sdm/algorithm.py @@ -0,0 +1,337 @@ +from __future__ import division +import numpy as np +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpofit.result import NonParametricAlgorithmResult + + +# TODO document me! +class CRAlgorithm(object): + r""" + """ + def train(self, images, gt_shapes, current_shapes, verbose=False, + **kwargs): + n_images = len(images) + n_samples_image = len(current_shapes[0]) + + self._features_patch_length = compute_features_info( + images[0], gt_shapes[0], self.features, + patch_shape=self.patch_shape)[1] + + # obtain delta_x and gt_x + delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) + + # initialize iteration counter and list of regressors + k = 0 + self.regressors = [] + + # Cascaded Regression loop + while k < self.iterations: + # generate regression data + features = obtain_patch_features( + images, current_shapes, self.patch_shape, self.features, + features_patch_length=self._features_patch_length) + + # perform regression + if verbose: + print_dynamic('- Performing regression...') + regressor = self._perform_regression(features, delta_x, **kwargs) + # add regressor to list + self.regressors.append(regressor) + + # estimate delta_points + estimated_delta_x = regressor(features) + if verbose: + error = _compute_rmse(delta_x, estimated_delta_x) + print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + + j = 0 + for shapes in current_shapes: + for s in shapes: + # update current x + current_x = s.as_vector() + estimated_delta_x[j] + # update current shape inplace + s.from_vector_inplace(current_x) + # update delta_x + delta_x[j] = gt_x[j] - current_x + # increase index + j += 1 + # increase iteration counter + k += 1 + + # rearrange current shapes into their original list of list form + return current_shapes + + def run(self, image, initial_shape, gt_shape=None, **kwargs): + # set current shape and initialize list of shapes + current_shape = initial_shape + shapes = [initial_shape] + + # Cascaded Regression loop + for r in self.regressors: + # compute regression features + features = compute_patch_features( + image, current_shape, self.patch_shape, self.features, + features_patch_length=self._features_patch_length) + + # solve for increments on the shape vector + dx = r(features) + + # update current shape + current_shape = current_shape.from_vector( + current_shape.as_vector() + dx) + shapes.append(current_shape) + + # return algorithm result + return NonParametricAlgorithmResult(image, self, shapes, + gt_shape=gt_shape) + + +# TODO: document me! +class SN(CRAlgorithm): + r""" + Supervised Newton. + + This class implements the Supervised Descent Method technique, proposed + by Xiong and De la Torre in [XiongD13]. + + References + ---------- + .. [XiongD13] Supervised Descent Method and its Applications to + Face Alignment + Xuehan Xiong and Fernando De la Torre Fernando + IEEE International Conference on Computer Vision and Pattern Recognition + May, 2013 + """ + def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + eps=10 ** -5): + self.patch_shape = patch_shape + self.features = features + self.patch_shape = patch_shape + self.iterations = iterations + self.eps = eps + # wire regression callable + self._perform_regression = _supervised_newton + + +# TODO: document me! +class SGN(CRAlgorithm): + r""" + Supervised Gauss-Newton + + This class implements a variation of the Supervised Descent Method + [XiongD13] by some of the ideas incorporating ideas... + + References + ---------- + .. [XiongD13] Supervised Descent Method and its Applications to + Face Alignment + Xuehan Xiong and Fernando De la Torre Fernando + IEEE International Conference on Computer Vision and Pattern Recognition + May, 2013 + .. [Tzimiropoulos15] Supervised Descent Method and its Applications to + Face Alignment + Xuehan Xiong and Fernando De la Torre Fernando + IEEE International Conference on Computer Vision and Pattern Recognition + May, 2013 + """ + def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + eps=10 ** -5): + self.patch_shape = patch_shape + self.features = features + self.patch_shape = patch_shape + self.iterations = iterations + self.eps = eps + # wire regression callable + self._perform_regression = _supervised_gauss_newton + + +# TODO: document me! +class _supervised_newton(object): + r""" + """ + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = features.T.dot(features) + XT = features.T.dot(deltas) + if gamma: + XX += gamma * np.eye(features.shape[1]) + # descent direction + self.R = np.linalg.solve(XX, XT) + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +class _supervised_gauss_newton(object): + r""" + """ + + def __init__(self, features, deltas, gamma=None): + # ridge regression + XX = deltas.T.dot(deltas) + XT = deltas.T.dot(features) + if gamma: + XX += gamma * np.eye(deltas.shape[1]) + # average Jacobian + self.J = np.linalg.solve(XX, XT) + # average Hessian + self.H = self.J.dot(self.J.T) + # descent direction + self.R = np.linalg.solve(self.H, self.J).T + + def __call__(self, features): + return np.dot(features, self.R) + + +# TODO: document me! +def _compute_rmse(x1, x2): + return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + + +# TODO: docment me! +def compute_patch_features(image, shape, patch_shape, features_callable, + features_patch_length=None): + """r + """ + patches = image.extract_patches(shape, patch_size=patch_shape, + as_single_array=True) + + if features_patch_length: + patch_features = np.empty((shape.n_points, features_patch_length)) + for j, p in enumerate(patches): + patch_features[j] = features_callable(p[0]).ravel() + else: + patch_features = [] + for j, p in enumerate(patches): + patch_features.append(features_callable(p[0]).ravel()) + patch_features = np.asarray(patch_features) + + return patch_features.ravel() + + +# TODO: docment me! +def generate_patch_features(image, shapes, patch_shape, features_callable, + features_patch_length=None): + """r + """ + if features_patch_length: + patch_features = np.empty((len(shapes), + shapes[0].n_points * features_patch_length)) + for j, s in enumerate(shapes): + patch_features[j] = compute_patch_features( + image, s, patch_shape, features_callable, + features_patch_length=features_patch_length) + else: + patch_features = [] + for j, s in enumerate(shapes): + patch_features.append(compute_patch_features( + image, s, patch_shape, features_callable, + features_patch_length=features_patch_length)) + patch_features = np.asarray(patch_features) + + return patch_features.ravel() + + +# TODO: docment me! +def obtain_patch_features(images, shapes, patch_shape, features_callable, + features_patch_length=None): + """r + """ + n_images = len(images) + n_shapes = len(shapes[0]) + n_points = shapes[0][0].n_points + + if features_patch_length: + + patch_features = np.empty((n_images, (n_shapes * n_points * + features_patch_length))) + for j, i in enumerate(images): + patch_features[j] = generate_patch_features( + i, shapes[j], patch_shape, features_callable, + features_patch_length=features_patch_length) + else: + patch_features = [] + for j, i in images: + patch_features.append(generate_patch_features( + i, shapes[j], patch_shape, features_callable, + features_patch_length=features_patch_length)) + patch_features = np.asarray(patch_features) + + return patch_features.reshape((-1, n_points * features_patch_length)) + + +def compute_delta_x(gt_shape, current_shapes): + r""" + """ + n_x = gt_shape.n_parameters + n_current_shapes = len(current_shapes) + + # initialize ground truth and delta shape vectors + gt_x = np.empty((n_current_shapes, n_x)) + delta_x = np.empty((n_current_shapes, n_x)) + + for j, s in enumerate(current_shapes): + # compute ground truth shape vector + gt_x[j] = gt_shape.as_vector() + # compute delta shape vector + delta_x[j] = gt_x[j] - s.as_vector() + + return delta_x, gt_x + + +def obtain_delta_x(gt_shapes, current_shapes): + r""" + """ + n_x = gt_shapes[0].n_parameters + n_gt_shapes = len(gt_shapes) + n_current_shapes = len(current_shapes[0]) + + # initialize current, ground truth and delta parameters + gt_x = np.empty((n_gt_shapes, n_current_shapes, n_x)) + delta_x = np.empty((n_gt_shapes, n_current_shapes, n_x)) + + # obtain ground truth points and compute delta points + for j, (gt_s, shapes) in enumerate(zip(gt_shapes, current_shapes)): + # compute ground truth par + delta_x[j], gt_x[j] = compute_delta_x(gt_s, shapes) + + return delta_x.reshape((-1, n_x)), gt_x.reshape((-1, n_x)) + + +def compute_features_info(image, shape, features_callable, + patch_shape=(17, 17)): + # TODO: include offsets support? + patches = image.extract_patches(shape, patch_size=patch_shape, + as_single_array=True) + + # TODO: include offsets support? + features_patch_shape = features_callable(patches[0, 0]).shape + features_patch_length = np.prod(features_patch_shape) + features_shape = patches.shape[:1] + features_patch_shape + features_length = np.prod(features_shape) + + return (features_patch_shape, features_patch_length, + features_shape, features_length) + +# def initialize_sampling(self, image, group=None, label=None): +# if self._sampling is None: +# sampling = np.ones(self.patch_shape, dtype=np.bool) +# else: +# sampling = self._sampling +# +# # TODO: include offsets support? +# patches = image.extract_patches_around_landmarks( +# group=group, label=label, patch_size=self.patch_shape, +# as_single_array=True) +# +# # TODO: include offsets support? +# features_patch_shape = self.features(patches[0, 0]).shape +# self._features_patch_length = np.prod(features_patch_shape) +# self._features_shape = (patches.shape[0], features_patch_shape) +# self._features_length = np.prod(self._features_shape) +# +# feature_mask = np.tile(sampling[None, None, None, ...], +# self._feature_shape[:3] + (1, 1)) +# self._feature_mask = np.nonzero(feature_mask.flatten())[0] From 1ea756cf7b97ba734579046b594eb73713c29775 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 17 Jun 2015 14:23:45 +0100 Subject: [PATCH 271/423] Update sdm/__init__.py --- menpofit/sdm/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/__init__.py b/menpofit/sdm/__init__.py index 9661b9d..9180e28 100644 --- a/menpofit/sdm/__init__.py +++ b/menpofit/sdm/__init__.py @@ -1,2 +1,2 @@ -from .trainer import SDMTrainer, SDAAMTrainer, SDCLMTrainer -from .fitter import SDMFitter, SDAAMFitter, SDCLMFitter +from .algorithm import SN, SGN +from .fitter import CRFitter, SDMFitter From c286a5ccca311b8cd4f3c4c25833958c05cb4893 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 22 Jun 2015 16:31:51 +0100 Subject: [PATCH 272/423] Slight modifications to SDM --- menpofit/result.py | 40 +++++++++++++++++++++++++++++++++------ menpofit/sdm/algorithm.py | 8 ++------ menpofit/sdm/fitter.py | 2 +- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 531f759..6fbe755 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -459,6 +459,33 @@ def initial_shape(self): return self.initial_transform.target +# TODO: document me! +class NonParametricAlgorithmResult(IterativeResult): + r""" + """ + def __init__(self, image, fitter, shapes, gt_shape=None): + self.image = image + self.fitter = fitter + self._shapes = shapes + self._gt_shape = gt_shape + + @property + def n_iters(self): + return len(self.shapes) - 1 + + @property + def shapes(self): + return self._shapes + + @property + def final_shape(self): + return self.shapes[-1] + + @property + def initial_shape(self): + return self.shapes[0] + + # TODO: document me! class MultiFitterResult(IterativeResult): r""" @@ -584,7 +611,7 @@ def compute_error(target, ground_truth, error_type='me_norm'): target_points = target.points if error_type == 'me_norm': - return _compute_me_norm(target_points, gt_points) + return _compute_norm_p2p_error(target_points, gt_points) elif error_type == 'me': return _compute_me(target_points, gt_points) elif error_type == 'rmse': @@ -611,13 +638,14 @@ def _compute_rmse(target, ground_truth): # TODO: Document me! -# TODO: rename to more descriptive name -def _compute_me_norm(target, ground_truth): +def _compute_norm_p2p_error(target, source, ground_truth=None): r""" """ + if ground_truth is None: + ground_truth = source normalizer = np.mean(np.max(ground_truth, axis=0) - np.min(ground_truth, axis=0)) - return _compute_me(target, ground_truth) / normalizer + return _compute_me(target, source) / normalizer # TODO: Document me! @@ -628,8 +656,8 @@ def compute_cumulative_error(errors, x_axis): return [np.count_nonzero([errors <= x]) / n_errors for x in x_axis] -def plot_cumulative_error_distribution(errors, error_range=None, figure_id=None, - new_figure=False, +def plot_cumulative_error_distribution(errors, error_range=None, + figure_id=None, new_figure=False, title='Cumulative Error Distribution', x_label='Normalized Point-to-Point Error', y_label='Images Proportion', diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index ce921c1..db5a195 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -11,9 +11,6 @@ class CRAlgorithm(object): """ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): - n_images = len(images) - n_samples_image = len(current_shapes[0]) - self._features_patch_length = compute_features_info( images[0], gt_shapes[0], self.features, patch_shape=self.patch_shape)[1] @@ -155,7 +152,7 @@ def __init__(self, features, deltas, gamma=None): XX = features.T.dot(features) XT = features.T.dot(deltas) if gamma: - XX += gamma * np.eye(features.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # descent direction self.R = np.linalg.solve(XX, XT) @@ -167,13 +164,12 @@ def __call__(self, features): class _supervised_gauss_newton(object): r""" """ - def __init__(self, features, deltas, gamma=None): # ridge regression XX = deltas.T.dot(deltas) XT = deltas.T.dot(features) if gamma: - XX += gamma * np.eye(deltas.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # average Jacobian self.J = np.linalg.solve(XX, XT) # average Hessian diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2230555..2c844e5 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -26,7 +26,7 @@ def __init__(self, cr_algorithm_cls=SN, features=no_op, self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations - self.iterations = checks.check_iterations(iterations, n_levels) + self.iterations = checks.check_max_iters(iterations, n_levels) # set up algorithms self._set_up(cr_algorithm_cls, features, patch_shape, **kwargs) From 465e413e0ff8cfba3f4e6bee275ae92544a9c151 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 22 Jun 2015 16:32:55 +0100 Subject: [PATCH 273/423] Delete sdm/trainer.py --- menpofit/sdm/trainer.py | 981 ---------------------------------------- 1 file changed, 981 deletions(-) delete mode 100644 menpofit/sdm/trainer.py diff --git a/menpofit/sdm/trainer.py b/menpofit/sdm/trainer.py deleted file mode 100644 index 83a9ec2..0000000 --- a/menpofit/sdm/trainer.py +++ /dev/null @@ -1,981 +0,0 @@ -from __future__ import division, print_function -import abc -import numpy as np -from menpo.transform import Scale -from menpo.shape import mean_pointcloud -from menpo.feature import sparse_hog, no_op -from menpofit.modelinstance import PDM, OrthoPDM -from menpo.visualize import print_dynamic, progress_bar_str - -from menpofit import checks -from menpofit.transform import (ModelDrivenTransform, OrthoMDTransform, - DifferentiableAlignmentSimilarity) -from menpofit.regression.trainer import ( - NonParametricRegressorTrainer, ParametricRegressorTrainer, - SemiParametricClassifierBasedRegressorTrainer) -from menpofit.regression.regressors import mlr -from menpofit.regression.parametricfeatures import weights -from menpofit.base import DeformableModel, create_pyramid -from .fitter import SDMFitter, SDAAMFitter, SDCLMFitter - - -def check_regression_features(regression_features, n_levels): - try: - return checks.check_list_callables(regression_features, n_levels) - except ValueError: - raise ValueError("regression_features must be a callable or a list of " - "{} callables".format(n_levels)) - - -def check_regression_type(regression_type, n_levels): - r""" - Checks the regression type (method) per level. - - It must be a callable or a list of those from the family of - functions defined in :ref:`regression_functions` - - Parameters - ---------- - regression_type : `function` or list of those - The regression type to check. - - n_levels : `int` - The number of pyramid levels. - - Returns - ------- - regression_type_list : `list` - A list of regression types that has length ``n_levels``. - """ - try: - return checks.check_list_callables(regression_type, n_levels) - except ValueError: - raise ValueError("regression_type must be a callable or a list of " - "{} callables".format(n_levels)) - - -def check_n_permutations(n_permutations): - if n_permutations < 1: - raise ValueError("n_permutations must be > 0") - - -def apply_pyramid_on_images(generators, n_levels, verbose=False): - r""" - Exhausts the pyramid generators verbosely - """ - all_images = [] - for j in range(n_levels): - - if verbose: - level_str = '- Apply pyramid: ' - if n_levels > 1: - level_str = '- Apply pyramid: [Level {} - '.format(j + 1) - - level_images = [] - for c, g in enumerate(generators): - if verbose: - print_dynamic( - '{}Computing feature space/rescaling - {}'.format( - level_str, - progress_bar_str((c + 1.) / len(generators), - show_bar=False))) - level_images.append(next(g)) - all_images.append(level_images) - if verbose: - print_dynamic('- Apply pyramid: Done\n') - return all_images - - -class SDTrainer(DeformableModel): - r""" - Mixin for Supervised Descent Trainers. - - Parameters - ---------- - regression_type : `callable`, or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - Examples of such callables can be found in :ref:`regression_callables`. - regression_features :`` None`` or `callable` or `[callable]`, optional - The features that are used during the regression. - - If `list`, a regression feature is defined per level. - - If not list or list with length ``1``, the specified regression feature - will be used for all levels. - - Depending on the :map:`SDTrainer` object, this parameter can take - different types. - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the - training shapes. - - Returns - ------- - fitter : :map:`MultilevelFitter` - The fitter object. - - Raises - ------ - ValueError - ``regression_type`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements - ValueError - n_levels must be `int` > ``0`` - ValueError - ``downscale`` must be >= ``1`` - ValueError - ``n_perturbations`` must be > 0 - ValueError - ``features`` must be a `string` or a `function` or a list of those - containing ``1`` or ``n_levels`` elements - """ - __metaclass__ = abc.ABCMeta - - def __init__(self, regression_type=mlr, regression_features=None, - features=no_op, n_levels=3, downscale=1.2, noise_std=0.04, - rotation=False, n_perturbations=10): - features = checks.check_features(features, n_levels) - DeformableModel.__init__(self, features) - - # general deformable model checks - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - - # SDM specific checks - regression_type_list = check_regression_type(regression_type, - n_levels) - regression_features = check_regression_features(regression_features, - n_levels) - check_n_permutations(n_perturbations) - - # store parameters - self.regression_type = regression_type_list - self.regression_features = regression_features - self.n_levels = n_levels - self.downscale = downscale - self.noise_std = noise_std - self.rotation = rotation - self.n_perturbations = n_perturbations - - def train(self, images, group=None, label=None, verbose=False, **kwargs): - r""" - Trains a Supervised Descent Regressor given a list of landmarked - images. - - Parameters - ---------- - images: list of :map:`MaskedImage` - The set of landmarked images from which to build the SD. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label: `string`, optional - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - verbose: `boolean`, optional - Flag that controls information and progress printing. - """ - if verbose: - print_dynamic('- Computing reference shape') - self.reference_shape = self._compute_reference_shape(images, group, - label) - # store number of training images - self.n_training_images = len(images) - - # normalize the scaling of all images wrt the reference_shape size - self._rescale_reference_shape() - normalized_images = self._normalization_wrt_reference_shape( - images, group, label, self.reference_shape, verbose=verbose) - - # create pyramid - generators = create_pyramid(normalized_images, self.n_levels, - self.downscale, self.features, - verbose=verbose) - - # get feature images of all levels - images = apply_pyramid_on_images(generators, self.n_levels, - verbose=verbose) - - # this .reverse sets the lowest resolution as the first level - images.reverse() - - # extract the ground truth shapes - gt_shapes = [[i.landmarks[group][label] for i in img] - for img in images] - - # build the regressors - if verbose: - if self.n_levels > 1: - print_dynamic('- Building regressors for each of the {} ' - 'pyramid levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building regressors\n') - - regressors = [] - # for each pyramid level (low --> high) - for j, (level_images, level_gt_shapes) in enumerate(zip(images, - gt_shapes)): - if verbose: - if self.n_levels == 1: - print_dynamic('\n') - elif self.n_levels > 1: - print_dynamic('\nLevel {}:\n'.format(j + 1)) - - # build regressor - trainer = self._set_regressor_trainer(j) - if j == 0: - regressor = trainer.train(level_images, level_gt_shapes, - verbose=verbose, **kwargs) - else: - regressor = trainer.train(level_images, level_gt_shapes, - level_shapes, verbose=verbose, - **kwargs) - - if verbose: - print_dynamic('- Perturbing shapes...') - level_shapes = trainer.perturb_shapes(gt_shapes[0]) - - regressors.append(regressor) - count = 0 - total = len(regressors) * len(images[0]) * len(level_shapes[0]) - for k, r in enumerate(regressors): - - test_images = images[k] - test_gt_shapes = gt_shapes[k] - - fitting_results = [] - for (i, gt_s, level_s) in zip(test_images, test_gt_shapes, - level_shapes): - fr_list = [] - for ls in level_s: - parameters = r.get_parameters(ls) - fr = r.fit(i, parameters) - fr.gt_shape = gt_s - fr_list.append(fr) - count += 1 - - fitting_results.append(fr_list) - if verbose: - print_dynamic('- Fitting shapes: {}'.format( - progress_bar_str((count + 1.) / total, - show_bar=False))) - - level_shapes = [[Scale(self.downscale, - n_dims=self.reference_shape.n_dims - ).apply(fr.final_shape) - for fr in fr_list] - for fr_list in fitting_results] - - if verbose: - print_dynamic('- Fitting shapes: computing mean error...') - mean_error = np.mean(np.array([fr.final_error() - for fr_list in fitting_results - for fr in fr_list])) - if verbose: - print_dynamic("- Fitting shapes: mean error " - "is {0:.6f}.\n".format(mean_error)) - - return self._build_supervised_descent_fitter(regressors) - - @classmethod - def _normalization_wrt_reference_shape(cls, images, group, label, - reference_shape, verbose=False): - r""" - Normalizes the images sizes with respect to the reference - shape (mean shape) scaling. This step is essential before building a - deformable model. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the model. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - reference_shape : :map:`PointCloud` - The reference shape that is used to resize all training images to - a consistent object size. - - verbose: bool, optional - Flag that controls information and progress printing. - - Returns - ------- - normalized_images : :map:`MaskedImage` list - A list with the normalized images. - """ - normalized_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic('- Normalizing images size: {}'.format( - progress_bar_str((c + 1.) / len(images), - show_bar=False))) - normalized_images.append(i.rescale_to_reference_shape( - reference_shape, group=group, label=label)) - - if verbose: - print_dynamic('- Normalizing images size: Done\n') - return normalized_images - - @abc.abstractmethod - def _compute_reference_shape(self, images, group, label): - r""" - Function that computes the reference shape, given a set of images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape computed based on the given images shapes. - """ - pass - - def _rescale_reference_shape(self): - r""" - Function that rescales the reference shape w.r.t. to - ``normalization_diagonal`` parameter. - """ - pass - - @abc.abstractmethod - def _set_regressor_trainer(self, **kwargs): - r""" - Function that sets the regression object to be one from - :map:`RegressorTrainer`, - """ - pass - - @abc.abstractmethod - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object. - - Parameters - ---------- - regressors : list of :map:`RegressorTrainer` - The list of regressors. - - Returns - ------- - fitter : :map:`SDMFitter` - The SDM fitter object. - """ - pass - - -class SDMTrainer(SDTrainer): - r""" - Class that trains Supervised Descent Method using Non-Parametric - Regression. - - Parameters - ---------- - regression_type : `callable` or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - The callable should be one of the methods defined in - :ref:`regression_callables` - - regression_features: ``None`` or `callable` or `[callable]`, optional - If list of length ``n_levels``, then a feature is defined per level. - - If not a list, then the specified feature will be applied to all - pyramid levels. - - Per level: - If ``None``, no features are extracted, thus specified - ``features`` is used in the regressor. - - It is recommended to set the desired features using this option, - leaving ``features`` equal to :map:`no_op`. This means that the - images will remain in the intensities space and the features will - be extracted by the regressor. - - patch_shape: tuple of `int` - The shape of the patches used by the SDM. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - initial shape. - - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the initial shape. - - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the shapes. - - normalization_diagonal : `int` >= ``20``, optional - During training, all images are rescaled to ensure that the scale of - their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the normalization_diagonal - value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - Raises - ------ - ValueError - ``regression_features`` must be ``None`` or a `string` or a `function` - or a list of those containing 1 or ``n_level`` elements - """ - def __init__(self, regression_type=mlr, regression_features=sparse_hog, - patch_shape=(16, 16), features=no_op, n_levels=3, - downscale=1.5, noise_std=0.04, - rotation=False, n_perturbations=10, - normalization_diagonal=None): - super(SDMTrainer, self).__init__( - regression_type=regression_type, - regression_features=regression_features, - features=features, n_levels=n_levels, downscale=downscale, - noise_std=noise_std, rotation=rotation, - n_perturbations=n_perturbations) - self.patch_shape = patch_shape - self.normalization_diagonal = normalization_diagonal - - def _compute_reference_shape(self, images, group, label): - r""" - Function that computes the reference shape, given a set of images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape computed based on the given images. - """ - shapes = [i.landmarks[group][label] for i in images] - return mean_pointcloud(shapes) - - def _rescale_reference_shape(self): - r""" - Function that rescales the reference shape w.r.t. to - ``normalization_diagonal`` parameter. - """ - if self.normalization_diagonal: - x, y = self.reference_shape.range() - scale = self.normalization_diagonal / np.sqrt(x**2 + y**2) - Scale(scale, self.reference_shape.n_dims).apply_inplace( - self.reference_shape) - - def _set_regressor_trainer(self, level): - r""" - Function that sets the regression class to be the - :map:`NonParametricRegressorTrainer`. - - Parameters - ---------- - level : `int` - The scale level. - - Returns - ------- - trainer : :map:`NonParametricRegressorTrainer` - The regressor object. - """ - return NonParametricRegressorTrainer( - self.reference_shape, regression_type=self.regression_type[level], - regression_features=self.regression_features[level], - patch_shape=self.patch_shape, noise_std=self.noise_std, - rotation=self.rotation, n_perturbations=self.n_perturbations) - - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object. - - Parameters - ---------- - regressors : list of :map:`RegressorTrainer` - The list of regressors. - - Returns - ------- - fitter : :map:`SDMFitter` - The SDM fitter object. - """ - return SDMFitter(regressors, self.n_training_images, self.features, - self.reference_shape, self.downscale) - - -class SDAAMTrainer(SDTrainer): - r""" - Class that trains Supervised Descent Regressor for a given Active - Appearance Model, thus uses Parametric Regression. - - Parameters - ---------- - aam : :map:`AAM` - The trained AAM object. - regression_type : `callable`, or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - Examples of such callables can be found in :ref:`regression_callables`. - regression_features: `function` or list of those, optional - If list of length ``n_levels``, then a feature is defined per level. - - If not a list or a list with length ``1``, then the specified feature - will be applied to all pyramid levels. - - The callable should be one of the methods defined in - :ref:`parametricfeatures`. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the - training shapes. - update : {'additive', 'compositional'} - Defines the way that the warp will be updated. - md_transform: :map:`ModelDrivenTransform`, optional - The model driven transform class to be used. - n_shape : `int` > ``1`` or ``0`` <= `float` <= ``1`` or ``None``, or a list of those, optional - The number of shape components to be used per fitting level. - - If list of length ``n_levels``, then a number of components is defined - per level. The first element of the list corresponds to the lowest - pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - components will be used for all levels. - - Per level: - If ``None``, all the available shape components - (``n_active_components``)will be used. - - If `int` > ``1``, a specific number of shape components is - specified. - - If ``0`` <= `float` <= ``1``, it specifies the variance percentage - that is captured by the components. - n_appearance : `int` > ``1`` or ``0`` <= `float` <= ``1`` or ``None``, or a list of those, optional - The number of appearance components to be used per fitting level. - - If list of length ``n_levels``, then a number of components is defined - per level. The first element of the list corresponds to the lowest - pyramidal level and so on. - - If not a list or a list with length 1, then the specified number of - components will be used for all levels. - - Per level: - If ``None``, all the available appearance components - (``n_active_components``) will be used. - - If `int > ``1``, a specific number of appearance components is - specified. - - If ``0`` <= `float` <= ``1``, it specifies the variance percentage - that is captured by the components. - - Raises - ------- - ValueError - n_shape can be an integer or a float or None or a list containing 1 - or ``n_levels`` of those - ValueError - n_appearance can be an integer or a float or None or a list containing - 1 or ``n_levels`` of those - ValueError - ``regression_features`` must be a `function` or a list of those - containing ``1`` or ``n_levels`` elements - """ - def __init__(self, aam, regression_type=mlr, regression_features=weights, - noise_std=0.04, rotation=False, n_perturbations=10, - update='compositional', md_transform=OrthoMDTransform, - n_shape=None, n_appearance=None): - super(SDAAMTrainer, self).__init__( - regression_type=regression_type, - regression_features=regression_features, - features=aam.features, n_levels=aam.n_levels, - downscale=aam.downscale, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.aam = aam - self.update = update - self.md_transform = md_transform - # hard coded for now as this is the only supported configuration. - self.global_transform = DifferentiableAlignmentSimilarity - - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.aam.n_levels > 1: - for sm in self.aam.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.aam.n_levels: - for sm, n in zip(self.aam.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float, ' - 'an integer or float list containing 1 ' - 'or {} elements or else ' - 'None'.format(self.aam.n_levels)) - - # check n_appearance parameter - if n_appearance is not None: - if type(n_appearance) is int or type(n_appearance) is float: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: - for am, n in zip(self.aam.appearance_models, n_appearance): - am.n_active_components = n - else: - raise ValueError('n_appearance can be an integer or a float, ' - 'an integer or float list containing 1 ' - 'or {} elements or else ' - 'None'.format(self.aam.n_levels)) - - def _compute_reference_shape(self, images, group, label): - r""" - Function that returns the reference shape computed during AAM building. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape computed based on. - """ - return self.aam.reference_shape - - def _normalize_object_size(self, images, group, label): - r""" - Function that normalizes the images sizes with respect to the reference - shape (mean shape) scaling. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the model. - - group : `string` - The key of the landmark set that should be used. If ```None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - normalized_images : :map:`MaskedImage` list - A list with the normalized images. - """ - return [i.rescale_to_reference_shape(self.reference_shape, - group=group, label=label) - for i in images] - - def _set_regressor_trainer(self, level): - r""" - Function that sets the regression class to be the - :map:`ParametricRegressorTrainer`. - - Parameters - ---------- - level : `int` - The scale level. - - Returns - ------- - trainer: :map:`ParametricRegressorTrainer` - The regressor object. - """ - am = self.aam.appearance_models[level] - sm = self.aam.shape_models[level] - - if self.md_transform is not ModelDrivenTransform: - md_transform = self.md_transform( - sm, self.aam.transform, self.global_transform, - source=am.mean().landmarks['source'].lms) - else: - md_transform = self.md_transform( - sm, self.aam.transform, - source=am.mean().landmarks['source'].lms) - - return ParametricRegressorTrainer( - am, md_transform, self.reference_shape, - regression_type=self.regression_type[level], - regression_features=self.regression_features[level], - update=self.update, noise_std=self.noise_std, - rotation=self.rotation, n_perturbations=self.n_perturbations) - - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object for AAMs. - - Parameters - ---------- - regressors : :map:`RegressorTrainer` - The regressor to build with. - - Returns - ------- - fitter : :map:`SDAAMFitter` - The SDM fitter object. - """ - return SDAAMFitter(self.aam, regressors, self.n_training_images) - - -class SDCLMTrainer(SDTrainer): - r""" - Class that trains Supervised Descent Regressor for a given Constrained - Local Model, thus uses Semi Parametric Classifier-Based Regression. - - Parameters - ---------- - clm : :map:`CLM` - The trained CLM object. - regression_type : `callable`, or list of those, optional - If list of length ``n_levels``, then a regression type is defined per - level. - - If not a list or a list with length ``1``, then the specified regression - type will be applied to all pyramid levels. - - Examples of such callables can be found in :ref:`regression_callables`. - noise_std: float, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int` > ``0``, optional - Defines the number of perturbations that will be applied to the - training shapes. - pdm_transform : :map:`ModelDrivenTransform`, optional - The point distribution transform class to be used. - n_shape : `int` > ``1`` or ``0`` <= `float` <= ``1`` or ``None``, or a list of those, optional - The number of shape components to be used per fitting level. - - If list of length ``n_levels``, then a number of components is defined - per level. The first element of the list corresponds to the lowest - pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - components will be used for all levels. - - Per level: - If ``None``, all the available shape components - (``n_active_components``) will be used. - - If `int` > ``1``, a specific number of shape components is - specified. - - If ``0`` <= `float` <= ``1``, it specifies the variance percentage - that is captured by the components. - - Raises - ------- - ValueError - ``n_shape`` can be an integer or a `float` or ``None`` or a list - containing ``1`` or ``n_levels`` of those. - """ - def __init__(self, clm, regression_type=mlr, noise_std=0.04, - rotation=False, n_perturbations=10, pdm_transform=OrthoPDM, - n_shape=None): - super(SDCLMTrainer, self).__init__( - regression_type=regression_type, - regression_features=[None] * clm.n_levels, - features=clm.features, n_levels=clm.n_levels, - downscale=clm.downscale, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.clm = clm - self.patch_shape = clm.patch_shape - self.pdm_transform = pdm_transform - # hard coded for now as this is the only supported configuration. - self.global_transform = DifferentiableAlignmentSimilarity - - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.clm.n_levels > 1: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.clm.n_levels: - for sm, n in zip(self.clm.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None' - 'or a list containing 1 or {} of ' - 'those'.format(self.clm.n_levels)) - - def _compute_reference_shape(self, images, group, label): - r""" - Function that returns the reference shape computed during CLM building. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - group : `string` - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - - label : `string` - The label of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - - Returns - ------- - reference_shape : :map:`PointCloud` - The reference shape. - """ - return self.clm.reference_shape - - def _set_regressor_trainer(self, level): - r""" - Function that sets the regression class to be the - :map:`SemiParametricClassifierBasedRegressorTrainer` - - Parameters - ---------- - level : `int` - The scale level. - - Returns - ------- - trainer: :map:`SemiParametricClassifierBasedRegressorTrainer` - The regressor object. - """ - clfs = self.clm.classifiers[level] - sm = self.clm.shape_models[level] - - if self.pdm_transform is not PDM: - pdm_transform = self.pdm_transform(sm, self.global_transform) - else: - pdm_transform = self.pdm_transform(sm) - - return SemiParametricClassifierBasedRegressorTrainer( - clfs, pdm_transform, self.reference_shape, - regression_type=self.regression_type[level], - patch_shape=self.patch_shape, update='additive', - noise_std=self.noise_std, rotation=self.rotation, - n_perturbations=self.n_perturbations) - - def _build_supervised_descent_fitter(self, regressors): - r""" - Builds an SDM fitter object for CLMs. - - Parameters - ---------- - regressors : :map:`RegressorTrainer` - Regressor to train with. - - Returns - ------- - fitter : :map:`SDCLMFitter` - The SDM fitter object. - """ - return SDCLMFitter(self.clm, regressors, self.n_training_images) From 70b370950faf54b3351bb00783e8cba0778df480 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 22 Jun 2015 18:21:36 +0100 Subject: [PATCH 274/423] Changed old ...inplace calls on lk/fitter.py and fitter.py --- menpofit/fitter.py | 3 +-- menpofit/lk/fitter.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 3d8ce11..3ca7817 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -157,8 +157,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # if specified, crop the image if crop_image: - image = image.copy() - image.crop_to_landmarks_proportion_inplace(crop_image, + image = image.crop_to_landmarks_proportion(crop_image, group='initial_shape') # rescale image wrt the scale factor between reference_shape and diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 73f6eb1..68f692b 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -62,10 +62,7 @@ def scale_features(self): return self._scale_features def _prepare_template(self, template, group=None, label=None): - # copy template - template = template.copy() - - template = template.crop_to_landmarks_inplace(group=group, label=label) + template = template.crop_to_landmarks(group=group, label=label) template = template.as_masked() # rescale template to diagonal range From 48270efa045befb1f496213970408e85422e967d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 23 Jun 2015 11:56:54 +0100 Subject: [PATCH 275/423] Update initialization method for ModelFitter --- menpofit/fitter.py | 106 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 3ca7817..b230e2b 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -2,7 +2,8 @@ import abc import numpy as np from menpo.shape import PointCloud -from menpo.transform import Scale, AlignmentAffine, AlignmentSimilarity +from menpo.transform import ( + Scale, Similarity, AlignmentAffine, AlignmentSimilarity) import menpofit.checks as checks @@ -317,23 +318,80 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - # TODO: Bounding boxes should be PointGraphs - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, - rotation=False): - transform = noisy_align(AlignmentSimilarity, - self.reference_bounding_box, bounding_box, - noise_std=noise_std, rotation=rotation) + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.5): + transform = noisy_params_alignment_similarity( + self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): + def noisy_shape_from_shape(self, shape, noise_std=0.5): return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std, rotation=rotation) + shape.bounding_box(), noise_std=noise_std) -# TODO: document me! -def noisy_align(alignment_transform_cls, source, target, noise_std=0.1, - **kwargs): +def noisy_params_alignment_similarity(source, target, noise_std=0.5): r""" + Constructs and perturbs the optimal similarity transform between source + and target by adding white noise to its parameters. + Parameters + ---------- + source: :class:`menpo.shape.PointCloud` + The source pointcloud instance used in the alignment + target: :class:`menpo.shape.PointCloud` + The target pointcloud instance used in the alignment + noise_std: float or triplet of floats, optional + The standard deviation of the white noise. If float the same amount + of noise is applied to the scale, rotation and translation + parameters of the true similarity transform. If triplet of + floats, the first, second and third elements denote the amount of + noise to be applied to the scale, rotation and translation + parameters respectively. + Returns + ------- + noisy_transform : :class: `menpo.transform.Similarity` + The noisy Similarity Transform + """ + if isinstance(noise_std, float): + noise_std = [noise_std] * 3 + elif len(noise_std) == 1: + noise_std *= 3 + + transform = AlignmentSimilarity(source, target, rotation=True) + parameters = transform.as_vector() + + scale = noise_std[0] * parameters[0] + rotation = noise_std[1] * parameters[1] + translation = noise_std[2] * target.range() + + noise = (([scale, rotation] + list(translation)) * + np.random.randn(transform.n_parameters)) + return Similarity.init_identity(source.n_dims).from_vector( + parameters + noise) + + +def noisy_target_alignment_transform(source, target, + alignment_transform_cls=AlignmentAffine, + noise_std=0.1, **kwargs): + r""" + Constructs and the optimal alignment transform between the source and + a noisy version of the target obtained by adding white noise to each of + its points. + + Parameters + ---------- + source: :class:`menpo.shape.PointCloud` + The source pointcloud instance used in the alignment + target: :class:`menpo.shape.PointCloud` + The target pointcloud instance used in the alignment + alignment_transform_cls: :class:`menpo.transform.Alignment`, optional + The alignment transform class used to perform the alignment. + noise_std: float or triplet of floats, optional + The standard deviation of the white noise to be added to each one of + the target points. + + Returns + ------- + noisy_transform : :class: `menpo.transform.Alignment` + The noisy Similarity Transform """ noise = noise_std * target.range() * np.random.randn(target.n_points, target.n_dims) @@ -341,11 +399,27 @@ def noisy_align(alignment_transform_cls, source, target, noise_std=0.1, return alignment_transform_cls(source, noisy_target, **kwargs) -# TODO: document me! -def align_shape_with_bounding_box(alignment_transform_cls, shape, - bounding_box, **kwargs): +def align_shape_with_bounding_box(shape, bounding_box, + alignment_transform_cls=AlignmentSimilarity, + **kwargs): r""" + Aligns the shape with the bounding box using a particular ali . + + Parameters + ---------- + source: :class:`menpo.shape.PointCloud` + The shape instance used in the alignment. + bounding_box: :class:`menpo.shape.PointCloud` + The bounding box instance used in the alignment. + alignment_transform_cls: :class:`menpo.transform.Alignment`, optional + The class of the alignment transform used to perform the alignment. + + Returns + ------- + noisy_transform : :class: `menpo.transform.Alignment` + The noisy Alignment Transform """ shape_bb = shape.bounding_box() - return alignment_transform_cls(shape_bb, bounding_box, **kwargs) + transform = alignment_transform_cls(shape_bb, bounding_box, **kwargs) + return transform.apply(shape) From c36037c3c5394d6e941fb75c18d3befb23d9e2b9 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 23 Jun 2015 13:42:19 +0100 Subject: [PATCH 276/423] Make Linear(AAM, ATM) fitting results return point graph shapes --- menpofit/aam/fitter.py | 4 ++-- menpofit/atm/fitter.py | 2 +- menpofit/transform/modeldriven.py | 11 ++++++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index de7bed1..c6c477b 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -83,7 +83,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self.aam.n_landmarks) + sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( LinearLKAAMInterface, am, md_transform, sampling=s, @@ -140,7 +140,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self.aam.n_landmarks) + sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = cr_algorithm_cls( CRLinearAAMInterface, am, md_transform, sampling=s, diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 53f8820..702a008 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -36,7 +36,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): type(self.atm) is LinearPatchATM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( - sm, self.atm.n_landmarks) + sm, self.atm.reference_shape) # set up algorithm using linear aam interface algorithm = algorithm_cls(LKLinearATMInterface, wt, md_transform, sampling=sampling, diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index fc5fd8a..0238db5 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -529,21 +529,26 @@ def __init__(self, model, transform_cls, source=None): class LinearOrthoMDTransform(OrthoPDM, Transform): r""" """ - def __init__(self, model, n_landmarks): + def __init__(self, model, sparse_instance): super(LinearOrthoMDTransform, self).__init__(model) - self.n_landmarks = n_landmarks + self._sparse_instance = sparse_instance self.W = np.vstack((self.similarity_model.components, self.model.components)) V = self.W[:, :self.n_dims*self.n_landmarks] self.pinv_V = np.linalg.pinv(V) + @property + def n_landmarks(self): + return self._sparse_instance.n_points + @property def dense_target(self): return PointCloud(self.target.points[self.n_landmarks:]) @property def sparse_target(self): - return PointCloud(self.target.points[:self.n_landmarks]) + sparse_target = PointCloud(self.target.points[:self.n_landmarks]) + return self._sparse_instance.from_vector(sparse_target.as_vector()) def set_target(self, target): if target.n_points == self.n_landmarks: From 3b98d0c088041ac6595661e0eff00ec3677092a1 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 29 Jun 2015 12:09:12 +0100 Subject: [PATCH 277/423] Update checks for AAMs, ATMs and LK --- menpofit/aam/builder.py | 22 ++++---- menpofit/aam/fitter.py | 31 +++++------ menpofit/atm/builder.py | 11 ++-- menpofit/atm/fitter.py | 18 +++---- menpofit/checks.py | 116 +++++++++++++++------------------------- menpofit/fitter.py | 34 +----------- menpofit/lk/fitter.py | 52 +++++++----------- 7 files changed, 106 insertions(+), 178 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index ff8a783..9ef763b 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -13,8 +13,6 @@ DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -# TODO: fix features checker -# TODO: implement checker for conflict between features and scale_features # TODO: document me! class AAMBuilder(object): r""" @@ -129,8 +127,9 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -190,7 +189,7 @@ def build(self, images, group=None, label=None, verbose=False): # obtain image representation if j == 0: # compute features at highest level - feature_images = compute_features(images, self.features, + feature_images = compute_features(images, self.features[j], level_str=level_str, verbose=verbose) level_images = feature_images @@ -203,7 +202,8 @@ def build(self, images, group=None, label=None, verbose=False): # scale images and compute features at other levels scaled_images = scale_images(images, s, level_str=level_str, verbose=verbose) - level_images = compute_features(scaled_images, self.features, + level_images = compute_features(scaled_images, + self.features[j], level_str=level_str, verbose=verbose) @@ -381,8 +381,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -523,8 +524,9 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -675,8 +677,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters @@ -830,8 +833,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') + max_shape_components, n_levels, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_levels, 'max_appearance_components') # set parameters diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index c6c477b..a4ee35e 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -18,20 +18,10 @@ class AAMFitter(ModelFitter): r""" """ - def __init__(self, aam, n_shape=None, n_appearance=None): - super(AAMFitter, self).__init__(aam) - self._algorithms = [] - self._check_n_shape(n_shape) - self._check_n_appearance(n_appearance) - @property def aam(self): return self._model - @property - def algorithms(self): - return self._algorithms - def _check_n_appearance(self, n_appearance): if n_appearance is not None: if type(n_appearance) is int or type(n_appearance) is float: @@ -60,8 +50,10 @@ class LKAAMFitter(AAMFitter): """ def __init__(self, aam, n_shape=None, n_appearance=None, lk_algorithm_cls=AIC, sampling=None, **kwargs): - super(LKAAMFitter, self).__init__( - aam, n_shape=n_shape, n_appearance=n_appearance) + self._model = aam + self.algorithms = [] + self._check_n_shape(n_shape) + self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self._set_up(lk_algorithm_cls, sampling, **kwargs) @@ -105,7 +97,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) # TODO: document me! @@ -115,8 +107,10 @@ class CRAAMFitter(AAMFitter): def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, max_iters=6, **kwargs): - super(CRAAMFitter, self).__init__( - aam, n_shape=n_shape, n_appearance=n_appearance) + self._model = aam + self.algorithms = [] + self._check_n_shape(n_shape) + self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self.n_perturbations = n_perturbations self.max_iters = checks.check_max_iters(max_iters, self.n_levels) @@ -163,7 +157,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): LinearPatchAAM, PartsAAM)) # append algorithms to list - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) def train(self, images, group=None, label=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam @@ -172,7 +166,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): if self.scale_features: # compute features at highest level - feature_images = compute_features(images, self.features, + feature_images = compute_features(images, self.features[0], verbose=verbose) # for each pyramid level (low --> high) @@ -195,7 +189,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): # scale images and compute features at other levels scaled_images = scale_images(images, s, level_str=level_str, verbose=verbose) - level_images = compute_features(scaled_images, self.features, + level_images = compute_features(scaled_images, + self.features[j], level_str=level_str, verbose=verbose) diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 523ecdb..42c95f9 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -111,6 +111,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -180,7 +181,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): if j == 0 or self.scale_shapes: if j == 0: level_shapes = shapes - level_reference_shape= reference_shape + level_reference_shape = reference_shape else: scale_transform = Scale(scale_factor=s, n_dims=2) level_shapes = [scale_transform.apply(s) for s in shapes] @@ -202,7 +203,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): # obtain template representation if j == 0: # compute features at highest level - feature_template = self.features(template) + feature_template = self.features[j](template) level_template = feature_template elif self.scale_features: # scale features at other levels @@ -210,7 +211,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): else: # scale template and compute features at other levels scaled_template = template.rescale(s) - level_template = self.features(scaled_template) + level_template = self.features[j](scaled_template) # extract potentially rescaled template shape level_template_shape = level_template.landmarks[group][label] @@ -349,6 +350,7 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -478,6 +480,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -613,6 +616,7 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters @@ -742,6 +746,7 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scales, n_levels = checks.check_scales(scales) patch_shape = checks.check_patch_shape(patch_shape, n_levels) features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') # set parameters diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 702a008..3cd7d97 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -14,11 +14,15 @@ class LKATMFitter(ModelFitter): """ def __init__(self, atm, algorithm_cls=IC, n_shape=None, sampling=None, **kwargs): - super(LKATMFitter, self).__init__(atm) - self._algorithms = [] + self._model = atm + self.algorithms = [] self._check_n_shape(n_shape) self._set_up(algorithm_cls, sampling, **kwargs) + @property + def atm(self): + return self._model + def _set_up(self, algorithm_cls, sampling, **kwargs): for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, self.atm.shape_models)): @@ -58,15 +62,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): LinearPatchATM, PartsATM)) # append algorithms to list - self._algorithms.append(algorithm) - - @property - def atm(self): - return self._model - - @property - def algorithms(self): - return self._algorithms + self.algorithms.append(algorithm) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): diff --git a/menpofit/checks.py b/menpofit/checks.py index 2ac6e66..0c5dc0c 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,52 +1,5 @@ import numpy as np -from menpofit.base import is_pyramid_on_features - - -def check_features(features, n_levels): - r""" - Checks the feature type per level. - - Parameters - ---------- - features : callable or list of callables - The features to apply to the images. - n_levels : int - The number of pyramid levels. - - Returns - ------- - feature_list : list - A list of feature function. - """ - # Firstly, make sure we have a list of callables of the right length - if is_pyramid_on_features(features): - return features - else: - try: - all_callables = check_list_callables(features, n_levels, - allow_single=False) - except ValueError: - raise ValueError("features must be a callable or a list of " - "{} callables".format(n_levels)) - return all_callables - - -def check_list_callables(callables, n_callables, allow_single=True): - if not isinstance(callables, list): - if allow_single: - # expand to a list of callables for them - callables = [callables] * n_callables - else: - raise ValueError("Expected a list of callables " - "(allow_single=False)") - # must have a list by now - for c in callables: - if not callable(c): - raise ValueError("All items must be callables") - if len(callables) != n_callables: - raise ValueError("List of callables must be {} " - "long".format(n_callables)) - return callables +import warnings def check_diagonal(diagonal): @@ -73,6 +26,47 @@ def check_scales(scales): "int/float") +def check_features(features, n_levels): + r""" + Checks the feature type per level. + + Parameters + ---------- + features : callable or list of callables + The features to apply to the images. + n_levels : int + The number of pyramid levels. + + Returns + ------- + feature_list : list + A list of feature function. + """ + if callable(features): + return [features] * n_levels + elif len(features) == 1 and np.alltrue([callable(f) for f in features]): + return list(features) * n_levels + elif len(features) == n_levels and np.alltrue([callable(f) + for f in features]): + return list(features) + else: + raise ValueError("features must be a callable or a list/tuple of " + "callables with the same length as scales") + + +# TODO: document me! +def check_scale_features(scale_features, features): + r""" + """ + if np.alltrue([f == features[0] for f in features]): + return scale_features + else: + warnings.warn('scale_features has been automatically set to False ' + 'because different types of features are used at each ' + 'level.') + return False + + # TODO: document me! def check_patch_shape(patch_shape, n_levels): if len(patch_shape) == 2 and isinstance(patch_shape[0], int): @@ -98,7 +92,7 @@ def check_max_components(max_components, n_levels, var_name): str_error = ("{} must be None or an int > 0 or a 0 <= float <= 1 or " "a list of those containing 1 or {} elements").format( var_name, n_levels) - if not isinstance(max_components, list): + if not isinstance(max_components, (list, tuple)): max_components_list = [max_components] * n_levels elif len(max_components) == 1: max_components_list = [max_components[0]] * n_levels @@ -146,27 +140,3 @@ def check_sampling(sampling, n_levels): 'None'.format(n_levels)) return sampling - -# def check_n_levels(n_levels): -# r""" -# Checks the number of pyramid levels - must be int > 0. -# """ -# if not isinstance(n_levels, int) or n_levels < 1: -# raise ValueError("n_levels must be int > 0") -# -# -# def check_downscale(downscale): -# r""" -# Checks the downscale factor of the pyramid that must be >= 1. -# """ -# if downscale < 1: -# raise ValueError("downscale must be >= 1") -# -# -# def check_boundary(boundary): -# r""" -# Checks the boundary added around the reference shape that must be -# int >= 0. -# """ -# if not isinstance(boundary, int) or boundary < 0: -# raise ValueError("boundary must be >= 0") diff --git a/menpofit/fitter.py b/menpofit/fitter.py index b230e2b..4ec63ef 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.shape import PointCloud from menpo.transform import ( @@ -20,26 +19,6 @@ def n_levels(self): """ return len(self.scales) - @abc.abstractproperty - def algorithms(self): - pass - - # @abc.abstractproperty - # def reference_shape(self): - # pass - # - # @abc.abstractproperty - # def features(self): - # pass - # - # @abc.abstractproperty - # def scales(self): - # pass - # - # @abc.abstractproperty - # def scale_features(self): - # pass - def fit(self, image, initial_shape, max_iters=50, gt_shape=None, crop_image=0.5, **kwargs): r""" @@ -174,14 +153,14 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, for j, s in enumerate(scales): if j == 0: # compute features at highest level - feature_image = self.features(image) + feature_image = self.features[j](image) elif self.scale_features: # scale features at other levels feature_image = images[0].rescale(s) else: # scale image and compute features at other levels scaled_image = image.rescale(s) - feature_image = self.features(scaled_image) + feature_image = self.features[j](scaled_image) images.append(feature_image) images.reverse() @@ -247,20 +226,11 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, return algorithm_results - @abc.abstractmethod - def _fitter_result(self, image, algorithm_results, affine_correction, - gt_shape=None): - pass - -# TODO: correctly implement initialization from bounding box # TODO: document me! class ModelFitter(MultiFitter): r""" """ - def __init__(self, model): - self._model = model - @property def reference_shape(self): r""" diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 68f692b..e2a91e0 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -1,7 +1,8 @@ from __future__ import division from menpo.feature import no_op from menpofit.transform import DifferentiableAlignmentAffine -from menpofit.fitter import MultiFitter, noisy_align +from menpofit.fitter import MultiFitter, noisy_target_alignment_transform +from menpofit import checks from .algorithm import IC from .residual import SSD, FourierSSD from .result import LKFitterResult @@ -15,19 +16,25 @@ def __init__(self, template, group=None, label=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, scales=(1, .5), scale_features=True, algorithm_cls=IC, residual_cls=SSD, **kwargs): - self._features = features + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) + # set parameters + self.features = features self.transform_cls = transform_cls self.diagonal = diagonal - self._scales = list(scales) - self._scales.reverse() - self._scale_features = scale_features + self.scales = list(scales) + self.scales.reverse() + self.scale_features = scale_features self.templates, self.sources = self._prepare_template( template, group=group, label=label) - self._reference_shape = self.sources[0] + self.reference_shape = self.sources[0] - self._algorithms = [] + self.algorithms = [] for j, (t, s) in enumerate(zip(self.templates, self.sources)): transform = self.transform_cls(s, s) if ('kernel_func' in kwargs and @@ -39,27 +46,7 @@ def __init__(self, template, group=None, label=None, features=no_op, else: residual = residual_cls() algorithm = algorithm_cls(t, transform, residual, **kwargs) - self._algorithms.append(algorithm) - - @property - def algorithms(self): - return self._algorithms - - @property - def reference_shape(self): - return self._reference_shape - - @property - def features(self): - return self._features - - @property - def scales(self): - return self._scales - - @property - def scale_features(self): - return self._scale_features + self.algorithms.append(algorithm) def _prepare_template(self, template, group=None, label=None): template = template.crop_to_landmarks(group=group, label=label) @@ -78,14 +65,14 @@ def _prepare_template(self, template, group=None, label=None): for j, s in enumerate(scales): if j == 0: # compute features at highest level - feature_template = self.features(template) + feature_template = self.features[j](template) elif self.scale_features: # scale features at other levels feature_template = templates[0].rescale(s) else: # scale image and compute features at other levels scaled_template = template.rescale(s) - feature_template = self.features(scaled_template) + feature_template = self.features[j](scaled_template) templates.append(feature_template) templates.reverse() @@ -95,8 +82,9 @@ def _prepare_template(self, template, group=None, label=None): return templates, sources def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): - transform = noisy_align(self.transform_cls, self.reference_shape, - gt_shape, noise_std=noise_std) + transform = noisy_target_alignment_transform( + self.transform_cls, self.reference_shape, gt_shape, + noise_std=noise_std) return transform.apply(self.reference_shape) def _fitter_result(self, image, algorithm_results, affine_correction, From 28061cae9477d60f7d291f3b144560286bb21826 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 29 Jun 2015 16:18:59 +0100 Subject: [PATCH 278/423] Fixed noise_std default value - Allow nise_std to be set in CRAAMs --- menpofit/aam/fitter.py | 5 +++-- menpofit/fitter.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a4ee35e..a9e0cad 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -106,13 +106,14 @@ class CRAAMFitter(AAMFitter): """ def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, - max_iters=6, **kwargs): + noise_std=0.05, max_iters=6, **kwargs): self._model = aam self.algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self.n_perturbations = n_perturbations + self.noise_std = noise_std self.max_iters = checks.check_max_iters(max_iters, self.n_levels) self._set_up(cr_algorithm_cls, sampling, **kwargs) @@ -203,7 +204,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape(gt_s) + p_s = self.noisy_shape_from_shape(gt_s, self.noise_std) perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 4ec63ef..06209da 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -288,17 +288,17 @@ def _check_n_shape(self, n_shape): 'or a list containing 1 or {} of ' 'those'.format(self._model.n_levels)) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.5): + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): transform = noisy_params_alignment_similarity( self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.5): + def noisy_shape_from_shape(self, shape, noise_std=0.05): return self.noisy_shape_from_bounding_box( shape.bounding_box(), noise_std=noise_std) -def noisy_params_alignment_similarity(source, target, noise_std=0.5): +def noisy_params_alignment_similarity(source, target, noise_std=0.05): r""" Constructs and perturbs the optimal similarity transform between source and target by adding white noise to its parameters. From 47c66dcc2b823e142d36e9bb37b7716154881023 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 29 Jun 2015 23:33:11 +0100 Subject: [PATCH 279/423] Add psi parameter to CRAAM --- menpofit/aam/algorithm/cr.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/cr.py index f9c5d7c..26f78a7 100644 --- a/menpofit/aam/algorithm/cr.py +++ b/menpofit/aam/algorithm/cr.py @@ -326,28 +326,35 @@ def _compute_features2(self, image): class PSD(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None): - return _supervised_descent(features, deltas, gamma=gamma) + def _perform_regression(self, features, deltas, gamma=None, + dtype=np.float64): + regressor = _supervised_newton(features, deltas, gamma=gamma, + dtype=dtype) + regressor.R = self.project_out(regressor.R) + return regressor # TODO: document me! class PAJ(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None): - return _average_jacobian(features, deltas, gamma=gamma) + def _perform_regression(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + return _supervised_gauss_newton(features, deltas, gamma=gamma, + psi=psi, dtype=dtype) # TODO: document me! -class _supervised_descent(object): +class _supervised_newton(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression + def __init__(self, features, deltas, gamma=None, dtype=np.float64): + features = features.astype(dtype) + deltas = deltas.astype(dtype) XX = features.T.dot(features) XT = features.T.dot(deltas) if gamma: - XX += gamma * np.eye(features.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # descent direction self.R = np.linalg.solve(XX, XT) @@ -356,19 +363,23 @@ def __call__(self, features): # TODO: document me! -class _average_jacobian(object): +class _supervised_gauss_newton(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression + def __init__(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + features = features.astype(dtype) + deltas = deltas.astype(dtype) XX = deltas.T.dot(deltas) XT = deltas.T.dot(features) if gamma: - XX += gamma * np.eye(deltas.shape[1]) + np.fill_diagonal(XX, gamma + np.diag(XX)) # average Jacobian self.J = np.linalg.solve(XX, XT) # average Hessian self.H = self.J.dot(self.J.T) + if psi: + np.fill_diagonal(self.H, psi + np.diag(self.H)) # descent direction self.R = np.linalg.solve(self.H, self.J).T @@ -379,4 +390,3 @@ def __call__(self, features): # TODO: document me! def _compute_rmse(x1, x2): return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) - From e3b0d8c03087ddec666c959fd799df229f1c90d8 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 02:26:58 +0100 Subject: [PATCH 280/423] Rename AAM, ATM, LK Fitters, Algorithms and Results. - Names are now explicit --- menpofit/aam/__init__.py | 21 ++- menpofit/aam/algorithm/__init__.py | 19 ++- menpofit/aam/algorithm/lk.py | 179 ++++++++++----------- menpofit/aam/algorithm/{cr.py => sd.py} | 200 ++++++++++++++++-------- menpofit/aam/fitter.py | 39 ++--- menpofit/aam/result.py | 8 +- menpofit/atm/__init__.py | 4 +- menpofit/atm/algorithm.py | 68 ++++---- menpofit/atm/fitter.py | 18 ++- menpofit/atm/result.py | 6 +- menpofit/lk/__init__.py | 8 +- menpofit/lk/algorithm.py | 39 ++--- menpofit/lk/fitter.py | 15 +- menpofit/lk/residual.py | 18 ++- menpofit/lk/result.py | 8 +- 15 files changed, 378 insertions(+), 272 deletions(-) rename menpofit/aam/algorithm/{cr.py => sd.py} (71%) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 32a3556..65d6960 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,12 +1,17 @@ from .builder import ( AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, LinearPatchAAMBuilder, PartsAAMBuilder) -from .fitter import LKAAMFitter, CRAAMFitter +from .fitter import LucasKanadeAAMFitter, SupervisedDescentAAMFitter from .algorithm import ( - PFC, PIC, - SFC, SIC, - AFC, AIC, - MAFC, MAIC, - WFC, WIC, - PSD, PAJ) - + ProjectOutForwardCompositional, ProjectOutInverseCompositional, + SimultaneousForwardCompositional, SimultaneousInverseCompositional, + AlternatingForwardCompositional, AlternatingInverseCompositional, + ModifiedAlternatingForwardCompositional, + ModifiedAlternatingInverseCompositional, + WibergForwardCompositional, WibergInverseCompositional, + SumOfSquaresSupervisedNewtonDescent, + SumOfSquaresSupervisedGaussNewtonDescent, + ProjectOutSupervisedNewtonDescent, + ProjectOutSupervisedGaussNewtonDescent, + AppearanceWeightsSupervisedNewtonDescent, + AppearanceWeightsSupervisedDescent) diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py index 3416b8c..40cf345 100644 --- a/menpofit/aam/algorithm/__init__.py +++ b/menpofit/aam/algorithm/__init__.py @@ -1,7 +1,14 @@ from .lk import ( - PFC, PIC, - SFC, SIC, - AFC, AIC, - MAFC, MAIC, - WFC, WIC) -from .cr import PSD, PAJ + ProjectOutForwardCompositional, ProjectOutInverseCompositional, + SimultaneousForwardCompositional, SimultaneousInverseCompositional, + AlternatingForwardCompositional, AlternatingInverseCompositional, + ModifiedAlternatingForwardCompositional, + ModifiedAlternatingInverseCompositional, + WibergForwardCompositional, WibergInverseCompositional) +from .sd import ( + SumOfSquaresSupervisedNewtonDescent, + SumOfSquaresSupervisedGaussNewtonDescent, + ProjectOutSupervisedNewtonDescent, + ProjectOutSupervisedGaussNewtonDescent, + AppearanceWeightsSupervisedNewtonDescent, + AppearanceWeightsSupervisedDescent) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 1241d88..5b3d3fc 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -1,15 +1,15 @@ from __future__ import division -import abc import numpy as np from menpo.image import Image from menpo.feature import gradient as fast_gradient, no_op from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: needs to use interfaces in menpofit.algorithm.py -# TODO: implement more clever sampling? -class LKAAMInterface(object): - +# TODO: implement more clever sampling for the standard interface? +# TODO document me! +class LucasKanadeStandardInterface(object): + r""" + """ def __init__(self, aam_algorithm, sampling=None): self.algorithm = aam_algorithm @@ -130,8 +130,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class LinearLKAAMInterface(LKAAMInterface): - +# TODO document me! +class LucasKanaddLinearInterface(LucasKanadeStandardInterface): + r""" + """ @property def shape_model(self): return self.transform.model @@ -143,8 +145,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class PartsLKAAMInterface(LKAAMInterface): - +# TODO document me! +class LucasKanadePartsInterface(LucasKanadeStandardInterface): + r""" + """ def __init__(self, aam_algorithm, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): self.algorithm = aam_algorithm @@ -200,10 +204,10 @@ def steepest_descent_images(self, nabla, dw_dp): return sdi.reshape((-1, sdi.shape[-1])) -# TODO: handle costs for all LKAAMAlgorithms # TODO document me! -class LKAAMAlgorithm(object): - +class LucasKanade(object): + r""" + """ def __init__(self, aam_interface, appearance_model, transform, eps=10**-5, **kwargs): # set common state for all AAM algorithms @@ -214,9 +218,9 @@ def __init__(self, aam_interface, appearance_model, transform, # set interface self.interface = aam_interface(self, **kwargs) # perform pre-computations - self.precompute() + self._precompute() - def precompute(self, **kwargs): + def _precompute(self): # grab number of shape and appearance parameters self.n = self.transform.n_parameters self.m = self.appearance_model.n_active_components @@ -245,13 +249,10 @@ def precompute(self, **kwargs): S = self.appearance_model.eigenvalues self.s2_inv_S = s2 / S - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None, - map_inference=False): - pass - -class ProjectOut(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class ProjectOut(LucasKanade): r""" Abstract Interface for Project-out AAM algorithms """ @@ -280,11 +281,11 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self.e_m = i_m - self.a_bar_m # solve for increments on the shape parameters - self.dp = self.solve(map_inference) + self.dp = self._solve(map_inference) # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -297,20 +298,14 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, gt_shape=gt_shape) - @abc.abstractmethod - def solve(self, map_inference): - pass - @abc.abstractmethod - def update_warp(self): - pass - - -class PFC(ProjectOut): +# TODO: handle costs! +# TODO: Document me! +class ProjectOutForwardCompositional(ProjectOut): r""" Project-out Forward Compositional (PFC) Gauss-Newton algorithm """ - def solve(self, map_inference): + def _solve(self, map_inference): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # compute masked forward Jacobian @@ -327,13 +322,15 @@ def solve(self, map_inference): else: return self.interface.solve_shape_ml(JQJ_m, QJ_m, self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class PIC(ProjectOut): +# TODO: handle costs! +# TODO: Document me! +class ProjectOutInverseCompositional(ProjectOut): r""" Project-out Inverse Compositional (PIC) Gauss-Newton algorithm """ @@ -351,7 +348,7 @@ def precompute(self): # compute masked Jacobian pseudo-inverse self.pinv_QJ_m = np.linalg.solve(self.JQJ_m, self.QJ_m.T) - def solve(self, map_inference): + def _solve(self, map_inference): # solve for increments on the shape parameters if map_inference: return self.interface.solve_shape_map( @@ -360,13 +357,15 @@ def solve(self, map_inference): else: return -self.pinv_QJ_m.dot(self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) -class Simultaneous(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class Simultaneous(LucasKanade): r""" Abstract Interface for Simultaneous AAM algorithms """ @@ -400,7 +399,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # solve for increments on the appearance and shape parameters # simultaneously - dc, self.dp = self.solve(map_inference) + dc, self.dp = self._solve(map_inference) # update appearance parameters self.c += dc @@ -410,7 +409,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -423,11 +422,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - @abc.abstractmethod - def compute_jacobian(self): - pass - - def solve(self, map_inference): + def _solve(self, map_inference): # compute masked Jacobian J_m = self.compute_jacobian() # assemble masked simultaneous Jacobian @@ -443,48 +438,50 @@ def solve(self, map_inference): else: return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) - @abc.abstractmethod - def update_warp(self): - pass - -class SFC(Simultaneous): +# TODO: handle costs! +# TODO: Document me! +class SimultaneousForwardCompositional(Simultaneous): r""" Simultaneous Forward Compositional (SFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class SIC(Simultaneous): +# TODO: handle costs! +# TODO: Document me! +class SimultaneousInverseCompositional(Simultaneous): r""" Simultaneous Inverse Compositional (SIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) -class Alternating(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class Alternating(LucasKanade): r""" Abstract Interface for Alternating AAM algorithms """ - def precompute(self, **kwargs): + def _precompute(self, **kwargs): # call super method super(Alternating, self).precompute() # compute MAP appearance Hessian @@ -529,7 +526,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, dc = self.pinv_A_m.dot(e_m + Jdp) # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # compute masked Hessian H_m = J_m.T.dot(J_m) # solve for increments on the shape parameters @@ -549,7 +546,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -562,47 +559,45 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) - @abc.abstractmethod - def compute_jacobian(self): - pass - - @abc.abstractmethod - def update_warp(self): - pass - -class AFC(Alternating): +# TODO: handle costs! +# TODO: Document me! +class AlternatingForwardCompositional(Alternating): r""" Alternating Forward Compositional (AFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class AIC(Alternating): +# TODO: handle costs! +# TODO: Document me! +class AlternatingInverseCompositional(Alternating): r""" Alternating Inverse Compositional (AIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) +# TODO: handle costs! +# TODO: Document me! class ModifiedAlternating(Alternating): r""" Abstract Interface for Modified Alternating AAM algorithms @@ -635,7 +630,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, e_m = i_m - a_m # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # compute masked Hessian H_m = J_m.T.dot(J_m) # solve for increments on the shape parameters @@ -647,7 +642,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -661,23 +656,27 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) -class MAFC(ModifiedAlternating): +# TODO: handle costs! +# TODO: Document me! +class ModifiedAlternatingForwardCompositional(ModifiedAlternating): r""" Modified Alternating Forward Compositional (MAFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class MAIC(ModifiedAlternating): +# TODO: handle costs! +# TODO: Document me! +class ModifiedAlternatingInverseCompositional(ModifiedAlternating): r""" Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm """ @@ -693,7 +692,9 @@ def update_warp(self): self.transform.as_vector() - self.dp) -class Wiberg(LKAAMAlgorithm): +# TODO: handle costs! +# TODO: Document me! +class Wiberg(LucasKanade): r""" Abstract Interface for Wiberg AAM algorithms """ @@ -735,7 +736,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, e_m = i_m - self.a_bar_m # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # project out appearance models QJ_m = self.project_out(J_m) # compute masked Hessian @@ -750,7 +751,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -764,33 +765,37 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) -class WFC(Wiberg): +# TODO: handle costs! +# TODO: Document me! +class WibergForwardCompositional(Wiberg): r""" Wiberg Forward Compositional (WFC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # return forward Jacobian return self.interface.steepest_descent_images(nabla_i, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class WIC(Wiberg): +# TODO: handle costs! +# TODO: Document me! +class WibergInverseCompositional(Wiberg): r""" Wiberg Inverse Compositional (WIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) diff --git a/menpofit/aam/algorithm/cr.py b/menpofit/aam/algorithm/sd.py similarity index 71% rename from menpofit/aam/algorithm/cr.py rename to menpofit/aam/algorithm/sd.py index 26f78a7..a6a75b2 100644 --- a/menpofit/aam/algorithm/cr.py +++ b/menpofit/aam/algorithm/sd.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.image import Image from menpo.feature import no_op @@ -7,9 +6,11 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: implement more clever sampling? -class CRAAMInterface(object): - +# TODO: implement more clever sampling for the standard interface? +# TODO document me! +class SupervisedDescentStandardInterface(object): + r""" + """ def __init__(self, cr_aam_algorithm, sampling=None): self.algorithm = cr_aam_algorithm @@ -60,8 +61,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class CRLinearAAMInterface(CRAAMInterface): - +# TODO document me! +class SupervisedDescentLinearInterface(SupervisedDescentStandardInterface): + r""" + """ @property def shape_model(self): return self.transform.model @@ -73,8 +76,10 @@ def algorithm_result(self, image, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) -class CRPartsAAMInterface(CRAAMInterface): - +# TODO document me! +class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): + r""" + """ def __init__(self, cr_aam_algorithm, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): self.algorithm = cr_aam_algorithm @@ -102,8 +107,9 @@ def warp(self, image): # TODO document me! -class CRAAMAlgorithm(object): - +class SupervisedDescent(object): + r""" + """ def __init__(self, aam_interface, appearance_model, transform, max_iters=3, eps=10**-5, **kwargs): # set common state for all AAM algorithms @@ -111,42 +117,21 @@ def __init__(self, aam_interface, appearance_model, transform, max_iters=3, self.template = appearance_model.mean() self.transform = transform self.max_iters = max_iters + # TODO: Make use of eps in self.train? self.eps = eps # set interface self.interface = aam_interface(self, **kwargs) # perform pre-computations - self.precompute() - - def precompute(self): - # grab number of shape and appearance parameters - self.n = self.transform.n_parameters - self.m = self.appearance_model.n_active_components - - # grab appearance model components - self.A = self.appearance_model.components - # mask them - self.A_m = self.A.T[self.interface.i_mask, :] - # compute their pseudoinverse - self.pinv_A_m = np.linalg.pinv(self.A_m) + self._precompute() + def _precompute(self): # grab appearance model mean - self.a_bar = self.appearance_model.mean() + a_bar = self.appearance_model.mean() # vectorize it and mask it - self.a_bar_m = self.a_bar.as_vector()[self.interface.i_mask] - - # compute shape model prior - s2 = (self.appearance_model.noise_variance() / - self.interface.shape_model.noise_variance()) - L = self.interface.shape_model.eigenvalues - self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) - # compute appearance model prior - S = self.appearance_model.eigenvalues - self.s2_inv_S = s2 / S - - def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): - # check training data - self._check_training_data(images, gt_shapes, current_shapes) + self.a_bar_m = a_bar.as_vector()[self.interface.i_mask] + def train(self, images, gt_shapes, current_shapes, verbose=False, + **kwargs): n_images = len(images) n_samples_image = len(current_shapes[0]) @@ -173,8 +158,10 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): **kwargs) # add regressor to list self.regressors.append(regressor) + # compute regression rmse estimated_delta_params = regressor(features) + # TODO: Should print a more informative error here? rmse = _compute_rmse(delta_params, estimated_delta_params) if verbose: print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(rmse)) @@ -198,15 +185,6 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): final_shapes.append(current_shapes[k:l]) return final_shapes - @staticmethod - def _check_training_data(images, gt_shapes, current_shapes): - if len(images) != len(gt_shapes): - raise ValueError("The number of shapes must be equal to " - "the number of images.") - elif len(images) != len(current_shapes): - raise ValueError("The number of current shapes must be " - "equal or multiple to the number of images.") - def _generate_params(self, gt_shapes, current_shapes): # initialize current and delta parameters arrays n_samples = len(gt_shapes) * len(current_shapes[0]) @@ -252,7 +230,7 @@ def _generate_features(self, images, current_params, verbose=False): # set transform self.transform.from_vector_inplace(current_params[k]) # compute regression features - f = self._compute_features(i) + f = self._compute_train_features(i) # add to features array features[k] = f # increment counter @@ -260,14 +238,6 @@ def _generate_features(self, images, current_params, verbose=False): return features - @abc.abstractmethod - def _compute_features(self, image): - pass - - @abc.abstractmethod - def _perform_regression(self, features, deltas, gamma=None): - pass - def run(self, image, initial_shape, gt_shape=None, **kwargs): # initialize transform self.transform.set_target(initial_shape) @@ -279,7 +249,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # Cascaded Regression loop while k < self.max_iters: # compute regression features - features = self._compute_features2(image) + features = self._compute_test_features(image) # solve for increments on the shape parameters dp = self.regressors[k](features) @@ -297,14 +267,64 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class ProjectOut(CRAAMAlgorithm): +class SumOfSquaresSupervisedDescent(SupervisedDescent): r""" """ + def _compute_train_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + def _compute_test_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # compute masked error + return i_m - self.a_bar_m + + +# TODO: document me! +class SumOfSquaresSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, + dtype=np.float64): + return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) + + +# TODO: document me! +class SumOfSquaresSupervisedGaussNewtonDescent(SumOfSquaresSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + return _supervised_gauss_newton(features, deltas, gamma=gamma, + psi=psi, dtype=dtype) + + +# TODO: document me! +class ProjectOutSupervisedDescent(SupervisedDescent): + r""" + """ + def _precompute(self): + # call super method + super(ProjectOutSupervisedNewtonDescent)._precompute() + # grab appearance model components + A = self.appearance_model.components + # mask them + self.A_m = A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) + def project_out(self, J): # project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) - def _compute_features(self, image): + def _compute_train_features(self, image): # warp image i = self.interface.warp(image) # vectorize it and mask it @@ -313,7 +333,7 @@ def _compute_features(self, image): e_m = i_m - self.a_bar_m return self.project_out(e_m) - def _compute_features2(self, image): + def _compute_test_features(self, image): # warp image i = self.interface.warp(image) # vectorize it and mask it @@ -323,7 +343,7 @@ def _compute_features2(self, image): # TODO: document me! -class PSD(ProjectOut): +class ProjectOutSupervisedNewtonDescent(ProjectOutSupervisedDescent): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -335,7 +355,62 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class PAJ(ProjectOut): +class ProjectOutSupervisedGaussNewtonDescent(ProjectOutSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, psi=None, + dtype=np.float64): + return _supervised_gauss_newton(features, deltas, gamma=gamma, + psi=psi, dtype=dtype) + + +# TODO: document me! +class AppearanceWeightsSupervisedDescent(SupervisedDescent): + r""" + """ + def _precompute(self): + # call super method + super(ProjectOutSupervisedNewtonDescent)._precompute() + # grab appearance model components + A = self.appearance_model.components + # mask them + A_m = A.T[self.interface.i_mask, :] + # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(A_m) + + def project(self, J): + # project a particular vector or matrix onto the appearance bases + return self.pinv_A_m.dot(J - self.a_bar_m) + + def _compute_train_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # project it onto the appearance model + return self.project(i_m) + + def _compute_test_features(self, image): + # warp image + i = self.interface.warp(image) + # vectorize it and mask it + i_m = i.as_vector()[self.interface.i_mask] + # project it onto the appearance model + return self.project(i_m) + + +# TODO: document me! +class AppearanceWeightsSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): + r""" + """ + def _perform_regression(self, features, deltas, gamma=None, + dtype=np.float64): + return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) + + +# TODO: document me! +class AppearanceWeightsSupervisedGaussNewtonDescent( + AppearanceWeightsSupervisedDescent): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, @@ -369,6 +444,7 @@ class _supervised_gauss_newton(object): def __init__(self, features, deltas, gamma=None, psi=None, dtype=np.float64): features = features.astype(dtype) + # ridge regression deltas = deltas.astype(dtype) XX = deltas.T.dot(deltas) XT = deltas.T.dot(features) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a9e0cad..ddd46cc 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -8,9 +8,11 @@ import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM from .algorithm.lk import ( - LKAAMInterface, LinearLKAAMInterface, PartsLKAAMInterface, AIC) -from .algorithm.cr import ( - CRAAMInterface, CRLinearAAMInterface, CRPartsAAMInterface, PAJ) + LucasKanadeStandardInterface, LucasKanaddLinearInterface, + LucasKanadePartsInterface, WibergInverseCompositional) +from .algorithm.sd import ( + SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, + SupervisedDescentPartsInterface, ProjectOutSupervisedNewtonDescent) from .result import AAMFitterResult @@ -45,11 +47,11 @@ def _fitter_result(self, image, algorithm_results, affine_correction, # TODO: document me! -class LKAAMFitter(AAMFitter): +class LucasKanadeAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, n_shape=None, n_appearance=None, - lk_algorithm_cls=AIC, sampling=None, **kwargs): + def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, + n_shape=None, n_appearance=None, sampling=None, **kwargs): self._model = aam self.algorithms = [] self._check_n_shape(n_shape) @@ -68,7 +70,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = lk_algorithm_cls( - LKAAMInterface, am, md_transform, sampling=s, + LucasKanadeStandardInterface, am, md_transform, sampling=s, **kwargs) elif (type(self.aam) is LinearAAM or @@ -78,7 +80,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( - LinearLKAAMInterface, am, md_transform, sampling=s, + LucasKanaddLinearInterface, am, md_transform, sampling=s, **kwargs) elif type(self.aam) is PartsAAM: @@ -86,7 +88,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = lk_algorithm_cls( - PartsLKAAMInterface, am, pdm, sampling=s, + LucasKanadePartsInterface, am, pdm, sampling=s, patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) @@ -101,12 +103,12 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): # TODO: document me! -class CRAAMFitter(AAMFitter): +class SupervisedDescentAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, cr_algorithm_cls=PAJ, n_shape=None, - n_appearance=None, sampling=None, n_perturbations=10, - noise_std=0.05, max_iters=6, **kwargs): + def __init__(self, aam, cr_algorithm_cls=ProjectOutSupervisedNewtonDescent, + n_shape=None,n_appearance=None, sampling=None, + n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam self.algorithms = [] self._check_n_shape(n_shape) @@ -128,8 +130,8 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface algorithm = cr_algorithm_cls( - CRAAMInterface, am, md_transform, sampling=s, - max_iters=self.max_iters[j], **kwargs) + SupervisedDescentStandardInterface, am, md_transform, + sampling=s, max_iters=self.max_iters[j], **kwargs) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -138,15 +140,15 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = cr_algorithm_cls( - CRLinearAAMInterface, am, md_transform, sampling=s, - max_iters=self.max_iters[j], **kwargs) + SupervisedDescentLinearInterface, am, md_transform, + sampling=s, max_iters=self.max_iters[j], **kwargs) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = cr_algorithm_cls( - CRPartsAAMInterface, am, pdm, + SupervisedDescentPartsInterface, am, pdm, sampling=s, max_iters=self.max_iters[j], patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts, **kwargs) @@ -160,6 +162,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): # append algorithms to list self.algorithms.append(algorithm) + # TODO: Allow training from bounding boxes def train(self, images, group=None, label=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index afd516c..7c57ce0 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -1,10 +1,9 @@ from __future__ import division -from menpofit.result import ( - ParametricAlgorithmResult, MultiFitterResult, SerializableIterativeResult) +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class AAMAlgorithmResult(ParametricAlgorithmResult): r""" """ @@ -15,6 +14,7 @@ def __init__(self, image, fitter, shape_parameters, self.appearance_parameters = appearance_parameters +# TODO: handle costs! # TODO: document me! class LinearAAMAlgorithmResult(AAMAlgorithmResult): r""" @@ -33,8 +33,8 @@ def initial_shape(self): return self.initial_transform.sparse_target +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class AAMFitterResult(MultiFitterResult): r""" """ diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index 12bd84b..cea05ea 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,5 +1,5 @@ from .builder import ( ATMBuilder, PatchATMBuilder, LinearATMBuilder, LinearPatchATMBuilder, PartsATMBuilder) -from .fitter import LKATMFitter -from .algorithm import FC, IC \ No newline at end of file +from .fitter import LucasKanadeATMFitter +from .algorithm import ForwardCompositional, InverseCompositional diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 2193d70..a171c50 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,5 +1,4 @@ from __future__ import division -import abc import numpy as np from menpo.image import Image from menpo.feature import no_op @@ -7,9 +6,10 @@ from .result import ATMAlgorithmResult, LinearATMAlgorithmResult -# TODO: implement more clever sampling? -class LKATMInterface(object): - +# TODO: implement more clever sampling for the standard interface? +class LucasKanadeStandardInterface(object): + r""" + """ def __init__(self, lk_algorithm, sampling=None): self.algorithm = lk_algorithm @@ -103,8 +103,10 @@ def algorithm_result(self, image, shape_parameters, gt_shape=None): image, self.algorithm, shape_parameters, gt_shape=gt_shape) -class LKLinearATMInterface(LKATMInterface): - +# TODO document me! +class LucasKanadeLinearInterface(LucasKanadeStandardInterface): + r""" + """ @property def shape_model(self): return self.transform.model @@ -114,8 +116,10 @@ def algorithm_result(self, image, shape_parameters, gt_shape=None): image, self.algorithm, shape_parameters, gt_shape=gt_shape) -class LKPartsATMInterface(LKATMInterface): - +# TODO document me! +class LucasKanadePartsInterface(LucasKanadeStandardInterface): + r""" + """ def __init__(self, lk_algorithm, patch_shape=(17, 17), normalize_parts=no_op, sampling=None): self.algorithm = lk_algorithm @@ -174,9 +178,8 @@ def steepest_descent_images(self, nabla, dw_dp): return sdi.reshape((-1, sdi.shape[-1])) -# TODO: handle costs for all LKAAMAlgorithms # TODO document me! -class LKATMAlgorithm(object): +class LucasKanade(object): def __init__(self, lk_atm_interface_cls, template, transform, eps=10**-5, **kwargs): @@ -187,9 +190,9 @@ def __init__(self, lk_atm_interface_cls, template, transform, # set interface self.interface = lk_atm_interface_cls(self, **kwargs) # perform pre-computations - self.precompute() + self._precompute() - def precompute(self, **kwargs): + def _precompute(self, **kwargs): # grab number of shape and appearance parameters self.n = self.transform.n_parameters @@ -204,13 +207,10 @@ def precompute(self, **kwargs): L = self.interface.shape_model.eigenvalues self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None, - map_inference=False): - pass - -class Compositional(LKATMAlgorithm): +# TODO: handle costs! +# TODO document me! +class Compositional(LucasKanade): r""" Abstract Interface for Compositional ATM algorithms """ @@ -235,11 +235,11 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self.e_m = i_m - self.t_m # solve for increments on the shape parameters - self.dp = self.solve(map_inference) + self.dp = self._solve(map_inference) # update warp s_k = self.transform.target.points - self.update_warp() + self._update_warp() p_list.append(self.transform.as_vector()) # test convergence @@ -252,20 +252,14 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, return self.interface.algorithm_result( image, p_list, gt_shape=gt_shape) - @abc.abstractmethod - def solve(self, map_inference): - pass - @abc.abstractmethod - def update_warp(self): - pass - - -class FC(Compositional): +# TODO: handle costs! +# TODO document me! +class ForwardCompositional(Compositional): r""" Forward Compositional (FC) Gauss-Newton algorithm """ - def solve(self, map_inference): + def _solve(self, map_inference): # compute warped image gradient nabla_i = self.interface.gradient(self.i) # compute masked forward Jacobian @@ -280,19 +274,21 @@ def solve(self, map_inference): else: return self.interface.solve_shape_ml(JJ_m, J_m, self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on forward composition self.transform.from_vector_inplace( self.transform.as_vector() + self.dp) -class IC(Compositional): +# TODO: handle costs! +# TODO document me! +class InverseCompositional(Compositional): r""" Inverse Compositional (IC) Gauss-Newton algorithm """ - def precompute(self): + def _precompute(self): # call super method - super(IC, self).precompute() + super(InverseCompositional, self).precompute() # compute appearance model mean gradient nabla_t = self.interface.gradient(self.template) # compute masked inverse Jacobian @@ -302,7 +298,7 @@ def precompute(self): # compute masked Jacobian pseudo-inverse self.pinv_J_m = np.linalg.solve(self.JJ_m, self.J_m.T) - def solve(self, map_inference): + def _solve(self, map_inference): # solve for increments on the shape parameters if map_inference: return self.interface.solve_shape_map( @@ -311,7 +307,7 @@ def solve(self, map_inference): else: return -self.pinv_J_m.dot(self.e_m) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 3cd7d97..8dbb453 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -4,16 +4,17 @@ from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM from .algorithm import ( - LKATMInterface, LKLinearATMInterface, LKPartsATMInterface, IC) + LucasKanadeStandardInterface, LucasKanadeLinearInterface, + LucasKanadePartsInterface, InverseCompositional) from .result import ATMFitterResult # TODO: document me! -class LKATMFitter(ModelFitter): +class LucasKanadeATMFitter(ModelFitter): r""" """ - def __init__(self, atm, algorithm_cls=IC, n_shape=None, sampling=None, - **kwargs): + def __init__(self, atm, algorithm_cls=InverseCompositional, + n_shape=None, sampling=None, **kwargs): self._model = atm self.algorithms = [] self._check_n_shape(n_shape) @@ -33,8 +34,9 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): sm, self.atm.transform, source=wt.landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = algorithm_cls(LKATMInterface, wt, md_transform, - sampling=sampling, **kwargs) + algorithm = algorithm_cls(LucasKanadeStandardInterface, wt, + md_transform, sampling=sampling, + **kwargs) elif (type(self.atm) is LinearATM or type(self.atm) is LinearPatchATM): @@ -42,7 +44,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.atm.reference_shape) # set up algorithm using linear aam interface - algorithm = algorithm_cls(LKLinearATMInterface, wt, + algorithm = algorithm_cls(LucasKanadeLinearInterface, wt, md_transform, sampling=sampling, **kwargs) @@ -51,7 +53,7 @@ def _set_up(self, algorithm_cls, sampling, **kwargs): pdm = OrthoPDM(sm) # set up algorithm using parts aam interface algorithm = algorithm_cls( - LKPartsATMInterface, wt, pdm, sampling=sampling, + LucasKanadePartsInterface, wt, pdm, sampling=sampling, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 4ee4cab..f285f76 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -2,12 +2,14 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class ATMAlgorithmResult(ParametricAlgorithmResult): r""" """ + +# TODO: handle costs! # TODO: document me! class LinearATMAlgorithmResult(ATMAlgorithmResult): r""" @@ -26,8 +28,8 @@ def initial_shape(self): return self.initial_transform.sparse_target +# TODO: handle costs! # TODO: document me! -# TODO: handle costs class ATMFitterResult(MultiFitterResult): r""" """ diff --git a/menpofit/lk/__init__.py b/menpofit/lk/__init__.py index b01bf94..7a1abbc 100644 --- a/menpofit/lk/__init__.py +++ b/menpofit/lk/__init__.py @@ -1,3 +1,5 @@ -from .fitter import LKFitter -from .algorithm import FA, FC, IC -from .residual import SSD, FourierSSD, ECC, GradientImages, GradientCorrelation +from .fitter import LucasKanadeFitter +from .algorithm import ( + ForwardAdditive, ForwardCompositional, InverseCompositional) +from .residual import ( + SSD, FourierSSD, ECC, GradientImages, GradientCorrelation) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 3034c08..9671e94 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -1,15 +1,12 @@ from scipy.linalg import norm -import abc import numpy as np -from .result import LKAlgorithmResult +from .result import LucasKanadeAlgorithmResult # TODO: implement Inverse Additive Algorithm? -# TODO: implement Linear, Parts interfaces? Will they play nice with residuals? # TODO: implement sampling? -# TODO: handle costs for all LKAlgorithms # TODO: document me! -class LKAlgorithm(object): +class LucasKanade(object): r""" """ def __init__(self, template, transform, residual, eps=10**-10): @@ -18,12 +15,10 @@ def __init__(self, template, transform, residual, eps=10**-10): self.residual = residual self.eps = eps - @abc.abstractmethod - def run(self, image, initial_shape, max_iters=20, gt_shape=None): - pass - -class FA(LKAlgorithm): +# TODO: handle costs! +# TODO: document me! +class ForwardAdditive(LucasKanade): r""" Forward Additive Lucas-Kanade algorithm """ @@ -69,18 +64,21 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LKAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, gt_shape=None) -class FC(LKAlgorithm): +# TODO: handle costs! +# TODO: document me! +class ForwardCompositional(LucasKanade): r""" Forward Compositional Lucas-Kanade algorithm """ def __init__(self, template, transform, residual, eps=10**-10): - super(FC, self).__init__(template, transform, residual, eps=eps) - self.precompute() + super(ForwardCompositional, self).__init__( + template, transform, residual, eps=eps) + self._precompute() - def precompute(self): + def _precompute(self): # compute warp jacobian self.dW_dp = np.rollaxis( self.transform.d_dp(self.template.indices()), -1) @@ -126,15 +124,18 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): return LKAlgorithmResult(image, self, p_list, gt_shape=None) -class IC(LKAlgorithm): +# TODO: handle costs! +# TODO: document me! +class InverseCompositional(LucasKanade): r""" Inverse Compositional Lucas-Kanade algorithm """ def __init__(self, template, transform, residual, eps=10**-10): - super(IC, self).__init__(template, transform, residual, eps=eps) - self.precompute() + super(InverseCompositional, self).__init__( + template, transform, residual, eps=eps) + self._precompute() - def precompute(self): + def _precompute(self): # compute warp jacobian dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index e2a91e0..2983299 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -3,19 +3,20 @@ from menpofit.transform import DifferentiableAlignmentAffine from menpofit.fitter import MultiFitter, noisy_target_alignment_transform from menpofit import checks -from .algorithm import IC +from .algorithm import InverseCompositional from .residual import SSD, FourierSSD -from .result import LKFitterResult +from .result import LucasKanadeFitterResult # TODO: document me! -class LKFitter(MultiFitter): +class LucasKanadeFitter(MultiFitter): r""" """ def __init__(self, template, group=None, label=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, - scales=(1, .5), scale_features=True, algorithm_cls=IC, - residual_cls=SSD, **kwargs): + scales=(1, .5), scale_features=True, + algorithm_cls=InverseCompositional, residual_cls=SSD, + **kwargs): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -89,5 +90,5 @@ def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): - return LKFitterResult(image, self, algorithm_results, - affine_correction, gt_shape=gt_shape) \ No newline at end of file + return LucasKanadeFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index 401b1e5..5f808d3 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -4,7 +4,7 @@ import scipy.linalg from menpo.feature import gradient - +# TODO: Do we want residuals to support masked templates? class Residual(object): """ An abstract base class for calculating the residual between two images @@ -132,7 +132,8 @@ def steepest_descent_update(self, sdi, image, template): class SSD(Residual): - + r""" + """ def __init__(self, kernel=None): self.kernel = kernel @@ -188,8 +189,10 @@ def steepest_descent_update(self, sdi, image, template): return sdi.T.dot(error_img) +# TODO: Does not support masked templates at the moment class FourierSSD(Residual): - + r""" + """ def __init__(self, kernel=None): self.kernel = kernel @@ -254,7 +257,8 @@ def steepest_descent_update(self, sdi, image, template): class ECC(Residual): - + r""" + """ def _normalise_images(self, image): # TODO: do we need to copy the image? # TODO: is this supposed to be per channel normalization? @@ -327,7 +331,8 @@ def steepest_descent_update(self, sdi, image, template): class GradientImages(Residual): - + r""" + """ def _regularise_gradients(self, grad): pixels = grad.pixels ab = np.sqrt(np.sum(pixels**2, axis=0)) @@ -391,7 +396,8 @@ def steepest_descent_update(self, sdi, image, template): class GradientCorrelation(Residual): - + r""" + """ def steepest_descent_images(self, image, dW_dp, forward=None): n_dims = image.n_dims n_channels = image.n_channels diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py index 1eddf5d..6674a63 100644 --- a/menpofit/lk/result.py +++ b/menpofit/lk/result.py @@ -2,17 +2,17 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: document me! # TODO: handle costs! -class LKAlgorithmResult(ParametricAlgorithmResult): +# TODO: document me! +class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): r""" """ pass +# TODO: handle costs! # TODO: document me! -# TODO: handle costs -class LKFitterResult(MultiFitterResult): +class LucasKanadeFitterResult(MultiFitterResult): r""" """ pass From 4e2ad7c19909f9311ffc6db627e1f3a4ea007b6e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 04:27:02 +0100 Subject: [PATCH 281/423] Small fixes related to the previous commit --- menpofit/aam/__init__.py | 9 +++------ menpofit/aam/algorithm/__init__.py | 12 ++++++------ menpofit/aam/algorithm/lk.py | 10 +++++----- menpofit/aam/algorithm/sd.py | 23 +++++++++++------------ menpofit/aam/fitter.py | 4 ++-- menpofit/atm/algorithm.py | 2 +- menpofit/lk/algorithm.py | 9 ++++++--- menpofit/lk/fitter.py | 4 ++-- 8 files changed, 36 insertions(+), 37 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 65d6960..76765db 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -9,9 +9,6 @@ ModifiedAlternatingForwardCompositional, ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional, - SumOfSquaresSupervisedNewtonDescent, - SumOfSquaresSupervisedGaussNewtonDescent, - ProjectOutSupervisedNewtonDescent, - ProjectOutSupervisedGaussNewtonDescent, - AppearanceWeightsSupervisedNewtonDescent, - AppearanceWeightsSupervisedDescent) + SumOfSquaresNewton, SumOfSquaresGaussNewton, + ProjectOutNewton, ProjectOutGaussNewton, + AppearanceWeightsNewton, AppearanceWeightsGaussNewton) diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py index 40cf345..4c054af 100644 --- a/menpofit/aam/algorithm/__init__.py +++ b/menpofit/aam/algorithm/__init__.py @@ -6,9 +6,9 @@ ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional) from .sd import ( - SumOfSquaresSupervisedNewtonDescent, - SumOfSquaresSupervisedGaussNewtonDescent, - ProjectOutSupervisedNewtonDescent, - ProjectOutSupervisedGaussNewtonDescent, - AppearanceWeightsSupervisedNewtonDescent, - AppearanceWeightsSupervisedDescent) + SumOfSquaresNewton, + SumOfSquaresGaussNewton, + ProjectOutNewton, + ProjectOutGaussNewton, + AppearanceWeightsNewton, + AppearanceWeightsGaussNewton) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 5b3d3fc..33cd8c9 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -334,9 +334,9 @@ class ProjectOutInverseCompositional(ProjectOut): r""" Project-out Inverse Compositional (PIC) Gauss-Newton algorithm """ - def precompute(self): + def _precompute(self): # call super method - super(PIC, self).precompute() + super(ProjectOutInverseCompositional, self)._precompute() # compute appearance model mean gradient nabla_a = self.interface.gradient(self.a_bar) # compute masked inverse Jacobian @@ -483,7 +483,7 @@ class Alternating(LucasKanade): """ def _precompute(self, **kwargs): # call super method - super(Alternating, self).precompute() + super(Alternating, self)._precompute() # compute MAP appearance Hessian self.AA_m_map = self.A_m.T.dot(self.A_m) + np.diag(self.s2_inv_S) @@ -680,13 +680,13 @@ class ModifiedAlternatingInverseCompositional(ModifiedAlternating): r""" Modified Alternating Inverse Compositional (MAIC) Gauss-Newton algorithm """ - def compute_jacobian(self): + def _compute_jacobian(self): # compute warped appearance model gradient nabla_a = self.interface.gradient(self.a) # return inverse Jacobian return self.interface.steepest_descent_images(-nabla_a, self.dW_dp) - def update_warp(self): + def _update_warp(self): # update warp based on inverse composition self.transform.from_vector_inplace( self.transform.as_vector() - self.dp) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index a6a75b2..e8934f7 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -267,7 +267,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class SumOfSquaresSupervisedDescent(SupervisedDescent): +class SumOfSquares(SupervisedDescent): r""" """ def _compute_train_features(self, image): @@ -288,7 +288,7 @@ def _compute_test_features(self, image): # TODO: document me! -class SumOfSquaresSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): +class SumOfSquaresNewton(SumOfSquares): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -297,7 +297,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class SumOfSquaresSupervisedGaussNewtonDescent(SumOfSquaresSupervisedDescent): +class SumOfSquaresGaussNewton(SumOfSquares): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, @@ -307,12 +307,12 @@ def _perform_regression(self, features, deltas, gamma=None, psi=None, # TODO: document me! -class ProjectOutSupervisedDescent(SupervisedDescent): +class ProjectOut(SupervisedDescent): r""" """ def _precompute(self): # call super method - super(ProjectOutSupervisedNewtonDescent)._precompute() + super(ProjectOut, self)._precompute() # grab appearance model components A = self.appearance_model.components # mask them @@ -343,7 +343,7 @@ def _compute_test_features(self, image): # TODO: document me! -class ProjectOutSupervisedNewtonDescent(ProjectOutSupervisedDescent): +class ProjectOutNewton(ProjectOut): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -355,7 +355,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class ProjectOutSupervisedGaussNewtonDescent(ProjectOutSupervisedDescent): +class ProjectOutGaussNewton(ProjectOut): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, @@ -365,12 +365,12 @@ def _perform_regression(self, features, deltas, gamma=None, psi=None, # TODO: document me! -class AppearanceWeightsSupervisedDescent(SupervisedDescent): +class AppearanceWeights(SupervisedDescent): r""" """ def _precompute(self): # call super method - super(ProjectOutSupervisedNewtonDescent)._precompute() + super(AppearanceWeights, self)._precompute() # grab appearance model components A = self.appearance_model.components # mask them @@ -400,7 +400,7 @@ def _compute_test_features(self, image): # TODO: document me! -class AppearanceWeightsSupervisedNewtonDescent(SumOfSquaresSupervisedDescent): +class AppearanceWeightsNewton(AppearanceWeights): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -409,8 +409,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class AppearanceWeightsSupervisedGaussNewtonDescent( - AppearanceWeightsSupervisedDescent): +class AppearanceWeightsGaussNewton(AppearanceWeights): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index ddd46cc..232f6d4 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -12,7 +12,7 @@ LucasKanadePartsInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, - SupervisedDescentPartsInterface, ProjectOutSupervisedNewtonDescent) + SupervisedDescentPartsInterface, ProjectOutNewton) from .result import AAMFitterResult @@ -106,7 +106,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): class SupervisedDescentAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, cr_algorithm_cls=ProjectOutSupervisedNewtonDescent, + def __init__(self, aam, cr_algorithm_cls=ProjectOutNewton, n_shape=None,n_appearance=None, sampling=None, n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index a171c50..5deda18 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -288,7 +288,7 @@ class InverseCompositional(Compositional): """ def _precompute(self): # call super method - super(InverseCompositional, self).precompute() + super(InverseCompositional, self)._precompute() # compute appearance model mean gradient nabla_t = self.interface.gradient(self.template) # compute masked inverse Jacobian diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 9671e94..37325f6 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -64,7 +64,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LucasKanadeAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, + gt_shape=gt_shape) # TODO: handle costs! @@ -121,7 +122,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LKAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, + gt_shape=gt_shape) # TODO: handle costs! @@ -178,4 +180,5 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # increase iteration counter k += 1 - return LKAlgorithmResult(image, self, p_list, gt_shape=None) + return LucasKanadeAlgorithmResult(image, self, p_list, + gt_shape=gt_shape) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 2983299..593af6c 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -84,8 +84,8 @@ def _prepare_template(self, template, group=None, label=None): def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): transform = noisy_target_alignment_transform( - self.transform_cls, self.reference_shape, gt_shape, - noise_std=noise_std) + self.reference_shape, gt_shape, + alignment_transform_cls=self.transform_cls, noise_std=noise_std) return transform.apply(self.reference_shape) def _fitter_result(self, image, algorithm_results, affine_correction, From 52ba60e02d1e80eb5b4c7e5a746d9725126e238d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 14:05:29 +0100 Subject: [PATCH 282/423] Tidy up scales in prepare_image and prepare_template --- menpofit/fitter.py | 5 +---- menpofit/lk/fitter.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 06209da..c8fb9c8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -146,11 +146,8 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, group='initial_shape') # obtain image representation - from copy import deepcopy - scales = deepcopy(self.scales) - scales.reverse() images = [] - for j, s in enumerate(scales): + for j, s in enumerate(self.scales[::-1]): if j == 0: # compute features at highest level feature_image = self.features[j](image) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 593af6c..5e1cc72 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -59,11 +59,8 @@ def _prepare_template(self, template, group=None, label=None): self.diagonal, group=group, label=label) # obtain image representation - from copy import deepcopy - scales = deepcopy(self.scales) - scales.reverse() templates = [] - for j, s in enumerate(scales): + for j, s in enumerate(self.scales[::-1]): if j == 0: # compute features at highest level feature_template = self.features[j](template) From 52ae7c88c81dd4e5d3fe4288e5edc72457e85970 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 18:37:56 +0100 Subject: [PATCH 283/423] Add new holistic sampling methods --- menpofit/aam/__init__.py | 4 +++- menpofit/aam/algorithm/lk.py | 8 +++++--- menpofit/aam/fitter.py | 39 +++++++++++++++++++++++++++++++++++- menpofit/checks.py | 19 +++++++++++------- 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index 76765db..c06a2d2 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,7 +1,9 @@ from .builder import ( AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, LinearPatchAAMBuilder, PartsAAMBuilder) -from .fitter import LucasKanadeAAMFitter, SupervisedDescentAAMFitter +from .fitter import ( + LucasKanadeAAMFitter, SupervisedDescentAAMFitter, + holistic_sampling_from_scale, holistic_sampling_from_step) from .algorithm import ( ProjectOutForwardCompositional, ProjectOutInverseCompositional, SimultaneousForwardCompositional, SimultaneousInverseCompositional, diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 33cd8c9..73fe696 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -19,9 +19,11 @@ def __init__(self, aam_algorithm, sampling=None): sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 + sampling = xrange(0, n_true_pixels, 1) + elif isinstance(sampling, np.int): + sampling = xrange(0, n_true_pixels, sampling) + + sampling_mask[sampling] = 1 self.i_mask = np.nonzero(np.tile( sampling_mask[None, ...], (n_channels, 1)).flatten())[0] diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 232f6d4..e95b967 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,5 +1,8 @@ from __future__ import division -from menpo.transform import Scale +import numpy as np +from copy import deepcopy +from menpo.transform import Scale, AlignmentUniformScale +from menpo.image import BooleanImage from menpofit.builder import ( rescale_images_to_reference_shape, compute_features, scale_images) from menpofit.fitter import ModelFitter @@ -224,3 +227,37 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): transform.apply_inplace(shape) +# TODO: Document me! +def holistic_sampling_from_scale(aam, scale=0.35): + reference = aam.appearance_models[0].mean() + scaled_reference = reference.rescale(scale) + + t = AlignmentUniformScale(scaled_reference.landmarks['source'].lms, + reference.landmarks['source'].lms) + new_indices = np.require(np.round(t.apply( + scaled_reference.mask.true_indices())), dtype=np.int) + + modified_mask = deepcopy(reference.mask.pixels) + modified_mask[:] = False + modified_mask[:, new_indices[:, 0], new_indices[:, 1]] = True + + true_positions = np.nonzero( + modified_mask[:, reference.mask.mask].ravel())[0] + + return true_positions, BooleanImage(modified_mask[0]) + + +def holistic_sampling_from_step(aam, step=8): + reference = aam.appearance_models[0].mean() + + n_true_pixels = reference.n_true_pixels() + true_positions = np.zeros(n_true_pixels, dtype=np.bool) + sampling = xrange(0, n_true_pixels, step) + true_positions[sampling] = True + + modified_mask = reference.mask.copy() + new_indices = modified_mask.true_indices()[sampling, :] + modified_mask.mask[:] = False + modified_mask.mask[new_indices[:, 0], new_indices[:, 1]] = True + + return true_positions, modified_mask diff --git a/menpofit/checks.py b/menpofit/checks.py index 0c5dc0c..4d06498 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -125,18 +125,23 @@ def check_max_iters(max_iters, n_levels): # TODO: document me! def check_sampling(sampling, n_levels): - if isinstance(sampling, (list, tuple)): + if (isinstance(sampling, (list, tuple)) and + np.alltrue([isinstance(s, (np.ndarray, np.int, None)) + for s in sampling])): if len(sampling) == 1: - sampling = sampling * n_levels - elif len(sampling) != n_levels: + return sampling * n_levels + elif len(sampling) == n_levels: + return sampling + else: raise ValueError('A sampling list can only ' 'contain 1 element or {} ' 'elements'.format(n_levels)) - elif isinstance(sampling, np.ndarray): - sampling = [sampling] * n_levels + elif isinstance(sampling, (np.ndarray, np.int, None)): + return [sampling] * n_levels else: - raise ValueError('sampling can be a ndarray, a ndarray list ' + raise ValueError('sampling can be an integer or ndarray, ' + 'a integer or ndarray list ' 'containing 1 or {} elements or ' 'None'.format(n_levels)) - return sampling + From aeaf2929c24fb0ae3bbc4c99c9de34f62120e47a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 19:09:51 +0100 Subject: [PATCH 284/423] Delete TODOs regarding sampling --- menpofit/aam/algorithm/lk.py | 1 - menpofit/aam/algorithm/sd.py | 1 - menpofit/atm/algorithm.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 73fe696..2144e14 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -5,7 +5,6 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: implement more clever sampling for the standard interface? # TODO document me! class LucasKanadeStandardInterface(object): r""" diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index e8934f7..e1f8e09 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -6,7 +6,6 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult -# TODO: implement more clever sampling for the standard interface? # TODO document me! class SupervisedDescentStandardInterface(object): r""" diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 5deda18..3fb1098 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -6,7 +6,7 @@ from .result import ATMAlgorithmResult, LinearATMAlgorithmResult -# TODO: implement more clever sampling for the standard interface? +# TODO document me! class LucasKanadeStandardInterface(object): r""" """ From 15c2bcefe65f22d71de2400d0e1cb90ab1f30fc1 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 30 Jun 2015 19:13:09 +0100 Subject: [PATCH 285/423] Update checks --- menpofit/checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/checks.py b/menpofit/checks.py index 4d06498..82f4978 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -126,7 +126,7 @@ def check_max_iters(max_iters, n_levels): # TODO: document me! def check_sampling(sampling, n_levels): if (isinstance(sampling, (list, tuple)) and - np.alltrue([isinstance(s, (np.ndarray, np.int, None)) + np.alltrue([isinstance(s, (np.ndarray, np.int)) or sampling is None for s in sampling])): if len(sampling) == 1: return sampling * n_levels @@ -136,7 +136,7 @@ def check_sampling(sampling, n_levels): raise ValueError('A sampling list can only ' 'contain 1 element or {} ' 'elements'.format(n_levels)) - elif isinstance(sampling, (np.ndarray, np.int, None)): + elif isinstance(sampling, (np.ndarray, np.int)) or sampling is None: return [sampling] * n_levels else: raise ValueError('sampling can be an integer or ndarray, ' From 42b43644abb96e28d76d864797dc6eeaee74fad2 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 1 Jul 2015 11:17:54 +0100 Subject: [PATCH 286/423] Correct typo in LucasKanadeLinearInterface --- menpofit/aam/algorithm/lk.py | 2 +- menpofit/aam/fitter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 2144e14..9a20c46 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -132,7 +132,7 @@ def algorithm_result(self, image, shape_parameters, # TODO document me! -class LucasKanaddLinearInterface(LucasKanadeStandardInterface): +class LucasKanadeLinearInterface(LucasKanadeStandardInterface): r""" """ @property diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index e95b967..e8039fe 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -11,7 +11,7 @@ import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM from .algorithm.lk import ( - LucasKanadeStandardInterface, LucasKanaddLinearInterface, + LucasKanadeStandardInterface, LucasKanadeLinearInterface, LucasKanadePartsInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, @@ -83,7 +83,7 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): sm, self.aam.reference_shape) # set up algorithm using linear aam interface algorithm = lk_algorithm_cls( - LucasKanaddLinearInterface, am, md_transform, sampling=s, + LucasKanadeLinearInterface, am, md_transform, sampling=s, **kwargs) elif type(self.aam) is PartsAAM: From 8162ac292a63fb03a0d1997ce16e52a77055c624 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 1 Jul 2015 17:20:15 +0100 Subject: [PATCH 287/423] Add costs to AAMs - Little renaming as well --- menpofit/aam/__init__.py | 2 +- menpofit/aam/algorithm/__init__.py | 4 +- menpofit/aam/algorithm/lk.py | 272 ++++++++++++++++++++--------- menpofit/aam/algorithm/sd.py | 6 +- menpofit/aam/fitter.py | 14 +- menpofit/aam/result.py | 80 ++++++++- menpofit/atm/result.py | 2 +- menpofit/result.py | 16 +- 8 files changed, 288 insertions(+), 108 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index c06a2d2..bb37752 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -11,6 +11,6 @@ ModifiedAlternatingForwardCompositional, ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional, - SumOfSquaresNewton, SumOfSquaresGaussNewton, + MeanTemplateNewton, MeanTemplateGaussNewton, ProjectOutNewton, ProjectOutGaussNewton, AppearanceWeightsNewton, AppearanceWeightsGaussNewton) diff --git a/menpofit/aam/algorithm/__init__.py b/menpofit/aam/algorithm/__init__.py index 4c054af..636c758 100644 --- a/menpofit/aam/algorithm/__init__.py +++ b/menpofit/aam/algorithm/__init__.py @@ -6,8 +6,8 @@ ModifiedAlternatingInverseCompositional, WibergForwardCompositional, WibergInverseCompositional) from .sd import ( - SumOfSquaresNewton, - SumOfSquaresGaussNewton, + MeanTemplateNewton, + MeanTemplateGaussNewton, ProjectOutNewton, ProjectOutGaussNewton, AppearanceWeightsNewton, diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 9a20c46..d970387 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -124,10 +124,11 @@ def solve_all_ml(self, H, J, e): dq = - np.linalg.solve(H, J.T.dot(e)) return dq[:self.m], dq[self.m:] - def algorithm_result(self, image, shape_parameters, + def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( image, self.algorithm, shape_parameters, + cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -263,6 +264,10 @@ def project_out(self, J): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x, f): + return lambda: x.T.dot(f(x)) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -271,16 +276,20 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # vectorize it and mask it - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - # compute masked error - self.e_m = i_m - self.a_bar_m + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # update cost_functions + cost_functions = [cost_closure(self.e_m, self.project_out)] + while k < max_iters and eps > self.eps: # solve for increments on the shape parameters self.dp = self._solve(map_inference) @@ -289,6 +298,17 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.a_bar_m + + # update cost + cost_functions.append(cost_closure(self.e_m, self.project_out)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -297,7 +317,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) # TODO: handle costs! @@ -372,6 +392,10 @@ class Simultaneous(LucasKanade): """ def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x): + return lambda: x.T.dot(x) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -380,30 +404,33 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(self.c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [self.c] + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - self.e_m = i_m - a_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + self.c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(self.c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [self.c] + # compute masked error + self.e_m = i_m - a_m + + # update cost + cost_functions = [cost_closure(self.e_m)] + + while k < max_iters and eps > self.eps: # solve for increments on the appearance and shape parameters # simultaneously dc, self.dp = self._solve(map_inference) # update appearance parameters - self.c += dc + self.c = self.c + dc self.a = self.appearance_model.instance(self.c) a_m = self.a.as_vector()[self.interface.i_mask] c_list.append(self.c) @@ -413,6 +440,17 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - a_m + + # update cost + cost_functions.append(cost_closure(self.e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -421,11 +459,12 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) def _solve(self, map_inference): # compute masked Jacobian - J_m = self.compute_jacobian() + J_m = self._compute_jacobian() # assemble masked simultaneous Jacobian J_sim_m = np.hstack((-self.A_m, J_m)) # compute masked Hessian @@ -490,6 +529,10 @@ def _precompute(self, **kwargs): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x): + return lambda: x.T.dot(x) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -498,27 +541,28 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - Jdp = 0 - else: - Jdp = J_m.dot(self.dp) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - a_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + Jdp = 0 + + # compute masked error + e_m = i_m - a_m + # update cost + cost_functions = [cost_closure(e_m)] + + while k < max_iters and eps > self.eps: # solve for increment on the appearance parameters if map_inference: Ae_m_map = - self.s2_inv_S * c + self.A_m.dot(e_m + Jdp) @@ -540,7 +584,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, e_m - self.A_m.dot(dc)) # update appearance parameters - c += dc + c = c + dc self.a = self.appearance_model.instance(c) a_m = self.a.as_vector()[self.interface.i_mask] c_list.append(c) @@ -550,6 +594,20 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute Jdp + Jdp = J_m.dot(self.dp) + + # compute masked error + e_m = i_m - a_m + + # update cost + cost_functions.append(cost_closure(e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -558,7 +616,8 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) # TODO: handle costs! @@ -605,6 +664,10 @@ class ModifiedAlternating(Alternating): """ def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x): + return lambda: x.T.dot(x) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -615,21 +678,27 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - c = self.pinv_A_m.dot(i_m - a_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - a_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - a_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + # update cost + cost_functions = [cost_closure(e_m)] + + while k < max_iters and eps > self.eps: # compute masked Jacobian J_m = self._compute_jacobian() # compute masked Hessian @@ -646,6 +715,23 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # update appearance parameters + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - a_m + + # update cost + cost_functions.append(cost_closure(e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -654,7 +740,8 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) # TODO: handle costs! @@ -705,6 +792,10 @@ def project_out(self, J): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x, f): + return lambda: x.T.dot(f(x)) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -713,29 +804,27 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # mask warped image - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - if k == 0: - # initialize appearance parameters by projecting masked image - # onto masked appearance model - c = self.pinv_A_m.dot(i_m - self.a_bar_m) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list = [c] - else: - c = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) - self.a = self.appearance_model.instance(c) - a_m = self.a.as_vector()[self.interface.i_mask] - c_list.append(c) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] - # compute masked error - e_m = i_m - self.a_bar_m + # initialize appearance parameters by projecting masked image + # onto masked appearance model + c = self.pinv_A_m.dot(i_m - self.a_bar_m) + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list = [c] + # compute masked error + e_m = i_m - self.a_bar_m + + # update cost + cost_functions = [cost_closure(e_m, self.project_out)] + + while k < max_iters and eps > self.eps: # compute masked Jacobian J_m = self._compute_jacobian() # project out appearance models @@ -755,6 +844,24 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # mask warped image + i_m = self.i.as_vector()[self.interface.i_mask] + + # update appearance parameters + dc = self.pinv_A_m.dot(i_m - a_m + J_m.dot(self.dp)) + c = c + dc + self.a = self.appearance_model.instance(c) + a_m = self.a.as_vector()[self.interface.i_mask] + c_list.append(c) + + # compute masked error + e_m = i_m - self.a_bar_m + + # update cost + cost_functions.append(cost_closure(e_m, self.project_out)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -763,7 +870,8 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, appearance_parameters=c_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, + appearance_parameters=c_list, gt_shape=gt_shape) # TODO: handle costs! diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index e1f8e09..903f4bf 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -266,7 +266,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class SumOfSquares(SupervisedDescent): +class MeanTemplate(SupervisedDescent): r""" """ def _compute_train_features(self, image): @@ -287,7 +287,7 @@ def _compute_test_features(self, image): # TODO: document me! -class SumOfSquaresNewton(SumOfSquares): +class MeanTemplateNewton(MeanTemplate): r""" """ def _perform_regression(self, features, deltas, gamma=None, @@ -296,7 +296,7 @@ def _perform_regression(self, features, deltas, gamma=None, # TODO: document me! -class SumOfSquaresGaussNewton(SumOfSquares): +class MeanTemplateGaussNewton(MeanTemplate): r""" """ def _perform_regression(self, features, deltas, gamma=None, psi=None, diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index e8039fe..a985331 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -109,8 +109,8 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): class SupervisedDescentAAMFitter(AAMFitter): r""" """ - def __init__(self, aam, cr_algorithm_cls=ProjectOutNewton, - n_shape=None,n_appearance=None, sampling=None, + def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, + n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam self.algorithms = [] @@ -120,9 +120,9 @@ def __init__(self, aam, cr_algorithm_cls=ProjectOutNewton, self.n_perturbations = n_perturbations self.noise_std = noise_std self.max_iters = checks.check_max_iters(max_iters, self.n_levels) - self._set_up(cr_algorithm_cls, sampling, **kwargs) + self._set_up(sd_algorithm_cls, sampling, **kwargs) - def _set_up(self, cr_algorithm_cls, sampling, **kwargs): + def _set_up(self, sd_algorithm_cls, sampling, **kwargs): for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, sampling)): @@ -132,7 +132,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( SupervisedDescentStandardInterface, am, md_transform, sampling=s, max_iters=self.max_iters[j], **kwargs) @@ -142,7 +142,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) # set up algorithm using linear aam interface - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( SupervisedDescentLinearInterface, am, md_transform, sampling=s, max_iters=self.max_iters[j], **kwargs) @@ -150,7 +150,7 @@ def _set_up(self, cr_algorithm_cls, sampling, **kwargs): # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( SupervisedDescentPartsInterface, am, pdm, sampling=s, max_iters=self.max_iters[j], patch_shape=self.aam.patch_shape[j], diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 7c57ce0..38fabff 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -7,11 +7,50 @@ class AAMAlgorithmResult(ParametricAlgorithmResult): r""" """ - def __init__(self, image, fitter, shape_parameters, + def __init__(self, image, algorithm, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): super(AAMAlgorithmResult, self).__init__( - image, fitter, shape_parameters, gt_shape=gt_shape) + image, algorithm, shape_parameters, gt_shape=gt_shape) + self._cost_functions = cost_functions self.appearance_parameters = appearance_parameters + self._warped_images = None + self._appearance_reconstructions = None + self._costs = None + + @property + def warped_images(self): + if self._warped_images is None: + self._warped_images = [] + for p in self.shape_parameters: + self.algorithm.transform.from_vector_inplace(p) + self._warped_images.append( + self.algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def appearance_reconstructions(self): + if self.appearance_parameters is not None: + if self._appearance_reconstructions is None: + self._appearance_reconstructions = [] + for c in self.appearance_parameters: + instance = self.algorithm.appearance_model.instance(c) + self._appearance_reconstructions.append(instance) + return self._appearance_reconstructions + else: + raise ValueError('appearance_reconstructions is not well ' + 'defined for the chosen AAM algorithm: ' + '{}'.format(self.algorithm.__class__)) + + @property + def costs(self): + if self._cost_functions is not None: + if self._costs is None: + self._costs = [f() for f in self._cost_functions] + return self._costs + else: + raise ValueError('costs is not well ' + 'defined for the chosen AAM algorithm: ' + '{}'.format(self.algorithm.__class__)) # TODO: handle costs! @@ -21,7 +60,7 @@ class LinearAAMAlgorithmResult(AAMAlgorithmResult): """ @property def shapes(self, as_points=False): - return [self.fitter.transform.from_vector(p).sparse_target + return [self.algorithm.transform.from_vector(p).sparse_target for p in self.shape_parameters] @property @@ -38,4 +77,37 @@ def initial_shape(self): class AAMFitterResult(MultiFitterResult): r""" """ - pass + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(AAMFitterResult, self).__init__( + image, fitter, algorithm_results, affine_correction, + gt_shape=gt_shape) + self._warped_images = None + + @property + def warped_images(self): + if self._warped_images is None: + algorithm = self.algorithm_results[-1].algorithm + self._warped_images = [] + for s in self.shapes: + algorithm.transform.set_target(s) + self._warped_images.append( + algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def appearance_reconstructions(self): + reconstructions = self.algorithm_results[0].appearance_reconstructions + if reconstructions is not None: + for a in self.algorithm_results[1:]: + reconstructions = (reconstructions + + a.appearance_reconstructions) + return reconstructions + + @property + def costs(self): + costs = [] + for a in self.algorithm_results: + costs += a.costs + return costs + diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index f285f76..56b698e 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -16,7 +16,7 @@ class LinearATMAlgorithmResult(ATMAlgorithmResult): """ @property def shapes(self): - return [self.fitter.transform.from_vector(p).sparse_target + return [self.algorithm.transform.from_vector(p).sparse_target for p in self.shape_parameters] @property diff --git a/menpofit/result.py b/menpofit/result.py index 6fbe755..335a51b 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -412,9 +412,9 @@ def as_serializableresult(self): class ParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, fitter, shape_parameters, gt_shape=None): + def __init__(self, image, algorithm, shape_parameters, gt_shape=None): self.image = image - self.fitter = fitter + self.algorithm = algorithm self.shape_parameters = shape_parameters self._gt_shape = gt_shape @@ -428,7 +428,7 @@ def transforms(self): Generates a list containing the transforms obtained at each fitting iteration. """ - return [self.fitter.transform.from_vector(p) + return [self.algorithm.transform.from_vector(p) for p in self.shape_parameters] @property @@ -436,18 +436,18 @@ def final_transform(self): r""" Returns the final transform. """ - return self.fitter.transform.from_vector(self.shape_parameters[-1]) + return self.algorithm.transform.from_vector(self.shape_parameters[-1]) @property def initial_transform(self): r""" Returns the initial transform from which the fitting started. """ - return self.fitter.transform.from_vector(self.shape_parameters[0]) + return self.algorithm.transform.from_vector(self.shape_parameters[0]) @property def shapes(self): - return [self.fitter.transform.from_vector(p).target + return [self.algorithm.transform.from_vector(p).target for p in self.shape_parameters] @property @@ -463,9 +463,9 @@ def initial_shape(self): class NonParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, fitter, shapes, gt_shape=None): + def __init__(self, image, algorithm, shapes, gt_shape=None): self.image = image - self.fitter = fitter + self.algorithm = algorithm self._shapes = shapes self._gt_shape = gt_shape From 11076ad7ccbf693f99aca906b82f592608244372 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 1 Jul 2015 18:11:53 +0100 Subject: [PATCH 288/423] Add cost for ATMs --- menpofit/aam/algorithm/lk.py | 3 ++- menpofit/atm/algorithm.py | 49 ++++++++++++++++++++++++++---------- menpofit/atm/result.py | 48 ++++++++++++++++++++++++++++++++++- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index d970387..98eaa6d 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -140,10 +140,11 @@ class LucasKanadeLinearInterface(LucasKanadeStandardInterface): def shape_model(self): return self.transform.model - def algorithm_result(self, image, shape_parameters, + def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( image, self.algorithm, shape_parameters, + cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 3fb1098..423afab 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -98,9 +98,11 @@ def solve_shape_ml(cls, H, J, e): # compute and return ML solution return -np.linalg.solve(H, J.T.dot(e)) - def algorithm_result(self, image, shape_parameters, gt_shape=None): + def algorithm_result(self, image, shape_parameters, cost_functions=None, + gt_shape=None): return ATMAlgorithmResult( - image, self.algorithm, shape_parameters, gt_shape=gt_shape) + image, self.algorithm, shape_parameters, + cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! @@ -111,9 +113,11 @@ class LucasKanadeLinearInterface(LucasKanadeStandardInterface): def shape_model(self): return self.transform.model - def algorithm_result(self, image, shape_parameters, gt_shape=None): + def algorithm_result(self, image, shape_parameters, cost_functions=None, + gt_shape=None): return LinearATMAlgorithmResult( - image, self.algorithm, shape_parameters, gt_shape=gt_shape) + image, self.algorithm, shape_parameters, + cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! @@ -216,6 +220,10 @@ class Compositional(LucasKanade): """ def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): + # define cost closure + def cost_closure(x, f): + return lambda: x.T.dot(f(x)) + # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] @@ -224,16 +232,20 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, k = 0 eps = np.Inf - # Compositional Gauss-Newton loop - while k < max_iters and eps > self.eps: - # warp image - self.i = self.interface.warp(image) - # vectorize it and mask it - i_m = self.i.as_vector()[self.interface.i_mask] + # Compositional Gauss-Newton loop ------------------------------------- - # compute masked error - self.e_m = i_m - self.t_m + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.t_m + + # update cost + cost_functions = [cost_closure(self.e_m)] + while k < max_iters and eps > self.eps: # solve for increments on the shape parameters self.dp = self._solve(map_inference) @@ -242,6 +254,17 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, self._update_warp() p_list.append(self.transform.as_vector()) + # warp image + self.i = self.interface.warp(image) + # vectorize it and mask it + i_m = self.i.as_vector()[self.interface.i_mask] + + # compute masked error + self.e_m = i_m - self.t_m + + # update cost + cost_functions.append(cost_closure(self.e_m)) + # test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) @@ -250,7 +273,7 @@ def run(self, image, initial_shape, gt_shape=None, max_iters=20, # return algorithm result return self.interface.algorithm_result( - image, p_list, gt_shape=gt_shape) + image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) # TODO: handle costs! diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 56b698e..68571ac 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -7,6 +7,29 @@ class ATMAlgorithmResult(ParametricAlgorithmResult): r""" """ + def __init__(self, image, algorithm, shape_parameters, cost_functions=None, + gt_shape=None): + super(ATMAlgorithmResult, self).__init__( + image, algorithm, shape_parameters, gt_shape=gt_shape) + self._cost_functions = cost_functions + self._warped_images = None + self._costs = None + + @property + def warped_images(self): + if self._warped_images is None: + self._warped_images = [] + for p in self.shape_parameters: + self.algorithm.transform.from_vector_inplace(p) + self._warped_images.append( + self.algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def costs(self): + if self._costs is None: + self._costs = [f() for f in self._cost_functions] + return self._costs # TODO: handle costs! @@ -33,4 +56,27 @@ def initial_shape(self): class ATMFitterResult(MultiFitterResult): r""" """ - pass + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(ATMFitterResult, self).__init__( + image, fitter, algorithm_results, affine_correction, + gt_shape=gt_shape) + self._warped_images = None + + @property + def warped_images(self): + if self._warped_images is None: + algorithm = self.algorithm_results[-1].algorithm + self._warped_images = [] + for s in self.shapes: + algorithm.transform.set_target(s) + self._warped_images.append( + algorithm.interface.warp(self.image)) + return self._warped_images + + @property + def costs(self): + costs = [] + for a in self.algorithm_results: + costs += a.costs + return costs From 656d7cef717207967ac71e74ff7cde3817a7edcf Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 2 Jul 2015 13:19:27 +0100 Subject: [PATCH 289/423] Add cost for LK - Fixes a couple of bugs in the GradientImages and ECC residuals. --- menpofit/lk/algorithm.py | 18 +++++ menpofit/lk/residual.py | 138 +++++++++++++++++++++++++++------------ menpofit/lk/result.py | 53 +++++++++++++-- 3 files changed, 163 insertions(+), 46 deletions(-) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 37325f6..7a02a63 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -27,6 +27,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] + cost_functions = [] + # initialize iteration counter and epsilon k = 0 eps = np.Inf @@ -58,6 +60,9 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.from_vector_inplace(self.transform.as_vector() + dp) p_list.append(self.transform.as_vector()) + # update cost + cost_functions.append(self.residual.cost_closure()) + # test convergence eps = np.abs(norm(dp)) @@ -65,6 +70,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): k += 1 return LucasKanadeAlgorithmResult(image, self, p_list, + cost_functions=cost_functions, gt_shape=gt_shape) @@ -89,6 +95,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] + cost_functions = [] + # initialize iteration counter and epsilon k = 0 eps = np.Inf @@ -116,6 +124,9 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.compose_after_from_vector_inplace(dp) p_list.append(self.transform.as_vector()) + # update cost + cost_functions.append(self.residual.cost_closure()) + # test convergence eps = np.abs(norm(dp)) @@ -123,6 +134,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): k += 1 return LucasKanadeAlgorithmResult(image, self, p_list, + cost_functions=cost_functions, gt_shape=gt_shape) @@ -153,6 +165,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] + cost_functions = [] + # initialize iteration counter and epsilon k = 0 eps = np.Inf @@ -174,6 +188,9 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): self.transform.compose_after_from_vector_inplace(inv_dp) p_list.append(self.transform.as_vector()) + # update cost + cost_functions.append(self.residual.cost_closure()) + # test convergence eps = np.abs(norm(dp)) @@ -181,4 +198,5 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): k += 1 return LucasKanadeAlgorithmResult(image, self, p_list, + cost_functions=cost_functions, gt_shape=gt_shape) diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index 5f808d3..57b386b 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -1,3 +1,4 @@ +from __future__ import division import abc import numpy as np from numpy.fft import fftn, ifftn, fft2 @@ -130,12 +131,16 @@ def steepest_descent_update(self, sdi, image, template): """ pass + @abc.abstractmethod + def cost_closure(self): + pass + class SSD(Residual): r""" """ def __init__(self, kernel=None): - self.kernel = kernel + self._kernel = kernel def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient @@ -153,23 +158,23 @@ def steepest_descent_images(self, image, dW_dp, forward=None): for d in a: sdi += d - if self.kernel is not None: - # if required, filter steepest descent images - # fft_sdi: ch x h x w x params - filtered_sdi = ifftn(self.kernel[..., None] * - fftn(sdi, axes=(-3, -2)), - axes=(-3, -2)) + if self._kernel is None: # reshape steepest descent images # sdi: (ch x h x w) x params # filtered_sdi: (ch x h x w) x params sdi = sdi.reshape((-1, sdi.shape[-1])) - filtered_sdi = filtered_sdi.reshape(sdi.shape) + filtered_sdi = sdi else: + # if required, filter steepest descent images + # fft_sdi: ch x h x w x params + filtered_sdi = ifftn(self._kernel[..., None] * + fftn(sdi, axes=(-3, -2)), + axes=(-3, -2)) # reshape steepest descent images # sdi: (ch x h x w) x params # filtered_sdi: (ch x h x w) x params sdi = sdi.reshape((-1, sdi.shape[-1])) - filtered_sdi = sdi + filtered_sdi = filtered_sdi.reshape(sdi.shape) return filtered_sdi, sdi @@ -185,8 +190,19 @@ def hessian(self, sdi, sdi2=None): return H def steepest_descent_update(self, sdi, image, template): - error_img = image.as_vector() - template.as_vector() - return sdi.T.dot(error_img) + self._error_img = image.as_vector() - template.as_vector() + return sdi.T.dot(self._error_img) + + def cost_closure(self): + def cost_closure(x, k): + if k is None: + return lambda: x.T.dot(x) + else: + x = x.reshape((-1,) + k.shape[-2:]) + kx = ifftn(k[..., None] * fftn(x, axes=(-2, -1)), + axes=(-2, -1)) + return lambda: x.ravel().T.dot(kx.ravel()) + return cost_closure(self._error_img, self._kernel) # TODO: Does not support masked templates at the moment @@ -194,7 +210,7 @@ class FourierSSD(Residual): r""" """ def __init__(self, kernel=None): - self.kernel = kernel + self._kernel = kernel def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient @@ -216,20 +232,20 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # fft_sdi: ch x h x w x params fft_sdi = fftn(sdi, axes=(-3, -2)) - if self.kernel is not None: - # if required, filter steepest descent images - filtered_fft_sdi = self.kernel[..., None] * fft_sdi + if self._kernel is None: # reshape steepest descent images # fft_sdi: (ch x h x w) x params # filtered_fft_sdi: (ch x h x w) x params fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) - filtered_fft_sdi = filtered_fft_sdi.reshape(fft_sdi.shape) + filtered_fft_sdi = fft_sdi else: + # if required, filter steepest descent images + filtered_fft_sdi = self._kernel[..., None] * fft_sdi # reshape steepest descent images # fft_sdi: (ch x h x w) x params # filtered_fft_sdi: (ch x h x w) x params fft_sdi = fft_sdi.reshape((-1, fft_sdi.shape[-1])) - filtered_fft_sdi = fft_sdi + filtered_fft_sdi = filtered_fft_sdi.reshape(fft_sdi.shape) return filtered_fft_sdi, fft_sdi @@ -243,11 +259,11 @@ def hessian(self, sdi, sdi2=None): def steepest_descent_update(self, sdi, image, template): # compute error image # error_img: ch x h x w - error_img = image.pixels - template.pixels + self._error_img = image.pixels - template.pixels # compute error image fft # fft_error_img: ch x (h x w) - fft_error_img = fft2(error_img) + fft_error_img = fft2(self._error_img) # compute steepest descent update # fft_sdi: params x (ch x h x w) @@ -255,6 +271,16 @@ def steepest_descent_update(self, sdi, image, template): # fft_sdu: params return sdi.conjugate().T.dot(fft_error_img.ravel()) + def cost_closure(self): + def cost_closure(x, k): + if k is None: + return lambda: x.ravel().T.dot(x.ravel()) + else: + kx = ifftn(k[..., None] * fftn(x, axes=(-2, -1)), + axes=(-2, -1)) + return lambda: x.ravel().T.dot(kx.ravel()) + return cost_closure(self._error_img, self._kernel) + class ECC(Residual): r""" @@ -273,7 +299,8 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient # gradient: dims x ch x pixels grad = self.gradient(norm_image, forward=forward) - grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) + grad = grad.as_vector().reshape((image.n_dims, image.n_channels) + + image.shape) # compute steepest descent images # gradient: dims x ch x pixels @@ -286,32 +313,38 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # reshape steepest descent images # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) + sdi = sdi.reshape((-1, sdi.shape[-1])) - def hessian(self, sdi): + return sdi, sdi + + def hessian(self, sdi, sdi2=None): # compute hessian - # sdi.T: params x (ch x pixels) - # sdi: (ch x pixels) x params + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params # hessian: params x x params - H = sdi.T.dot(sdi) + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) self._H_inv = scipy.linalg.inv(H) return H def steepest_descent_update(self, sdi, image, template): - normalised_IWxp = self._normalise_images(image).as_vector() - normalised_template = self._normalise_images(template).as_vector() + self._normalised_IWxp = self._normalise_images(image).as_vector() + self._normalised_template = self._normalise_images( + template).as_vector() - Gt = sdi.T.dot(normalised_template) - Gw = sdi.T.dot(normalised_IWxp) + Gt = sdi.T.dot(self._normalised_template) + Gw = sdi.T.dot(self._normalised_IWxp) # Calculate the numerator - IWxp_norm = scipy.linalg.norm(normalised_IWxp) + IWxp_norm = scipy.linalg.norm(self._normalised_IWxp) num1 = IWxp_norm ** 2 num2 = np.dot(Gw.T, np.dot(self._H_inv, Gw)) num = num1 - num2 # Calculate the denominator - den1 = np.dot(normalised_template, normalised_IWxp) + den1 = np.dot(self._normalised_template, self._normalised_IWxp) den2 = np.dot(Gt.T, np.dot(self._H_inv, Gw)) den = den1 - den2 @@ -325,10 +358,15 @@ def steepest_descent_update(self, sdi, image, template): l2 = - den / den3 l = np.maximum(l1, l2) - self._error_img = l * normalised_IWxp - normalised_template + self._error_img = l * self._normalised_IWxp - self._normalised_template return sdi.T.dot(self._error_img) + def cost_closure(self): + def cost_closure(x, y): + return lambda: x.T.dot(y) + return cost_closure(self._normalised_IWxp, self._normalised_template) + class GradientImages(Residual): r""" @@ -353,7 +391,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # second_grad: dims x dims x ch x pixels second_grad = self.gradient(self._template_grad) second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels, -1)) + (n_dims, n_dims, n_channels) + image.shape) # Fix crossed derivatives: dydx = dxdy second_grad[1, 0, ...] = second_grad[0, 1, ...] @@ -368,15 +406,21 @@ def steepest_descent_images(self, image, dW_dp, forward=None): sdi += d # reshape steepest descent images - # sdi: (dims x ch x h x w) x params - return sdi.reshape((-1, sdi.shape[-1])) + # sdi: (ch x pixels) x params + sdi = sdi.reshape((-1, sdi.shape[-1])) - def hessian(self, sdi): + return sdi, sdi + + def hessian(self, sdi, sdi2=None): # compute hessian - # sdi.T: params x (dims x ch x pixels) - # sdi: (dims x ch x pixels) x params - # hessian: params x x params - return sdi.T.dot(sdi) + # sdi.T: params x (ch x h x w) + # sdi: (ch x h x w) x params + # hessian: params x x params + if sdi2 is None: + H = sdi.T.dot(sdi) + else: + H = sdi.T.dot(sdi2) + return H def steepest_descent_update(self, sdi, image, template): # compute image regularized gradient @@ -394,6 +438,11 @@ def steepest_descent_update(self, sdi, image, template): # sdu: params return sdi.T.dot(self._error_img) + def cost_closure(self): + def cost_closure(x): + return lambda: x.T.dot(x) + return cost_closure(self._error_img) + class GradientCorrelation(Residual): r""" @@ -502,5 +551,10 @@ def steepest_descent_update(self, sdi, image, template): # compute step size qp = np.sum(self._cos_phi * IWxp_cos_phi + self._sin_phi * IWxp_sin_phi) - l = self._N / qp - return l * sdu + self._l = self._N / qp + return self._l * sdu + + def cost_closure(self): + def cost_closure(x): + return lambda: 1/x + return cost_closure(self._l) diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py index 6674a63..9d6e4c5 100644 --- a/menpofit/lk/result.py +++ b/menpofit/lk/result.py @@ -5,9 +5,30 @@ # TODO: handle costs! # TODO: document me! class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): - r""" - """ - pass + def __init__(self, image, algorithm, shape_parameters, + cost_functions=None, gt_shape=None): + super(LucasKanadeAlgorithmResult, self).__init__( + image, algorithm, shape_parameters, gt_shape=gt_shape) + self._cost_functions = cost_functions + self._warped_images = None + self._costs = None + + @property + def warped_images(self): + if self._warped_images is None: + self._warped_images = [] + for p in self.shape_parameters: + self.algorithm.transform.from_vector_inplace(p) + self._warped_images.append( + self.image.warp_to_mask(self.algorithm.template.mask, + self.algorithm.transform)) + return self._warped_images + + @property + def costs(self): + if self._costs is None: + self._costs = [f() for f in self._cost_functions] + return self._costs # TODO: handle costs! @@ -15,4 +36,28 @@ class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): class LucasKanadeFitterResult(MultiFitterResult): r""" """ - pass + def __init__(self, image, fitter, algorithm_results, affine_correction, + gt_shape=None): + super(LucasKanadeFitterResult, self).__init__( + image, fitter, algorithm_results, affine_correction, + gt_shape=gt_shape) + self._warped_images = None + + @property + def warped_images(self): + if self._warped_images is None: + algorithm = self.algorithm_results[-1].algorithm + self._warped_images = [] + for s in self.shapes: + algorithm.transform.set_target(s) + self._warped_images.append( + self.image.warp_to_mask(algorithm.template.mask, + algorithm.transform)) + return self._warped_images + + @property + def costs(self): + costs = [] + for a in self.algorithm_results: + costs += a.costs + return costs From a1094c797f72231720d86c7b00660eba624e6f82 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Thu, 2 Jul 2015 19:58:48 +0100 Subject: [PATCH 290/423] Remove addressed TODOs --- menpofit/aam/algorithm/lk.py | 15 --------------- menpofit/aam/result.py | 3 --- menpofit/atm/algorithm.py | 3 --- menpofit/atm/result.py | 3 --- menpofit/lk/algorithm.py | 3 --- menpofit/lk/result.py | 2 -- 6 files changed, 29 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 98eaa6d..756fe2a 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -253,7 +253,6 @@ def _precompute(self): self.s2_inv_S = s2 / S -# TODO: handle costs! # TODO: Document me! class ProjectOut(LucasKanade): r""" @@ -321,7 +320,6 @@ def cost_closure(x, f): image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class ProjectOutForwardCompositional(ProjectOut): r""" @@ -350,7 +348,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class ProjectOutInverseCompositional(ProjectOut): r""" @@ -385,7 +382,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class Simultaneous(LucasKanade): r""" @@ -480,7 +476,6 @@ def _solve(self, map_inference): return self.interface.solve_all_ml(H_sim_m, J_sim_m, self.e_m) -# TODO: handle costs! # TODO: Document me! class SimultaneousForwardCompositional(Simultaneous): r""" @@ -498,7 +493,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class SimultaneousInverseCompositional(Simultaneous): r""" @@ -516,7 +510,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class Alternating(LucasKanade): r""" @@ -621,7 +614,6 @@ def cost_closure(x): appearance_parameters=c_list, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class AlternatingForwardCompositional(Alternating): r""" @@ -639,7 +631,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class AlternatingInverseCompositional(Alternating): r""" @@ -657,7 +648,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class ModifiedAlternating(Alternating): r""" @@ -745,7 +735,6 @@ def cost_closure(x): appearance_parameters=c_list, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class ModifiedAlternatingForwardCompositional(ModifiedAlternating): r""" @@ -763,7 +752,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class ModifiedAlternatingInverseCompositional(ModifiedAlternating): r""" @@ -781,7 +769,6 @@ def _update_warp(self): self.transform.as_vector() - self.dp) -# TODO: handle costs! # TODO: Document me! class Wiberg(LucasKanade): r""" @@ -875,7 +862,6 @@ def cost_closure(x, f): appearance_parameters=c_list, gt_shape=gt_shape) -# TODO: handle costs! # TODO: Document me! class WibergForwardCompositional(Wiberg): r""" @@ -893,7 +879,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO: Document me! class WibergInverseCompositional(Wiberg): r""" diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 38fabff..00a500a 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -2,7 +2,6 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: handle costs! # TODO: document me! class AAMAlgorithmResult(ParametricAlgorithmResult): r""" @@ -53,7 +52,6 @@ def costs(self): '{}'.format(self.algorithm.__class__)) -# TODO: handle costs! # TODO: document me! class LinearAAMAlgorithmResult(AAMAlgorithmResult): r""" @@ -72,7 +70,6 @@ def initial_shape(self): return self.initial_transform.sparse_target -# TODO: handle costs! # TODO: document me! class AAMFitterResult(MultiFitterResult): r""" diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 423afab..43e27a1 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -212,7 +212,6 @@ def _precompute(self, **kwargs): self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) -# TODO: handle costs! # TODO document me! class Compositional(LucasKanade): r""" @@ -276,7 +275,6 @@ def cost_closure(x, f): image, p_list, cost_functions=cost_functions, gt_shape=gt_shape) -# TODO: handle costs! # TODO document me! class ForwardCompositional(Compositional): r""" @@ -303,7 +301,6 @@ def _update_warp(self): self.transform.as_vector() + self.dp) -# TODO: handle costs! # TODO document me! class InverseCompositional(Compositional): r""" diff --git a/menpofit/atm/result.py b/menpofit/atm/result.py index 68571ac..b7aec3e 100644 --- a/menpofit/atm/result.py +++ b/menpofit/atm/result.py @@ -2,7 +2,6 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: handle costs! # TODO: document me! class ATMAlgorithmResult(ParametricAlgorithmResult): r""" @@ -32,7 +31,6 @@ def costs(self): return self._costs -# TODO: handle costs! # TODO: document me! class LinearATMAlgorithmResult(ATMAlgorithmResult): r""" @@ -51,7 +49,6 @@ def initial_shape(self): return self.initial_transform.sparse_target -# TODO: handle costs! # TODO: document me! class ATMFitterResult(MultiFitterResult): r""" diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index 7a02a63..b296300 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -16,7 +16,6 @@ def __init__(self, template, transform, residual, eps=10**-10): self.eps = eps -# TODO: handle costs! # TODO: document me! class ForwardAdditive(LucasKanade): r""" @@ -74,7 +73,6 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): gt_shape=gt_shape) -# TODO: handle costs! # TODO: document me! class ForwardCompositional(LucasKanade): r""" @@ -138,7 +136,6 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): gt_shape=gt_shape) -# TODO: handle costs! # TODO: document me! class InverseCompositional(LucasKanade): r""" diff --git a/menpofit/lk/result.py b/menpofit/lk/result.py index 9d6e4c5..874fbda 100644 --- a/menpofit/lk/result.py +++ b/menpofit/lk/result.py @@ -2,7 +2,6 @@ from menpofit.result import ParametricAlgorithmResult, MultiFitterResult -# TODO: handle costs! # TODO: document me! class LucasKanadeAlgorithmResult(ParametricAlgorithmResult): def __init__(self, image, algorithm, shape_parameters, @@ -31,7 +30,6 @@ def costs(self): return self._costs -# TODO: handle costs! # TODO: document me! class LucasKanadeFitterResult(MultiFitterResult): r""" From 50c49ba7eaefd4240db95d6ec697ca738d72f649 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 3 Jul 2015 14:47:59 +0100 Subject: [PATCH 291/423] Fix noisy_align method removed Replaced with new methods - simple refactoring bug. --- menpofit/sdm/fitter.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2c844e5..28af7f0 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -3,7 +3,7 @@ from menpo.transform import Scale, AlignmentSimilarity from menpo.feature import no_op from menpofit.builder import normalization_wrt_reference_shape, scale_images -from menpofit.fitter import MultiFitter, noisy_align +from menpofit.fitter import MultiFitter, noisy_target_alignment_transform from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import SN @@ -174,8 +174,9 @@ def _fitter_result(self, image, algorithm_results, affine_correction, def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, rotation=False): - transform = noisy_align(AlignmentSimilarity, + transform = noisy_target_alignment_transform( self.reference_bounding_box, bounding_box, + alignment_transform_cls=AlignmentSimilarity, noise_std=noise_std, rotation=rotation) return transform.apply(self.reference_shape) @@ -408,4 +409,4 @@ def __str__(self): # # feat_str = [feat_str] # # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out \ No newline at end of file +# # return out From a5a9ab7ee5da7b37f5edba6eb7ec8974956b991a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 3 Jul 2015 16:45:16 +0100 Subject: [PATCH 292/423] Add incremental sdm --- menpofit/sdm/__init__.py | 4 +- menpofit/sdm/algorithm.py | 162 +++++++++++++++++++++----------------- menpofit/sdm/fitter.py | 118 +++++++++++++++++++-------- 3 files changed, 179 insertions(+), 105 deletions(-) diff --git a/menpofit/sdm/__init__.py b/menpofit/sdm/__init__.py index 9180e28..16e88b4 100644 --- a/menpofit/sdm/__init__.py +++ b/menpofit/sdm/__init__.py @@ -1,2 +1,2 @@ -from .algorithm import SN, SGN -from .fitter import CRFitter, SDMFitter +from .algorithm import Newton, GaussNewton +from .fitter import SupervisedDescentFitter diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index db5a195..951cd97 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -5,8 +5,9 @@ from menpofit.result import NonParametricAlgorithmResult -# TODO document me! -class CRAlgorithm(object): +# TODO: compute more meaningful error +# TODO: document me! +class SupervisedDescentAlgorithm(object): r""" """ def train(self, images, gt_shapes, current_shapes, verbose=False, @@ -31,13 +32,14 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, # perform regression if verbose: - print_dynamic('- Performing regression...') - regressor = self._perform_regression(features, delta_x, **kwargs) + print_dynamic('- Performing regression.') + r = self._regressor_cls(**kwargs) + r.train(features, delta_x) # add regressor to list - self.regressors.append(regressor) + self.regressors.append(r) # estimate delta_points - estimated_delta_x = regressor(features) + estimated_delta_x = r.predict(features) if verbose: error = _compute_rmse(delta_x, estimated_delta_x) print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) @@ -59,6 +61,44 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, # rearrange current shapes into their original list of list form return current_shapes + def increment(self, images, gt_shapes, current_shapes, verbose=False, + **kwarg): + # obtain delta_x and gt_x + delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) + + # Cascaded Regression loop + for r in self.regressors: + # generate regression data + features = obtain_patch_features( + images, current_shapes, self.patch_shape, self.features, + features_patch_length=self._features_patch_length) + + # update regression + if verbose: + print_dynamic('- Updating regression') + r.increment(features, delta_x) + + # estimate delta_points + estimated_delta_x = r.predict(features) + if verbose: + error = _compute_rmse(delta_x, estimated_delta_x) + print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + + j = 0 + for shapes in current_shapes: + for s in shapes: + # update current x + current_x = s.as_vector() + estimated_delta_x[j] + # update current shape inplace + s.from_vector_inplace(current_x) + # update delta_x + delta_x[j] = gt_x[j] - current_x + # increase index + j += 1 + + # rearrange current shapes into their original list of list form + return current_shapes + def run(self, image, initial_shape, gt_shape=None, **kwargs): # set current shape and initialize list of shapes current_shape = initial_shape @@ -72,7 +112,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): features_patch_length=self._features_patch_length) # solve for increments on the shape vector - dx = r(features) + dx = r.predict(features) # update current shape current_shape = current_shape.from_vector( @@ -85,100 +125,80 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # TODO: document me! -class SN(CRAlgorithm): +class Newton(SupervisedDescentAlgorithm): r""" - Supervised Newton. - - This class implements the Supervised Descent Method technique, proposed - by Xiong and De la Torre in [XiongD13]. - - References - ---------- - .. [XiongD13] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10 ** -5): + eps=10**-5): self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - # wire regression callable - self._perform_regression = _supervised_newton + self._regressor_cls = _incremental_least_squares # TODO: document me! -class SGN(CRAlgorithm): +class GaussNewton(SupervisedDescentAlgorithm): r""" - Supervised Gauss-Newton - - This class implements a variation of the Supervised Descent Method - [XiongD13] by some of the ideas incorporating ideas... - - References - ---------- - .. [XiongD13] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 - .. [Tzimiropoulos15] Supervised Descent Method and its Applications to - Face Alignment - Xuehan Xiong and Fernando De la Torre Fernando - IEEE International Conference on Computer Vision and Pattern Recognition - May, 2013 """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10 ** -5): + eps=10**-5): self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - # wire regression callable - self._perform_regression = _supervised_gauss_newton + self._perform_regression = _incremental_indirect_least_squares # TODO: document me! -class _supervised_newton(object): +class _incremental_least_squares(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression - XX = features.T.dot(features) - XT = features.T.dot(deltas) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # descent direction - self.R = np.linalg.solve(XX, XT) + def __init__(self, l=0): + self.l = l + + def train(self, X, Y): + # regularized least squares + XX = X.T.dot(X) + np.fill_diagonal(XX, self.l + np.diag(XX)) + self.V = np.linalg.inv(XX) + self.W = self.V.dot(X.T.dot(Y)) + + def increment(self, X, Y): + # incremental regularized least squares + U = X.dot(self.V).dot(X.T) + np.fill_diagonal(U, 1 + np.diag(U)) + U = np.linalg.inv(U) + Q = self.V.dot(X.T).dot(U).dot(X) + self.V = self.V - Q.dot(self.V) + self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) - def __call__(self, features): - return np.dot(features, self.R) + def predict(self, x): + return np.dot(x, self.W) # TODO: document me! -class _supervised_gauss_newton(object): +class _incremental_indirect_least_squares(object): r""" """ - def __init__(self, features, deltas, gamma=None): - # ridge regression - XX = deltas.T.dot(deltas) - XT = deltas.T.dot(features) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # average Jacobian - self.J = np.linalg.solve(XX, XT) - # average Hessian - self.H = self.J.dot(self.J.T) - # descent direction - self.R = np.linalg.solve(self.H, self.J).T - - def __call__(self, features): - return np.dot(features, self.R) + def __init__(self, l=0, d=0): + self._ils = _incremental_least_squares(l) + self.d = d + + def train(self, X, Y): + # regularized least squares exchanging the roles of X and Y + self._ils.train(Y, X) + J = self._ils.W + # solve the original problem by computing the pseudo-inverse of the + # previous solution + H = J.T.dot(J) + np.fill_diagonal(H, self.d + np.diag(H)) + self.W = np.linalg.solve(H, J.T) + + def predict(self, x): + return np.dot(x, self.W) # TODO: document me! diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 28af7f0..5dd2994 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,49 +1,48 @@ from __future__ import division -from functools import partial +import numpy as np from menpo.transform import Scale, AlignmentSimilarity from menpo.feature import no_op -from menpofit.builder import normalization_wrt_reference_shape, scale_images -from menpofit.fitter import MultiFitter, noisy_target_alignment_transform +from menpofit.builder import ( + normalization_wrt_reference_shape, rescale_images_to_reference_shape, + scale_images) +from menpofit.fitter import MultiFitter, noisy_params_alignment_similarity from menpofit.result import MultiFitterResult import menpofit.checks as checks -from .algorithm import SN +from .algorithm import Newton # TODO: document me! -class CRFitter(MultiFitter): +class SupervisedDescentFitter(MultiFitter): r""" """ - def __init__(self, cr_algorithm_cls=SN, features=no_op, + def __init__(self, sd_algorithm_cls=Newton, features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), - iterations=6, n_perturbations=10, **kwargs): + iterations=6, n_perturbations=10, noise_std=0.05, **kwargs): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) features = checks.check_features(features, n_levels) patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters - self._algorithms = [] self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations + self.noise_std = noise_std self.iterations = checks.check_max_iters(iterations, n_levels) # set up algorithms - self._set_up(cr_algorithm_cls, features, patch_shape, **kwargs) - - @property - def algorithms(self): - return self._algorithms + self._set_up(sd_algorithm_cls, features, patch_shape, **kwargs) @property def reference_bounding_box(self): return self.reference_shape.bounding_box() - def _set_up(self, cr_algorithm_cls, features, patch_shape, **kwargs): + def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): + self.algorithms = [] for j in range(self.n_levels): - algorithm = cr_algorithm_cls( + algorithm = sd_algorithm_cls( features=features[j], patch_shape=patch_shape[j], iterations=self.iterations[j], **kwargs) - self._algorithms.append(algorithm) + self.algorithms.append(algorithm) def train(self, images, group=None, label=None, verbose=False, **kwargs): # normalize images and compute reference shape @@ -71,7 +70,8 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for gt_s in level_gt_shapes: perturbed_shapes = [] for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape(gt_s) + p_s = self.noisy_shape_from_shape( + gt_s, noise_std=self.noise_std) perturbed_shapes.append(p_s) current_shapes.append(perturbed_shapes) @@ -87,6 +87,68 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for shape in image_shapes: transform.apply_inplace(shape) + def increment(self, images, group=None, label=None, verbose=False, + **kwargs): + # normalize images with respect to reference shape of aam + images = rescale_images_to_reference_shape( + images, group, label, self.reference_shape, verbose=verbose) + + # for each pyramid level (low --> high) + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # scale images and compute features at other levels + level_images = scale_images(images, self.scales[j], + level_str=level_str, verbose=verbose) + + # extract ground truth shapes for current level + level_gt_shapes = [i.landmarks[group][label] for i in level_images] + + if j == 0: + # generate perturbed shapes + current_shapes = [] + for gt_s in level_gt_shapes: + perturbed_shapes = [] + for _ in range(self.n_perturbations): + p_s = self.noisy_shape_from_shape( + gt_s, noise_std=self.noise_std) + perturbed_shapes.append(p_s) + current_shapes.append(perturbed_shapes) + + # train cascaded regression algorithm + current_shapes = self.algorithms[j].increment( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + + # scale current shapes to next level resolution + if self.scales[j] != (1 or self.scales[-1]): + transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + + def train_incrementally(self, images, group=None, label=None, + batch_size=100, verbose=False, **kwargs): + n_batches = np.int(np.ceil(len(images) / batch_size)) + + # train first batch + print 'Training batch 1.' + self.train(images[:batch_size], group=group, label=label, + verbose=verbose, **kwargs) + + # train all other batches + start = batch_size + for j in range(1, n_batches): + print 'Training batch {}.'.format(j+1) + end = start + batch_size + self.increment(images[start:end], group=group, label=label, + verbose=verbose, **kwargs) + start = end + def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): r""" @@ -137,8 +199,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # if specified, crop the image if crop_image: - image = image.copy() - image.crop_to_landmarks_proportion_inplace(crop_image, + image = image.crop_to_landmarks_proportion(crop_image, group='initial_shape') # rescale image wrt the scale factor between reference_shape and @@ -172,17 +233,14 @@ def _fitter_result(self, image, algorithm_results, affine_correction, return MultiFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.04, - rotation=False): - transform = noisy_target_alignment_transform( - self.reference_bounding_box, bounding_box, - alignment_transform_cls=AlignmentSimilarity, - noise_std=noise_std, rotation=rotation) + def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): + transform = noisy_params_alignment_similarity( + self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.04, rotation=False): + def noisy_shape_from_shape(self, shape, noise_std=0.05): return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std, rotation=rotation) + shape.bounding_box(), noise_std=noise_std) # TODO: fix me! def __str__(self): @@ -257,10 +315,6 @@ def __str__(self): # return out -# TODO: document me! -SDMFitter = partial(CRFitter, cr_algorithm_cls=SN) - - # class CRFitter(MultiFitter): # r""" # """ @@ -409,4 +463,4 @@ def __str__(self): # # feat_str = [feat_str] # # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out +# # return out \ No newline at end of file From 3701b71cd93f57c1e6b9dba8e3bda0c038bfba0b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 10:03:46 +0100 Subject: [PATCH 293/423] Add incremental AAM --- menpofit/aam/builder.py | 109 +++++++++++++++++++++++++++++++++++++++- menpofit/aam/fitter.py | 4 +- menpofit/atm/fitter.py | 2 +- 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 9ef763b..373236a 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,4 +1,5 @@ from __future__ import division +import numpy as np from copy import deepcopy from menpo.model import PCAModel from menpo.shape import mean_pointcloud @@ -8,7 +9,8 @@ from menpofit.builder import ( normalization_wrt_reference_shape, compute_features, scale_images, warp_images, extract_patches, build_shape_model, align_shapes, - build_reference_frame, build_patch_reference_frame, densify_shapes) + build_reference_frame, build_patch_reference_frame, densify_shapes, + rescale_images_to_reference_shape) from menpofit.transform import ( DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) @@ -253,6 +255,111 @@ def build(self, images, group=None, label=None, verbose=False): return aam + def increment(self, aam, images, group=None, label=None, + forgetting_factor=1.0, verbose=False): + # normalize images with respect to reference shape of aam + images = rescale_images_to_reference_shape( + images, group, label, aam.reference_shape, verbose=verbose) + + # increment models at each scale + if verbose: + print_dynamic('- Incrementing models\n') + + # for each pyramid level (high --> low) + for j, s in enumerate(self.scales[::-1]): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + + # obtain image representation + if j == 0: + # compute features at highest level + feature_images = compute_features(images, self.features[j], + level_str=level_str, + verbose=verbose) + level_images = feature_images + elif self.scale_features: + # scale features at other levels + level_images = scale_images(feature_images, s, + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(images, s, level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, + self.features[j], + level_str=level_str, + verbose=verbose) + + # extract potentially rescaled shapes + level_shapes = [i.landmarks[group][label] + for i in level_images] + + # obtain shape representation + if j == 0 or self.scale_shapes: + if verbose: + print_dynamic('{}Incrementing shape model'.format( + level_str)) + # compute aligned shapes + aligned_shapes = align_shapes(level_shapes) + # increment shape model + aam.shape_models[j].increment( + aligned_shapes, forgetting_factor=forgetting_factor) + if self.max_shape_components is not None: + aam.shape_models[j].trim_components( + self.max_appearance_components[j]) + else: + # copy previous shape model + aam.shape_models[j] = deepcopy(aam.shape_models[j-1]) + + mean_shape = aam.appearance_models[j].mean().landmarks[ + 'source'].lms + + # obtain warped images + warped_images = self._warp_images(level_images, level_shapes, + mean_shape, j, + level_str, verbose) + + # obtain appearance representation + if verbose: + print_dynamic('{}Incrementing appearance model'.format( + level_str)) + # increment appearance model + aam.appearance_models[j].increment(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + aam.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + def build_incrementally(self, images, group=None, label=None, + forgetting_factor=1.0, batch_size=100, + verbose=False): + # number of batches + n_batches = np.int(np.ceil(len(images) / batch_size)) + + # train first batch + print 'Training batch 1.' + aam = self.build(images[:batch_size], group=group, label=label, + verbose=verbose) + + # train all other batches + start = batch_size + for j in range(1, n_batches): + print 'Training batch {}.'.format(j+1) + end = start + batch_size + self.increment(aam, images[start:end], group=group, label=label, + forgetting_factor=forgetting_factor, + verbose=verbose) + start = end + + return aam + @classmethod def _build_shape_model(cls, shapes, max_components, level): return build_shape_model(shapes, max_components=max_components) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a985331..a8fbc9c 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -56,13 +56,13 @@ class LucasKanadeAAMFitter(AAMFitter): def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=None, n_appearance=None, sampling=None, **kwargs): self._model = aam - self.algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): + self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, sampling)): @@ -113,7 +113,6 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, n_shape=None, n_appearance=None, sampling=None, n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): self._model = aam - self.algorithms = [] self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) sampling = checks.check_sampling(sampling, self.n_levels) @@ -123,6 +122,7 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, self._set_up(sd_algorithm_cls, sampling, **kwargs) def _set_up(self, sd_algorithm_cls, sampling, **kwargs): + self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, sampling)): diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 8dbb453..4a5ce71 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -16,7 +16,6 @@ class LucasKanadeATMFitter(ModelFitter): def __init__(self, atm, algorithm_cls=InverseCompositional, n_shape=None, sampling=None, **kwargs): self._model = atm - self.algorithms = [] self._check_n_shape(n_shape) self._set_up(algorithm_cls, sampling, **kwargs) @@ -25,6 +24,7 @@ def atm(self): return self._model def _set_up(self, algorithm_cls, sampling, **kwargs): + self.algorithms = [] for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, self.atm.shape_models)): From 89ec1a0985c5803643c2709c1f022bf8aa3a5b3e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 10:05:05 +0100 Subject: [PATCH 294/423] Small change in SDMFiter --- menpofit/sdm/fitter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 5dd2994..99a6530 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -133,6 +133,7 @@ def increment(self, images, group=None, label=None, verbose=False, def train_incrementally(self, images, group=None, label=None, batch_size=100, verbose=False, **kwargs): + # number of batches n_batches = np.int(np.ceil(len(images) / batch_size)) # train first batch From 6bdf976131d37b074d407503ab8171cca3c898d3 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 16:09:21 +0100 Subject: [PATCH 295/423] Add initialization from user provided shapes and bounding boxes. - Allows users to define their own perturb_from_shape and perturb_from_bounding_box functions - Fixes typo in GaussNewton --- menpofit/fitter.py | 12 ++++ menpofit/sdm/algorithm.py | 3 +- menpofit/sdm/fitter.py | 131 +++++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 38 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index c8fb9c8..848203d 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -366,6 +366,18 @@ def noisy_target_alignment_transform(source, target, return alignment_transform_cls(source, noisy_target, **kwargs) +def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05): + transform = noisy_params_alignment_similarity( + shape.bounding_box(), bounding_box, noise_std=noise_std) + return transform.apply(shape) + + +def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05): + transform = noisy_params_alignment_similarity( + reference_shape, shape, noise_std=noise_std) + return transform.apply(reference_shape) + + def align_shape_with_bounding_box(shape, bounding_box, alignment_transform_cls=AlignmentSimilarity, **kwargs): diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 951cd97..fe76cd4 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -149,7 +149,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._perform_regression = _incremental_indirect_least_squares + self._regressor_cls = _incremental_indirect_least_squares # TODO: document me! @@ -331,6 +331,7 @@ def compute_features_info(image, shape, features_callable, return (features_patch_shape, features_patch_length, features_shape, features_length) + # def initialize_sampling(self, image, group=None, label=None): # if self._sampling is None: # sampling = np.ones(self.patch_shape, dtype=np.bool) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 99a6530..3e9bb1b 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,11 +1,14 @@ from __future__ import division import numpy as np -from menpo.transform import Scale, AlignmentSimilarity +import warnings +from menpo.transform import Scale from menpo.feature import no_op from menpofit.builder import ( normalization_wrt_reference_shape, rescale_images_to_reference_shape, scale_images) -from menpofit.fitter import MultiFitter, noisy_params_alignment_similarity +from menpofit.fitter import ( + MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, + align_shape_with_bounding_box) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -17,7 +20,10 @@ class SupervisedDescentFitter(MultiFitter): """ def __init__(self, sd_algorithm_cls=Newton, features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), - iterations=6, n_perturbations=10, noise_std=0.05, **kwargs): + iterations=6, n_perturbations=30, + perturb_from_shape=noisy_shape_from_shape, + perturb_from_bounding_box=noisy_shape_from_bounding_box, + **kwargs): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -27,15 +33,12 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations - self.noise_std = noise_std self.iterations = checks.check_max_iters(iterations, n_levels) + self._perturb_from_shape = perturb_from_shape + self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._set_up(sd_algorithm_cls, features, patch_shape, **kwargs) - @property - def reference_bounding_box(self): - return self.reference_shape.bounding_box() - def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): self.algorithms = [] for j in range(self.n_levels): @@ -44,11 +47,42 @@ def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): iterations=self.iterations[j], **kwargs) self.algorithms.append(algorithm) - def train(self, images, group=None, label=None, verbose=False, **kwargs): + def perturb_from_shape(self, shape, **kwargs): + return self._perturb_from_shape(self.reference_shape, shape, **kwargs) + + def perturb_from_bounding_box(self, bounding_box, **kwargs): + return self._perturb_from_bounding_box(self.reference_shape, + bounding_box, **kwargs) + + def train(self, images, group=None, label=None, + perturbation_group=None, verbose=False, **kwargs): # normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) + # handle perturbations + if perturbation_group is None: + perturbation_group = 'perturbed_' + # generate perturbations by perturbing ground truth shapes + for i in images: + gt_s = i.landmarks[group][label] + for j in range(self.n_perturbations): + p_s = self.perturb_from_shape(gt_s) + p_group = perturbation_group + '{}'.format(j) + i.landmarks[p_group] = p_s + else: + # reset number of perturbations + n_perturbations = 0 + for k in images[0].landmarks.keys(): + if perturbation_group in k: + n_perturbations += 1 + if n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided initialization_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + # for each pyramid level (low --> high) for j in range(self.n_levels): if verbose: @@ -65,17 +99,21 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - # generate perturbed shapes + # extract perturbations at the very bottom level current_shapes = [] - for gt_s in level_gt_shapes: - perturbed_shapes = [] - for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape( - gt_s, noise_std=self.noise_std) - perturbed_shapes.append(p_s) - current_shapes.append(perturbed_shapes) - - # train cascaded regression algorithm + for i in level_images: + c_shapes = [] + for k in range(self.n_perturbations): + p_group = perturbation_group + '{}'.format(k) + c_s = i.landmarks[p_group].lms + if c_s.n_points != level_gt_shapes[0].n_points: + # assume c_s is bounding box + c_s = align_shape_with_bounding_box( + self.reference_shape, c_s) + c_shapes.append(c_s) + current_shapes.append(c_shapes) + + # train supervised descent algorithm current_shapes = self.algorithms[j].train( level_images, level_gt_shapes, current_shapes, verbose=verbose, **kwargs) @@ -87,12 +125,36 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): for shape in image_shapes: transform.apply_inplace(shape) - def increment(self, images, group=None, label=None, verbose=False, + def increment(self, images, group=None, label=None, + perturbation_group=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( images, group, label, self.reference_shape, verbose=verbose) + # handle perturbations + if perturbation_group is None: + perturbation_group = 'perturbed_' + # generate perturbations by perturbing ground truth shapes + for i in images: + gt_s = i.landmarks[group][label] + for j in range(self.n_perturbations): + p_s = self.perturb_from_shape(gt_s) + p_group = perturbation_group + '{}'.format(j) + i.landmarks[p_group] = p_s + else: + # reset number of perturbations + n_perturbations = 0 + for k in images[0].landmarks.keys(): + if perturbation_group in k: + n_perturbations += 1 + if n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided initialization_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + # for each pyramid level (low --> high) for j in range(self.n_levels): if verbose: @@ -109,15 +171,19 @@ def increment(self, images, group=None, label=None, verbose=False, level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - # generate perturbed shapes + # extract perturbations at the very bottom level current_shapes = [] - for gt_s in level_gt_shapes: - perturbed_shapes = [] - for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape( - gt_s, noise_std=self.noise_std) - perturbed_shapes.append(p_s) - current_shapes.append(perturbed_shapes) + for i in level_images: + c_shapes = [] + for k in range(self.n_perturbations): + p_group = perturbation_group + '{}'.format(k) + c_s = i.landmarks[p_group].lms + if c_s.n_points != level_gt_shapes[0].n_points: + # assume c_s is bounding box + c_s = align_shape_with_bounding_box( + self.reference_shape, c_s) + c_shapes.append(c_s) + current_shapes.append(c_shapes) # train cascaded regression algorithm current_shapes = self.algorithms[j].increment( @@ -234,15 +300,6 @@ def _fitter_result(self, image, algorithm_results, affine_correction, return MultiFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): - transform = noisy_params_alignment_similarity( - self.reference_bounding_box, bounding_box, noise_std=noise_std) - return transform.apply(self.reference_shape) - - def noisy_shape_from_shape(self, shape, noise_std=0.05): - return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std) - # TODO: fix me! def __str__(self): pass From e96964663cb94a424eaba9ca8e8687d626293967 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 18:43:23 +0100 Subject: [PATCH 296/423] Add new math subpackage - At the moment this subpackage contains regression techniques used by sdm. - This package might be moved to menpo core in the long run. - Fixes small bugs on fitting results and fitters. --- menpofit/fitter.py | 4 +-- menpofit/math/__init__.py | 2 ++ menpofit/math/least_squares.py | 60 ++++++++++++++++++++++++++++++++++ menpofit/result.py | 10 +++--- menpofit/sdm/algorithm.py | 56 +++---------------------------- 5 files changed, 73 insertions(+), 59 deletions(-) create mode 100644 menpofit/math/__init__.py create mode 100644 menpofit/math/least_squares.py diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 848203d..231dba8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -218,8 +218,8 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, shape = algorithm_result.final_shape if s != self.scales[-1]: - Scale(self.scales[j+1]/s, - n_dims=shape.n_dims).apply_inplace(shape) + shape = Scale(self.scales[j+1]/s, + n_dims=shape.n_dims).apply(shape) return algorithm_results diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py new file mode 100644 index 0000000..1fec9a8 --- /dev/null +++ b/menpofit/math/__init__.py @@ -0,0 +1,2 @@ +from least_squares import ( + incremental_least_squares, incremental_indirect_least_squares) \ No newline at end of file diff --git a/menpofit/math/least_squares.py b/menpofit/math/least_squares.py new file mode 100644 index 0000000..4f6cbf2 --- /dev/null +++ b/menpofit/math/least_squares.py @@ -0,0 +1,60 @@ +import numpy as np + + +# TODO: document me! +class incremental_least_squares(object): + r""" + """ + def __init__(self, l=0): + self.l = l + + def train(self, X, Y): + # regularized least squares + XX = X.T.dot(X) + np.fill_diagonal(XX, self.l + np.diag(XX)) + self.V = np.linalg.inv(XX) + self.W = self.V.dot(X.T.dot(Y)) + + def increment(self, X, Y): + # incremental regularized least squares + U = X.dot(self.V).dot(X.T) + np.fill_diagonal(U, 1 + np.diag(U)) + U = np.linalg.inv(U) + Q = self.V.dot(X.T).dot(U).dot(X) + self.V = self.V - Q.dot(self.V) + self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) + + def predict(self, x): + return np.dot(x, self.W) + + +# TODO: document me! +class incremental_indirect_least_squares(object): + r""" + """ + def __init__(self, l=0, d=0): + self._ils = incremental_least_squares(l) + self.d = d + + def train(self, X, Y): + # regularized least squares exchanging the roles of X and Y + self._ils.train(Y, X) + J = self._ils.W + # solve the original problem by computing the pseudo-inverse of the + # previous solution + H = J.T.dot(J) + np.fill_diagonal(H, self.d + np.diag(H)) + self.W = np.linalg.solve(H, J.T) + + def increment(self, X, Y): + # incremental least squares exchanging the roles of X and Y + self._ils.increment(Y, X) + J = self._ils.W + # solve the original problem by computing the pseudo-inverse of the + # previous solution + H = J.T.dot(J) + np.fill_diagonal(H, self.d + np.diag(H)) + self.W = np.linalg.solve(H, J.T) + + def predict(self, x): + return np.dot(x, self.W) \ No newline at end of file diff --git a/menpofit/result.py b/menpofit/result.py index 335a51b..81d12ba 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -595,11 +595,11 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): r""" """ shapes = [] - for j, (alg, s) in enumerate(zip(algorithm_results, scales)): - transform = Scale(scales[-1]/s, alg.final_shape.n_dims) - for t in alg.shapes: - t = transform.apply(t) - shapes.append(affine_correction.apply(t)) + for j, (alg, scale) in enumerate(zip(algorithm_results, scales)): + transform = Scale(scales[-1]/scale, alg.final_shape.n_dims) + for shape in alg.shapes: + shape = transform.apply(shape) + shapes.append(affine_correction.apply(shape)) return shapes diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index fe76cd4..d9e66c6 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -3,7 +3,8 @@ from menpo.feature import no_op from menpo.visualize import print_dynamic from menpofit.result import NonParametricAlgorithmResult - +from menpofit.math import ( + incremental_least_squares, incremental_indirect_least_squares) # TODO: compute more meaningful error # TODO: document me! @@ -135,7 +136,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = _incremental_least_squares + self._regressor_cls = incremental_least_squares # TODO: document me! @@ -149,56 +150,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = _incremental_indirect_least_squares - - -# TODO: document me! -class _incremental_least_squares(object): - r""" - """ - def __init__(self, l=0): - self.l = l - - def train(self, X, Y): - # regularized least squares - XX = X.T.dot(X) - np.fill_diagonal(XX, self.l + np.diag(XX)) - self.V = np.linalg.inv(XX) - self.W = self.V.dot(X.T.dot(Y)) - - def increment(self, X, Y): - # incremental regularized least squares - U = X.dot(self.V).dot(X.T) - np.fill_diagonal(U, 1 + np.diag(U)) - U = np.linalg.inv(U) - Q = self.V.dot(X.T).dot(U).dot(X) - self.V = self.V - Q.dot(self.V) - self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) - - def predict(self, x): - return np.dot(x, self.W) - - -# TODO: document me! -class _incremental_indirect_least_squares(object): - r""" - """ - def __init__(self, l=0, d=0): - self._ils = _incremental_least_squares(l) - self.d = d - - def train(self, X, Y): - # regularized least squares exchanging the roles of X and Y - self._ils.train(Y, X) - J = self._ils.W - # solve the original problem by computing the pseudo-inverse of the - # previous solution - H = J.T.dot(J) - np.fill_diagonal(H, self.d + np.diag(H)) - self.W = np.linalg.solve(H, J.T) - - def predict(self, x): - return np.dot(x, self.W) + self._regressor_cls = incremental_indirect_least_squares # TODO: document me! From 7922f588014f8169c3c7f51d88635eb473d7e4d6 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 18:52:00 +0100 Subject: [PATCH 297/423] Small fix in result.py --- menpofit/result.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 81d12ba..2d346f0 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -558,8 +558,8 @@ def final_shape(self): @property def initial_shape(self): initial_shape = self.algorithm_results[0].initial_shape - Scale(self.scales[-1]/self.scales[0], - initial_shape.n_dims).apply_inplace(initial_shape) + initial_shape = Scale(self.scales[-1]/self.scales[0], + initial_shape.n_dims).apply_inplace(initial_shape) return self._affine_correction.apply(initial_shape) From 134f9cd857ecc56c82793e5a1289145149646b1c Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 6 Jul 2015 18:58:41 +0100 Subject: [PATCH 298/423] Fix for results.py --- menpofit/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/result.py b/menpofit/result.py index 2d346f0..4306185 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -559,7 +559,7 @@ def final_shape(self): def initial_shape(self): initial_shape = self.algorithm_results[0].initial_shape initial_shape = Scale(self.scales[-1]/self.scales[0], - initial_shape.n_dims).apply_inplace(initial_shape) + initial_shape.n_dims).apply(initial_shape) return self._affine_correction.apply(initial_shape) From 2551e901267449ad3a23dc8d4ab5140834873f46 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:26:53 +0100 Subject: [PATCH 299/423] Update errors in results. - results now receive a function to compute errors. - previous error funtions are slightly modified. --- menpofit/result.py | 96 +++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 4306185..2f2e1de 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -2,6 +2,7 @@ import abc import numpy as np from menpo.transform import Scale +from menpo.shape import PointCloud from menpo.image import Image @@ -48,44 +49,48 @@ def fitted_image(self): image.landmarks['ground'] = self.gt_shape return image - def final_error(self, error_type='me_norm'): + def final_error(self, compute_error=None): r""" Returns the final fitting error. Parameters ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. + compute_error: `callable`, optional + Callable that computes the error between the fitted and + ground truth shapes. Returns ------- final_error : `float` The final error at the end of the fitting procedure. """ + if compute_error is None: + compute_error = compute_normalise_point_to_point_error if self.gt_shape is not None: - return compute_error(self.final_shape, self.gt_shape, error_type) + return compute_error(self.final_shape, self.gt_shape) else: raise ValueError('Ground truth has not been set, final error ' 'cannot be computed') - def initial_error(self, error_type='me_norm'): + def initial_error(self, compute_error=None): r""" Returns the initial fitting error. Parameters ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. + compute_error: `callable`, optional + Callable that computes the error between the fitted and + ground truth shapes. Returns ------- initial_error : `float` The initial error at the start of the fitting procedure. """ + if compute_error is None: + compute_error = compute_normalise_point_to_point_error if self.gt_shape is not None: - return compute_error(self.initial_shape, self.gt_shape, error_type) + return compute_error(self.initial_shape, self.gt_shape) else: raise ValueError('Ground truth has not been set, final error ' 'cannot be computed') @@ -141,23 +146,25 @@ def iter_image(self): image.landmarks['iter_'+str(j)] = s return image - def errors(self, error_type='me_norm'): + def errors(self, compute_error=None): r""" Returns a list containing the error at each fitting iteration. Parameters ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. + compute_error: `callable`, optional + Callable that computes the error between the fitted and + ground truth shapes. Returns ------- errors : `list` of `float` The errors at each iteration of the fitting process. """ + if compute_error is None: + compute_error = compute_normalise_point_to_point_error if self.gt_shape is not None: - return [compute_error(t, self.gt_shape, error_type) + return [compute_error(t, self.gt_shape) for t in self.shapes] else: raise ValueError('Ground truth has not been set, errors cannot ' @@ -604,48 +611,57 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): # TODO: Document me! -def compute_error(target, ground_truth, error_type='me_norm'): +def pointcloud_to_points(func): + def func_wrapper(*args, **kwargs): + args = list(args) + for index, arg in enumerate(args): + if isinstance(arg, PointCloud): + args[index] = arg.points + for key in kwargs: + if isinstance(kwargs[key], PointCloud): + kwargs[key] = kwargs[key].points + return func(*args, **kwargs) + return func_wrapper + + +# TODO: Document me! +@pointcloud_to_points +def compute_root_mean_square_error(shape, gt_shape): r""" """ - gt_points = ground_truth.points - target_points = target.points - - if error_type == 'me_norm': - return _compute_norm_p2p_error(target_points, gt_points) - elif error_type == 'me': - return _compute_me(target_points, gt_points) - elif error_type == 'rmse': - return _compute_rmse(target_points, gt_points) - else: - raise ValueError("Unknown error_type string selected. Valid options " - "are: me_norm, me, rmse'") + return np.sqrt(np.mean((shape.flatten() - gt_shape.flatten()) ** 2)) # TODO: Document me! -# TODO: rename to more descriptive name -def _compute_me(target, ground_truth): +@pointcloud_to_points +def compute_point_to_point_error(shape, gt_shape): r""" """ - return np.mean(np.sqrt(np.sum((target - ground_truth) ** 2, axis=-1))) + return np.mean(np.sqrt(np.sum((shape - gt_shape) ** 2, axis=-1))) # TODO: Document me! -# TODO: rename to more descriptive name -def _compute_rmse(target, ground_truth): +@pointcloud_to_points +def compute_normalise_root_mean_square_error(shape, gt_shape, norm_shape=None): r""" """ - return np.sqrt(np.mean((target.flatten() - ground_truth.flatten()) ** 2)) + if norm_shape is None: + norm_shape = gt_shape + normalizer = np.mean(np.max(norm_shape, axis=0) - + np.min(norm_shape, axis=0)) + return compute_root_mean_square_error(shape, gt_shape) / normalizer # TODO: Document me! -def _compute_norm_p2p_error(target, source, ground_truth=None): +@pointcloud_to_points +def compute_normalise_point_to_point_error(shape, gt_shape, norm_shape=None): r""" """ - if ground_truth is None: - ground_truth = source - normalizer = np.mean(np.max(ground_truth, axis=0) - - np.min(ground_truth, axis=0)) - return _compute_me(target, source) / normalizer + if norm_shape is None: + norm_shape = gt_shape + normalizer = np.mean(np.max(norm_shape, axis=0) - + np.min(norm_shape, axis=0)) + return compute_point_to_point_error(shape, gt_shape) / normalizer # TODO: Document me! From fac06d565f36b0129fb463f9ee7ec8c8fd84c1e5 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:30:22 +0100 Subject: [PATCH 300/423] Update sdm algorithms to print more informative errors. - algorithms (optionally) receive a function to compute the training error. --- menpofit/sdm/algorithm.py | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index d9e66c6..2be9d1d 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -2,17 +2,21 @@ import numpy as np from menpo.feature import no_op from menpo.visualize import print_dynamic -from menpofit.result import NonParametricAlgorithmResult +from menpofit.result import ( + NonParametricAlgorithmResult, compute_normalise_point_to_point_error) from menpofit.math import ( incremental_least_squares, incremental_indirect_least_squares) -# TODO: compute more meaningful error + # TODO: document me! class SupervisedDescentAlgorithm(object): r""" """ def train(self, images, gt_shapes, current_shapes, verbose=False, **kwargs): + + n_perturbations = len(current_shapes[0]) + template_shape = gt_shapes[0] self._features_patch_length = compute_features_info( images[0], gt_shapes[0], self.features, patch_shape=self.patch_shape)[1] @@ -42,8 +46,18 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, # estimate delta_points estimated_delta_x = r.predict(features) if verbose: - error = _compute_rmse(delta_x, estimated_delta_x) - print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + s1 = template_shape.from_vector(dx) + s2 = template_shape.from_vector(edx) + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('- Training error -> mean: {0:.4f}, ' + 'std: {1:.4f}, median: {2:.4f}.\n'. + format(mean, std, median)) j = 0 for shapes in current_shapes: @@ -64,6 +78,10 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, def increment(self, images, gt_shapes, current_shapes, verbose=False, **kwarg): + + n_perturbations = len(current_shapes[0]) + template_shape = gt_shapes[0] + # obtain delta_x and gt_x delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) @@ -82,8 +100,18 @@ def increment(self, images, gt_shapes, current_shapes, verbose=False, # estimate delta_points estimated_delta_x = r.predict(features) if verbose: - error = _compute_rmse(delta_x, estimated_delta_x) - print_dynamic('- Training Error is {0:.4f}.\n'.format(error)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + s1 = template_shape.from_vector(dx) + s2 = template_shape.from_vector(edx) + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('- Training error -> mean: {0:.4f}, ' + 'std: {1:.4f}, median: {2:.4f}.\n'. + format(mean, std, median)) j = 0 for shapes in current_shapes: @@ -130,13 +158,15 @@ class Newton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5): + eps=10**-5, + compute_error=compute_normalise_point_to_point_error): + self._regressor_cls = incremental_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = incremental_least_squares + self._compute_error = compute_error # TODO: document me! @@ -144,18 +174,15 @@ class GaussNewton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5): + eps=10**-5, + compute_error=compute_normalise_point_to_point_error): + self._regressor_cls = incremental_indirect_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations self.eps = eps - self._regressor_cls = incremental_indirect_least_squares - - -# TODO: document me! -def _compute_rmse(x1, x2): - return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + self._compute_error = compute_error # TODO: docment me! From 3bea70866d84c7c4f0cf79236aa1808e24ea4bb0 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:38:30 +0100 Subject: [PATCH 301/423] Add the ability to train sdmfitters using a single bounding box per image. --- menpofit/sdm/algorithm.py | 13 +++++----- menpofit/sdm/fitter.py | 54 +++++++++++++++++++++++++++------------ 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 2be9d1d..1f711e9 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -158,31 +158,30 @@ class Newton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5, - compute_error=compute_normalise_point_to_point_error): + compute_error=compute_normalise_point_to_point_error, + eps=10**-5): self._regressor_cls = incremental_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations - self.eps = eps self._compute_error = compute_error - + self.eps = eps # TODO: document me! class GaussNewton(SupervisedDescentAlgorithm): r""" """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, - eps=10**-5, - compute_error=compute_normalise_point_to_point_error): + compute_error=compute_normalise_point_to_point_error, + eps=10**-5): self._regressor_cls = incremental_indirect_least_squares self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape self.iterations = iterations - self.eps = eps self._compute_error = compute_error + self.eps = eps # TODO: docment me! diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 3e9bb1b..9f71488 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -54,32 +54,42 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): return self._perturb_from_bounding_box(self.reference_shape, bounding_box, **kwargs) - def train(self, images, group=None, label=None, - perturbation_group=None, verbose=False, **kwargs): + def train(self, images, group=None, label=None, bounding_box_group=None, + verbose=False, **kwargs): # normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) # handle perturbations - if perturbation_group is None: - perturbation_group = 'perturbed_' + if bounding_box_group is None: + bounding_box_group = 'bb_' # generate perturbations by perturbing ground truth shapes for i in images: gt_s = i.landmarks[group][label] for j in range(self.n_perturbations): p_s = self.perturb_from_shape(gt_s) - p_group = perturbation_group + '{}'.format(j) + p_group = bounding_box_group + '{}'.format(j) i.landmarks[p_group] = p_s else: # reset number of perturbations n_perturbations = 0 for k in images[0].landmarks.keys(): - if perturbation_group in k: + if bounding_box_group in k: n_perturbations += 1 - if n_perturbations != self.n_perturbations: + if n_perturbations == 1: + for i in images: + bb = i.landmarks[bounding_box_group].lms + p_s = align_shape_with_bounding_box( + self.reference_shape, bb) + i.landmarks[bounding_box_group + '0'] = p_s + for j in range(1, self.n_perturbations): + p_s = self.perturb_from_bounding_box(bb) + p_group = bounding_box_group + '{}'.format(j) + i.landmarks[p_group] = p_s + elif n_perturbations != self.n_perturbations: warnings.warn('The original value of n_perturbation {} ' 'will be reset to {} in order to agree with ' - 'the provided initialization_group.'. + 'the provided bounding_box_group.'. format(self.n_perturbations, n_perturbations)) self.n_perturbations = n_perturbations @@ -104,7 +114,7 @@ def train(self, images, group=None, label=None, for i in level_images: c_shapes = [] for k in range(self.n_perturbations): - p_group = perturbation_group + '{}'.format(k) + p_group = bounding_box_group + '{}'.format(k) c_s = i.landmarks[p_group].lms if c_s.n_points != level_gt_shapes[0].n_points: # assume c_s is bounding box @@ -126,32 +136,42 @@ def train(self, images, group=None, label=None, transform.apply_inplace(shape) def increment(self, images, group=None, label=None, - perturbation_group=None, verbose=False, + bounding_box_group=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( images, group, label, self.reference_shape, verbose=verbose) # handle perturbations - if perturbation_group is None: - perturbation_group = 'perturbed_' + if bounding_box_group is None: + bounding_box_group = 'bb_' # generate perturbations by perturbing ground truth shapes for i in images: gt_s = i.landmarks[group][label] for j in range(self.n_perturbations): p_s = self.perturb_from_shape(gt_s) - p_group = perturbation_group + '{}'.format(j) + p_group = bounding_box_group + '{}'.format(j) i.landmarks[p_group] = p_s else: # reset number of perturbations n_perturbations = 0 for k in images[0].landmarks.keys(): - if perturbation_group in k: + if bounding_box_group in k: n_perturbations += 1 - if n_perturbations != self.n_perturbations: + if n_perturbations == 1: + for i in images: + bb = i.landmarks[bounding_box_group].lms + p_s = align_shape_with_bounding_box( + self.reference_shape, bb) + i.landmarks[bounding_box_group + '0'] = p_s + for j in range(1, self.n_perturbations): + p_s = self.perturb_from_bounding_box(bb) + p_group = bounding_box_group + '{}'.format(j) + i.landmarks[p_group] = p_s + elif n_perturbations != self.n_perturbations: warnings.warn('The original value of n_perturbation {} ' 'will be reset to {} in order to agree with ' - 'the provided initialization_group.'. + 'the provided bounding_box_group.'. format(self.n_perturbations, n_perturbations)) self.n_perturbations = n_perturbations @@ -176,7 +196,7 @@ def increment(self, images, group=None, label=None, for i in level_images: c_shapes = [] for k in range(self.n_perturbations): - p_group = perturbation_group + '{}'.format(k) + p_group = bounding_box_group + '{}'.format(k) c_s = i.landmarks[p_group].lms if c_s.n_points != level_gt_shapes[0].n_points: # assume c_s is bounding box From 7748f8d2c2c3afcfd99a73b3c1db476e0ec38949 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 7 Jul 2015 15:39:21 +0100 Subject: [PATCH 302/423] Update widget to play well with changes on error computation - This is a temporal fix. --- menpofit/visualize/widgets/base.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 8dd5b22..cb634bf 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -2448,11 +2448,20 @@ def update_info(name, value): # Create output str if fitting_results[im].gt_shape is not None: + from menpofit.result import ( + compute_root_mean_square_error, compute_point_to_point_error, + compute_normalise_point_to_point_error) + if value is 'me_norm': + func = compute_normalise_point_to_point_error + elif value is 'me': + func = compute_point_to_point_error + elif value is 'rmse': + func = compute_root_mean_square_error text_per_line = [ "> Initial error: {:.4f}".format( - fitting_results[im].initial_error(error_type=value)), + fitting_results[im].initial_error(compute_error=func)), "> Final error: {:.4f}".format( - fitting_results[im].final_error(error_type=value)), + fitting_results[im].final_error(compute_error=func)), "> {} iterations".format(fitting_results[im].n_iters)] else: text_per_line = [ @@ -2511,10 +2520,20 @@ def plot_ced_fun(name): # Get error type error_type = error_type_wid.value + from menpofit.result import ( + compute_root_mean_square_error, compute_point_to_point_error, + compute_normalise_point_to_point_error) + if error_type is 'me_norm': + func = compute_normalise_point_to_point_error + elif error_type is 'me': + func = compute_point_to_point_error + elif error_type is 'rmse': + func = compute_root_mean_square_error + # Create errors list - fit_errors = [f.final_error(error_type=error_type) + fit_errors = [f.final_error(compute_error=func) for f in fitting_results] - initial_errors = [f.initial_error(error_type=error_type) + initial_errors = [f.initial_error(compute_error=func) for f in fitting_results] errors = [fit_errors, initial_errors] From ed743fd2b302af7a444fcd5931b53f0144316028 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 8 Jul 2015 15:32:58 +0100 Subject: [PATCH 303/423] Add back ability to not use rotation on initialization. --- menpofit/builder.py | 2 +- menpofit/fitter.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index b4cfa57..a309b4e 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -150,7 +150,7 @@ def scale_images(images, scale, level_str='', verbose=None): for c, i in enumerate(images): if verbose: print_dynamic( - '{}Scaling features: {}'.format( + '{}Scaling images: {}'.format( level_str, progress_bar_str((c + 1.) / len(images), show_bar=False))) scaled_images.append(i.rescale(scale)) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 231dba8..808bceb 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -295,7 +295,8 @@ def noisy_shape_from_shape(self, shape, noise_std=0.05): shape.bounding_box(), noise_std=noise_std) -def noisy_params_alignment_similarity(source, target, noise_std=0.05): +def noisy_params_alignment_similarity(source, target, noise_std=0.05, + rotation=True): r""" Constructs and perturbs the optimal similarity transform between source and target by adding white noise to its parameters. @@ -322,7 +323,7 @@ def noisy_params_alignment_similarity(source, target, noise_std=0.05): elif len(noise_std) == 1: noise_std *= 3 - transform = AlignmentSimilarity(source, target, rotation=True) + transform = AlignmentSimilarity(source, target, rotation=rotation) parameters = transform.as_vector() scale = noise_std[0] * parameters[0] @@ -366,15 +367,18 @@ def noisy_target_alignment_transform(source, target, return alignment_transform_cls(source, noisy_target, **kwargs) -def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05): +def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, + rotation=True): transform = noisy_params_alignment_similarity( - shape.bounding_box(), bounding_box, noise_std=noise_std) + shape.bounding_box(), bounding_box, noise_std=noise_std, + rotation=rotation) return transform.apply(shape) -def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05): +def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05, + rotation=True): transform = noisy_params_alignment_similarity( - reference_shape, shape, noise_std=noise_std) + reference_shape, shape, noise_std=noise_std, rotation=rotation) return transform.apply(reference_shape) From 62ac2036e743401a9c49aab6b3614a9cb3dad58a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 13 Jul 2015 14:24:09 +0100 Subject: [PATCH 304/423] Add the ability to perturb without rotation --- menpofit/fitter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 808bceb..94f814d 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -1,5 +1,6 @@ from __future__ import division import numpy as np +from copy import deepcopy from menpo.shape import PointCloud from menpo.transform import ( Scale, Similarity, AlignmentAffine, AlignmentSimilarity) @@ -296,7 +297,7 @@ def noisy_shape_from_shape(self, shape, noise_std=0.05): def noisy_params_alignment_similarity(source, target, noise_std=0.05, - rotation=True): + rotation=False): r""" Constructs and perturbs the optimal similarity transform between source and target by adding white noise to its parameters. @@ -324,7 +325,7 @@ def noisy_params_alignment_similarity(source, target, noise_std=0.05, noise_std *= 3 transform = AlignmentSimilarity(source, target, rotation=rotation) - parameters = transform.as_vector() + parameters = deepcopy(transform.as_vector()) scale = noise_std[0] * parameters[0] rotation = noise_std[1] * parameters[1] @@ -368,7 +369,7 @@ def noisy_target_alignment_transform(source, target, def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, - rotation=True): + rotation=False): transform = noisy_params_alignment_similarity( shape.bounding_box(), bounding_box, noise_std=noise_std, rotation=rotation) @@ -376,7 +377,7 @@ def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05, - rotation=True): + rotation=False): transform = noisy_params_alignment_similarity( reference_shape, shape, noise_std=noise_std, rotation=rotation) return transform.apply(reference_shape) @@ -405,4 +406,3 @@ def align_shape_with_bounding_box(shape, bounding_box, shape_bb = shape.bounding_box() transform = alignment_transform_cls(shape_bb, bounding_box, **kwargs) return transform.apply(shape) - From b899d5520785fb071c1bd350950acf45420dfa53 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 13 Jul 2015 14:27:30 +0100 Subject: [PATCH 305/423] Small refactoring of regression classes --- menpofit/math/__init__.py | 3 +- .../math/{least_squares.py => regression.py} | 43 ++++++++++++------- menpofit/sdm/algorithm.py | 8 ++-- 3 files changed, 33 insertions(+), 21 deletions(-) rename menpofit/math/{least_squares.py => regression.py} (54%) diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py index 1fec9a8..d916940 100644 --- a/menpofit/math/__init__.py +++ b/menpofit/math/__init__.py @@ -1,2 +1 @@ -from least_squares import ( - incremental_least_squares, incremental_indirect_least_squares) \ No newline at end of file +from regression import IRLRegression, IIRLRegression \ No newline at end of file diff --git a/menpofit/math/least_squares.py b/menpofit/math/regression.py similarity index 54% rename from menpofit/math/least_squares.py rename to menpofit/math/regression.py index 4f6cbf2..58db19f 100644 --- a/menpofit/math/least_squares.py +++ b/menpofit/math/regression.py @@ -2,21 +2,31 @@ # TODO: document me! -class incremental_least_squares(object): +class IRLRegression(object): r""" + Incremental Regularized Linear Regression """ - def __init__(self, l=0): + def __init__(self, l=0, bias=True): self.l = l + self.bias = bias def train(self, X, Y): - # regularized least squares + if self.bias: + # add bias + X = np.hstack((X, np.ones((X.shape[0], 1)))) + + # regularized linear regression XX = X.T.dot(X) np.fill_diagonal(XX, self.l + np.diag(XX)) self.V = np.linalg.inv(XX) self.W = self.V.dot(X.T.dot(Y)) def increment(self, X, Y): - # incremental regularized least squares + if self.bias: + # add bias + X = np.hstack((X, np.ones((X.shape[0], 1)))) + + # incremental regularized linear regression U = X.dot(self.V).dot(X.T) np.fill_diagonal(U, 1 + np.diag(U)) U = np.linalg.inv(U) @@ -25,21 +35,27 @@ def increment(self, X, Y): self.W = self.W - Q.dot(self.W) + self.V.dot(X.T.dot(Y)) def predict(self, x): + if self.bias: + if len(x.shape) == 1: + x = np.hstack((x, np.ones(1))) + else: + x = np.hstack((x, np.ones((x.shape[0], 1)))) return np.dot(x, self.W) # TODO: document me! -class incremental_indirect_least_squares(object): +class IIRLRegression(IRLRegression): r""" + Indirect Incremental Regularized Linear Regression """ - def __init__(self, l=0, d=0): - self._ils = incremental_least_squares(l) + def __init__(self, l=0, bias=True, d=0): + super(IIRLRegression, self).__init__(l=l, bias=bias) self.d = d def train(self, X, Y): - # regularized least squares exchanging the roles of X and Y - self._ils.train(Y, X) - J = self._ils.W + # regularized linear regression exchanging the roles of X and Y + super(IIRLRegression, self).train(Y, X) + J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) @@ -48,13 +64,10 @@ def train(self, X, Y): def increment(self, X, Y): # incremental least squares exchanging the roles of X and Y - self._ils.increment(Y, X) - J = self._ils.W + super(IIRLRegression, self).increment(Y, X) + J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) np.fill_diagonal(H, self.d + np.diag(H)) self.W = np.linalg.solve(H, J.T) - - def predict(self, x): - return np.dot(x, self.W) \ No newline at end of file diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 1f711e9..2c433ad 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -4,8 +4,7 @@ from menpo.visualize import print_dynamic from menpofit.result import ( NonParametricAlgorithmResult, compute_normalise_point_to_point_error) -from menpofit.math import ( - incremental_least_squares, incremental_indirect_least_squares) +from menpofit.math import IRLRegression, IIRLRegression # TODO: document me! @@ -160,7 +159,7 @@ class Newton(SupervisedDescentAlgorithm): def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5): - self._regressor_cls = incremental_least_squares + self._regressor_cls = IRLRegression self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape @@ -168,6 +167,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, self._compute_error = compute_error self.eps = eps + # TODO: document me! class GaussNewton(SupervisedDescentAlgorithm): r""" @@ -175,7 +175,7 @@ class GaussNewton(SupervisedDescentAlgorithm): def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5): - self._regressor_cls = incremental_indirect_least_squares + self._regressor_cls = IIRLRegression self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape From d9ba91020bc594fb4025d9f329e2ddfd3f794d5e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Wed, 15 Jul 2015 17:15:44 +0100 Subject: [PATCH 306/423] Fix LK --- menpofit/lk/algorithm.py | 10 +++++++--- menpofit/lk/residual.py | 12 ++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/menpofit/lk/algorithm.py b/menpofit/lk/algorithm.py index b296300..49bcd5b 100644 --- a/menpofit/lk/algorithm.py +++ b/menpofit/lk/algorithm.py @@ -40,6 +40,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): # compute warp jacobian dW_dp = np.rollaxis( self.transform.d_dp(self.template.indices()), -1) + dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + + dW_dp.shape[-1:]) # compute steepest descent images filtered_J, J = self.residual.steepest_descent_images( @@ -53,7 +55,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): filtered_J, IWxp, self.template) # compute gradient descent parameter updates - dp = np.real(np.linalg.solve(H, sd_dp)) + dp = -np.real(np.linalg.solve(H, sd_dp)) # Update warp weights self.transform.from_vector_inplace(self.transform.as_vector() + dp) @@ -85,8 +87,10 @@ def __init__(self, template, transform, residual, eps=10**-10): def _precompute(self): # compute warp jacobian - self.dW_dp = np.rollaxis( + dW_dp = np.rollaxis( self.transform.d_dp(self.template.indices()), -1) + self.dW_dp = dW_dp.reshape(dW_dp.shape[:1] + self.template.shape + + dW_dp.shape[-1:]) def run(self, image, initial_shape, max_iters=20, gt_shape=None): # initialize transform @@ -116,7 +120,7 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None): filtered_J, IWxp, self.template) # compute gradient descent parameter updates - dp = np.real(np.linalg.solve(H, sd_dp)) + dp = -np.real(np.linalg.solve(H, sd_dp)) # Update warp weights self.transform.compose_after_from_vector_inplace(dp) diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index 57b386b..d406c0f 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -147,7 +147,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # grad: dims x ch x h x w nabla = self.gradient(image, forward=forward) nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + - image.shape) + nabla.shape) # compute steepest descent images # gradient: dims x ch x h x w @@ -217,7 +217,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # grad: dims x ch x h x w nabla = self.gradient(image, forward=forward) nabla = nabla.as_vector().reshape((image.n_dims, image.n_channels) + - image.shape) + nabla.shape) # compute steepest descent images # gradient: dims x ch x h x w @@ -300,7 +300,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # gradient: dims x ch x pixels grad = self.gradient(norm_image, forward=forward) grad = grad.as_vector().reshape((image.n_dims, image.n_channels) + - image.shape) + grad.shape) # compute steepest descent images # gradient: dims x ch x pixels @@ -391,7 +391,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # second_grad: dims x dims x ch x pixels second_grad = self.gradient(self._template_grad) second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels) + image.shape) + (n_dims, n_dims, n_channels) + second_grad.shape) # Fix crossed derivatives: dydx = dxdy second_grad[1, 0, ...] = second_grad[0, 1, ...] @@ -454,7 +454,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # compute gradient # grad: dims x ch x pixels grad = self.gradient(image, forward=forward) - grad2 = grad.as_vector().reshape((n_dims, n_channels) + image.shape) + grad2 = grad.as_vector().reshape((n_dims, n_channels) + grad.shape) # compute IGOs (remember axis 0 is y, axis 1 is x) # grad: dims x ch x pixels @@ -479,7 +479,7 @@ def steepest_descent_images(self, image, dW_dp, forward=None): # second_grad: dims x dims x ch x pixels second_grad = self.gradient(grad) second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels) + image.shape) + (n_dims, n_dims, n_channels) + second_grad.shape) # Fix crossed derivatives: dydx = dxdy second_grad[1, 0, ...] = second_grad[0, 1, ...] From 86f54e4448d0ef3c3d57cbabc5ca300a48dc474f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 20 Jul 2015 08:28:37 +0100 Subject: [PATCH 307/423] Add new perturbation procedure. -Perturbations are can be now generated from uniform and gaussian distributions. --- menpofit/fitter.py | 90 +++++++++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 94f814d..ab69006 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -3,6 +3,7 @@ from copy import deepcopy from menpo.shape import PointCloud from menpo.transform import ( + scale_about_centre, rotate_ccw_about_centre, Translation, Scale, Similarity, AlignmentAffine, AlignmentSimilarity) import menpofit.checks as checks @@ -287,7 +288,7 @@ def _check_n_shape(self, n_shape): 'those'.format(self._model.n_levels)) def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): - transform = noisy_params_alignment_similarity( + transform = noisy_alignment_similarity_transform( self.reference_bounding_box, bounding_box, noise_std=noise_std) return transform.apply(self.reference_shape) @@ -296,45 +297,67 @@ def noisy_shape_from_shape(self, shape, noise_std=0.05): shape.bounding_box(), noise_std=noise_std) -def noisy_params_alignment_similarity(source, target, noise_std=0.05, - rotation=False): +def noisy_alignment_similarity_transform(source, target, noise_type='uniform', + noise_percentage=0.1, rotation=False): r""" Constructs and perturbs the optimal similarity transform between source - and target by adding white noise to its parameters. + and target by adding noise to its parameters. + Parameters ---------- source: :class:`menpo.shape.PointCloud` The source pointcloud instance used in the alignment target: :class:`menpo.shape.PointCloud` The target pointcloud instance used in the alignment - noise_std: float or triplet of floats, optional - The standard deviation of the white noise. If float the same amount + noise_type: str, optional + The type of noise to be added, 'uniform' or 'gaussian'. + noise_percentage: 0 < float < 1 or triplet of 0 < float < 1, optional + The standard percentage of noise to be added. If float the same amount of noise is applied to the scale, rotation and translation parameters of the true similarity transform. If triplet of floats, the first, second and third elements denote the amount of noise to be applied to the scale, rotation and translation parameters respectively. + rotation: boolean, optional + If False rotation is not considered when computing the optimal + similarity transform between source and target. + Returns ------- - noisy_transform : :class: `menpo.transform.Similarity` - The noisy Similarity Transform + noisy_alignment_similarity_transform : :class: `menpo.transform.Similarity` + The noisy Similarity Transform between source and target. """ - if isinstance(noise_std, float): - noise_std = [noise_std] * 3 - elif len(noise_std) == 1: - noise_std *= 3 - - transform = AlignmentSimilarity(source, target, rotation=rotation) - parameters = deepcopy(transform.as_vector()) - - scale = noise_std[0] * parameters[0] - rotation = noise_std[1] * parameters[1] - translation = noise_std[2] * target.range() - - noise = (([scale, rotation] + list(translation)) * - np.random.randn(transform.n_parameters)) - return Similarity.init_identity(source.n_dims).from_vector( - parameters + noise) + if isinstance(noise_percentage, float): + noise_percentage= [noise_percentage] * 3 + elif len(noise_percentage) == 1: + noise_percentage *= 3 + + similarity = AlignmentSimilarity(source, target, rotation=rotation) + + if noise_type is 'normal': + # + s = noise_percentage[0] * (0.5 / 3) * np.asscalar(np.random.randn(1)) + # + r = noise_percentage[1] * (180 / 3) * np.asscalar(np.random.randn(1)) + # + t = noise_percentage[2] * (target.range() / 3) * np.random.randn(2) + + s = scale_about_centre(target, 1 + s) + r = rotate_ccw_about_centre(target, r) + t = Translation(t, source.n_dims) + elif noise_type is 'uniform': + # + s = noise_percentage[0] * 0.5 * (2 * np.asscalar(np.random.randn(1)) - 1) + # + r = noise_percentage[1] * 180 * (2 * np.asscalar(np.random.rand(1)) - 1) + # + t = noise_percentage[2] * target.range() * (2 * np.random.rand(2) - 1) + + s = scale_about_centre(target, 1. + s) + r = rotate_ccw_about_centre(target, r) + t = Translation(t, source.n_dims) + + return similarity.compose_after(t.compose_after(s.compose_after(r))) def noisy_target_alignment_transform(source, target, @@ -368,18 +391,19 @@ def noisy_target_alignment_transform(source, target, return alignment_transform_cls(source, noisy_target, **kwargs) -def noisy_shape_from_bounding_box(shape, bounding_box, noise_std=0.05, - rotation=False): - transform = noisy_params_alignment_similarity( - shape.bounding_box(), bounding_box, noise_std=noise_std, - rotation=rotation) +def noisy_shape_from_bounding_box(shape, bounding_box, noise_type='uniform', + noise_percentage=0.1, rotation=False): + transform = noisy_alignment_similarity_transform( + shape.bounding_box(), bounding_box, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) return transform.apply(shape) -def noisy_shape_from_shape(reference_shape, shape, noise_std=0.05, - rotation=False): - transform = noisy_params_alignment_similarity( - reference_shape, shape, noise_std=noise_std, rotation=rotation) +def noisy_shape_from_shape(reference_shape, shape, noise_type='uniform', + noise_percentage=0.1, rotation=False): + transform = noisy_alignment_similarity_transform( + reference_shape, shape, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) return transform.apply(reference_shape) From d00174f4522707506e5ed902f8373d6465c5265f Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 16:19:43 +0100 Subject: [PATCH 308/423] Fix pickling of SDM --- menpofit/result.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index 2f2e1de..f19c516 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -1,5 +1,6 @@ from __future__ import division import abc +from functools import wraps import numpy as np from menpo.transform import Scale from menpo.shape import PointCloud @@ -611,8 +612,10 @@ def _rescale_shapes_to_reference(algorithm_results, scales, affine_correction): # TODO: Document me! -def pointcloud_to_points(func): - def func_wrapper(*args, **kwargs): +def pointcloud_to_points(wrapped): + + @wraps(wrapped) + def wrapper(*args, **kwargs): args = list(args) for index, arg in enumerate(args): if isinstance(arg, PointCloud): @@ -620,8 +623,8 @@ def func_wrapper(*args, **kwargs): for key in kwargs: if isinstance(kwargs[key], PointCloud): kwargs[key] = kwargs[key].points - return func(*args, **kwargs) - return func_wrapper + return wrapped(*args, **kwargs) + return wrapper # TODO: Document me! From 097aebc6601b824a766ca84d56705a9e6d56a030 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 16:21:26 +0100 Subject: [PATCH 309/423] Fix the None bounding box key error --- menpofit/sdm/fitter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 9f71488..07287fa 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -56,11 +56,17 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, **kwargs): - # normalize images and compute reference shape + # Normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) - # handle perturbations + # In the case where group is None, we need to get the only key so that + # we can add landmarks below and not get a complaint about using None + first_image = images[0] + if group is None: + group = first_image.landmarks.group_labels[0] + + # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: bounding_box_group = 'bb_' # generate perturbations by perturbing ground truth shapes From 4cb37a086ed42782660684424257f19f8c9facd9 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:25:38 +0100 Subject: [PATCH 310/423] Fixing verbose mode of helper functions Use the print_progress method --- menpofit/builder.py | 85 +++++++++++++++++++++------------------------ 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index a309b4e..291f46d 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -1,11 +1,12 @@ from __future__ import division +from functools import partial import numpy as np from menpo.shape import mean_pointcloud, PointCloud, TriMesh from menpo.image import Image, MaskedImage from menpo.feature import no_op from menpo.transform import Scale, Translation, GeneralizedProcrustesAnalysis from menpo.model.pca import PCAModel -from menpo.visualize import print_dynamic, progress_bar_str +from menpo.visualize import print_dynamic, print_progress def compute_reference_shape(shapes, normalization_diagonal, verbose=False): @@ -51,18 +52,15 @@ def rescale_images_to_reference_shape(images, group, label, reference_shape, verbose=False): r""" """ - # normalize the scaling of all images wrt the reference_shape size - normalized_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic('- Normalizing images size: {}'.format( - progress_bar_str((c + 1.) / len(images), - show_bar=False))) - normalized_images.append(i.rescale_to_reference_shape( - reference_shape, group=group, label=label)) - if verbose: - print_dynamic('- Normalizing images size: Done\n') + wrap = partial(print_progress, prefix='- Normalizing images size') + else: + wrap = lambda x: x + + # Normalize the scaling of all images wrt the reference_shape size + normalized_images = [i.rescale_to_reference_shape(reference_shape, + group=group, label=label) + for i in wrap(images)] return normalized_images @@ -119,42 +117,37 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, shapes = [i.landmarks[group][label] for i in images] # compute the reference shape and fix its diagonal length - reference_shape = compute_reference_shape(shapes, diagonal, - verbose=verbose) + reference_shape = compute_reference_shape(shapes, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size normalized_images = rescale_images_to_reference_shape( - images, group, label, reference_shape, verbose=False) + images, group, label, reference_shape, verbose=verbose) return reference_shape, normalized_images # TODO: document me! -def compute_features(images, features, level_str='', verbose=None): - feature_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic( - '{}Computing feature space: {}'.format( - level_str, progress_bar_str((c + 1.) / len(images), - show_bar=False))) - i = features(i) - feature_images.append(i) +def compute_features(images, features, level_str='', verbose=False): + if verbose: + wrap = partial(print_progress, + prefix='{}Computing feature space'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x - return feature_images + return [features(i) for i in wrap(images)] # TODO: document me! -def scale_images(images, scale, level_str='', verbose=None): - if scale != 1: - scaled_images = [] - for c, i in enumerate(images): - if verbose: - print_dynamic( - '{}Scaling images: {}'.format( - level_str, progress_bar_str((c + 1.) / len(images), - show_bar=False))) - scaled_images.append(i.rescale(scale)) - return scaled_images +def scale_images(images, scale, level_str='', verbose=False): + if verbose: + wrap = partial(print_progress, + prefix='{}Scaling images'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + + if not np.allclose(scale, 1): + return [i.rescale(scale) for i in wrap(images)] else: return images @@ -182,14 +175,16 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! def extract_patches(images, shapes, patch_shape, normalize_function=no_op, - level_str='', verbose=None): + level_str='', verbose=False): + if verbose: + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + parts_images = [] - for c, (i, s) in enumerate(zip(images, shapes)): - if verbose: - print_dynamic('{}Warping images - {}'.format( - level_str, - progress_bar_str(float(c + 1) / len(images), - show_bar=False))) + for i, s in wrap(zip(images, shapes)): parts = i.extract_patches(s, patch_size=patch_shape, as_single_array=True) parts = normalize_function(parts) @@ -341,4 +336,4 @@ def build_shape_model(shapes, max_components=None): # trim shape model if required shape_model.trim_components(max_components) - return shape_model \ No newline at end of file + return shape_model From 5daa854ae40e796a3621c4b2861d92cac0b0334c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:26:03 +0100 Subject: [PATCH 311/423] Update aam warp_images set_target and improve printing --- menpofit/builder.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 291f46d..3ae8322 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -152,21 +152,25 @@ def scale_images(images, scale, level_str='', verbose=False): return images -# TODO: Can be done more efficiently for PWA defining a dummy transform # TODO: document me! def warp_images(images, shapes, reference_frame, transform, level_str='', verbose=None): + if verbose: + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + warped_images = [] - for c, (i, s) in enumerate(zip(images, shapes)): - if verbose: - print_dynamic('{}Warping images - {}'.format( - level_str, - progress_bar_str(float(c + 1) / len(images), - show_bar=False))) - # compute transforms - t = transform(reference_frame.landmarks['source'].lms, s) + # Build a dummy transform, use set_target for efficiency + warp_transform = transform(reference_frame.landmarks['source'].lms, + reference_frame.landmarks['source'].lms) + for i, s in wrap(zip(images, shapes)): + # Update Transform Target + warp_transform.set_target(s) # warp images - warped_i = i.warp_to_mask(reference_frame.mask, t) + warped_i = i.warp_to_mask(reference_frame.mask, warp_transform) # attach reference frame landmarks to images warped_i.landmarks['source'] = reference_frame.landmarks['source'] warped_images.append(warped_i) From 18f3b2febac5e5ce88510ae83ed5c38493781c61 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:26:46 +0100 Subject: [PATCH 312/423] Update regression - better printing Also, use a range rather than a while --- menpofit/sdm/algorithm.py | 52 +++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 2c433ad..24719bb 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -1,7 +1,8 @@ from __future__ import division +from functools import partial import numpy as np from menpo.feature import no_op -from menpo.visualize import print_dynamic +from menpo.visualize import print_dynamic, print_progress from menpofit.result import ( NonParametricAlgorithmResult, compute_normalise_point_to_point_error) from menpofit.math import IRLRegression, IIRLRegression @@ -11,8 +12,8 @@ class SupervisedDescentAlgorithm(object): r""" """ - def train(self, images, gt_shapes, current_shapes, verbose=False, - **kwargs): + def train(self, images, gt_shapes, current_shapes, level_str='', + verbose=False, **kwargs): n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] @@ -24,27 +25,32 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) # initialize iteration counter and list of regressors - k = 0 self.regressors = [] # Cascaded Regression loop - while k < self.iterations: + for k in range(self.iterations): # generate regression data features = obtain_patch_features( images, current_shapes, self.patch_shape, self.features, - features_patch_length=self._features_patch_length) + features_patch_length=self._features_patch_length, + level_str='{}(Iteration {}) - '.format(level_str, k), + verbose=verbose) - # perform regression + # Perform regression if verbose: - print_dynamic('- Performing regression.') + print_dynamic('{}(Iteration {}) - Performing regression'.format( + level_str, k)) r = self._regressor_cls(**kwargs) r.train(features, delta_x) # add regressor to list self.regressors.append(r) - # estimate delta_points + # Estimate delta_points estimated_delta_x = r.predict(features) + if verbose: + print_dynamic('{}(Iteration {}) - Calculating errors'.format( + level_str, k)) errors = [] for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): s1 = template_shape.from_vector(dx) @@ -54,9 +60,9 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, mean = np.mean(errors) std = np.std(errors) median = np.median(errors) - print_dynamic('- Training error -> mean: {0:.4f}, ' - 'std: {1:.4f}, median: {2:.4f}.\n'. - format(mean, std, median)) + print_dynamic('{}(Iteration {}) - Training error -> ' + 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. + format(level_str, k, mean, std, median)) j = 0 for shapes in current_shapes: @@ -69,10 +75,7 @@ def train(self, images, gt_shapes, current_shapes, verbose=False, delta_x[j] = gt_x[j] - current_x # increase index j += 1 - # increase iteration counter - k += 1 - # rearrange current shapes into their original list of list form return current_shapes def increment(self, images, gt_shapes, current_shapes, verbose=False, @@ -198,7 +201,7 @@ def compute_patch_features(image, shape, patch_shape, features_callable, patch_features[j] = features_callable(p[0]).ravel() else: patch_features = [] - for j, p in enumerate(patches): + for p in patches: patch_features.append(features_callable(p[0]).ravel()) patch_features = np.asarray(patch_features) @@ -219,7 +222,7 @@ def generate_patch_features(image, shapes, patch_shape, features_callable, features_patch_length=features_patch_length) else: patch_features = [] - for j, s in enumerate(shapes): + for s in shapes: patch_features.append(compute_patch_features( image, s, patch_shape, features_callable, features_patch_length=features_patch_length)) @@ -230,24 +233,31 @@ def generate_patch_features(image, shapes, patch_shape, features_callable, # TODO: docment me! def obtain_patch_features(images, shapes, patch_shape, features_callable, - features_patch_length=None): + features_patch_length=None, level_str='', + verbose=False): """r """ + if verbose: + wrap = partial(print_progress, + prefix='{}Extracting patches'.format(level_str), + end_with_newline=not level_str) + else: + wrap = lambda x: x + n_images = len(images) n_shapes = len(shapes[0]) n_points = shapes[0][0].n_points if features_patch_length: - patch_features = np.empty((n_images, (n_shapes * n_points * features_patch_length))) - for j, i in enumerate(images): + for j, i in enumerate(wrap(images)): patch_features[j] = generate_patch_features( i, shapes[j], patch_shape, features_callable, features_patch_length=features_patch_length) else: patch_features = [] - for j, i in images: + for j, i in enumerate(wrap(images)): patch_features.append(generate_patch_features( i, shapes[j], patch_shape, features_callable, features_patch_length=features_patch_length)) From e49a718cfc928eecc6ae55d08ff65f70397dde92 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 20 Jul 2015 17:27:08 +0100 Subject: [PATCH 313/423] Big change to how bboxes are created May need updating a bit, but the logic works for noisy bboxes. Essentially, unify the logic for creating perturbations so that the noisy perturbations are generated based on the difference between the ground truth and the given bounding box. --- menpofit/sdm/fitter.py | 113 +++++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 44 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 07287fa..387205f 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,14 +1,16 @@ from __future__ import division +from functools import partial import numpy as np import warnings from menpo.transform import Scale from menpo.feature import no_op +from menpo.visualize import print_progress from menpofit.builder import ( normalization_wrt_reference_shape, rescale_images_to_reference_shape, scale_images) from menpofit.fitter import ( MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, - align_shape_with_bounding_box) + align_shape_with_bounding_box, noisy_params_alignment_similarity) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -68,36 +70,52 @@ def train(self, images, group=None, label=None, bounding_box_group=None, # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: - bounding_box_group = 'bb_' - # generate perturbations by perturbing ground truth shapes + bounding_box_group = '__gt_bb_' for i in images: gt_s = i.landmarks[group][label] - for j in range(self.n_perturbations): - p_s = self.perturb_from_shape(gt_s) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - else: - # reset number of perturbations - n_perturbations = 0 - for k in images[0].landmarks.keys(): - if bounding_box_group in k: - n_perturbations += 1 - if n_perturbations == 1: - for i in images: - bb = i.landmarks[bounding_box_group].lms - p_s = align_shape_with_bounding_box( - self.reference_shape, bb) - i.landmarks[bounding_box_group + '0'] = p_s - for j in range(1, self.n_perturbations): - p_s = self.perturb_from_bounding_box(bb) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) - self.n_perturbations = n_perturbations + perturb_bbox_group = bounding_box_group + '0' + i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + + # Find all bounding boxes on the images with the given bounding box key + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) + n_perturbations = len(all_bb_keys) + + # If there is only one example bounding box, then we will generate + # more perturbations based on the bounding box. + if n_perturbations == 1: + if verbose: + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg) + else: + wrap = lambda x: x + + for i in wrap(images): + # We assume that the first bounding box is a valid perturbation + # thus create n_perturbations - 1 new bounding boxes + for j in range(1, self.n_perturbations): + # TODO: This should use the new logic that @jalabort + # has come up with. Also, would it be good if this was + # customizable? As in, the ability to pass some kind of + # probability distribution to draw from? + gt_s = i.landmarks[group][label].bounding_box() + bb = i.landmarks[all_bb_keys[0]].lms + # TODO: Noisy align given bb to gt_s - is this correct? + p_s = noisy_params_alignment_similarity(bb, gt_s).apply(bb) + perturb_bbox_group = bounding_box_group + '_{}'.format(j) + i.landmarks[perturb_bbox_group] = p_s + elif n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided bounding_box_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + + # Re-grab all the bounding box keys for iterating over when calculating + # perturbations + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) # for each pyramid level (low --> high) for j in range(self.n_levels): @@ -106,37 +124,44 @@ def train(self, images, group=None, label=None, bounding_box_group=None, level_str = ' - Level {}: '.format(j) else: level_str = ' - ' + else: + level_str = None - # scale images and compute features at other levels + # Scale images and compute features at other levels level_images = scale_images(images, self.scales[j], level_str=level_str, verbose=verbose) - # extract ground truth shapes for current level + # Extract scaled ground truth shapes for current level level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - # extract perturbations at the very bottom level + if verbose: + msg = '{}Generating {} perturbations per image'.format( + level_str, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False) + else: + wrap = lambda x: x + + # Extract perturbations at the very bottom level current_shapes = [] - for i in level_images: + for i in wrap(level_images): c_shapes = [] - for k in range(self.n_perturbations): - p_group = bounding_box_group + '{}'.format(k) - c_s = i.landmarks[p_group].lms - if c_s.n_points != level_gt_shapes[0].n_points: - # assume c_s is bounding box - c_s = align_shape_with_bounding_box( - self.reference_shape, c_s) + for perturb_bbox_group in all_bb_keys: + bbox = i.landmarks[perturb_bbox_group].lms + c_s = align_shape_with_bounding_box( + self.reference_shape, bbox) c_shapes.append(c_s) current_shapes.append(c_shapes) # train supervised descent algorithm current_shapes = self.algorithms[j].train( level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) + level_str=level_str, verbose=verbose, **kwargs) - # scale current shapes to next level resolution + # Scale current shapes to next level resolution if self.scales[j] != (1 or self.scales[-1]): - transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) + transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) for image_shapes in current_shapes: for shape in image_shapes: transform.apply_inplace(shape) @@ -547,4 +572,4 @@ def __str__(self): # # feat_str = [feat_str] # # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out \ No newline at end of file +# # return out From 8bd851065a192fcc013d109140d0b13ff9fb917b Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Tue, 21 Jul 2015 09:02:16 +0100 Subject: [PATCH 314/423] Small changes --- menpofit/fitter.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ab69006..72fd8fd 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -328,29 +328,24 @@ def noisy_alignment_similarity_transform(source, target, noise_type='uniform', The noisy Similarity Transform between source and target. """ if isinstance(noise_percentage, float): - noise_percentage= [noise_percentage] * 3 + noise_percentage = [noise_percentage] * 3 elif len(noise_percentage) == 1: noise_percentage *= 3 similarity = AlignmentSimilarity(source, target, rotation=rotation) - if noise_type is 'normal': - # + if noise_type is 'gaussian': s = noise_percentage[0] * (0.5 / 3) * np.asscalar(np.random.randn(1)) - # r = noise_percentage[1] * (180 / 3) * np.asscalar(np.random.randn(1)) - # t = noise_percentage[2] * (target.range() / 3) * np.random.randn(2) s = scale_about_centre(target, 1 + s) r = rotate_ccw_about_centre(target, r) t = Translation(t, source.n_dims) + elif noise_type is 'uniform': - # s = noise_percentage[0] * 0.5 * (2 * np.asscalar(np.random.randn(1)) - 1) - # r = noise_percentage[1] * 180 * (2 * np.asscalar(np.random.rand(1)) - 1) - # t = noise_percentage[2] * target.range() * (2 * np.random.rand(2) - 1) s = scale_about_centre(target, 1. + s) From ec5a16ae5a2e390e7f8df4bdf3b5639ef14479f5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 22 Jul 2015 14:36:38 +0100 Subject: [PATCH 315/423] Call reset_algorithms when recalling train --- menpofit/sdm/fitter.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 387205f..3ff4a4b 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -32,6 +32,11 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, features = checks.check_features(features, n_levels) patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters + self.algorithms = [] + self.reference_shape = None + self._sd_algorithm_cls = sd_algorithm_cls + self._features = features + self._patch_shape = patch_shape self.diagonal = diagonal self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations @@ -39,15 +44,16 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self._perturb_from_shape = perturb_from_shape self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms - self._set_up(sd_algorithm_cls, features, patch_shape, **kwargs) + self._reset_algorithms(**kwargs) - def _set_up(self, sd_algorithm_cls, features, patch_shape, **kwargs): - self.algorithms = [] + def _reset_algorithms(self, **kwargs): + if len(self.algorithms) > 0: + for j in range(len(self.algorithms) - 1, -1, -1): + del self.algorithms[j] for j in range(self.n_levels): - algorithm = sd_algorithm_cls( - features=features[j], patch_shape=patch_shape[j], - iterations=self.iterations[j], **kwargs) - self.algorithms.append(algorithm) + self.algorithms.append(self._sd_algorithm_cls( + features=self._features[j], patch_shape=self._patch_shape[j], + iterations=self.iterations[j], **kwargs)) def perturb_from_shape(self, shape, **kwargs): return self._perturb_from_shape(self.reference_shape, shape, **kwargs) @@ -58,6 +64,9 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, **kwargs): + # Reset the algorithm classes + self._reset_algorithms() + # Normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( images, group, label, self.diagonal, verbose=verbose) @@ -102,7 +111,8 @@ def train(self, images, group=None, label=None, bounding_box_group=None, gt_s = i.landmarks[group][label].bounding_box() bb = i.landmarks[all_bb_keys[0]].lms # TODO: Noisy align given bb to gt_s - is this correct? - p_s = noisy_params_alignment_similarity(bb, gt_s).apply(bb) + p_s = noisy_params_alignment_similarity( + bb, gt_s, noise_std=0.03).apply(bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: @@ -442,13 +452,13 @@ def __str__(self): # self.n_perturbations = n_perturbations # self.iterations = checks.check_iterations(iterations, n_levels) # # set up algorithms -# self._set_up(cr_algorithm_cls, features, sampling, **kwargs) +# self._reset_algorithms(cr_algorithm_cls, features, sampling, **kwargs) # # @property # def algorithms(self): # return self._algorithms # -# def _set_up(self, cr_algorithm_cls, features, sampling, **kwargs): +# def _reset_algorithms(self, cr_algorithm_cls, features, sampling, **kwargs): # for j, s in range(self.n_levels): # algorithm = cr_algorithm_cls( # features=features[j], sampling=sampling[j], From 012f024edbb8e4a5c9fddab48d9addaed9edfc61 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 22 Jul 2015 15:29:35 +0100 Subject: [PATCH 316/423] Properly use the perturb method for SDM --- menpofit/sdm/fitter.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 3ff4a4b..d16c9c1 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -10,7 +10,7 @@ scale_images) from menpofit.fitter import ( MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, - align_shape_with_bounding_box, noisy_params_alignment_similarity) + align_shape_with_bounding_box) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -104,15 +104,11 @@ def train(self, images, group=None, label=None, bounding_box_group=None, # We assume that the first bounding box is a valid perturbation # thus create n_perturbations - 1 new bounding boxes for j in range(1, self.n_perturbations): - # TODO: This should use the new logic that @jalabort - # has come up with. Also, would it be good if this was - # customizable? As in, the ability to pass some kind of - # probability distribution to draw from? gt_s = i.landmarks[group][label].bounding_box() bb = i.landmarks[all_bb_keys[0]].lms - # TODO: Noisy align given bb to gt_s - is this correct? - p_s = noisy_params_alignment_similarity( - bb, gt_s, noise_std=0.03).apply(bb) + + # This is customizable by passing in the correct method + p_s = self._perturb_from_bounding_box(gt_s, bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: From 4d4259b6b79f32c976fc3630865ceb338928cd1a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 22 Jul 2015 15:44:24 +0100 Subject: [PATCH 317/423] Update increment on SDM to same as train Use one function since they were identical. Just call added a kwarg for incrementing. --- menpofit/sdm/fitter.py | 107 ++++++++--------------------------------- 1 file changed, 21 insertions(+), 86 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index d16c9c1..429e0ca 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -63,9 +63,13 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): bounding_box, **kwargs) def train(self, images, group=None, label=None, bounding_box_group=None, - verbose=False, **kwargs): - # Reset the algorithm classes - self._reset_algorithms() + verbose=False, increment=False, **kwargs): + if not increment: + # Reset the algorithm classes + self._reset_algorithms() + else: + if len(self.algorithms) == 0: + raise ValueError('Must train before training incrementally.') # Normalize images and compute reference shape self.reference_shape, images = normalization_wrt_reference_shape( @@ -161,9 +165,14 @@ def train(self, images, group=None, label=None, bounding_box_group=None, current_shapes.append(c_shapes) # train supervised descent algorithm - current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose, **kwargs) + if increment: + current_shapes = self.algorithms[j].increment( + level_images, level_gt_shapes, current_shapes, + verbose=verbose, **kwargs) + else: + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + level_str=level_str, verbose=verbose, **kwargs) # Scale current shapes to next level resolution if self.scales[j] != (1 or self.scales[-1]): @@ -175,84 +184,10 @@ def train(self, images, group=None, label=None, bounding_box_group=None, def increment(self, images, group=None, label=None, bounding_box_group=None, verbose=False, **kwargs): - # normalize images with respect to reference shape of aam - images = rescale_images_to_reference_shape( - images, group, label, self.reference_shape, verbose=verbose) - - # handle perturbations - if bounding_box_group is None: - bounding_box_group = 'bb_' - # generate perturbations by perturbing ground truth shapes - for i in images: - gt_s = i.landmarks[group][label] - for j in range(self.n_perturbations): - p_s = self.perturb_from_shape(gt_s) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - else: - # reset number of perturbations - n_perturbations = 0 - for k in images[0].landmarks.keys(): - if bounding_box_group in k: - n_perturbations += 1 - if n_perturbations == 1: - for i in images: - bb = i.landmarks[bounding_box_group].lms - p_s = align_shape_with_bounding_box( - self.reference_shape, bb) - i.landmarks[bounding_box_group + '0'] = p_s - for j in range(1, self.n_perturbations): - p_s = self.perturb_from_bounding_box(bb) - p_group = bounding_box_group + '{}'.format(j) - i.landmarks[p_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) - self.n_perturbations = n_perturbations - - # for each pyramid level (low --> high) - for j in range(self.n_levels): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # scale images and compute features at other levels - level_images = scale_images(images, self.scales[j], - level_str=level_str, verbose=verbose) - - # extract ground truth shapes for current level - level_gt_shapes = [i.landmarks[group][label] for i in level_images] - - if j == 0: - # extract perturbations at the very bottom level - current_shapes = [] - for i in level_images: - c_shapes = [] - for k in range(self.n_perturbations): - p_group = bounding_box_group + '{}'.format(k) - c_s = i.landmarks[p_group].lms - if c_s.n_points != level_gt_shapes[0].n_points: - # assume c_s is bounding box - c_s = align_shape_with_bounding_box( - self.reference_shape, c_s) - c_shapes.append(c_s) - current_shapes.append(c_shapes) - - # train cascaded regression algorithm - current_shapes = self.algorithms[j].increment( - level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) - - # scale current shapes to next level resolution - if self.scales[j] != (1 or self.scales[-1]): - transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) + return self.train(images, group=group, label=label, + bounding_box_group=bounding_box_group, + verbose=verbose, + increment=True, **kwargs) def train_incrementally(self, images, group=None, label=None, batch_size=100, verbose=False, **kwargs): @@ -260,14 +195,14 @@ def train_incrementally(self, images, group=None, label=None, n_batches = np.int(np.ceil(len(images) / batch_size)) # train first batch - print 'Training batch 1.' + print('Training batch 1.') self.train(images[:batch_size], group=group, label=label, verbose=verbose, **kwargs) # train all other batches start = batch_size for j in range(1, n_batches): - print 'Training batch {}.'.format(j+1) + print('Training batch {}.'.format(j+1)) end = start + batch_size self.increment(images[start:end], group=group, label=label, verbose=verbose, **kwargs) From 8d22660b6690ba9362f92bbf03f6f2ea0cbeef10 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 23 Jul 2015 10:43:59 +0100 Subject: [PATCH 318/423] Fix rescaling to reference shape for incremental sdm Also, add verbose guard to printing --- menpofit/sdm/fitter.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 429e0ca..231460d 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -5,9 +5,8 @@ from menpo.transform import Scale from menpo.feature import no_op from menpo.visualize import print_progress -from menpofit.builder import ( - normalization_wrt_reference_shape, rescale_images_to_reference_shape, - scale_images) +from menpofit.builder import (normalization_wrt_reference_shape, scale_images, + rescale_images_to_reference_shape) from menpofit.fitter import ( MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, align_shape_with_bounding_box) @@ -64,22 +63,25 @@ def perturb_from_bounding_box(self, bounding_box, **kwargs): def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, increment=False, **kwargs): + # In the case where group is None, we need to get the only key so that + # we can add landmarks below and not get a complaint about using None + first_image = images[0] + if group is None: + group = first_image.landmarks.group_labels[0] + if not increment: # Reset the algorithm classes self._reset_algorithms() + # Normalize images and compute reference shape + self.reference_shape, images = normalization_wrt_reference_shape( + images, group, label, self.diagonal, verbose=verbose) else: if len(self.algorithms) == 0: raise ValueError('Must train before training incrementally.') - - # Normalize images and compute reference shape - self.reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) - - # In the case where group is None, we need to get the only key so that - # we can add landmarks below and not get a complaint about using None - first_image = images[0] - if group is None: - group = first_image.landmarks.group_labels[0] + # We are incrementing, so rescale to existing reference shape + images = rescale_images_to_reference_shape(images, group, label, + self.reference_shape, + verbose=verbose) # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: @@ -195,14 +197,16 @@ def train_incrementally(self, images, group=None, label=None, n_batches = np.int(np.ceil(len(images) / batch_size)) # train first batch - print('Training batch 1.') + if verbose: + print('Training batch 1.') self.train(images[:batch_size], group=group, label=label, verbose=verbose, **kwargs) # train all other batches start = batch_size for j in range(1, n_batches): - print('Training batch {}.'.format(j+1)) + if verbose: + print('Training batch {}.'.format(j + 1)) end = start + batch_size self.increment(images[start:end], group=group, label=label, verbose=verbose, **kwargs) From cb210472a77df70c3f30633630596d7c2a453211 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 14:06:21 +0100 Subject: [PATCH 319/423] Remove perturb_from_shape We will just let people do that themselves explicitly --- menpofit/sdm/fitter.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 231460d..044709b 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -22,7 +22,6 @@ class SupervisedDescentFitter(MultiFitter): def __init__(self, sd_algorithm_cls=Newton, features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), iterations=6, n_perturbations=30, - perturb_from_shape=noisy_shape_from_shape, perturb_from_bounding_box=noisy_shape_from_bounding_box, **kwargs): # check parameters @@ -40,7 +39,6 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self.scales = list(scales)[::-1] self.n_perturbations = n_perturbations self.iterations = checks.check_max_iters(iterations, n_levels) - self._perturb_from_shape = perturb_from_shape self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._reset_algorithms(**kwargs) @@ -51,15 +49,12 @@ def _reset_algorithms(self, **kwargs): del self.algorithms[j] for j in range(self.n_levels): self.algorithms.append(self._sd_algorithm_cls( - features=self._features[j], patch_shape=self._patch_shape[j], + features=self._holistic_features[j], patch_shape=self._patch_shape[j], iterations=self.iterations[j], **kwargs)) - def perturb_from_shape(self, shape, **kwargs): - return self._perturb_from_shape(self.reference_shape, shape, **kwargs) - - def perturb_from_bounding_box(self, bounding_box, **kwargs): + def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, - bounding_box, **kwargs) + bounding_box) def train(self, images, group=None, label=None, bounding_box_group=None, verbose=False, increment=False, **kwargs): @@ -114,7 +109,7 @@ def train(self, images, group=None, label=None, bounding_box_group=None, bb = i.landmarks[all_bb_keys[0]].lms # This is customizable by passing in the correct method - p_s = self._perturb_from_bounding_box(gt_s, bb) + p_s = self.perturb_from_bounding_box(gt_s, bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: @@ -227,13 +222,10 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, ---------- image : :map:`Image` or subclass The image to be fitted. - initial_shape : :map:`PointCloud` The initial shape from which the fitting will start. - - gt_shape : class : :map:`PointCloud`, optional + gt_shape : :map:`PointCloud`, optional The original ground truth shape associated to the image. - crop_image: `None` or float`, optional If `float`, it specifies the proportion of the border wrt the initial shape to which the image will be internally cropped around @@ -248,10 +240,8 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, ------- images : `list` of :map:`Image` or subclass The list of images that will be fitted by the fitters. - initial_shapes : `list` of :map:`PointCloud` The initial shape for each one of the previous images. - gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ From 79b84edbff028894e98336012949642d0f1ed9f7 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 16:45:12 +0100 Subject: [PATCH 320/423] Make train_incremental work on generators Use a batch method to allow ingesting a generator rather than a list. --- menpofit/base.py | 11 ++++++++++- menpofit/sdm/fitter.py | 40 ++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index 175054e..a350256 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -1,5 +1,5 @@ from __future__ import division -from menpo.transform import AlignmentSimilarity, Similarity +import itertools import numpy as np from menpo.visualize import progress_bar_str, print_dynamic, print_progress @@ -11,6 +11,15 @@ def name_of_callable(c): return c.__class__.__name__ # callable class +def batch(iterable, n): + it = iter(iterable) + while True: + chunk = tuple(itertools.islice(it, n)) + if not chunk: + return + yield chunk + + def is_pyramid_on_features(features): r""" True if feature extraction happens once and then a gaussian pyramid diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 044709b..109a4be 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,15 +1,14 @@ from __future__ import division from functools import partial -import numpy as np import warnings from menpo.transform import Scale from menpo.feature import no_op from menpo.visualize import print_progress +from menpofit.base import batch from menpofit.builder import (normalization_wrt_reference_shape, scale_images, rescale_images_to_reference_shape) -from menpofit.fitter import ( - MultiFitter, noisy_shape_from_shape, noisy_shape_from_bounding_box, - align_shape_with_bounding_box) +from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, + align_shape_with_bounding_box) from menpofit.result import MultiFitterResult import menpofit.checks as checks from .algorithm import Newton @@ -109,7 +108,7 @@ def train(self, images, group=None, label=None, bounding_box_group=None, bb = i.landmarks[all_bb_keys[0]].lms # This is customizable by passing in the correct method - p_s = self.perturb_from_bounding_box(gt_s, bb) + p_s = self._perturb_from_bounding_box(gt_s, bb) perturb_bbox_group = bounding_box_group + '_{}'.format(j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: @@ -134,7 +133,7 @@ def train(self, images, group=None, label=None, bounding_box_group=None, else: level_str = None - # Scale images and compute features at other levels + # Scale images level_images = scale_images(images, self.scales[j], level_str=level_str, verbose=verbose) @@ -188,24 +187,21 @@ def increment(self, images, group=None, label=None, def train_incrementally(self, images, group=None, label=None, batch_size=100, verbose=False, **kwargs): - # number of batches - n_batches = np.int(np.ceil(len(images) / batch_size)) - - # train first batch - if verbose: - print('Training batch 1.') - self.train(images[:batch_size], group=group, label=label, - verbose=verbose, **kwargs) - - # train all other batches - start = batch_size - for j in range(1, n_batches): + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + + # Train all batches + for k, image_batch in enumerate(image_batches): + n_images = len(image_batch) if verbose: - print('Training batch {}.'.format(j + 1)) - end = start + batch_size - self.increment(images[start:end], group=group, label=label, + print('Training batch {} of {} images.'.format(k, n_images)) + if k == 0: + self.train(image_batch, group=group, label=label, verbose=verbose, **kwargs) - start = end + else: + self.increment(image_batch, group=group, label=label, + verbose=verbose, **kwargs) def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): From d617574efa377fc409fdaf8245beb0640bbbb743 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 16:46:17 +0100 Subject: [PATCH 321/423] Fix non-verbose mode for SDM Add a new little print_progress method that takes a verbose flag and can ignore verbosity. --- menpofit/base.py | 2 +- menpofit/builder.py | 45 ++++++++++++--------------------- menpofit/sdm/algorithm.py | 12 ++++----- menpofit/sdm/fitter.py | 24 +++++++----------- menpofit/visualize/__init__.py | 1 + menpofit/visualize/textutils.py | 24 ++++++++++++++++++ 6 files changed, 56 insertions(+), 52 deletions(-) create mode 100644 menpofit/visualize/textutils.py diff --git a/menpofit/base.py b/menpofit/base.py index a350256..5f641cd 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -114,4 +114,4 @@ def build_sampling_grid(patch_shape): start = -patch_half_shape end = patch_half_shape + 1 sampling_grid = np.mgrid[start[0]:end[0], start[1]:end[1]] - return sampling_grid.swapaxes(0, 2).swapaxes(0, 1) \ No newline at end of file + return sampling_grid.swapaxes(0, 2).swapaxes(0, 1) diff --git a/menpofit/builder.py b/menpofit/builder.py index 3ae8322..b1f4b2a 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -6,7 +6,8 @@ from menpo.feature import no_op from menpo.transform import Scale, Translation, GeneralizedProcrustesAnalysis from menpo.model.pca import PCAModel -from menpo.visualize import print_dynamic, print_progress +from menpo.visualize import print_dynamic +from menpofit.visualize import print_progress def compute_reference_shape(shapes, normalization_diagonal, verbose=False): @@ -52,10 +53,8 @@ def rescale_images_to_reference_shape(images, group, label, reference_shape, verbose=False): r""" """ - if verbose: - wrap = partial(print_progress, prefix='- Normalizing images size') - else: - wrap = lambda x: x + wrap = partial(print_progress, prefix='- Normalizing images size', + verbose=verbose) # Normalize the scaling of all images wrt the reference_shape size normalized_images = [i.rescale_to_reference_shape(reference_shape, @@ -127,24 +126,18 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, # TODO: document me! def compute_features(images, features, level_str='', verbose=False): - if verbose: - wrap = partial(print_progress, - prefix='{}Computing feature space'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Computing feature space'.format(level_str), + end_with_newline=not level_str, verbose=verbose) return [features(i) for i in wrap(images)] # TODO: document me! def scale_images(images, scale, level_str='', verbose=False): - if verbose: - wrap = partial(print_progress, - prefix='{}Scaling images'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Scaling images'.format(level_str), + end_with_newline=not level_str, verbose=verbose) if not np.allclose(scale, 1): return [i.rescale(scale) for i in wrap(images)] @@ -155,12 +148,9 @@ def scale_images(images, scale, level_str='', verbose=False): # TODO: document me! def warp_images(images, shapes, reference_frame, transform, level_str='', verbose=None): - if verbose: - wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str, verbose=verbose) warped_images = [] # Build a dummy transform, use set_target for efficiency @@ -180,12 +170,9 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! def extract_patches(images, shapes, patch_shape, normalize_function=no_op, level_str='', verbose=False): - if verbose: - wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Warping images'.format(level_str), + end_with_newline=not level_str, verbose=verbose) parts_images = [] for i, s in wrap(zip(images, shapes)): diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 24719bb..92c54b4 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -2,7 +2,8 @@ from functools import partial import numpy as np from menpo.feature import no_op -from menpo.visualize import print_dynamic, print_progress +from menpo.visualize import print_dynamic +from menpofit.visualize import print_progress from menpofit.result import ( NonParametricAlgorithmResult, compute_normalise_point_to_point_error) from menpofit.math import IRLRegression, IIRLRegression @@ -237,12 +238,9 @@ def obtain_patch_features(images, shapes, patch_shape, features_callable, verbose=False): """r """ - if verbose: - wrap = partial(print_progress, - prefix='{}Extracting patches'.format(level_str), - end_with_newline=not level_str) - else: - wrap = lambda x: x + wrap = partial(print_progress, + prefix='{}Extracting patches'.format(level_str), + end_with_newline=not level_str, verbose=verbose) n_images = len(images) n_shapes = len(shapes[0]) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 109a4be..0422a28 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -3,8 +3,8 @@ import warnings from menpo.transform import Scale from menpo.feature import no_op -from menpo.visualize import print_progress -from menpofit.base import batch +from menpofit.visualize import print_progress +from menpofit.base import batch, name_of_callable from menpofit.builder import (normalization_wrt_reference_shape, scale_images, rescale_images_to_reference_shape) from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, @@ -93,12 +93,9 @@ def train(self, images, group=None, label=None, bounding_box_group=None, # If there is only one example bounding box, then we will generate # more perturbations based on the bounding box. if n_perturbations == 1: - if verbose: - msg = '- Generating {} new initial bounding boxes ' \ - 'per image'.format(self.n_perturbations) - wrap = partial(print_progress, prefix=msg) - else: - wrap = lambda x: x + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg, verbose=verbose) for i in wrap(images): # We assume that the first bounding box is a valid perturbation @@ -141,13 +138,10 @@ def train(self, images, group=None, label=None, bounding_box_group=None, level_gt_shapes = [i.landmarks[group][label] for i in level_images] if j == 0: - if verbose: - msg = '{}Generating {} perturbations per image'.format( - level_str, self.n_perturbations) - wrap = partial(print_progress, prefix=msg, - end_with_newline=False) - else: - wrap = lambda x: x + msg = '{}Generating {} perturbations per image'.format( + level_str, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False, verbose=verbose) # Extract perturbations at the very bottom level current_shapes = [] diff --git a/menpofit/visualize/__init__.py b/menpofit/visualize/__init__.py index 6aaea70..30039c3 100644 --- a/menpofit/visualize/__init__.py +++ b/menpofit/visualize/__init__.py @@ -5,3 +5,4 @@ visualize_fitting_result) except ImportError: pass +from .textutils import print_progress diff --git a/menpofit/visualize/textutils.py b/menpofit/visualize/textutils.py new file mode 100644 index 0000000..ce49bad --- /dev/null +++ b/menpofit/visualize/textutils.py @@ -0,0 +1,24 @@ +from menpo.visualize import print_progress as menpo_print_progress + + +def print_progress(iterable, prefix='', n_items=None, offset=0, + show_bar=True, show_count=True, show_eta=True, + end_with_newline=True, verbose=True): + r""" + Please see the menpo ``print_progress`` documentation. + + This method is identical to the print progress method, but adds a verbose + flag which allows the printing to be skipped if necessary. + """ + if verbose: + # Yield the images from the menpo print_progress (yield from would + # be perfect here :( ) + for i in menpo_print_progress(iterable, prefix=prefix, n_items=n_items, + offset=offset, show_bar=show_bar, + show_count=show_count, show_eta=show_eta, + end_with_newline=end_with_newline): + yield i + else: + # Skip the verbosity! + for i in iterable: + yield i From f194d4caed92cec2202d3546913185240ad58fef Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 16:47:13 +0100 Subject: [PATCH 322/423] Add the idea of holistic features Seperate features into patch and holistic. Holisitic gets computed on the whole image, before scaling. patch_features get computed inside each patch - and are equivalent to the old features. --- menpofit/sdm/fitter.py | 195 ++++++----------------------------------- 1 file changed, 27 insertions(+), 168 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 0422a28..567d40a 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -18,21 +18,21 @@ class SupervisedDescentFitter(MultiFitter): r""" """ - def __init__(self, sd_algorithm_cls=Newton, features=no_op, - patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), - iterations=6, n_perturbations=30, - perturb_from_bounding_box=noisy_shape_from_bounding_box, - **kwargs): + def __init__(self, sd_algorithm_cls=Newton, holistic_feature=no_op, + patch_features=no_op, patch_shape=(17, 17), diagonal=None, + scales=(1, 0.5), iterations=6, n_perturbations=30, + perturb_from_bounding_box=noisy_shape_from_bounding_box): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + patch_features = checks.check_features(patch_features, n_levels) patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters self.algorithms = [] self.reference_shape = None self._sd_algorithm_cls = sd_algorithm_cls - self._features = features + self._holistic_feature = holistic_feature + self._patch_features = patch_features self._patch_shape = patch_shape self.diagonal = diagonal self.scales = list(scales)[::-1] @@ -40,16 +40,17 @@ def __init__(self, sd_algorithm_cls=Newton, features=no_op, self.iterations = checks.check_max_iters(iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms - self._reset_algorithms(**kwargs) + self._reset_algorithms() - def _reset_algorithms(self, **kwargs): + def _reset_algorithms(self): if len(self.algorithms) > 0: for j in range(len(self.algorithms) - 1, -1, -1): del self.algorithms[j] for j in range(self.n_levels): self.algorithms.append(self._sd_algorithm_cls( - features=self._holistic_features[j], patch_shape=self._patch_shape[j], - iterations=self.iterations[j], **kwargs)) + features=self._patch_features[j], + patch_shape=self._patch_shape[j], + iterations=self.iterations[j])) def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, @@ -120,6 +121,12 @@ def train(self, images, group=None, label=None, bounding_box_group=None, all_bb_keys = list(first_image.landmarks.keys_matching( '*{}*'.format(bounding_box_group))) + # Before scaling, we compute the holistic feature on the whole image + msg = '- Computing holistic features ({})'.format( + name_of_callable(self._holistic_feature)) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + images = [self._holistic_feature(im) for im in wrap(images)] + # for each pyramid level (low --> high) for j in range(self.n_levels): if verbose: @@ -235,22 +242,25 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ - # attach landmarks to the image + # Attach landmarks to the image image.landmarks['initial_shape'] = initial_shape if gt_shape: image.landmarks['gt_shape'] = gt_shape - # if specified, crop the image + # If specified, crop the image if crop_image: image = image.crop_to_landmarks_proportion(crop_image, group='initial_shape') - # rescale image wrt the scale factor between reference_shape and + # Rescale image w.r.t the scale factor between reference_shape and # initial_shape image = image.rescale_to_reference_shape(self.reference_shape, group='initial_shape') - # obtain image representation + # Compute the holistic feature on the normalized image + image = self._holistic_feature(image) + + # Obtain image representation images = [] for s in self.scales: if s != 1: @@ -260,10 +270,10 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, scaled_image = image images.append(scaled_image) - # get initial shapes per level + # Get initial shapes per level initial_shapes = [i.landmarks['initial_shape'].lms for i in images] - # get ground truth shapes per level + # Get ground truth shapes per level if gt_shape: gt_shapes = [i.landmarks['gt_shape'].lms for i in images] else: @@ -347,154 +357,3 @@ def __str__(self): # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( # out, feat_str[0], n_channels[0], ch_str[0]) # return out - - -# class CRFitter(MultiFitter): -# r""" -# """ -# def __init__(self, cr_algorithm_cls=SN, features=no_op, diagonal=None, -# scales=(1, 0.5), sampling=None, n_perturbations=10, -# iterations=6, **kwargs): -# # check parameters -# checks.check_diagonal(diagonal) -# scales, n_levels = checks.check_scales(scales) -# features = checks.check_features(features, n_levels) -# sampling = checks.check_sampling(sampling, n_levels) -# # set parameters -# self._algorithms = [] -# self.diagonal = diagonal -# self.scales = list(scales) -# self.n_perturbations = n_perturbations -# self.iterations = checks.check_iterations(iterations, n_levels) -# # set up algorithms -# self._reset_algorithms(cr_algorithm_cls, features, sampling, **kwargs) -# -# @property -# def algorithms(self): -# return self._algorithms -# -# def _reset_algorithms(self, cr_algorithm_cls, features, sampling, **kwargs): -# for j, s in range(self.n_levels): -# algorithm = cr_algorithm_cls( -# features=features[j], sampling=sampling[j], -# max_iters=self.iterations[j], **kwargs) -# self._algorithms.append(algorithm) -# -# def train(self, images, group=None, label=None, verbose=False, **kwargs): -# # normalize images and compute reference shape -# reference_shape, images = normalization_wrt_reference_shape( -# images, group, label, self.diagonal, verbose=verbose) -# -# # for each pyramid level (low --> high) -# for j in range(self.n_levels): -# if verbose: -# if len(self.scales) > 1: -# level_str = ' - Level {}: '.format(j) -# else: -# level_str = ' - ' -# -# # scale images and compute features at other levels -# level_images = scale_images(images, self.scales[j], -# level_str=level_str, verbose=verbose) -# -# # extract ground truth shapes for current level -# level_gt_shapes = [i.landmarks[group][label] for i in level_images] -# -# if j == 0: -# # generate perturbed shapes -# current_shapes = [] -# for gt_s in level_gt_shapes: -# perturbed_shapes = [] -# for _ in range(self.n_perturbations): -# p_s = self.noisy_shape_from_shape(gt_s) -# perturbed_shapes.append(p_s) -# current_shapes.append(perturbed_shapes) -# -# # train cascaded regression algorithm -# current_shapes = self.algorithms[j].train( -# level_images, level_gt_shapes, current_shapes, -# verbose=verbose, **kwargs) -# -# # scale current shapes to next level resolution -# if self.scales[j] != self.scales[-1]: -# transform = Scale(self.scales[j+1]/self.scales[j], n_dims=2) -# for image_shapes in current_shapes: -# for shape in image_shapes: -# transform.apply_inplace(shape) -# -# def _fitter_result(self, image, algorithm_results, affine_correction, -# gt_shape=None): -# return MultiFitterResult(image, algorithm_results, affine_correction, -# gt_shape=gt_shape) -# -# # TODO: fix me! -# def __str__(self): -# pass -# # out = "Supervised Descent Method\n" \ -# # " - Non-Parametric '{}' Regressor\n" \ -# # " - {} training images.\n".format( -# # name_of_callable(self._fitters[0].regressor), -# # self._n_training_images) -# # # small strings about number of channels, channels string and downscale -# # down_str = [] -# # for j in range(self.n_levels): -# # if j == self.n_levels - 1: -# # down_str.append('(no downscale)') -# # else: -# # down_str.append('(downscale by {})'.format( -# # self.downscale**(self.n_levels - j - 1))) -# # temp_img = Image(image_data=np.random.rand(40, 40)) -# # if self.pyramid_on_features: -# # temp = self.features(temp_img) -# # n_channels = [temp.n_channels] * self.n_levels -# # else: -# # n_channels = [] -# # for j in range(self.n_levels): -# # temp = self.features[j](temp_img) -# # n_channels.append(temp.n_channels) -# # # string about features and channels -# # if self.pyramid_on_features: -# # feat_str = "- Feature is {} with ".format( -# # name_of_callable(self.features)) -# # if n_channels[0] == 1: -# # ch_str = ["channel"] -# # else: -# # ch_str = ["channels"] -# # else: -# # feat_str = [] -# # ch_str = [] -# # for j in range(self.n_levels): -# # if isinstance(self.features[j], str): -# # feat_str.append("- Feature is {} with ".format( -# # self.features[j])) -# # elif self.features[j] is None: -# # feat_str.append("- No features extracted. ") -# # else: -# # feat_str.append("- Feature is {} with ".format( -# # self.features[j].__name__)) -# # if n_channels[j] == 1: -# # ch_str.append("channel") -# # else: -# # ch_str.append("channels") -# # if self.n_levels > 1: -# # out = "{} - Gaussian pyramid with {} levels and downscale " \ -# # "factor of {}.\n".format(out, self.n_levels, -# # self.downscale) -# # if self.pyramid_on_features: -# # out = "{} - Pyramid was applied on feature space.\n " \ -# # "{}{} {} per image.\n".format(out, feat_str, -# # n_channels[0], ch_str[0]) -# # else: -# # out = "{} - Features were extracted at each pyramid " \ -# # "level.\n".format(out) -# # for i in range(self.n_levels - 1, -1, -1): -# # out = "{} - Level {} {}: \n {}{} {} per " \ -# # "image.\n".format( -# # out, self.n_levels - i, down_str[i], feat_str[i], -# # n_channels[i], ch_str[i]) -# # else: -# # if self.pyramid_on_features: -# # feat_str = [feat_str] -# # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( -# # out, feat_str[0], n_channels[0], ch_str[0]) -# # return out From 2dd1fe5b4f0e79a5dc028e8e651b8bb136c8e9d3 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:17:08 +0100 Subject: [PATCH 323/423] Try creating a sensible __str__ for sdm --- menpofit/sdm/fitter.py | 108 +++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 70 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 567d40a..5f548d3 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,4 +1,5 @@ from __future__ import division +import numpy as np from functools import partial import warnings from menpo.transform import Scale @@ -286,74 +287,41 @@ def _fitter_result(self, image, algorithm_results, affine_correction, return MultiFitterResult(image, self, algorithm_results, affine_correction, gt_shape=gt_shape) - # TODO: fix me! def __str__(self): - pass - # out = "Supervised Descent Method\n" \ - # " - Non-Parametric '{}' Regressor\n" \ - # " - {} training images.\n".format( - # name_of_callable(self._fitters[0].regressor), - # self._n_training_images) - # # small strings about number of channels, channels string and downscale - # down_str = [] - # for j in range(self.n_levels): - # if j == self.n_levels - 1: - # down_str.append('(no downscale)') - # else: - # down_str.append('(downscale by {})'.format( - # self.downscale**(self.n_levels - j - 1))) - # temp_img = Image(image_data=np.random.rand(40, 40)) - # if self.pyramid_on_features: - # temp = self.features(temp_img) - # n_channels = [temp.n_channels] * self.n_levels - # else: - # n_channels = [] - # for j in range(self.n_levels): - # temp = self.features[j](temp_img) - # n_channels.append(temp.n_channels) - # # string about features and channels - # if self.pyramid_on_features: - # feat_str = "- Feature is {} with ".format( - # name_of_callable(self.features)) - # if n_channels[0] == 1: - # ch_str = ["channel"] - # else: - # ch_str = ["channels"] - # else: - # feat_str = [] - # ch_str = [] - # for j in range(self.n_levels): - # if isinstance(self.features[j], str): - # feat_str.append("- Feature is {} with ".format( - # self.features[j])) - # elif self.features[j] is None: - # feat_str.append("- No features extracted. ") - # else: - # feat_str.append("- Feature is {} with ".format( - # self.features[j].__name__)) - # if n_channels[j] == 1: - # ch_str.append("channel") - # else: - # ch_str.append("channels") - # if self.n_levels > 1: - # out = "{} - Gaussian pyramid with {} levels and downscale " \ - # "factor of {}.\n".format(out, self.n_levels, - # self.downscale) - # if self.pyramid_on_features: - # out = "{} - Pyramid was applied on feature space.\n " \ - # "{}{} {} per image.\n".format(out, feat_str, - # n_channels[0], ch_str[0]) - # else: - # out = "{} - Features were extracted at each pyramid " \ - # "level.\n".format(out) - # for i in range(self.n_levels - 1, -1, -1): - # out = "{} - Level {} {}: \n {}{} {} per " \ - # "image.\n".format( - # out, self.n_levels - i, down_str[i], feat_str[i], - # n_channels[i], ch_str[i]) - # else: - # if self.pyramid_on_features: - # feat_str = [feat_str] - # out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format( - # out, feat_str[0], n_channels[0], ch_str[0]) - # return out + if self.diagonal is not None: + diagonal = self.diagonal + else: + diagonal = np.sqrt(np.sum(np.asarray(self.reference_shape.bounds()) + ** 2)) + is_custom_perturb_func = (self._perturb_from_bounding_box != + noisy_shape_from_bounding_box) + regressor_cls = self.algorithms[0]._regressor_cls + + # Compute level info strings + level_info = [] + lvl_str_tmplt = r""" - Level {} (Scale {}) + - {} iterations + - Patch shape: {}""" + for k, s in enumerate(self.scales): + level_info.append(lvl_str_tmplt.format(k, s, + self.iterations[k], + self._patch_shape[k])) + level_info = '\n'.join(level_info) + + cls_str = r"""Supervised Descent Method + - Regression performed using the {reg_alg} algorithm + - Regression class: {reg_cls} + - Levels: {levels} +{level_info} + - Perturbations generated per shape: {n_perturbations} + - Images scaled to diagonal: {diagonal:.2f} + - Custom perturbation scheme used: {is_custom_perturb_func}""".format( + reg_alg=name_of_callable(self._sd_algorithm_cls), + reg_cls=name_of_callable(regressor_cls), + n_levels=len(self.scales), + levels=self.scales, + level_info=level_info, + n_perturbations=self.n_perturbations, + diagonal=diagonal, + is_custom_perturb_func=is_custom_perturb_func) + return cls_str From 32cffd4aa0f300d86ef870260876df44daf7747c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:20:20 +0100 Subject: [PATCH 324/423] Calculate diagonal correctly. --- menpofit/sdm/fitter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 5f548d3..2b3fc9a 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -291,8 +291,8 @@ def __str__(self): if self.diagonal is not None: diagonal = self.diagonal else: - diagonal = np.sqrt(np.sum(np.asarray(self.reference_shape.bounds()) - ** 2)) + y, x = self.reference_shape.range() + diagonal = np.sqrt(x ** 2 + y ** 2) is_custom_perturb_func = (self._perturb_from_bounding_box != noisy_shape_from_bounding_box) regressor_cls = self.algorithms[0]._regressor_cls @@ -317,11 +317,11 @@ def __str__(self): - Images scaled to diagonal: {diagonal:.2f} - Custom perturbation scheme used: {is_custom_perturb_func}""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), - reg_cls=name_of_callable(regressor_cls), - n_levels=len(self.scales), - levels=self.scales, - level_info=level_info, - n_perturbations=self.n_perturbations, - diagonal=diagonal, - is_custom_perturb_func=is_custom_perturb_func) + reg_cls=name_of_callable(regressor_cls), + n_levels=len(self.scales), + levels=self.scales, + level_info=level_info, + n_perturbations=self.n_perturbations, + diagonal=diagonal, + is_custom_perturb_func=is_custom_perturb_func) return cls_str From 914e296602498ab42b70b56f9b2c2ff496d0ee7c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:41:04 +0100 Subject: [PATCH 325/423] Update name_of_callable to support partial Return the name of the partially wrapped function rather than 'partial' --- menpofit/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index 5f641cd..179e6d4 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -1,4 +1,5 @@ from __future__ import division +from functools import partial import itertools import numpy as np from menpo.visualize import progress_bar_str, print_dynamic, print_progress @@ -6,7 +7,10 @@ def name_of_callable(c): try: - return c.__name__ # function + if isinstance(c, partial): # partial + return c.func.__name__ + else: + return c.__name__ # function except AttributeError: return c.__class__.__name__ # callable class From 636ceb380f8a8c8ccb39a8455475c0261630c526 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 24 Jul 2015 17:42:57 +0100 Subject: [PATCH 326/423] Partial commit before squash Have to leave to play squash, need to turn the PC off for the weekend due to power outage. Was working on removing the kwargs. --- menpofit/math/regression.py | 18 ++++++++++-------- menpofit/sdm/algorithm.py | 9 +++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/menpofit/math/regression.py b/menpofit/math/regression.py index 58db19f..82dcce0 100644 --- a/menpofit/math/regression.py +++ b/menpofit/math/regression.py @@ -6,9 +6,11 @@ class IRLRegression(object): r""" Incremental Regularized Linear Regression """ - def __init__(self, l=0, bias=True): - self.l = l + def __init__(self, alpha=0, bias=True): + self.alpha = alpha self.bias = bias + self.V = None + self.W = None def train(self, X, Y): if self.bias: @@ -17,7 +19,7 @@ def train(self, X, Y): # regularized linear regression XX = X.T.dot(X) - np.fill_diagonal(XX, self.l + np.diag(XX)) + np.fill_diagonal(XX, self.alpha + np.diag(XX)) self.V = np.linalg.inv(XX) self.W = self.V.dot(X.T.dot(Y)) @@ -48,9 +50,9 @@ class IIRLRegression(IRLRegression): r""" Indirect Incremental Regularized Linear Regression """ - def __init__(self, l=0, bias=True, d=0): - super(IIRLRegression, self).__init__(l=l, bias=bias) - self.d = d + def __init__(self, alpha=0, bias=True, alpha2=0): + super(IIRLRegression, self).__init__(alpha=alpha, bias=bias) + self.alpha2 = alpha2 def train(self, X, Y): # regularized linear regression exchanging the roles of X and Y @@ -59,7 +61,7 @@ def train(self, X, Y): # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) - np.fill_diagonal(H, self.d + np.diag(H)) + np.fill_diagonal(H, self.alpha2 + np.diag(H)) self.W = np.linalg.solve(H, J.T) def increment(self, X, Y): @@ -69,5 +71,5 @@ def increment(self, X, Y): # solve the original problem by computing the pseudo-inverse of the # previous solution H = J.T.dot(J) - np.fill_diagonal(H, self.d + np.diag(H)) + np.fill_diagonal(H, self.alpha2 + np.diag(H)) self.W = np.linalg.solve(H, J.T) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 92c54b4..e25fd3d 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -162,8 +162,8 @@ class Newton(SupervisedDescentAlgorithm): """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, - eps=10**-5): - self._regressor_cls = IRLRegression + eps=10**-5, alpha=0, bias=True): + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape @@ -178,8 +178,9 @@ class GaussNewton(SupervisedDescentAlgorithm): """ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, compute_error=compute_normalise_point_to_point_error, - eps=10**-5): - self._regressor_cls = IIRLRegression + eps=10**-5, alpha=0, bias=True, alpha2=0): + self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, + alpha2=alpha2) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape From 6beb4d52e8463967047abc16da624b1132b11c70 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Sun, 26 Jul 2015 15:53:15 +0100 Subject: [PATCH 327/423] Add recursive call to name_of_callable --- menpofit/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index 179e6d4..246e415 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -8,7 +8,9 @@ def name_of_callable(c): try: if isinstance(c, partial): # partial - return c.func.__name__ + # Recursively call as partial may be wrapping either a callable + # or a function (or another partial for some reason!) + return name_of_callable(c) else: return c.__name__ # function except AttributeError: From 3b45c7d6fbfe6fe19ccc8f2ccac2dffb381e22ec Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:54:41 +0100 Subject: [PATCH 328/423] Fix error in name_of_callable Meant to walk down the partial functions, forgot to do that and so was an infinite recursion. --- menpofit/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index 246e415..6ced9b5 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -10,7 +10,7 @@ def name_of_callable(c): if isinstance(c, partial): # partial # Recursively call as partial may be wrapping either a callable # or a function (or another partial for some reason!) - return name_of_callable(c) + return name_of_callable(c.func) else: return c.__name__ # function except AttributeError: From ffc3d2e27cbc23c32cd9079c3b93d77d8175bbbb Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:55:14 +0100 Subject: [PATCH 329/423] Incorrect indent in batch --- menpofit/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/base.py b/menpofit/base.py index 6ced9b5..8d84311 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -22,7 +22,7 @@ def batch(iterable, n): while True: chunk = tuple(itertools.islice(it, n)) if not chunk: - return + return yield chunk From 68704f8b353c0fa09260d41f862f97f11880c946 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:55:49 +0100 Subject: [PATCH 330/423] SDM refactor training Move training into the constructor. This makes it easier to serialize SDMs. Now, you build a single model which gets trained inside the constructor. Therefore, I also refactored the train_incrementally function to inside the train method - so all training can be done incrementally. --- menpofit/sdm/fitter.py | 302 +++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 147 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2b3fc9a..f79d2ab 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,4 +1,5 @@ from __future__ import division +from itertools import chain import numpy as np from functools import partial import warnings @@ -19,10 +20,12 @@ class SupervisedDescentFitter(MultiFitter): r""" """ - def __init__(self, sd_algorithm_cls=Newton, holistic_feature=no_op, + def __init__(self, images, group=None, bounding_box_group=None, + sd_algorithm_cls=Newton, holistic_feature=no_op, patch_features=no_op, patch_shape=(17, 17), diagonal=None, scales=(1, 0.5), iterations=6, n_perturbations=30, - perturb_from_bounding_box=noisy_shape_from_bounding_box): + perturb_from_bounding_box=noisy_shape_from_bounding_box, + batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -41,12 +44,13 @@ def __init__(self, sd_algorithm_cls=Newton, holistic_feature=no_op, self.iterations = checks.check_max_iters(iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms - self._reset_algorithms() + self._setup_algorithms() - def _reset_algorithms(self): - if len(self.algorithms) > 0: - for j in range(len(self.algorithms) - 1, -1, -1): - del self.algorithms[j] + # Now, train the model! + self._train(images, group=group, bounding_box_group=bounding_box_group, + verbose=verbose, increment=False, batch_size=batch_size) + + def _setup_algorithms(self): for j in range(self.n_levels): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], @@ -57,153 +61,157 @@ def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, bounding_box) - def train(self, images, group=None, label=None, bounding_box_group=None, - verbose=False, increment=False, **kwargs): + def _train(self, images, group=None, bounding_box_group=None, + verbose=False, increment=False, batch_size=None): + + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + first_batch = next(image_batches) + else: + image_batches = [] + first_batch = list(images) + # In the case where group is None, we need to get the only key so that - # we can add landmarks below and not get a complaint about using None - first_image = images[0] + # we can attach landmarks below and not get a complaint about using None + first_image = first_batch[0] if group is None: group = first_image.landmarks.group_labels[0] - if not increment: - # Reset the algorithm classes - self._reset_algorithms() - # Normalize images and compute reference shape - self.reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) - else: - if len(self.algorithms) == 0: - raise ValueError('Must train before training incrementally.') - # We are incrementing, so rescale to existing reference shape - images = rescale_images_to_reference_shape(images, group, label, - self.reference_shape, - verbose=verbose) - - # No bounding box is given, so we will use the ground truth box - if bounding_box_group is None: - bounding_box_group = '__gt_bb_' - for i in images: - gt_s = i.landmarks[group][label] - perturb_bbox_group = bounding_box_group + '0' - i.landmarks[perturb_bbox_group] = gt_s.bounding_box() - - # Find all bounding boxes on the images with the given bounding box key - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) - n_perturbations = len(all_bb_keys) - - # If there is only one example bounding box, then we will generate - # more perturbations based on the bounding box. - if n_perturbations == 1: - msg = '- Generating {} new initial bounding boxes ' \ - 'per image'.format(self.n_perturbations) - wrap = partial(print_progress, prefix=msg, verbose=verbose) + for k, image_batch in enumerate(chain([first_batch], image_batches)): + # After the first batch, we are incrementing the model + if k > 0: + increment = True - for i in wrap(images): - # We assume that the first bounding box is a valid perturbation - # thus create n_perturbations - 1 new bounding boxes - for j in range(1, self.n_perturbations): - gt_s = i.landmarks[group][label].bounding_box() - bb = i.landmarks[all_bb_keys[0]].lms - - # This is customizable by passing in the correct method - p_s = self._perturb_from_bounding_box(gt_s, bb) - perturb_bbox_group = bounding_box_group + '_{}'.format(j) - i.landmarks[perturb_bbox_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) - self.n_perturbations = n_perturbations - - # Re-grab all the bounding box keys for iterating over when calculating - # perturbations - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) - - # Before scaling, we compute the holistic feature on the whole image - msg = '- Computing holistic features ({})'.format( - name_of_callable(self._holistic_feature)) - wrap = partial(print_progress, prefix=msg, verbose=verbose) - images = [self._holistic_feature(im) for im in wrap(images)] - - # for each pyramid level (low --> high) - for j in range(self.n_levels): if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - else: - level_str = None - - # Scale images - level_images = scale_images(images, self.scales[j], - level_str=level_str, verbose=verbose) - - # Extract scaled ground truth shapes for current level - level_gt_shapes = [i.landmarks[group][label] for i in level_images] - - if j == 0: - msg = '{}Generating {} perturbations per image'.format( - level_str, self.n_perturbations) - wrap = partial(print_progress, prefix=msg, - end_with_newline=False, verbose=verbose) - - # Extract perturbations at the very bottom level - current_shapes = [] - for i in wrap(level_images): - c_shapes = [] - for perturb_bbox_group in all_bb_keys: - bbox = i.landmarks[perturb_bbox_group].lms - c_s = align_shape_with_bounding_box( - self.reference_shape, bbox) - c_shapes.append(c_s) - current_shapes.append(c_shapes) - - # train supervised descent algorithm - if increment: - current_shapes = self.algorithms[j].increment( - level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) - else: - current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose, **kwargs) - - # Scale current shapes to next level resolution - if self.scales[j] != (1 or self.scales[-1]): - transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) - - def increment(self, images, group=None, label=None, - bounding_box_group=None, verbose=False, - **kwargs): - return self.train(images, group=group, label=label, - bounding_box_group=bounding_box_group, - verbose=verbose, - increment=True, **kwargs) - - def train_incrementally(self, images, group=None, label=None, - batch_size=100, verbose=False, **kwargs): - # Create a generator of fixed sized batches. Will still work even - # on an infinite list. - image_batches = batch(images, batch_size) - - # Train all batches - for k, image_batch in enumerate(image_batches): - n_images = len(image_batch) - if verbose: - print('Training batch {} of {} images.'.format(k, n_images)) - if k == 0: - self.train(image_batch, group=group, label=label, - verbose=verbose, **kwargs) + print('Computing batch {}'.format(k)) + + if not increment: + # Normalize images and compute reference shape + self.reference_shape, image_batch = normalization_wrt_reference_shape( + image_batch, group, self.diagonal, verbose=verbose) else: - self.increment(image_batch, group=group, label=label, - verbose=verbose, **kwargs) + # We are incrementing, so rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # No bounding box is given, so we will use the ground truth box + if bounding_box_group is None: + bounding_box_group = '__gt_bb_' + for i in image_batch: + gt_s = i.landmarks[group].lms + perturb_bbox_group = bounding_box_group + '0' + i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + + # Find all bounding boxes on the images with the given bounding + # box key + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) + n_perturbations = len(all_bb_keys) + + # If there is only one example bounding box, then we will generate + # more perturbations based on the bounding box. + if n_perturbations == 1: + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + + for i in wrap(image_batch): + # We assume that the first bounding box is a valid + # perturbation thus create n_perturbations - 1 new bounding + # boxes + for j in range(1, self.n_perturbations): + gt_s = i.landmarks[group].lms.bounding_box() + bb = i.landmarks[all_bb_keys[0]].lms + + # This is customizable by passing in the correct method + p_s = self._perturb_from_bounding_box(gt_s, bb) + perturb_bbox_group = '{}_{}'.format(bounding_box_group, + j) + i.landmarks[perturb_bbox_group] = p_s + elif n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided bounding_box_group.'. + format(self.n_perturbations, n_perturbations)) + self.n_perturbations = n_perturbations + + # Re-grab all the bounding box keys for iterating over when + # calculating perturbations + all_bb_keys = list(first_image.landmarks.keys_matching( + '*{}*'.format(bounding_box_group))) + + # Before scaling, we compute the holistic feature on the whole image + msg = '- Computing holistic features ({})'.format( + name_of_callable(self._holistic_feature)) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + image_batch = [self._holistic_feature(im) + for im in wrap(image_batch)] + + # for each pyramid level (low --> high) + current_shapes = [] + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + else: + level_str = None + + # Scale images + level_images = scale_images(image_batch, self.scales[j], + level_str=level_str, + verbose=verbose) + + # Extract scaled ground truth shapes for current level + level_gt_shapes = [i.landmarks[group].lms + for i in level_images] + + if j == 0: + msg = '{}Generating {} perturbations per image'.format( + level_str, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False, verbose=verbose) + + # Extract perturbations at the very bottom level + for i in wrap(level_images): + c_shapes = [] + for perturb_bbox_group in all_bb_keys: + bbox = i.landmarks[perturb_bbox_group].lms + c_s = align_shape_with_bounding_box( + self.reference_shape, bbox) + c_shapes.append(c_s) + current_shapes.append(c_shapes) + + # train supervised descent algorithm + if increment: + current_shapes = self.algorithms[j].increment( + level_images, level_gt_shapes, current_shapes, + verbose=verbose) + else: + current_shapes = self.algorithms[j].train( + level_images, level_gt_shapes, current_shapes, + level_str=level_str, verbose=verbose) + + # Scale current shapes to next level resolution + if self.scales[j] != (1 or self.scales[-1]): + transform = Scale(self.scales[j + 1] / self.scales[j], + n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) + + def increment(self, images, group=None, bounding_box_group=None, + verbose=False): + return self._train(images, group=group, + bounding_box_group=bounding_box_group, + verbose=verbose, + increment=True) def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): From 300759c46a19f40739718911db5d78d0f4011a90 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:58:22 +0100 Subject: [PATCH 331/423] Allow batch training of incremental SDM --- menpofit/sdm/fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index f79d2ab..2dcf5e4 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -207,11 +207,11 @@ def _train(self, images, group=None, bounding_box_group=None, transform.apply_inplace(shape) def increment(self, images, group=None, bounding_box_group=None, - verbose=False): + verbose=False, batch_size=None): return self._train(images, group=group, bounding_box_group=bounding_box_group, verbose=verbose, - increment=True) + increment=True, batch_size=batch_size) def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): From c52424fe5e9bff3bd558dca50870c6ae7d464e2a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 10:59:39 +0100 Subject: [PATCH 332/423] Remove label kwarg from builder functions --- menpofit/builder.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index b1f4b2a..ae2fb48 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -49,7 +49,7 @@ def compute_reference_shape(shapes, normalization_diagonal, verbose=False): # TODO: document me! -def rescale_images_to_reference_shape(images, group, label, reference_shape, +def rescale_images_to_reference_shape(images, group, reference_shape, verbose=False): r""" """ @@ -58,13 +58,12 @@ def rescale_images_to_reference_shape(images, group, label, reference_shape, # Normalize the scaling of all images wrt the reference_shape size normalized_images = [i.rescale_to_reference_shape(reference_shape, - group=group, label=label) + group=group) for i in wrap(images)] return normalized_images -def normalization_wrt_reference_shape(images, group, label, diagonal, - verbose=False): +def normalization_wrt_reference_shape(images, group, diagonal, verbose=False): r""" Function that normalizes the images sizes with respect to the reference shape (mean shape) scaling. This step is essential before building a @@ -81,15 +80,9 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, ---------- images : list of :class:`menpo.image.MaskedImage` The set of landmarked images to normalize. - group : `str` The key of the landmark set that should be used. If None, and if there is only one set of landmarks, this set will be used. - - label : `str` - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - diagonal: `int` If int, it ensures that the mean shape is scaled so that the diagonal of the bounding box containing it matches the @@ -100,7 +93,6 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, landmarks, this kwarg also specifies the diagonal length of the reference frame (provided that features computation does not change the image size). - verbose : `bool`, Optional Flag that controls information and progress printing. @@ -113,14 +105,14 @@ def normalization_wrt_reference_shape(images, group, label, diagonal, A list with the normalized images. """ # get shapes - shapes = [i.landmarks[group][label] for i in images] + shapes = [i.landmarks[group].lms for i in images] # compute the reference shape and fix its diagonal length reference_shape = compute_reference_shape(shapes, diagonal, verbose=verbose) # normalize the scaling of all images wrt the reference_shape size normalized_images = rescale_images_to_reference_shape( - images, group, label, reference_shape, verbose=verbose) + images, group, reference_shape, verbose=verbose) return reference_shape, normalized_images From bdcecce3a104281a62720e30e4a2f5089714ec27 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:01:24 +0100 Subject: [PATCH 333/423] Remove label kwarg from AAMs --- menpofit/aam/builder.py | 25 +++++++++---------------- menpofit/aam/fitter.py | 6 +++--- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 373236a..717a813 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -145,7 +145,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - def build(self, images, group=None, label=None, verbose=False): + def build(self, images, group=None, verbose=False): r""" Builds an Active Appearance Model from a list of landmarked images. @@ -153,15 +153,9 @@ def build(self, images, group=None, label=None, verbose=False): ---------- images : list of :map:`MaskedImage` The set of landmarked images from which to build the AAM. - group : `string`, optional The key of the landmark set that should be used. If ``None``, and if there is only one set of landmarks, this set will be used. - - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - verbose : `boolean`, optional Flag that controls information and progress printing. @@ -173,7 +167,7 @@ def build(self, images, group=None, label=None, verbose=False): """ # normalize images and compute reference shape reference_shape, images = normalization_wrt_reference_shape( - images, group, label, self.diagonal, verbose=verbose) + images, group, self.diagonal, verbose=verbose) # build models at each scale if verbose: @@ -210,7 +204,7 @@ def build(self, images, group=None, label=None, verbose=False): verbose=verbose) # extract potentially rescaled shapes - level_shapes = [i.landmarks[group][label] + level_shapes = [i.landmarks[group].lms for i in level_images] # obtain shape representation @@ -255,11 +249,11 @@ def build(self, images, group=None, label=None, verbose=False): return aam - def increment(self, aam, images, group=None, label=None, + def increment(self, aam, images, group=None, forgetting_factor=1.0, verbose=False): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( - images, group, label, aam.reference_shape, verbose=verbose) + images, group, aam.reference_shape, verbose=verbose) # increment models at each scale if verbose: @@ -295,7 +289,7 @@ def increment(self, aam, images, group=None, label=None, verbose=verbose) # extract potentially rescaled shapes - level_shapes = [i.landmarks[group][label] + level_shapes = [i.landmarks[group].lms for i in level_images] # obtain shape representation @@ -337,7 +331,7 @@ def increment(self, aam, images, group=None, label=None, if verbose: print_dynamic('{}Done\n'.format(level_str)) - def build_incrementally(self, images, group=None, label=None, + def build_incrementally(self, images, group=None, forgetting_factor=1.0, batch_size=100, verbose=False): # number of batches @@ -345,15 +339,14 @@ def build_incrementally(self, images, group=None, label=None, # train first batch print 'Training batch 1.' - aam = self.build(images[:batch_size], group=group, label=label, - verbose=verbose) + aam = self.build(images[:batch_size], group=group, verbose=verbose) # train all other batches start = batch_size for j in range(1, n_batches): print 'Training batch {}.'.format(j+1) end = start + batch_size - self.increment(aam, images[start:end], group=group, label=label, + self.increment(aam, images[start:end], group=group, forgetting_factor=forgetting_factor, verbose=verbose) start = end diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index a8fbc9c..24fe85c 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -166,10 +166,10 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): self.algorithms.append(algorithm) # TODO: Allow training from bounding boxes - def train(self, images, group=None, label=None, verbose=False, **kwargs): + def train(self, images, group=None, verbose=False, **kwargs): # normalize images with respect to reference shape of aam images = rescale_images_to_reference_shape( - images, group, label, self.reference_shape, verbose=verbose) + images, group, self.reference_shape, verbose=verbose) if self.scale_features: # compute features at highest level @@ -202,7 +202,7 @@ def train(self, images, group=None, label=None, verbose=False, **kwargs): verbose=verbose) # extract ground truth shapes for current level - level_gt_shapes = [i.landmarks[group][label] for i in level_images] + level_gt_shapes = [i.landmarks[group].lms for i in level_images] if j == 0: # generate perturbed shapes From f6bd1e95a8602919178741dd835f530939e46f05 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:02:09 +0100 Subject: [PATCH 334/423] Remove label kwarg from ATM --- menpofit/atm/builder.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 42c95f9..0f44545 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -124,7 +124,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, self.scale_features = scale_features self.max_shape_components = max_shape_components - def build(self, shapes, template, group=None, label=None, verbose=False): + def build(self, shapes, template, group=None, verbose=False): r""" Builds a Multilevel Active Template Model given a list of shapes and a template image. @@ -133,21 +133,13 @@ def build(self, shapes, template, group=None, label=None, verbose=False): ---------- shapes : list of :map:`PointCloud` The set of shapes from which to build the shape model of the ATM. - template : :map:`Image` or subclass The image to be used as template. - - group : `string`, optional + group : `str`, optional The key of the landmark set of the template that should be used. If ``None``, and if there is only one set of landmarks, this set will be used. - - label : `string`, optional - The label of the landmark manager of the template that you wish to - use. If ``None`` is passed, the convex hull of all landmarks is - used. - - verbose : `boolean`, optional + verbose : `bool`, optional Flag that controls information and progress printing. Returns @@ -162,7 +154,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): # normalize the template size using the reference_shape scaling template = template.rescale_to_reference_shape( - reference_shape, group=group, label=label) + reference_shape, group=group) # build models at each scale if verbose: @@ -214,7 +206,7 @@ def build(self, shapes, template, group=None, label=None, verbose=False): level_template = self.features[j](scaled_template) # extract potentially rescaled template shape - level_template_shape = level_template.landmarks[group][label] + level_template_shape = level_template.landmarks[group].lms # obtain warped template warped_template = self._warp_template(level_template, From cca54dd8f9d5e3ecc98002cc7e114eadb6eb581b Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:02:42 +0100 Subject: [PATCH 335/423] Remove label kwarg from LK package --- menpofit/lk/fitter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 5e1cc72..e00501c 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -12,7 +12,7 @@ class LucasKanadeFitter(MultiFitter): r""" """ - def __init__(self, template, group=None, label=None, features=no_op, + def __init__(self, template, group=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, scales=(1, .5), scale_features=True, algorithm_cls=InverseCompositional, residual_cls=SSD, @@ -31,7 +31,7 @@ def __init__(self, template, group=None, label=None, features=no_op, self.scale_features = scale_features self.templates, self.sources = self._prepare_template( - template, group=group, label=label) + template, group=group) self.reference_shape = self.sources[0] @@ -49,14 +49,14 @@ def __init__(self, template, group=None, label=None, features=no_op, algorithm = algorithm_cls(t, transform, residual, **kwargs) self.algorithms.append(algorithm) - def _prepare_template(self, template, group=None, label=None): - template = template.crop_to_landmarks(group=group, label=label) + def _prepare_template(self, template, group=None): + template = template.crop_to_landmarks(group=group) template = template.as_masked() # rescale template to diagonal range if self.diagonal: template = template.rescale_landmarks_to_diagonal_range( - self.diagonal, group=group, label=label) + self.diagonal, group=group) # obtain image representation templates = [] @@ -75,7 +75,7 @@ def _prepare_template(self, template, group=None, label=None): templates.reverse() # get sources per level - sources = [i.landmarks[group][label] for i in templates] + sources = [i.landmarks[group].lms for i in templates] return templates, sources From 75f57afc0c9b280f15da5b3f91016fea22285c3a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:04:01 +0100 Subject: [PATCH 336/423] Remove label kwarg from visualize package --- menpofit/visualize/widgets/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index cb634bf..302a5e2 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -213,7 +213,7 @@ def render_function(name, value): axes_font_weight=tmp3['axes_font_weight'], axes_x_limits=tmp3['axes_x_limits'], axes_y_limits=tmp3['axes_y_limits'], - figure_size=new_figure_size, label=None) + figure_size=new_figure_size) # Invert y axis if needed if mean_wid.value and axes_mode_wid.value == 1: @@ -247,7 +247,7 @@ def render_function(name, value): axes_font_weight=tmp3['axes_font_weight'], axes_x_limits=tmp3['axes_x_limits'], axes_y_limits=tmp3['axes_y_limits'], - figure_size=new_figure_size, label=None) + figure_size=new_figure_size) # Render vectors ax = plt.gca() From 50237afc1c1e9392279533f58b5b246f180412e9 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 27 Jul 2015 11:54:38 +0100 Subject: [PATCH 337/423] Cleanup the feature/patch extraction code Now just uses the asarray codepath since the performance difference is negligable. Also, renamed those methods. --- menpofit/sdm/algorithm.py | 92 ++++++++++----------------------------- 1 file changed, 23 insertions(+), 69 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index e25fd3d..29081af 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -14,13 +14,10 @@ class SupervisedDescentAlgorithm(object): r""" """ def train(self, images, gt_shapes, current_shapes, level_str='', - verbose=False, **kwargs): + verbose=False): n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] - self._features_patch_length = compute_features_info( - images[0], gt_shapes[0], self.features, - patch_shape=self.patch_shape)[1] # obtain delta_x and gt_x delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) @@ -31,9 +28,8 @@ def train(self, images, gt_shapes, current_shapes, level_str='', # Cascaded Regression loop for k in range(self.iterations): # generate regression data - features = obtain_patch_features( + features = features_per_image( images, current_shapes, self.patch_shape, self.features, - features_patch_length=self._features_patch_length, level_str='{}(Iteration {}) - '.format(level_str, k), verbose=verbose) @@ -41,7 +37,7 @@ def train(self, images, gt_shapes, current_shapes, level_str='', if verbose: print_dynamic('{}(Iteration {}) - Performing regression'.format( level_str, k)) - r = self._regressor_cls(**kwargs) + r = self._regressor_cls() r.train(features, delta_x) # add regressor to list self.regressors.append(r) @@ -79,8 +75,7 @@ def train(self, images, gt_shapes, current_shapes, level_str='', return current_shapes - def increment(self, images, gt_shapes, current_shapes, verbose=False, - **kwarg): + def increment(self, images, gt_shapes, current_shapes, verbose=False): n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] @@ -91,9 +86,8 @@ def increment(self, images, gt_shapes, current_shapes, verbose=False, # Cascaded Regression loop for r in self.regressors: # generate regression data - features = obtain_patch_features( - images, current_shapes, self.patch_shape, self.features, - features_patch_length=self._features_patch_length) + features = features_per_image(images, current_shapes, + self.patch_shape, self.features) # update regression if verbose: @@ -139,9 +133,8 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # Cascaded Regression loop for r in self.regressors: # compute regression features - features = compute_patch_features( - image, current_shape, self.patch_shape, self.features, - features_patch_length=self._features_patch_length) + features = features_per_patch(image, current_shape, + self.patch_shape, self.features) # solve for increments on the shape vector dx = r.predict(features) @@ -190,79 +183,40 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, # TODO: docment me! -def compute_patch_features(image, shape, patch_shape, features_callable, - features_patch_length=None): +def features_per_patch(image, shape, patch_shape, features_callable): """r """ patches = image.extract_patches(shape, patch_size=patch_shape, as_single_array=True) - if features_patch_length: - patch_features = np.empty((shape.n_points, features_patch_length)) - for j, p in enumerate(patches): - patch_features[j] = features_callable(p[0]).ravel() - else: - patch_features = [] - for p in patches: - patch_features.append(features_callable(p[0]).ravel()) - patch_features = np.asarray(patch_features) - - return patch_features.ravel() + patch_features = [features_callable(p[0]).ravel() for p in patches] + return np.asarray(patch_features).ravel() # TODO: docment me! -def generate_patch_features(image, shapes, patch_shape, features_callable, - features_patch_length=None): +def features_per_shape(image, shapes, patch_shape, features_callable): """r """ - if features_patch_length: - patch_features = np.empty((len(shapes), - shapes[0].n_points * features_patch_length)) - for j, s in enumerate(shapes): - patch_features[j] = compute_patch_features( - image, s, patch_shape, features_callable, - features_patch_length=features_patch_length) - else: - patch_features = [] - for s in shapes: - patch_features.append(compute_patch_features( - image, s, patch_shape, features_callable, - features_patch_length=features_patch_length)) - patch_features = np.asarray(patch_features) - - return patch_features.ravel() + patch_features = [features_per_patch(image, s, patch_shape, + features_callable) + for s in shapes] + + return np.asarray(patch_features).ravel() # TODO: docment me! -def obtain_patch_features(images, shapes, patch_shape, features_callable, - features_patch_length=None, level_str='', - verbose=False): +def features_per_image(images, shapes, patch_shape, features_callable, + level_str='', verbose=False): """r """ wrap = partial(print_progress, prefix='{}Extracting patches'.format(level_str), end_with_newline=not level_str, verbose=verbose) - n_images = len(images) - n_shapes = len(shapes[0]) - n_points = shapes[0][0].n_points - - if features_patch_length: - patch_features = np.empty((n_images, (n_shapes * n_points * - features_patch_length))) - for j, i in enumerate(wrap(images)): - patch_features[j] = generate_patch_features( - i, shapes[j], patch_shape, features_callable, - features_patch_length=features_patch_length) - else: - patch_features = [] - for j, i in enumerate(wrap(images)): - patch_features.append(generate_patch_features( - i, shapes[j], patch_shape, features_callable, - features_patch_length=features_patch_length)) - patch_features = np.asarray(patch_features) - - return patch_features.reshape((-1, n_points * features_patch_length)) + patch_features = [features_per_shape(i, shapes[j], patch_shape, + features_callable) + for j, i in enumerate(wrap(images))] + return np.asarray(patch_features) def compute_delta_x(gt_shape, current_shapes): From 6e8dec48fe3a6cd45793c39904ec1709ab65a633 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 27 Jul 2015 16:25:55 +0100 Subject: [PATCH 338/423] Fixes some errors introduced by sdm_refactor - Also define SDM alias --- menpofit/sdm/__init__.py | 2 +- menpofit/sdm/algorithm.py | 5 +++-- menpofit/sdm/fitter.py | 10 ++++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/menpofit/sdm/__init__.py b/menpofit/sdm/__init__.py index 16e88b4..8a616c7 100644 --- a/menpofit/sdm/__init__.py +++ b/menpofit/sdm/__init__.py @@ -1,2 +1,2 @@ from .algorithm import Newton, GaussNewton -from .fitter import SupervisedDescentFitter +from .fitter import SupervisedDescentFitter, SDM diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 29081af..b4fc7e7 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -201,7 +201,7 @@ def features_per_shape(image, shapes, patch_shape, features_callable): features_callable) for s in shapes] - return np.asarray(patch_features).ravel() + return np.asarray(patch_features) # TODO: docment me! @@ -216,7 +216,8 @@ def features_per_image(images, shapes, patch_shape, features_callable, patch_features = [features_per_shape(i, shapes[j], patch_shape, features_callable) for j, i in enumerate(wrap(images))] - return np.asarray(patch_features) + patch_features = np.asarray(patch_features) + return patch_features.reshape((-1, patch_features.shape[-1])) def compute_delta_x(gt_shape, current_shapes): diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 2dcf5e4..ab42b8c 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -77,9 +77,8 @@ def _train(self, images, group=None, bounding_box_group=None, # In the case where group is None, we need to get the only key so that # we can attach landmarks below and not get a complaint about using None - first_image = first_batch[0] if group is None: - group = first_image.landmarks.group_labels[0] + group = first_batch[0].landmarks.group_labels[0] for k, image_batch in enumerate(chain([first_batch], image_batches)): # After the first batch, we are incrementing the model @@ -109,7 +108,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Find all bounding boxes on the images with the given bounding # box key - all_bb_keys = list(first_image.landmarks.keys_matching( + all_bb_keys = list(image_batch[0].landmarks.keys_matching( '*{}*'.format(bounding_box_group))) n_perturbations = len(all_bb_keys) @@ -142,7 +141,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Re-grab all the bounding box keys for iterating over when # calculating perturbations - all_bb_keys = list(first_image.landmarks.keys_matching( + all_bb_keys = list(image_batch[0].landmarks.keys_matching( '*{}*'.format(bounding_box_group))) # Before scaling, we compute the holistic feature on the whole image @@ -333,3 +332,6 @@ def __str__(self): diagonal=diagonal, is_custom_perturb_func=is_custom_perturb_func) return cls_str + + +SDM = partial(SupervisedDescentFitter, sd_algorithm_cls=Newton) From 9ba57b5c7b558044b9bc67147fbe7f4562067fa2 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 10:44:22 +0100 Subject: [PATCH 339/423] Get rid of the first_batch thing That was a bit confusing and @jalabort rightly pointed out that it didn't work correctly anyway. --- menpofit/sdm/fitter.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index ab42b8c..72a09e8 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -70,17 +70,10 @@ def _train(self, images, group=None, bounding_box_group=None, # Create a generator of fixed sized batches. Will still work even # on an infinite list. image_batches = batch(images, batch_size) - first_batch = next(image_batches) else: - image_batches = [] - first_batch = list(images) + image_batches = [list(images)] - # In the case where group is None, we need to get the only key so that - # we can attach landmarks below and not get a complaint about using None - if group is None: - group = first_batch[0].landmarks.group_labels[0] - - for k, image_batch in enumerate(chain([first_batch], image_batches)): + for k, image_batch in enumerate(image_batches): # After the first batch, we are incrementing the model if k > 0: increment = True @@ -88,6 +81,13 @@ def _train(self, images, group=None, bounding_box_group=None, if verbose: print('Computing batch {}'.format(k)) + # In the case where group is None, we need to get the only key so + # that we can attach landmarks below and not get a complaint about + # using None + first_image = image_batch[0] + if group is None: + group = first_image.landmarks.group_labels[0] + if not increment: # Normalize images and compute reference shape self.reference_shape, image_batch = normalization_wrt_reference_shape( @@ -108,7 +108,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Find all bounding boxes on the images with the given bounding # box key - all_bb_keys = list(image_batch[0].landmarks.keys_matching( + all_bb_keys = list(first_image.landmarks.keys_matching( '*{}*'.format(bounding_box_group))) n_perturbations = len(all_bb_keys) @@ -141,7 +141,7 @@ def _train(self, images, group=None, bounding_box_group=None, # Re-grab all the bounding box keys for iterating over when # calculating perturbations - all_bb_keys = list(image_batch[0].landmarks.keys_matching( + all_bb_keys = list(first_image.landmarks.keys_matching( '*{}*'.format(bounding_box_group))) # Before scaling, we compute the holistic feature on the whole image From 9a7c3bb8797a5eee382a098cfbdf278e48adea05 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 10:49:17 +0100 Subject: [PATCH 340/423] SDM, flip the scales logic so that its increasing Go from smallest to largest scales rather than reversing the list. --- menpofit/sdm/fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 72a09e8..ec60492 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -23,7 +23,7 @@ class SupervisedDescentFitter(MultiFitter): def __init__(self, images, group=None, bounding_box_group=None, sd_algorithm_cls=Newton, holistic_feature=no_op, patch_features=no_op, patch_shape=(17, 17), diagonal=None, - scales=(1, 0.5), iterations=6, n_perturbations=30, + scales=(0.5, 1.0), iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): # check parameters @@ -39,7 +39,7 @@ def __init__(self, images, group=None, bounding_box_group=None, self._patch_features = patch_features self._patch_shape = patch_shape self.diagonal = diagonal - self.scales = list(scales)[::-1] + self.scales = scales self.n_perturbations = n_perturbations self.iterations = checks.check_max_iters(iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box From e8b829c3e96753448785a7d087a6c086610a343a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 10:50:45 +0100 Subject: [PATCH 341/423] Remove useless tests and benchmark packages Benchmark has been totally superseded by menpobench and the failing tests were annoying and pointless. --- menpofit/benchmark/__init__.py | 1 - menpofit/benchmark/base.py | 699 ---------------- menpofit/benchmark/io.py | 114 --- menpofit/benchmark/predefined.py | 779 ------------------ menpofit/test/__init__.py | 0 menpofit/test/aam_builder_test.py | 199 ----- menpofit/test/aam_fitter_test.py | 451 ---------- menpofit/test/atm_builder_test.py | 178 ---- menpofit/test/atm_fitter_test.py | 441 ---------- menpofit/test/clm_builder_test.py | 197 ----- menpofit/test/clm_fitter_test.py | 373 --------- menpofit/test/fitmulitlevel_base_test.py | 29 - menpofit/test/fittingresult_test.py | 108 --- .../test/multilevel_fittingresult_test.py | 18 - menpofit/test/sdm_test.py | 114 --- 15 files changed, 3701 deletions(-) delete mode 100644 menpofit/benchmark/__init__.py delete mode 100644 menpofit/benchmark/base.py delete mode 100644 menpofit/benchmark/io.py delete mode 100644 menpofit/benchmark/predefined.py delete mode 100644 menpofit/test/__init__.py delete mode 100644 menpofit/test/aam_builder_test.py delete mode 100644 menpofit/test/aam_fitter_test.py delete mode 100644 menpofit/test/atm_builder_test.py delete mode 100644 menpofit/test/atm_fitter_test.py delete mode 100644 menpofit/test/clm_builder_test.py delete mode 100644 menpofit/test/clm_fitter_test.py delete mode 100644 menpofit/test/fitmulitlevel_base_test.py delete mode 100644 menpofit/test/fittingresult_test.py delete mode 100644 menpofit/test/multilevel_fittingresult_test.py delete mode 100644 menpofit/test/sdm_test.py diff --git a/menpofit/benchmark/__init__.py b/menpofit/benchmark/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/menpofit/benchmark/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/menpofit/benchmark/base.py b/menpofit/benchmark/base.py deleted file mode 100644 index c87ef2d..0000000 --- a/menpofit/benchmark/base.py +++ /dev/null @@ -1,699 +0,0 @@ -import os - -import numpy as np - -import menpo.io as mio -from menpo.visualize.text_utils import print_dynamic, progress_bar_str -from menpo.landmark import labeller -from menpo.visualize.base import GraphPlotter -from menpofit.aam import AAMBuilder, LucasKanadeAAMFitter -from menpofit.clm import CLMBuilder, GradientDescentCLMFitter -from menpofit.sdm import SDMTrainer - - -def aam_fit_benchmark(fitting_images, aam, fitting_options=None, - perturb_options=None, verbose=False): - r""" - Fits a trained AAM model to a database. - - Parameters - ---------- - fitting_images: list of :class:MaskedImage objects - A list of the fitting images. - aam: :class:menpo.fitmultilevel.aam.AAM object - The trained AAM object. It can be generated from the - aam_build_benchmark() method. - fitting_options: dictionary, optional - A dictionary with the parameters that will be passed in the - LucasKanadeAAMFitter (:class:menpo.fitmultilevel.aam.base). - If None, the default options will be used. - This is an example of the dictionary with the default options: - fitting_options = {'algorithm': AlternatingInverseCompositional, - 'md_transform': OrthoMDTransform, - 'global_transform': AlignmentSimilarity, - 'n_shape': None, - 'n_appearance': None, - 'max_iters': 50, - 'error_type': 'me_norm' - } - For an explanation of the options, please refer to the - LucasKanadeAAMFitter documentation. - - Default: None - bounding_boxes: list of (2, 2) ndarray, optional - If provided, fits will be initialized from a bounding box. If - None, perturbation of ground truth will be used instead. - can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. - perturb_options: dictionary, optional - A dictionary with parameters that control the perturbation on the - ground truth shape with noise of specified std. Note that if - bounding_box is provided perturb_options is ignored and not used. - If None, the default options will be used. - This is an example of the dictionary with the default options: - initialization_options = {'noise_std': 0.04, - 'rotation': False - } - For an explanation of the options, please refer to the perturb_shape() - method documentation of :map:`MultilevelFitter`. - verbose: bool, optional - If True, it prints information regarding the AAM fitting including - progress bar, current image error and percentage of images with errors - less or equal than a value. - - Default: False - - Returns - ------- - fitting_results: :map:`FittingResultList` - A list with the :map:`FittingResult` object per image. - """ - if verbose: - print('AAM Fitting:') - perc1 = 0. - perc2 = 0. - - # parse options - if fitting_options is None: - fitting_options = {} - if perturb_options is None: - perturb_options = {} - - # extract some options - group = fitting_options.pop('gt_group', 'PTS') - max_iters = fitting_options.pop('max_iters', 50) - error_type = fitting_options.pop('error_type', 'me_norm') - - # create fitter - fitter = LucasKanadeAAMFitter(aam, **fitting_options) - - # fit images - n_images = len(fitting_images) - fitting_results = [] - for j, i in enumerate(fitting_images): - # perturb shape - gt_s = i.landmarks[group].lms - if 'bbox' in i.landmarks: - # shape from bounding box - s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) - else: - # shape from perturbation - s = fitter.perturb_shape(gt_s, **perturb_options) - # fit - fr = fitter.fit(i, s, gt_shape=gt_s, max_iters=max_iters) - fitting_results.append(fr) - - # print - final_error = fr.final_error(error_type=error_type) - initial_error = fr.initial_error(error_type=error_type) - if verbose: - if error_type == 'me_norm': - if final_error <= 0.03: - perc1 += 1. - if final_error <= 0.04: - perc2 += 1. - elif error_type == 'rmse': - if final_error <= 0.05: - perc1 += 1. - if final_error <= 0.06: - perc2 += 1. - print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' - 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( - progress_bar_str(float(j + 1) / n_images, show_bar=False), - perc1 * 100. / n_images, perc2 * 100. / n_images, j + 1, - n_images, initial_error, final_error)) - if verbose: - print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' - '{1:.1f}%]\n'.format(perc1 * 100. / n_images, - perc2 * 100. / n_images)) - - return fitting_results - - -def aam_build_benchmark(training_images, training_options=None, verbose=False): - r""" - Builds an AAM model. - - Parameters - ---------- - training_images: list of :class:MaskedImage objects - A list of the training images. - training_options: dictionary, optional - A dictionary with the parameters that will be passed in the AAMBuilder - (:class:menpo.fitmultilevel.aam.AAMBuilder). - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'group': 'PTS', - 'features': 'igo', - 'transform': PiecewiseAffine, - 'trilist': None, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'max_appearance_components': None, - 'boundary': 3 - } - For an explanation of the options, please refer to the AAMBuilder - documentation. - - Default: None - verbose: boolean, optional - If True, it prints information regarding the AAM training. - - Default: False - - Returns - ------- - aam: :class:menpo.fitmultilevel.aam.AAM object - The trained AAM model. - """ - if verbose: - print('AAM Training:') - - # parse options - if training_options is None: - training_options = {} - - # group option - group = training_options.pop('group', None) - - # trilist option - trilist = training_options.pop('trilist', None) - if trilist is not None: - labeller(training_images[0], 'PTS', trilist) - training_options['trilist'] = \ - training_images[0].landmarks[trilist.__name__].lms.trilist - - # build aam - aam = AAMBuilder(**training_options).build(training_images, group=group, - verbose=verbose) - - return aam - - -def clm_fit_benchmark(fitting_images, clm, fitting_options=None, - perturb_options=None, verbose=False): - r""" - Fits a trained CLM model to a database. - - Parameters - ---------- - fitting_images: list of :class:MaskedImage objects - A list of the fitting images. - clm: :class:menpo.fitmultilevel.clm.CLM object - The trained CLM object. It can be generated from the - clm_build_benchmark() method. - fitting_options: dictionary, optional - A dictionary with the parameters that will be passed in the - GradientDescentCLMFitter (:class:menpo.fitmultilevel.clm.base). - If None, the default options will be used. - This is an example of the dictionary with the default options: - fitting_options = {'algorithm': RegularizedLandmarkMeanShift, - 'pdm_transform': OrthoPDM, - 'global_transform': AlignmentSimilarity, - 'n_shape': None, - 'max_iters': 50, - 'error_type': 'me_norm' - } - For an explanation of the options, please refer to the - GradientDescentCLMFitter documentation. - - Default: None - bounding_boxes: list of (2, 2) ndarray, optional - If provided, fits will be initialized from a bounding box. If - None, perturbation of ground truth will be used instead. - can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. - perturb_options: dictionary, optional - A dictionary with parameters that control the perturbation on the - ground truth shape with noise of specified std. Note that if - bounding_box is provided perturb_options is ignored and not used. - verbose: boolean, optional - If True, it prints information regarding the AAM fitting including - progress bar, current image error and percentage of images with errors - less or equal than a value. - - Default: False - - Returns - ------- - fitting_results: :class:menpo.fit.fittingresult.FittingResultList object - A list with the FittingResult object per image. - """ - if verbose: - print('CLM Fitting:') - perc1 = 0. - perc2 = 0. - - # parse options - if fitting_options is None: - fitting_options = {} - - # extract some options - group = fitting_options.pop('gt_group', 'PTS') - max_iters = fitting_options.pop('max_iters', 50) - error_type = fitting_options.pop('error_type', 'me_norm') - - # create fitter - fitter = GradientDescentCLMFitter(clm, **fitting_options) - - # fit images - n_images = len(fitting_images) - fitting_results = [] - for j, i in enumerate(fitting_images): - # perturb shape - gt_s = i.landmarks[group].lms - if 'bbox' in i.landmarks: - # shape from bounding box - s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) - else: - # shape from perturbation - s = fitter.perturb_shape(gt_s, **perturb_options) - # fit - fr = fitter.fit(i, s, gt_shape=gt_s, max_iters=max_iters) - fitting_results.append(fr) - - # print - final_error = fr.final_error(error_type=error_type) - initial_error = fr.initial_error(error_type=error_type) - if verbose: - if error_type == 'me_norm': - if final_error <= 0.03: - perc1 += 1. - if final_error <= 0.04: - perc2 += 1. - elif error_type == 'rmse': - if final_error <= 0.05: - perc1 += 1. - if final_error <= 0.06: - perc2 += 1. - print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' - 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( - progress_bar_str(float(j + 1) / n_images, - show_bar=False), - perc1 * 100. / n_images, perc2 * 100. / n_images, - j + 1, n_images, initial_error, final_error)) - if verbose: - print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' - '{1:.1f}%]\n'.format(perc1 * 100. / n_images, - perc2 * 100. / n_images)) - - return fitting_results - - -def clm_build_benchmark(training_images, training_options=None, verbose=False): - r""" - Builds an CLM model. - - Parameters - ---------- - training_images: list of :class:MaskedImage objects - A list of the training images. - training_options: dictionary, optional - A dictionary with the parameters that will be passed in the CLMBuilder - (:class:menpo.fitmultilevel.clm.CLMBuilder). - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': sparse_hog, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'boundary': 3 - } - For an explanation of the options, please refer to the CLMBuilder - documentation. - - Default: None - verbose: boolean, optional - If True, it prints information regarding the CLM training. - - Default: False - - Returns - ------- - clm: :class:menpo.fitmultilevel.clm.CLM object - The trained CLM model. - """ - if verbose: - print('CLM Training:') - - # parse options - if training_options is None: - training_options = {} - - # group option - group = training_options.pop('group', None) - - # build aam - aam = CLMBuilder(**training_options).build(training_images, group=group, - verbose=verbose) - - return aam - - -def sdm_fit_benchmark(fitting_images, fitter, perturb_options=None, - fitting_options=None, verbose=False): - r""" - Fits a trained SDM to a database. - - Parameters - ---------- - fitting_images: list of :class:MaskedImage objects - A list of the fitting images. - fitter: :map:`SDMFitter` - The trained AAM object. It can be generated from the - aam_build_benchmark() method. - fitting_options: dictionary, optional - A dictionary with the parameters that will be passed in the - LucasKanadeAAMFitter (:class:menpo.fitmultilevel.sdm.base). - If None, the default options will be used. - This is an example of the dictionary with the default options: - fitting_options = {'algorithm': AlternatingInverseCompositional, - 'md_transform': OrthoMDTransform, - 'global_transform': AlignmentSimilarity, - 'n_shape': None, - 'n_appearance': None, - 'max_iters': 50, - 'error_type': 'me_norm' - } - For an explanation of the options, please refer to the - LucasKanadeAAMFitter documentation. - - Default: None - bounding_boxes: list of (2, 2) ndarray, optional - If provided, fits will be initialized from a bounding box. If - None, perturbation of ground truth will be used instead. - can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. - perturb_options: dictionary, optional - A dictionary with parameters that control the perturbation on the - ground truth shape with noise of specified std. Note that if - bounding_box is provided perturb_options is ignored and not used. - If None, the default options will be used. - This is an example of the dictionary with the default options: - initialization_options = {'noise_std': 0.04, - 'rotation': False - } - For an explanation of the options, please refer to the perturb_shape() - method documentation of :map:`MultilevelFitter`. - verbose: bool, optional - If True, it prints information regarding the AAM fitting including - progress bar, current image error and percentage of images with errors - less or equal than a value. - - Default: False - - Returns - ------- - fitting_results: :map:`FittingResultList` - A list with the :map:`FittingResult` object per image. - """ - if verbose: - print('SDM Fitting:') - perc1 = 0. - perc2 = 0. - - # parse options - if fitting_options is None: - fitting_options = {} - if perturb_options is None: - perturb_options = {} - - # extract some options - group = fitting_options.pop('gt_group', 'PTS') - error_type = fitting_options.pop('error_type', 'me_norm') - - # fit images - n_images = len(fitting_images) - fitting_results = [] - for j, i in enumerate(fitting_images): - # perturb shape - gt_s = i.landmarks[group].lms - if 'bbox' in i.landmarks: - # shape from bounding box - s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) - else: - # shape from perturbation - s = fitter.perturb_shape(gt_s, **perturb_options) - # fit - fr = fitter.fit(i, s, gt_shape=gt_s) - fitting_results.append(fr) - - # print - final_error = fr.final_error(error_type=error_type) - initial_error = fr.initial_error(error_type=error_type) - if verbose: - if error_type == 'me_norm': - if final_error <= 0.03: - perc1 += 1. - if final_error <= 0.04: - perc2 += 1. - elif error_type == 'rmse': - if final_error <= 0.05: - perc1 += 1. - if final_error <= 0.06: - perc2 += 1. - print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' - 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( - progress_bar_str(float(j + 1) / n_images, show_bar=False), - perc1 * 100. / n_images, perc2 * 100. / n_images, j + 1, - n_images, initial_error, final_error)) - if verbose: - print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' - '{1:.1f}%]\n'.format(perc1 * 100. / n_images, - perc2 * 100. / n_images)) - - return fitting_results - - -def sdm_build_benchmark(training_images, training_options=None, verbose=False): - r""" - Builds an SDM model. - - Parameters - ---------- - training_images: list of :class:MaskedImage objects - A list of the training images. - training_options: dictionary, optional - A dictionary with the parameters that will be passed in the AAMBuilder - (:class:menpo.fitmultilevel.aam.AAMBuilder). - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'group': 'PTS', - 'features': 'igo', - 'transform': PiecewiseAffine, - 'trilist': None, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'max_appearance_components': None, - 'boundary': 3 - } - For an explanation of the options, please refer to the AAMBuilder - documentation. - - Default: None - verbose: boolean, optional - If True, it prints information regarding the AAM training. - - Default: False - - Returns - ------- - aam: :class:menpo.fitmultilevel.aam.AAM object - The trained AAM model. - """ - if verbose: - print('SDM Training:') - - # parse options - if training_options is None: - training_options = {} - - # group option - group = training_options.pop('group', None) - - # build sdm - sdm = SDMTrainer(**training_options).train(training_images, group=group, - verbose=verbose) - return sdm - - -def load_database(database_path, bounding_boxes=None, - db_loading_options=None, verbose=False): - r""" - Loads the database images, crops them and converts them. - - Parameters - ---------- - database_path: str - The path of the database images. - db_loading_options: dictionary, optional - A dictionary with options related to image loading. - If None, the default options will be used. - This is an example of the dictionary with the default options: - training_options = {'crop_proportion': 0.1, - 'convert_to_grey': True, - } - - crop_proportion (float) defines the additional padding to be added all - around the landmarks bounds when the images are cropped. It is defined - as a proportion of the landmarks' range. - - convert_to_grey (boolean)defines whether the images will be converted - to greyscale. - - Default: None - verbose: boolean, optional - If True, it prints a progress percentage bar. - - Default: False - - Returns - ------- - images: list of :class:MaskedImage objects - A list of the loaded images. - - Raises - ------ - ValueError - Invalid path given - ValueError - No {files_extension} files in given path - """ - # check input options - if db_loading_options is None: - db_loading_options = {} - - # check given path - database_path = os.path.abspath(os.path.expanduser(database_path)) - if os.path.isdir(database_path) is not True: - raise ValueError('Invalid path given') - - # create final path - final_path = os.path.join(database_path, '*') - - # get options - crop_proportion = db_loading_options.pop('crop_proportion', 0.5) - convert_to_grey = db_loading_options.pop('convert_to_grey', True) - - # load images - images = [] - for i in mio.import_images(final_path, verbose=verbose): - # If we have bounding boxes then we need to make sure we crop to them! - # If we don't crop to the bounding box then we might crop out part of - # the image the bounding box belongs to. - landmark_group_label = None - if bounding_boxes is not None: - fname = i.path.name - landmark_group_label = 'bbox' - i.landmarks[landmark_group_label] = bounding_boxes[fname].detector - - # crop image - i.crop_to_landmarks_proportion_inplace(crop_proportion, - group=landmark_group_label) - - # convert it to greyscale if needed - if convert_to_grey and i.n_channels == 3: - i = i.as_greyscale(mode='luminosity') - - # append it to the list - images.append(i) - if verbose: - print("\nAssets loaded.") - return images - - -def convert_fitting_results_to_ced(fitting_results, max_error_bin=0.05, - bins_error_step=0.005, error_type='me_norm'): - r""" - Method that given a fitting_result object, it converts it to the - cumulative error distribution values that can be used for plotting. - - Parameters - ---------- - fitting_results: :class:menpo.fit.fittingresult.FittingResultList object - A list with the FittingResult object per image. - max_error_bin: float, Optional - The maximum error of the distribution. - - Default: 0.05 - bins_error_step: float, Optional - The sampling step of the distribution values. - - Default: 0.005 - - Returns - ------- - final_error_dist: list - Cumulative distribution values of the final errors. - initial_error_dist: list - Cumulative distribution values of the initial errors. - """ - error_bins = np.arange(0., max_error_bin + bins_error_step, - bins_error_step) - final_error = [f.final_error(error_type=error_type) - for f in fitting_results] - initial_error = [f.initial_error(error_type=error_type) - for f in fitting_results] - - final_error_dist = np.array( - [float(np.sum(final_error <= k)) / - len(final_error) for k in error_bins]) - initial_error_dist = np.array( - [float(np.sum(initial_error <= k)) / - len(final_error) for k in error_bins]) - return final_error_dist, initial_error_dist, error_bins - - -def plot_fitting_curves(x_axis, ceds, title, figure_id=None, new_figure=False, - x_label='Point-to-Point Normalized RMS Error', - y_limit=1, x_limit=0.05, legend_entries=None, **kwargs): - r""" - Method that plots Cumulative Error Distributions in a single figure. - - Parameters - ---------- - x_axis : ndarray - The horizontal axis values (errors). - ceds : list of ndarrays - The vertical axis values (percentages). - title : string - The plot title. - figure_id : Optional - A figure handle. - new_figure : boolean, Optional - If True, a new figure window will be created. - y_limit : float, Optional - The maximum value of the vertical axis. - x_limit : float, Optional - The maximum value of the vertical axis. - x_label : string - The label of the horizontal axis. - legend_entries : list of strings or None - The legend of the plot. If None, the legend will include an incremental - number per curve. - - Returns - ------- - final_error_dist : list - Cumulative distribution values of the final errors. - initial_error_dist : list - Cumulative distribution values of the initial errors. - """ - if legend_entries is None: - legend_entries = [str(i + 1) for i in range(len(ceds))] - y_label = 'Proportion of images' - x_axis_limits = [0, x_limit] - y_axis_limits = [0, y_limit] - return GraphPlotter(figure_id, new_figure, x_axis, ceds, title=title, - legend_entries=legend_entries, x_label=x_label, - y_label=y_label, - x_axis_limits=x_axis_limits, - y_axis_limits=y_axis_limits).render(**kwargs) diff --git a/menpofit/benchmark/io.py b/menpofit/benchmark/io.py deleted file mode 100644 index c9b9ff4..0000000 --- a/menpofit/benchmark/io.py +++ /dev/null @@ -1,114 +0,0 @@ -import urllib2 -try: - from StringIO import StringIO -except ImportError: - from io import StringIO -import os -import scipy.io as sio -import glob -import tempfile -import shutil -import zipfile -from collections import namedtuple - -# Container for bounding box -from menpo.shape import PointCloud - -BoundingBox = namedtuple('BoundingBox', ['detector', 'groundtruth']) -# Where the bounding boxes should be fetched from -bboxes_url = 'http://ibug.doc.ic.ac.uk/media/uploads/competitions/bounding_boxes.zip' - - -def download_ibug_bounding_boxes(path=None, verbose=False): - r"""Downloads the bounding box information provided on the iBUG website - and unzips it to the path. - - Parameters - ---------- - path : `str`, optional - The path that the bounding box files should be extracted to. - If None, the current directory will be used. - """ - if path is None: - path = os.getcwd() - else: - path = os.path.abspath(os.path.expanduser(path)) - if verbose: - print('Acquiring bounding box information from iBUG website...') - try: - remotezip = urllib2.urlopen(bboxes_url) - zipinmemory = StringIO(remotezip.read()) - ziplocal = zipfile.ZipFile(zipinmemory) - except Exception as e: - print('Unable to grab bounding boxes (are you online?)') - raise e - if verbose: - print('Extracting to {}'.format(os.path.join(path, 'Bounding Boxes'))) - try: - ziplocal.extractall(path=path) - if verbose: - print('Done.') - except Exception as e: - if verbose: - print('Unable to save.'.format(e)) - raise e - - -def import_bounding_boxes(boxes_path): - r""" - Imports the bounding boxes at boxes_path, returning a dict - where the key is a filename and the value is a BoundingBox. - - Parameters - ---------- - boxes_path : str - A path to a bounding box .mat file downloaded from the - iBUG website. - - Returns - ------- - dict: - Mapping of filenames to bounding boxes - - """ - bboxes_mat = sio.loadmat(boxes_path) - bboxes = {} - for bb in bboxes_mat['bounding_boxes'][0, :]: - fname, detector_bb, gt_bb = bb[0, 0] - bboxes[str(fname[0])] = BoundingBox( - PointCloud(detector_bb.reshape([2, 2])[:, ::-1]), - PointCloud(gt_bb.reshape([2, 2])[:, ::-1])) - return bboxes - - -def import_all_bounding_boxes(boxes_dir_path=None, verbose=True): - r""" - Imports all the bounding boxes contained in boxes_dir_path. - If the path is False, the bounding boxes are downloaded from the - iBUG website directly. - - - """ - temp_path = None - if boxes_dir_path is None: - print('No path provided - acuqiring zip to tmp dir...') - temp_path = tempfile.mkdtemp() - download_ibug_bounding_boxes(path=temp_path, verbose=verbose) - boxes_dir_path = os.path.join(temp_path, 'Bounding Boxes') - prefix = 'bounding_boxes_' - bbox_paths = glob.glob(os.path.join(boxes_dir_path, prefix + '*.mat')) - bboxes = {} - for bbox_path in bbox_paths: - db = os.path.splitext(os.path.split(bbox_path)[-1])[0][len(prefix):] - if verbose: - print('Importing {}'.format(db)) - bboxes[db] = import_bounding_boxes(bbox_path) - if verbose: - print('Cleaning up...') - if temp_path: - # If we downloaded, clean it up! - shutil.rmtree(temp_path) - if verbose: - print('Done.') - return bboxes - diff --git a/menpofit/benchmark/predefined.py b/menpofit/benchmark/predefined.py deleted file mode 100644 index 3780f41..0000000 --- a/menpofit/benchmark/predefined.py +++ /dev/null @@ -1,779 +0,0 @@ -from menpo.landmark import ibug_face_68_trimesh -from menpo.feature import sparse_hog, igo - -from menpofit.lucaskanade import AIC -from menpofit.transform import OrthoMDTransform, DifferentiablePiecewiseAffine -from menpofit.modelinstance import OrthoPDM -from menpofit.gradientdescent import RLMS -from menpofit.clm.classifier import linear_svm_lr - -from .io import import_bounding_boxes -from .base import (aam_build_benchmark, aam_fit_benchmark, - clm_build_benchmark, clm_fit_benchmark, - sdm_build_benchmark, sdm_fit_benchmark, - load_database, convert_fitting_results_to_ced, - plot_fitting_curves) - - -def aam_fastest_alternating_noise(training_db_path, fitting_db_path, - features=igo, noise_std=0.04, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.2, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': 'me_norm' - } - perturb_options = {'noise_std': 0.04, - 'rotation': False} - - # set passed parameters - training_options['features'] = features - perturb_options['noise_std'] = noise_std - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_results = aam_fit_benchmark(fitting_images, aam, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_fastest_alternating_bbox(training_db_path, fitting_db_path, - fitting_bboxes_path, features=igo, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.1, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': [igo] * 3, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': 'me_norm' - } - - # set passed parameters - training_options['features'] = features - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = aam_fit_benchmark(fitting_images, aam, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_best_performance_alternating_noise(training_db_path, fitting_db_path, - features=igo, noise_std=0.04, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.2, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.2, - 'scaled_shape_models': False, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': error_type - } - perturb_options = {'noise_std': 0.04, - 'rotation': False} - - # set passed parameters - training_options['features'] = features - perturb_options['noise_std'] = noise_std - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_results = aam_fit_benchmark(fitting_images, aam, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_best_performance_alternating_bbox(training_db_path, fitting_db_path, - fitting_bboxes_path, - features=igo, verbose=False, - plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.5, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': 200, - 'n_levels': 3, - 'downscale': 2, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 100, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': error_type - } - - # set passed parameters - training_options['features'] = features - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = aam_fit_benchmark(fitting_images, aam, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "AAMs using {} and Alternating IC".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def clm_basic_noise(training_db_path, fitting_db_path, - features=sparse_hog, classifier_trainers=linear_svm_lr, - noise_std=0.04, verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.4, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': [sparse_hog] * 3, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'boundary': 3 - } - fitting_options = {'algorithm': RLMS, - 'pdm_transform': OrthoPDM, - 'n_shape': [3, 6, 12], - 'max_iters': 50, - 'error_type': error_type - } - perturb_options = {'noise_std': 0.01, - 'rotation': False} - - # set passed parameters - training_options['features'] = features - training_options['classifier_trainers'] = classifier_trainers - perturb_options['noise_std'] = noise_std - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - clm = clm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_results = clm_fit_benchmark(fitting_images, clm, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "CLMs with {} and {} classifier using RLMS".format( - training_options['features'].__name__, - training_options['classifier_trainers']) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def clm_basic_bbox(training_db_path, fitting_db_path, fitting_bboxes_path, - features=sparse_hog, classifier_trainers=linear_svm_lr, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.5, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': [sparse_hog] * 3, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': None, - 'boundary': 3 - } - fitting_options = {'algorithm': RLMS, - 'pdm_transform': OrthoPDM, - 'n_shape': [3, 6, 12], - 'max_iters': 50, - 'error_type': error_type - } - - # set passed parameters - training_options['features'] = features - training_options['classifier_trainers'] = classifier_trainers - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - - clm = clm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = clm_fit_benchmark(fitting_images, clm, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "CLMs with {} and {} classifier using RLMS".format( - training_options['features'].__name__, - training_options['classifier_trainers']) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def sdm_fastest_bbox(training_db_path, fitting_db_path, - fitting_bboxes_path, features=None, - verbose=False, plot=False): - - # predefined options - error_type = 'me_norm' - db_loading_options = {'crop_proportion': 0.8, - 'convert_to_grey': True - } - training_options = {'group': 'PTS', - 'normalization_diagonal': 200, - 'n_levels': 4, - 'downscale': 1.01, - 'noise_std': 0.08, - 'patch_shape': (16, 16), - 'n_perturbations': 15, - } - fitting_options = { - 'error_type': error_type - } - - # run experiment - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - sdm = sdm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # import bounding boxes - bboxes_list = import_bounding_boxes(fitting_bboxes_path) - - # for all fittings, we crop to 0.5 - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - bounding_boxes=bboxes_list, - verbose=verbose) - - fitting_results = sdm_fit_benchmark(fitting_images, sdm, - fitting_options=fitting_options, - verbose=verbose) - - # convert results - max_error_bin = 0.05 - bins_error_step = 0.005 - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced(fitting_results, - max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - - # plot results - if plot: - title = "SDMs using default (sparse hogs)".format( - training_options['features'].__name__) - y_axis = [final_error_curve, initial_error_curve] - legend = ['Fitting', 'Initialization'] - plot_fitting_curves(error_bins, y_axis, title, new_figure=True, - x_limit=max_error_bin, legend_entries=legend, - line_colour=['r', 'b'], - marker_face_colour=['r', 'b'], - marker_style=['o', 'x']) - return fitting_results, final_error_curve, initial_error_curve, error_bins - - -def aam_params_combinations_noise(training_db_path, fitting_db_path, - n_experiments=1, features=None, - scaled_shape_models=None, - n_shape=None, - n_appearance=None, noise_std=None, - rotation=None, verbose=False, plot=False): - - # parse input - if features is None: - features = [igo] * n_experiments - elif len(features) is not n_experiments: - raise ValueError("features has wrong length") - if scaled_shape_models is None: - scaled_shape_models = [True] * n_experiments - elif len(scaled_shape_models) is not n_experiments: - raise ValueError("scaled_shape_models has wrong length") - if n_shape is None: - n_shape = [[3, 6, 12]] * n_experiments - elif len(n_shape) is not n_experiments: - raise ValueError("n_shape has wrong length") - if n_appearance is None: - n_appearance = [50] * n_experiments - elif len(n_appearance) is not n_experiments: - raise ValueError("n_appearance has wrong length") - if noise_std is None: - noise_std = [0.04] * n_experiments - elif len(noise_std) is not n_experiments: - raise ValueError("noise_std has wrong length") - if rotation is None: - rotation = [False] * n_experiments - elif len(rotation) is not n_experiments: - raise ValueError("rotation has wrong length") - - # load images - db_loading_options = {'crop_proportion': 0.1, - 'convert_to_grey': True - } - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - - # run experiments - max_error_bin = 0.05 - bins_error_step = 0.005 - curves_to_plot = [] - all_fitting_results = [] - for i in range(n_experiments): - if verbose: - print("\nEXPERIMENT {}/{}:".format(i + 1, n_experiments)) - print("- features: {}\n- scaled_shape_models: {}\n" - "- n_shape: {}\n" - "- n_appearance: {}\n- noise_std: {}\n" - "- rotation: {}".format( - features[i], scaled_shape_models[i], - n_shape[i], n_appearance[i], noise_std[i], rotation[i])) - - # predefined option dictionaries - error_type = 'me_norm' - training_options = {'group': 'PTS', - 'features': igo, - 'transform': DifferentiablePiecewiseAffine, - 'trilist': ibug_face_68_trimesh, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': True, - 'max_shape_components': 25, - 'max_appearance_components': 250, - 'boundary': 3 - } - fitting_options = {'algorithm': AIC, - 'md_transform': OrthoMDTransform, - 'n_shape': [3, 6, 12], - 'n_appearance': 50, - 'max_iters': 50, - 'error_type': error_type - } - pertrub_options = {'noise_std': 0.04, - 'rotation': False} - - # training - training_options['features'] = features[i] - training_options['scaled_shape_models'] = scaled_shape_models[i] - aam = aam_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # fitting - fitting_options['n_shape'] = n_shape[i] - fitting_options['n_appearance'] = n_appearance[i] - pertrub_options['noise_std'] = noise_std[i] - pertrub_options['rotation'] = rotation[i] - fitting_results = aam_fit_benchmark(fitting_images, aam, - perturb_options=pertrub_options, - fitting_options=fitting_options, - verbose=verbose) - all_fitting_results.append(fitting_results) - - # convert results - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced( - fitting_results, max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - curves_to_plot.append(final_error_curve) - if i == n_experiments - 1: - curves_to_plot.append(initial_error_curve) - - # plot results - if plot: - title = "AAMs using Alternating IC" - colour_list = ['r', 'b', 'g', 'y', 'c'] * n_experiments - marker_list = ['o', 'x', 'v', 'd'] * n_experiments - plot_fitting_curves(error_bins, curves_to_plot, title, new_figure=True, - x_limit=max_error_bin, line_colour=colour_list, - marker_face_colour=colour_list, - marker_style=marker_list) - return all_fitting_results - - -def clm_params_combinations_noise(training_db_path, fitting_db_path, - n_experiments=1, classifier_trainers=None, - patch_shape=None, features=None, - scaled_shape_models=None, n_shape=None, - noise_std=None, rotation=None, verbose=False, - plot=False): - - # parse input - if classifier_trainers is None: - classifier_trainers = [linear_svm_lr] * n_experiments - elif len(classifier_trainers) is not n_experiments: - raise ValueError("classifier_trainers has wrong length") - if patch_shape is None: - patch_shape = [(5, 5)] * n_experiments - elif len(patch_shape) is not n_experiments: - raise ValueError("patch_shape has wrong length") - if features is None: - features = [igo] * n_experiments - elif len(features) is not n_experiments: - raise ValueError("features has wrong length") - if scaled_shape_models is None: - scaled_shape_models = [True] * n_experiments - elif len(scaled_shape_models) is not n_experiments: - raise ValueError("scaled_shape_models has wrong length") - if n_shape is None: - n_shape = [[3, 6, 12]] * n_experiments - elif len(n_shape) is not n_experiments: - raise ValueError("n_shape has wrong length") - if noise_std is None: - noise_std = [0.04] * n_experiments - elif len(noise_std) is not n_experiments: - raise ValueError("noise_std has wrong length") - if rotation is None: - rotation = [False] * n_experiments - elif len(rotation) is not n_experiments: - raise ValueError("rotation has wrong length") - - # load images - db_loading_options = {'crop_proportion': 0.4, - 'convert_to_grey': True - } - training_images = load_database(training_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - fitting_images = load_database(fitting_db_path, - db_loading_options=db_loading_options, - verbose=verbose) - - # run experiments - max_error_bin = 0.05 - bins_error_step = 0.005 - curves_to_plot = [] - all_fitting_results = [] - for i in range(n_experiments): - if verbose: - print("\nEXPERIMENT {}/{}:".format(i + 1, n_experiments)) - print("- classifiers: {}\n- patch_shape: {}\n" - "- features: {}\n- scaled_shape_models: {}\n" - "- n_shape: {}\n" - "- noise_std: {}\n- rotation: {}".format( - classifier_trainers[i], patch_shape[i], features[i], - scaled_shape_models[i], n_shape[i], - noise_std[i], rotation[i])) - - # predefined option dictionaries - error_type = 'me_norm' - training_options = {'group': 'PTS', - 'classifier_trainers': linear_svm_lr, - 'patch_shape': (5, 5), - 'features': sparse_hog, - 'normalization_diagonal': None, - 'n_levels': 3, - 'downscale': 1.1, - 'scaled_shape_models': False, - 'max_shape_components': None, - 'boundary': 3 - } - fitting_options = {'algorithm': RLMS, - 'pdm_transform': OrthoPDM, - 'n_shape': [3, 6, 12], - 'max_iters': 50, - 'error_type': error_type - } - perturb_options = {'noise_std': 0.01, - 'rotation': False} - - # training - training_options['classifier_trainers'] = classifier_trainers[i] - training_options['patch_shape'] = patch_shape[i] - training_options['features'] = features[i] - training_options['scaled_shape_models'] = scaled_shape_models[i] - clm = clm_build_benchmark(training_images, - training_options=training_options, - verbose=verbose) - - # fitting - fitting_options['n_shape'] = n_shape[i] - perturb_options['noise_std'] = noise_std[i] - perturb_options['rotation'] = rotation[i] - fitting_results = clm_fit_benchmark(fitting_images, clm, - perturb_options=perturb_options, - fitting_options=fitting_options, - verbose=verbose) - all_fitting_results.append(fitting_results) - - # convert results - final_error_curve, initial_error_curve, error_bins = \ - convert_fitting_results_to_ced( - fitting_results, max_error_bin=max_error_bin, - bins_error_step=bins_error_step, - error_type=error_type) - curves_to_plot.append(final_error_curve) - if i == n_experiments - 1: - curves_to_plot.append(initial_error_curve) - - # plot results - if plot: - title = "CLMs using RLMS" - colour_list = ['r', 'b', 'g', 'y', 'c'] * n_experiments - marker_list = ['o', 'x', 'v', 'd'] * n_experiments - plot_fitting_curves(error_bins, curves_to_plot, title, new_figure=True, - x_limit=max_error_bin, line_colour=colour_list, - marker_face_colour=colour_list, - marker_style=marker_list) - return all_fitting_results diff --git a/menpofit/test/__init__.py b/menpofit/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/menpofit/test/aam_builder_test.py b/menpofit/test/aam_builder_test.py deleted file mode 100644 index 6bcaf3a..0000000 --- a/menpofit/test/aam_builder_test.py +++ /dev/null @@ -1,199 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO -import platform - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.transform import PiecewiseAffine, ThinPlateSplines -from menpo.feature import sparse_hog, igo, lbp, no_op - -import menpo.io as mio -from menpo.landmark import ibug_face_68_trimesh -from menpofit.aam import AAMBuilder, PatchBasedAAMBuilder - - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training.append(im) - -# build aams -template_trilist_image = training[0].landmarks[None] -trilist = ibug_face_68_trimesh(template_trilist_image)[1].lms.trilist -aam1 = AAMBuilder(features=[igo, sparse_hog, no_op], - transform=PiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=False, - max_shape_components=[1, 2, 3], - max_appearance_components=[3, 3, 3], - boundary=3).build(training) - -aam2 = AAMBuilder(features=[no_op, no_op], - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=None, - max_appearance_components=1, - boundary=0).build(training) - -aam3 = AAMBuilder(features=igo, - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=1, - downscale=3, - scaled_shape_models=True, - max_shape_components=[2], - max_appearance_components=10, - boundary=2).build(training) - -aam4 = PatchBasedAAMBuilder(features=lbp, - patch_shape=(10, 13), - normalization_diagonal=200, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=1, - max_appearance_components=None, - boundary=2).build(training) - - -@raises(ValueError) -def test_features_exception(): - AAMBuilder(features=[igo, sparse_hog]).build(training) - - -@raises(ValueError) -def test_n_levels_exception(): - AAMBuilder(n_levels=0).build(training) - - -@raises(ValueError) -def test_downscale_exception(): - aam = AAMBuilder(downscale=1).build(training) - assert (aam.downscale == 1) - AAMBuilder(downscale=0).build(training) - - -@raises(ValueError) -def test_normalization_diagonal_exception(): - aam = AAMBuilder(normalization_diagonal=100).build(training) - assert (aam.appearance_models[0].n_features == 382) - AAMBuilder(normalization_diagonal=10).build(training) - - -@raises(ValueError) -def test_max_shape_components_exception(): - AAMBuilder(max_shape_components=[1, 0.2, 'a']).build(training) - - -@raises(ValueError) -def test_max_appearance_components_exception(): - AAMBuilder(max_appearance_components=[1, 2]).build(training) - - -@raises(ValueError) -def test_boundary_exception(): - AAMBuilder(boundary=-1).build(training) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - AAMBuilder().build(training, verbose=True) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(aam1) - print(aam2) - print(aam3) - print(aam4) - - -def test_aam_1(): - assert(aam1.n_training_images == 4) - assert(aam1.n_levels == 3) - assert(aam1.downscale == 2) - #assert(aam1.features[0] == igo and aam1.features[2] == no_op) - assert_allclose(np.around(aam1.reference_shape.range()), (109., 103.)) - assert(not aam1.scaled_shape_models) - assert(not aam1.pyramid_on_features) - assert_allclose([aam1.shape_models[j].n_components - for j in range(aam1.n_levels)], (1, 2, 3)) - assert (np.all([aam1.appearance_models[j].n_components == 3 - for j in range(aam1.n_levels)])) - assert_allclose([aam1.appearance_models[j].template_instance.n_channels - for j in range(aam1.n_levels)], (2, 36, 1)) - assert_allclose([aam1.appearance_models[j].components.shape[1] - for j in range(aam1.n_levels)], (14892, 268056, 7446)) - - -def test_aam_2(): - assert (aam2.n_training_images == 4) - assert (aam2.n_levels == 2) - assert (aam2.downscale == 1.2) - #assert (aam2.features[0] == no_op and aam2.features[1] == no_op) - assert_allclose(np.around(aam2.reference_shape.range()), (169., 161.)) - assert aam2.scaled_shape_models - assert (not aam2.pyramid_on_features) - assert (np.all([aam2.shape_models[j].n_components == 3 - for j in range(aam2.n_levels)])) - assert (np.all([aam2.appearance_models[j].n_components == 1 - for j in range(aam2.n_levels)])) - assert (np.all([aam2.appearance_models[j].template_instance.n_channels == 1 - for j in range(aam2.n_levels)])) - assert_allclose([aam2.appearance_models[j].components.shape[1] - for j in range(aam2.n_levels)], (12827, 18518)) - - -def test_aam_3(): - assert (aam3.n_training_images == 4) - assert (aam3.n_levels == 1) - assert (aam3.downscale == 3) - #assert (aam3.features[0] == igo and len(aam3.features) == 1) - assert_allclose(np.around(aam3.reference_shape.range()), (169., 161.)) - assert aam3.scaled_shape_models - assert aam3.pyramid_on_features - assert (np.all([aam3.shape_models[j].n_components == 2 - for j in range(aam3.n_levels)])) - assert (np.all([aam3.appearance_models[j].n_components == 3 - for j in range(aam3.n_levels)])) - assert (np.all([aam3.appearance_models[j].template_instance.n_channels == 2 - for j in range(aam3.n_levels)])) - assert_allclose([aam3.appearance_models[j].components.shape[1] - for j in range(aam3.n_levels)], 37036) - - -def test_aam_4(): - assert (aam4.n_training_images == 4) - assert (aam4.n_levels == 2) - assert (aam4.downscale == 1.2) - #assert (aam4.features[0] == lbp) - assert_allclose(np.around(aam4.reference_shape.range()), (145., 138.)) - assert aam4.scaled_shape_models - assert aam4.pyramid_on_features - assert (np.all([aam4.shape_models[j].n_components == 1 - for j in range(aam4.n_levels)])) - assert (np.all([aam4.appearance_models[j].n_components == 3 - for j in range(aam4.n_levels)])) - assert (np.all([aam4.appearance_models[j].template_instance.n_channels == 4 - for j in range(aam4.n_levels)])) - if platform.system() != 'Windows': - # https://github.com/menpo/menpo/issues/450 - assert_allclose([aam4.appearance_models[j].components.shape[1] - for j in range(aam4.n_levels)], (23656, 25988)) diff --git a/menpofit/test/aam_fitter_test.py b/menpofit/test/aam_fitter_test.py deleted file mode 100644 index 39f8fc9..0000000 --- a/menpofit/test/aam_fitter_test.py +++ /dev/null @@ -1,451 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from mock import patch -from nose.plugins.attrib import attr -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import igo -from menpofit.transform import DifferentiablePiecewiseAffine - - -import menpo.io as mio -from menpo.shape.pointcloud import PointCloud -from menpo.landmark import ibug_face_68_trimesh -from menpofit.aam import AAMBuilder, LucasKanadeAAMFitter -from menpofit.lucaskanade.appearance import (AFA, AFC, AIC, - SFA, SFC, SIC, - PIC) - - -initial_shape = [] -initial_shape.append(PointCloud(np.array([[150.9737801, 1.85331141], - [191.20452708, 1.86714624], - [237.5088486, 7.16836457], - [280.68439528, 19.1356864], - [319.00988383, 36.18921029], - [351.31395982, 61.11002727], - [375.83681819, 86.68264647], - [401.50706656, 117.12858347], - [408.46977018, 156.72258055], - [398.49810436, 197.95690492], - [375.44584527, 234.437902], - [342.35427495, 267.96920594], - [299.04149064, 309.66693535], - [250.84207113, 331.07734674], - [198.46150259, 339.47188196], - [144.62222804, 337.84178783], - [89.92321435, 327.81734317], - [101.22474793, 26.90269773], - [89.23456877, 44.52571118], - [84.04683242, 66.6369272], - [86.36993557, 88.61559027], - [94.88123162, 108.04971327], - [88.08448274, 152.88439191], - [68.71150917, 176.94681489], - [55.7165906, 204.86028035], - [53.9169657, 232.87050281], - [69.08534014, 259.8486207], - [121.82883888, 130.79001073], - [152.30894887, 128.91266055], - [183.36381228, 128.04534764], - [216.59234031, 125.86784329], - [235.18182671, 93.18819461], - [242.46006172, 117.24575711], - [246.52987701, 142.46262589], - [240.51603561, 160.38006297], - [232.61083444, 175.36132625], - [137.35714406, 56.53012228], - [124.42060774, 67.0342585], - [121.98869265, 87.71006061], - [130.4421354, 105.16741493], - [139.32511836, 89.65144616], - [144.17935107, 69.97931719], - [125.04221953, 174.72789706], - [103.0127825, 188.96555839], - [97.38196408, 210.70911033], - [107.31622619, 232.4487582], - [119.12835959, 215.57040617], - [124.80355957, 193.64317941], - [304.3174261, 101.83559243], - [293.08249678, 116.76961123], - [287.11523488, 132.55435452], - [289.39839945, 148.49971074], - [283.59574087, 162.33458018], - [286.76478391, 187.30470094], - [292.65033117, 211.98694428], - [310.75841097, 187.33036207], - [319.06250309, 165.27131484], - [321.3339324, 148.86793045], - [321.82844973, 133.03866904], - [316.60228316, 115.15885333], - [303.45716953, 109.59946563], - [301.58563675, 135.32572565], - [298.16531481, 148.240518], - [295.39615418, 162.35992687], - [293.63384823, 201.35617245], - [301.95207707, 163.05299135], - [305.27555828, 148.48478086], - [306.41382116, 133.02994058]]))) - -initial_shape.append(PointCloud(np.array([[33.08569962, 26.2373455], - [43.88613611, 26.24105964], - [56.31709803, 27.66423659], - [67.90810205, 30.87701063], - [78.19704859, 35.45523787], - [86.86947323, 42.14553624], - [93.45293474, 49.0108189], - [100.34442715, 57.18440338], - [102.21365016, 67.81389656], - [99.53663441, 78.88375569], - [93.34797327, 88.67752592], - [84.46413615, 97.67941492], - [72.83628901, 108.8736808], - [59.89656483, 114.62156782], - [45.83436002, 116.87518356], - [31.38054772, 116.43756484], - [16.69592792, 113.74637996], - [19.72996295, 32.96215989], - [16.51105259, 37.69327358], - [15.11834126, 43.62930018], - [15.74200674, 49.52974132], - [18.02696835, 54.74706954], - [16.20229791, 66.78348784], - [11.00138601, 73.24333984], - [7.51274105, 80.73705133], - [7.02960972, 88.25673842], - [11.10174551, 95.4993444], - [25.26138338, 60.85198075], - [33.44414202, 60.34798312], - [41.78120024, 60.11514235], - [50.70180534, 59.53056465], - [55.69238052, 50.75731293], - [57.6463118, 57.21586007], - [58.73890353, 63.98563718], - [57.12441419, 68.79579249], - [55.00216617, 72.817696], - [29.43014699, 40.91600468], - [25.95717546, 43.73596863], - [25.30429808, 49.2866408], - [27.57372827, 53.97328126], - [29.95847378, 49.80782952], - [31.26165197, 44.52660569], - [26.12405475, 72.64764418], - [20.20998272, 76.46991865], - [18.69832059, 82.30724133], - [21.36529486, 88.14351591], - [24.53640666, 83.6123157], - [26.05998356, 77.72568327], - [74.25267847, 53.07881273], - [71.23652416, 57.08803288], - [69.63453966, 61.32564044], - [70.24748314, 65.6063665], - [68.68968841, 69.32050656], - [69.54045681, 76.02404113], - [71.12050401, 82.6502915], - [75.9818397, 76.03093018], - [78.21117488, 70.10890893], - [78.82096788, 65.70521959], - [78.95372711, 61.4556606], - [77.55069872, 56.65560521], - [74.02173206, 55.16311953], - [73.51929617, 62.06964895], - [72.60106888, 65.53678304], - [71.85765381, 69.32731119], - [71.38454121, 79.79633067], - [73.61767156, 69.51337283], - [74.50990078, 65.60235839], - [74.81548138, 61.45331734]]))) - -initial_shape.append(PointCloud(np.array([[46.63369884, 44.08764686], - [65.31491309, 44.09407109], - [86.81640178, 46.55570064], - [106.86503868, 52.11274643], - [124.66154301, 60.0315786], - [139.66199441, 71.6036014], - [151.04922447, 83.47828965], - [162.96924699, 97.61591112], - [166.20238999, 116.0014495], - [161.57203038, 135.14867658], - [150.86767554, 152.08868824], - [135.50154984, 167.65900498], - [115.38918643, 187.02141497], - [93.00770583, 196.9633751], - [68.68470174, 200.86139148], - [43.68434508, 200.10445456], - [18.28476712, 195.44958702], - [23.53265303, 55.71937105], - [17.9649934, 63.90264665], - [15.55605939, 74.17002657], - [16.63479621, 84.37585532], - [20.58703068, 93.40012265], - [17.43094904, 114.21918023], - [8.43507654, 125.39260635], - [2.4008645, 138.35427044], - [1.56520568, 151.36086382], - [8.60866558, 163.88819772], - [33.10019692, 103.95961759], - [47.25368667, 103.08786691], - [61.67406413, 102.68512872], - [77.10378638, 101.67400095], - [85.7358453, 86.49915174], - [89.11550583, 97.67032089], - [91.00533132, 109.37981584], - [88.21279407, 117.69980754], - [84.54200076, 124.65638206], - [40.31079125, 69.47691491], - [34.3036891, 74.35452803], - [33.17442528, 83.95537112], - [37.09979548, 92.06172262], - [41.22462339, 84.85685672], - [43.47869442, 75.72207092], - [34.59233557, 124.36224816], - [24.36292985, 130.97352987], - [21.74824996, 141.07018437], - [26.36124109, 151.16502601], - [31.84622487, 143.32753518], - [34.48151342, 133.14559097], - [117.83907583, 90.5145853], - [112.62211772, 97.44922176], - [109.85120974, 104.77889356], - [110.911401, 112.18314623], - [108.21692684, 118.60739086], - [109.68847724, 130.20230795], - [112.4214409, 141.66354869], - [120.82995787, 130.21422374], - [124.68597685, 119.97106848], - [125.74071883, 112.35412967], - [125.97034877, 105.00378581], - [123.54356964, 96.70126365], - [117.43961426, 94.11975273], - [116.5705649, 106.06578435], - [114.98233273, 112.06278965], - [113.69646838, 118.61916064], - [112.87813868, 136.72713211], - [116.74072208, 118.94098628], - [118.2839861, 112.17621352], - [118.81254036, 104.99973274]]))) - -initial_shape.append(PointCloud(np.array([[29.30459178, 27.24534074], - [39.47004743, 24.38292299], - [51.54667438, 22.42372272], - [63.30767547, 22.37162616], - [74.20561385, 23.95008332], - [84.14265809, 27.94519239], - [92.16017681, 32.65929179], - [100.81474852, 38.52291926], - [105.39445843, 48.03051044], - [105.81247938, 59.1588891], - [102.5870203, 70.01814005], - [96.6149594, 80.84730771], - [88.64221584, 94.46788512], - [77.98963764, 103.31089364], - [65.35346377, 109.16323748], - [51.63461821, 112.58672956], - [37.10056847, 113.95059826], - [18.51972657, 37.11814141], - [16.7457652 , 42.42481409], - [17.01019564, 48.38086547], - [19.16282912, 53.76837796], - [22.69767086, 58.07217393], - [24.17432616, 69.88402627], - [20.99379373, 77.34357057], - [19.69904043, 85.32174442], - [21.23971857, 92.52684647], - [26.99391031, 98.26243543], - [31.12604697, 61.89794357], - [38.69324039, 59.25231487], - [46.47759964, 56.82093276], - [54.71781058, 53.90368008], - [57.08652729, 44.32277008], - [60.63919033, 49.88253722], - [63.46381778, 55.96376588], - [63.2207775 , 60.91909025], - [62.29071322, 65.26731234], - [29.75929632, 42.02967737], - [27.23910711, 45.60515084], - [28.09755316, 51.00222264], - [31.47695917, 54.81070084], - [32.61597345, 50.25772899], - [32.44103485, 44.94168113], - [35.06791957, 72.77012704], - [30.51633486, 77.93664152], - [30.64262749, 83.83136479], - [34.70122609, 88.61629379], - [36.4832508 , 83.51044643], - [36.35508694, 77.56615533], - [75.16994555, 41.58256719], - [73.39524567, 46.15605223], - [73.01204743, 50.56922423], - [74.72479626, 54.43524106], - [74.24428281, 58.34404327], - [76.82374875, 64.42709819], - [80.0690436 , 70.24390436], - [82.88766915, 62.72435028], - [83.41431565, 56.55948008], - [82.81967592, 52.25328539], - [81.81699053, 48.21872699], - [79.2228748 , 44.073611], - [75.50567221, 43.60542492], - [76.86548014, 50.2385966], - [76.9213308 , 53.74522715], - [77.22751327, 57.5098225], - [79.56023029, 67.48793174], - [78.93326695, 57.21790467], - [78.73516471, 53.30042959], - [77.92179698, 49.31461186]]))) - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# build aam -template_trilist_image = training_images[0].landmarks[None] -trilist = ibug_face_68_trimesh(template_trilist_image)[1].lms.trilist -aam = AAMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1, 2, 3], - max_appearance_components=[3, 2, 1], - boundary=3).build(training_images) - -aam2 = AAMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=1, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1], - max_appearance_components=[1], - boundary=3).build(training_images) - - -def test_aam(): - assert (aam.n_training_images == 4) - assert (aam.n_levels == 3) - assert (aam.downscale == 2) - #assert (aam.features[0] == igo and len(aam.features) == 1) - assert_allclose(np.around(aam.reference_shape.range()), (109., 103.)) - assert aam.scaled_shape_models - assert aam.pyramid_on_features - assert_allclose([aam.shape_models[j].n_components - for j in range(aam.n_levels)], (1, 2, 3)) - assert_allclose([aam.appearance_models[j].n_components - for j in range(aam.n_levels)], (3, 2, 1)) - assert_allclose([aam.appearance_models[j].template_instance.n_channels - for j in range(aam.n_levels)], (2, 2, 2)) - assert_allclose([aam.appearance_models[j].components.shape[1] - for j in range(aam.n_levels)], (884, 3652, 14892)) - - -@raises(TypeError, ValueError) -def test_n_shape_exception(): - fitter = LucasKanadeAAMFitter(aam, n_shape=[3, 6, 'a']) - - -@raises(ValueError) -def test_n_appearance_exception(): - fitter = LucasKanadeAAMFitter(aam, n_appearance=[10, 20]) - - -def test_pertrurb_shape(): - fitter = LucasKanadeAAMFitter(aam) - s = fitter.perturb_shape(training_images[0].landmarks[None].lms, - noise_std=0.08, rotation=False) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -def test_obtain_shape_from_bb(): - fitter = LucasKanadeAAMFitter(aam) - s = fitter.obtain_shape_from_bb(np.array([[53.916, 1.853], - [408.469, 339.471]])) - assert ((np.around(s.points) == np.around(initial_shape[0].points)).all()) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -@raises(ValueError) -def test_max_iters_exception(): - fitter = LucasKanadeAAMFitter(aam, - algorithm=AIC) - fitter.fit(training_images[0], initial_shape[0], - max_iters=[10, 20, 30, 40]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(aam) - fitter = LucasKanadeAAMFitter(aam, - algorithm=AIC) - print(fitter) - print(aam2) - fitter = LucasKanadeAAMFitter(aam2, - algorithm=SFA) - print(fitter) - - -def aam_helper(aam, algorithm, im_number, max_iters, initial_error, - final_error, error_type): - fitter = LucasKanadeAAMFitter(aam, algorithm=algorithm) - fitting_result = fitter.fit( - training_images[im_number], initial_shape[im_number], - gt_shape=training_images[im_number].landmarks[None].lms, - max_iters=max_iters) - assert_allclose( - np.around(fitting_result.initial_error(error_type=error_type), 5), - initial_error) - assert_allclose( - np.around(fitting_result.final_error(error_type=error_type), 5), - final_error) - - -@attr('fuzzy') -def test_alternating_ic(): - aam_helper(aam, AIC, 0, 6, 0.09062, 0.05607, 'me_norm') - - -@attr('fuzzy') -def test_simultaneous_ic(): - aam_helper(aam, SIC, 2, 7, 0.12616, 0.11152, 'me_norm') - - -@attr('fuzzy') -def test_projectout_ic(): - aam_helper(aam, PIC, 3, 6, 0.10796, 0.07346, 'me_norm') - - -@attr('fuzzy') -def test_alternating_fa(): - aam_helper(aam, AFA, 0, 8, 0.09062, 0.07225, 'me_norm') - - -@attr('fuzzy') -def test_simultaneous_fa(): - aam_helper(aam, SFA, 2, 5, 0.12616, 0.11151, 'me_norm') - - -@attr('fuzzy') -def test_alternating_fc(): - aam_helper(aam, AFC, 0, 6, 0.09062, 0.07129, 'me_norm') - - -@attr('fuzzy') -def test_simultaneous_fc(): - aam_helper(aam, SFC, 2, 5, 0.12616, 0.11738, 'me_norm') diff --git a/menpofit/test/atm_builder_test.py b/menpofit/test/atm_builder_test.py deleted file mode 100644 index 9b50b3a..0000000 --- a/menpofit/test/atm_builder_test.py +++ /dev/null @@ -1,178 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.transform import PiecewiseAffine, ThinPlateSplines -from menpo.feature import sparse_hog, igo, lbp, no_op - -import menpo.io as mio -from menpofit.atm import ATMBuilder, PatchBasedATMBuilder - - - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training = [] -templates = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training.append(im.landmarks[None].lms) - templates.append(im) - -# build atms -atm1 = ATMBuilder(features=[igo, sparse_hog, no_op], - transform=PiecewiseAffine, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=False, - max_shape_components=[1, 2, 3], - boundary=3).build(training, templates[0]) - -atm2 = ATMBuilder(features=[no_op, no_op], - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=None, - boundary=0).build(training, templates[1]) - -atm3 = ATMBuilder(features=igo, - transform=ThinPlateSplines, - trilist=None, - normalization_diagonal=None, - n_levels=1, - downscale=3, - scaled_shape_models=True, - max_shape_components=[2], - boundary=2).build(training, templates[2]) - -atm4 = PatchBasedATMBuilder(features=lbp, - patch_shape=(10, 13), - normalization_diagonal=200, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=1, - boundary=2).build(training, templates[3]) - - -@raises(ValueError) -def test_features_exception(): - ATMBuilder(features=[igo, sparse_hog]).build(training, templates[0]) - - -@raises(ValueError) -def test_n_levels_exception(): - ATMBuilder(n_levels=0).build(training, templates[1]) - - -@raises(ValueError) -def test_downscale_exception(): - atm = ATMBuilder(downscale=1).build(training, templates[2]) - assert (atm.downscale == 1) - ATMBuilder(downscale=0).build(training, templates[2]) - - -@raises(ValueError) -def test_normalization_diagonal_exception(): - atm = ATMBuilder(normalization_diagonal=100).build(training, templates[3]) - assert (atm.warped_templates[0].n_true_pixels() == 1246) - ATMBuilder(normalization_diagonal=10).build(training, templates[3]) - - -@raises(ValueError) -def test_max_shape_components_exception(): - ATMBuilder(max_shape_components=[1, 0.2, 'a']).build(training, templates[0]) - - -@raises(ValueError) -def test_max_shape_components_exception_2(): - ATMBuilder(max_shape_components=[1, 2]).build(training, templates[0]) - - -@raises(ValueError) -def test_boundary_exception(): - ATMBuilder(boundary=-1).build(training, templates[1]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - ATMBuilder().build(training, templates[2], verbose=True) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(atm1) - print(atm2) - print(atm3) - print(atm4) - - -def test_atm_1(): - assert(atm1.n_training_shapes == 4) - assert(atm1.n_levels == 3) - assert(atm1.downscale == 2) - assert_allclose(np.around(atm1.reference_shape.range()), (109., 103.)) - assert(not atm1.scaled_shape_models) - assert(not atm1.pyramid_on_features) - assert_allclose([atm1.shape_models[j].n_components - for j in range(atm1.n_levels)], (1, 2, 3)) - assert_allclose([atm1.warped_templates[j].n_channels - for j in range(atm1.n_levels)], (2, 36, 1)) - assert_allclose([atm1.warped_templates[j].shape[1] - for j in range(atm1.n_levels)], (164, 164, 164)) - - -def test_atm_2(): - assert (atm2.n_training_shapes == 4) - assert (atm2.n_levels == 2) - assert (atm2.downscale == 1.2) - assert_allclose(np.around(atm2.reference_shape.range()), (169., 161.)) - assert atm2.scaled_shape_models - assert (not atm2.pyramid_on_features) - assert (np.all([atm2.shape_models[j].n_components == 3 - for j in range(atm2.n_levels)])) - assert (np.all([atm2.warped_templates[j].n_channels == 1 - for j in range(atm2.n_levels)])) - assert_allclose([atm2.warped_templates[j].shape[1] - for j in range(atm2.n_levels)], (132, 158)) - - -def test_atm_3(): - assert (atm3.n_training_shapes == 4) - assert (atm3.n_levels == 1) - assert (atm3.downscale == 3) - assert_allclose(np.around(atm3.reference_shape.range()), (169., 161.)) - assert atm3.scaled_shape_models - assert atm3.pyramid_on_features - assert (np.all([atm3.shape_models[j].n_components == 2 - for j in range(atm3.n_levels)])) - assert (np.all([atm3.warped_templates[j].n_channels == 2 - for j in range(atm3.n_levels)])) - assert_allclose([atm3.warped_templates[j].shape[1] - for j in range(atm3.n_levels)], 162) - - -def test_atm_4(): - assert (atm4.n_training_shapes == 4) - assert (atm4.n_levels == 2) - assert (atm4.downscale == 1.2) - assert_allclose(np.around(atm4.reference_shape.range()), (145., 138.)) - assert atm4.scaled_shape_models - assert atm4.pyramid_on_features - assert (np.all([atm4.shape_models[j].n_components == 1 - for j in range(atm4.n_levels)])) - assert (np.all([atm4.warped_templates[j].n_channels == 4 - for j in range(atm4.n_levels)])) - assert_allclose([atm4.warped_templates[j].shape[1] - for j in range(atm4.n_levels)], (162, 188)) diff --git a/menpofit/test/atm_fitter_test.py b/menpofit/test/atm_fitter_test.py deleted file mode 100644 index 53f5975..0000000 --- a/menpofit/test/atm_fitter_test.py +++ /dev/null @@ -1,441 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from mock import patch -from nose.plugins.attrib import attr -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import igo -from menpofit.transform import DifferentiablePiecewiseAffine - -import menpo.io as mio -from menpo.shape.pointcloud import PointCloud -from menpofit.atm import ATMBuilder, LucasKanadeATMFitter -from menpofit.lucaskanade.image import FA, FC, IC - - -initial_shape = [] -initial_shape.append(PointCloud(np.array([[150.9737801, 1.85331141], - [191.20452708, 1.86714624], - [237.5088486, 7.16836457], - [280.68439528, 19.1356864], - [319.00988383, 36.18921029], - [351.31395982, 61.11002727], - [375.83681819, 86.68264647], - [401.50706656, 117.12858347], - [408.46977018, 156.72258055], - [398.49810436, 197.95690492], - [375.44584527, 234.437902], - [342.35427495, 267.96920594], - [299.04149064, 309.66693535], - [250.84207113, 331.07734674], - [198.46150259, 339.47188196], - [144.62222804, 337.84178783], - [89.92321435, 327.81734317], - [101.22474793, 26.90269773], - [89.23456877, 44.52571118], - [84.04683242, 66.6369272], - [86.36993557, 88.61559027], - [94.88123162, 108.04971327], - [88.08448274, 152.88439191], - [68.71150917, 176.94681489], - [55.7165906, 204.86028035], - [53.9169657, 232.87050281], - [69.08534014, 259.8486207], - [121.82883888, 130.79001073], - [152.30894887, 128.91266055], - [183.36381228, 128.04534764], - [216.59234031, 125.86784329], - [235.18182671, 93.18819461], - [242.46006172, 117.24575711], - [246.52987701, 142.46262589], - [240.51603561, 160.38006297], - [232.61083444, 175.36132625], - [137.35714406, 56.53012228], - [124.42060774, 67.0342585], - [121.98869265, 87.71006061], - [130.4421354, 105.16741493], - [139.32511836, 89.65144616], - [144.17935107, 69.97931719], - [125.04221953, 174.72789706], - [103.0127825, 188.96555839], - [97.38196408, 210.70911033], - [107.31622619, 232.4487582], - [119.12835959, 215.57040617], - [124.80355957, 193.64317941], - [304.3174261, 101.83559243], - [293.08249678, 116.76961123], - [287.11523488, 132.55435452], - [289.39839945, 148.49971074], - [283.59574087, 162.33458018], - [286.76478391, 187.30470094], - [292.65033117, 211.98694428], - [310.75841097, 187.33036207], - [319.06250309, 165.27131484], - [321.3339324, 148.86793045], - [321.82844973, 133.03866904], - [316.60228316, 115.15885333], - [303.45716953, 109.59946563], - [301.58563675, 135.32572565], - [298.16531481, 148.240518], - [295.39615418, 162.35992687], - [293.63384823, 201.35617245], - [301.95207707, 163.05299135], - [305.27555828, 148.48478086], - [306.41382116, 133.02994058]]))) - -initial_shape.append(PointCloud(np.array([[33.08569962, 26.2373455], - [43.88613611, 26.24105964], - [56.31709803, 27.66423659], - [67.90810205, 30.87701063], - [78.19704859, 35.45523787], - [86.86947323, 42.14553624], - [93.45293474, 49.0108189], - [100.34442715, 57.18440338], - [102.21365016, 67.81389656], - [99.53663441, 78.88375569], - [93.34797327, 88.67752592], - [84.46413615, 97.67941492], - [72.83628901, 108.8736808], - [59.89656483, 114.62156782], - [45.83436002, 116.87518356], - [31.38054772, 116.43756484], - [16.69592792, 113.74637996], - [19.72996295, 32.96215989], - [16.51105259, 37.69327358], - [15.11834126, 43.62930018], - [15.74200674, 49.52974132], - [18.02696835, 54.74706954], - [16.20229791, 66.78348784], - [11.00138601, 73.24333984], - [7.51274105, 80.73705133], - [7.02960972, 88.25673842], - [11.10174551, 95.4993444], - [25.26138338, 60.85198075], - [33.44414202, 60.34798312], - [41.78120024, 60.11514235], - [50.70180534, 59.53056465], - [55.69238052, 50.75731293], - [57.6463118, 57.21586007], - [58.73890353, 63.98563718], - [57.12441419, 68.79579249], - [55.00216617, 72.817696], - [29.43014699, 40.91600468], - [25.95717546, 43.73596863], - [25.30429808, 49.2866408], - [27.57372827, 53.97328126], - [29.95847378, 49.80782952], - [31.26165197, 44.52660569], - [26.12405475, 72.64764418], - [20.20998272, 76.46991865], - [18.69832059, 82.30724133], - [21.36529486, 88.14351591], - [24.53640666, 83.6123157], - [26.05998356, 77.72568327], - [74.25267847, 53.07881273], - [71.23652416, 57.08803288], - [69.63453966, 61.32564044], - [70.24748314, 65.6063665], - [68.68968841, 69.32050656], - [69.54045681, 76.02404113], - [71.12050401, 82.6502915], - [75.9818397, 76.03093018], - [78.21117488, 70.10890893], - [78.82096788, 65.70521959], - [78.95372711, 61.4556606], - [77.55069872, 56.65560521], - [74.02173206, 55.16311953], - [73.51929617, 62.06964895], - [72.60106888, 65.53678304], - [71.85765381, 69.32731119], - [71.38454121, 79.79633067], - [73.61767156, 69.51337283], - [74.50990078, 65.60235839], - [74.81548138, 61.45331734]]))) - -initial_shape.append(PointCloud(np.array([[46.63369884, 44.08764686], - [65.31491309, 44.09407109], - [86.81640178, 46.55570064], - [106.86503868, 52.11274643], - [124.66154301, 60.0315786], - [139.66199441, 71.6036014], - [151.04922447, 83.47828965], - [162.96924699, 97.61591112], - [166.20238999, 116.0014495], - [161.57203038, 135.14867658], - [150.86767554, 152.08868824], - [135.50154984, 167.65900498], - [115.38918643, 187.02141497], - [93.00770583, 196.9633751], - [68.68470174, 200.86139148], - [43.68434508, 200.10445456], - [18.28476712, 195.44958702], - [23.53265303, 55.71937105], - [17.9649934, 63.90264665], - [15.55605939, 74.17002657], - [16.63479621, 84.37585532], - [20.58703068, 93.40012265], - [17.43094904, 114.21918023], - [8.43507654, 125.39260635], - [2.4008645, 138.35427044], - [1.56520568, 151.36086382], - [8.60866558, 163.88819772], - [33.10019692, 103.95961759], - [47.25368667, 103.08786691], - [61.67406413, 102.68512872], - [77.10378638, 101.67400095], - [85.7358453, 86.49915174], - [89.11550583, 97.67032089], - [91.00533132, 109.37981584], - [88.21279407, 117.69980754], - [84.54200076, 124.65638206], - [40.31079125, 69.47691491], - [34.3036891, 74.35452803], - [33.17442528, 83.95537112], - [37.09979548, 92.06172262], - [41.22462339, 84.85685672], - [43.47869442, 75.72207092], - [34.59233557, 124.36224816], - [24.36292985, 130.97352987], - [21.74824996, 141.07018437], - [26.36124109, 151.16502601], - [31.84622487, 143.32753518], - [34.48151342, 133.14559097], - [117.83907583, 90.5145853], - [112.62211772, 97.44922176], - [109.85120974, 104.77889356], - [110.911401, 112.18314623], - [108.21692684, 118.60739086], - [109.68847724, 130.20230795], - [112.4214409, 141.66354869], - [120.82995787, 130.21422374], - [124.68597685, 119.97106848], - [125.74071883, 112.35412967], - [125.97034877, 105.00378581], - [123.54356964, 96.70126365], - [117.43961426, 94.11975273], - [116.5705649, 106.06578435], - [114.98233273, 112.06278965], - [113.69646838, 118.61916064], - [112.87813868, 136.72713211], - [116.74072208, 118.94098628], - [118.2839861, 112.17621352], - [118.81254036, 104.99973274]]))) - -initial_shape.append(PointCloud(np.array([[29.30459178, 27.24534074], - [39.47004743, 24.38292299], - [51.54667438, 22.42372272], - [63.30767547, 22.37162616], - [74.20561385, 23.95008332], - [84.14265809, 27.94519239], - [92.16017681, 32.65929179], - [100.81474852, 38.52291926], - [105.39445843, 48.03051044], - [105.81247938, 59.1588891], - [102.5870203, 70.01814005], - [96.6149594, 80.84730771], - [88.64221584, 94.46788512], - [77.98963764, 103.31089364], - [65.35346377, 109.16323748], - [51.63461821, 112.58672956], - [37.10056847, 113.95059826], - [18.51972657, 37.11814141], - [16.7457652 , 42.42481409], - [17.01019564, 48.38086547], - [19.16282912, 53.76837796], - [22.69767086, 58.07217393], - [24.17432616, 69.88402627], - [20.99379373, 77.34357057], - [19.69904043, 85.32174442], - [21.23971857, 92.52684647], - [26.99391031, 98.26243543], - [31.12604697, 61.89794357], - [38.69324039, 59.25231487], - [46.47759964, 56.82093276], - [54.71781058, 53.90368008], - [57.08652729, 44.32277008], - [60.63919033, 49.88253722], - [63.46381778, 55.96376588], - [63.2207775 , 60.91909025], - [62.29071322, 65.26731234], - [29.75929632, 42.02967737], - [27.23910711, 45.60515084], - [28.09755316, 51.00222264], - [31.47695917, 54.81070084], - [32.61597345, 50.25772899], - [32.44103485, 44.94168113], - [35.06791957, 72.77012704], - [30.51633486, 77.93664152], - [30.64262749, 83.83136479], - [34.70122609, 88.61629379], - [36.4832508 , 83.51044643], - [36.35508694, 77.56615533], - [75.16994555, 41.58256719], - [73.39524567, 46.15605223], - [73.01204743, 50.56922423], - [74.72479626, 54.43524106], - [74.24428281, 58.34404327], - [76.82374875, 64.42709819], - [80.0690436 , 70.24390436], - [82.88766915, 62.72435028], - [83.41431565, 56.55948008], - [82.81967592, 52.25328539], - [81.81699053, 48.21872699], - [79.2228748 , 44.073611], - [75.50567221, 43.60542492], - [76.86548014, 50.2385966], - [76.9213308 , 53.74522715], - [77.22751327, 57.5098225], - [79.56023029, 67.48793174], - [78.93326695, 57.21790467], - [78.73516471, 53.30042959], - [77.92179698, 49.31461186]]))) - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_shapes = [] -templates = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_shapes.append(im.landmarks[None].lms) - templates.append(im) - -# build atm -atm1 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1, 2, 3], - boundary=3).build(training_shapes, templates[0]) - -atm2 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=1, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1], - boundary=3).build(training_shapes, templates[1]) - -atm3 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1, 2, 3], - boundary=3).build(training_shapes, templates[2]) - -atm4 = ATMBuilder(features=igo, - transform=DifferentiablePiecewiseAffine, - normalization_diagonal=150, - n_levels=1, - downscale=2, - scaled_shape_models=True, - max_shape_components=[1], - boundary=3).build(training_shapes, templates[3]) - - -def test_atm1(): - assert (atm1.n_training_shapes == 4) - assert (atm1.n_levels == 3) - assert (atm1.downscale == 2) - assert_allclose(np.around(atm1.reference_shape.range()), (109., 103.)) - assert atm1.scaled_shape_models - assert atm1.pyramid_on_features - assert_allclose([atm1.shape_models[j].n_components - for j in range(atm1.n_levels)], (1, 2, 3)) - assert_allclose([atm1.warped_templates[j].n_channels - for j in range(atm1.n_levels)], (2, 2, 2)) - assert_allclose([atm1.warped_templates[j].shape[1] - for j in range(atm1.n_levels)], (46, 85, 164)) - - -@raises(TypeError, ValueError) -def test_n_shape_exception(): - fitter = LucasKanadeATMFitter(atm1, n_shape=[3, 6, 'a']) - - -@raises(ValueError) -def test_n_shape_exception_2(): - fitter = LucasKanadeATMFitter(atm1, n_shape=[10, 20]) - - -def test_pertrurb_shape(): - fitter = LucasKanadeATMFitter(atm1) - s = fitter.perturb_shape(templates[0].landmarks[None].lms, - noise_std=0.08, rotation=False) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -def test_obtain_shape_from_bb(): - fitter = LucasKanadeATMFitter(atm1) - s = fitter.obtain_shape_from_bb(np.array([[53.916, 1.853], - [408.469, 339.471]])) - assert ((np.around(s.points) == np.around(initial_shape[0].points)).all()) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -@raises(ValueError) -def test_max_iters_exception(): - fitter = LucasKanadeATMFitter(atm1, - algorithm=IC) - fitter.fit(templates[0], initial_shape[0], max_iters=[10, 20, 30, 40]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(atm1) - fitter = LucasKanadeATMFitter(atm1, - algorithm=IC) - print(fitter) - print(atm2) - fitter = LucasKanadeATMFitter(atm2, - algorithm=FA) - print(fitter) - - -def atm_helper(atm, algorithm, im_number, max_iters, initial_error, - final_error, error_type): - fitter = LucasKanadeATMFitter(atm, algorithm=algorithm) - fitting_result = fitter.fit( - templates[im_number], initial_shape[im_number], - gt_shape=templates[im_number].landmarks[None].lms, - max_iters=max_iters) - assert_allclose( - np.around(fitting_result.initial_error(error_type=error_type), 5), - initial_error) - assert_allclose( - np.around(fitting_result.final_error(error_type=error_type), 5), - final_error) - - -@attr('fuzzy') -def test_ic(): - atm_helper(atm1, IC, 0, 6, 0.09062, 0.06788, 'me_norm') - - -@attr('fuzzy') -def test_fa(): - atm_helper(atm2, FA, 1, 8, 0.09051, 0.08188, 'me_norm') - - -@attr('fuzzy') -def test_fc(): - atm_helper(atm3, FC, 2, 6, 0.12615, 0.08255, 'me_norm') - -@attr('fuzzy') -def test_ic_2(): - atm_helper(atm4, IC, 3, 7, 0.09748, 0.09511, 'me_norm') diff --git a/menpofit/test/clm_builder_test.py b/menpofit/test/clm_builder_test.py deleted file mode 100644 index 406222a..0000000 --- a/menpofit/test/clm_builder_test.py +++ /dev/null @@ -1,197 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO -from sklearn import qda - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import sparse_hog, igo, no_op - -import menpo.io as mio -from menpofit.clm import CLMBuilder -from menpofit.clm.classifier import linear_svm_lr -from menpofit.base import name_of_callable - - -def random_forest(X, t): - clf = qda.QDA() - clf.fit(X, t) - - def random_forest_predict(x): - return clf.predict_proba(x)[:, 1] - - return random_forest_predict - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# build clms -clm1 = CLMBuilder(classifier_trainers=[linear_svm_lr], - patch_shape=(5, 5), - features=[igo, sparse_hog, no_op], - normalization_diagonal=150, - n_levels=3, - downscale=2, - scaled_shape_models=False, - max_shape_components=[1, 2, 3], - boundary=3).build(training_images) - -clm2 = CLMBuilder(classifier_trainers=[random_forest, linear_svm_lr], - patch_shape=(3, 10), - features=[no_op, no_op], - normalization_diagonal=None, - n_levels=2, - downscale=1.2, - scaled_shape_models=True, - max_shape_components=None, - boundary=0).build(training_images) - -clm3 = CLMBuilder(classifier_trainers=[linear_svm_lr], - patch_shape=(2, 3), - features=igo, - normalization_diagonal=None, - n_levels=1, - downscale=3, - scaled_shape_models=True, - max_shape_components=[1], - boundary=2).build(training_images) - - -@raises(ValueError) -def test_classifier_type_1_exception(): - CLMBuilder(classifier_trainers=[linear_svm_lr, linear_svm_lr]).build( - training_images) - -@raises(ValueError) -def test_classifier_type_2_exception(): - CLMBuilder(classifier_trainers=['linear_svm_lr']).build(training_images) - -@raises(ValueError) -def test_patch_shape_1_exception(): - CLMBuilder(patch_shape=(5, 1)).build(training_images) - -@raises(ValueError) -def test_patch_shape_2_exception(): - CLMBuilder(patch_shape=(5, 6, 7)).build(training_images) - -@raises(ValueError) -def test_features_exception(): - CLMBuilder(features=[igo, sparse_hog]).build(training_images) - -@raises(ValueError) -def test_n_levels_exception(): - clm = CLMBuilder(n_levels=0).build(training_images) - - -@raises(ValueError) -def test_downscale_exception(): - clm = CLMBuilder(downscale=1).build(training_images) - assert (clm.downscale == 1) - CLMBuilder(downscale=0).build(training_images) - - -@raises(ValueError) -def test_normalization_diagonal_exception(): - CLMBuilder(normalization_diagonal=10).build(training_images) - - -@raises(ValueError) -def test_max_shape_components_1_exception(): - CLMBuilder(max_shape_components=[1, 0.2, 'a']).build(training_images) - - -@raises(ValueError) -def test_max_shape_components_2_exception(): - CLMBuilder(max_shape_components=[1, 2]).build(training_images) - - -@raises(ValueError) -def test_boundary_exception(): - CLMBuilder(boundary=-1).build(training_images) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - CLMBuilder().build(training_images, verbose=True) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(clm1) - print(clm2) - print(clm3) - - -def test_clm_1(): - assert (clm1.n_training_images == 4) - assert (clm1.n_levels == 3) - assert (clm1.downscale == 2) - #assert (clm1.features[0] == igo and clm1.features[2] is no_op) - assert_allclose(np.around(clm1.reference_shape.range()), (109., 103.)) - assert (not clm1.scaled_shape_models) - assert (not clm1.pyramid_on_features) - assert_allclose(clm1.patch_shape, (5, 5)) - assert_allclose([clm1.shape_models[j].n_components - for j in range(clm1.n_levels)], (1, 2, 3)) - assert_allclose(clm1.n_classifiers_per_level, [68, 68, 68]) - - ran_0 = np.random.randint(0, clm1.n_classifiers_per_level[0]) - ran_1 = np.random.randint(0, clm1.n_classifiers_per_level[1]) - ran_2 = np.random.randint(0, clm1.n_classifiers_per_level[2]) - - assert (name_of_callable(clm1.classifiers[0][ran_0]) - == 'linear_svm_lr') - assert (name_of_callable(clm1.classifiers[1][ran_1]) - == 'linear_svm_lr') - assert (name_of_callable(clm1.classifiers[2][ran_2]) - == 'linear_svm_lr') - - -def test_clm_2(): - assert (clm2.n_training_images == 4) - assert (clm2.n_levels == 2) - assert (clm2.downscale == 1.2) - #assert (clm2.features[0] is no_op and clm2.features[1] is no_op) - assert_allclose(np.around(clm2.reference_shape.range()), (169., 161.)) - assert clm2.scaled_shape_models - assert (not clm2.pyramid_on_features) - assert_allclose(clm2.patch_shape, (3, 10)) - assert (np.all([clm2.shape_models[j].n_components == 3 - for j in range(clm2.n_levels)])) - assert_allclose(clm2.n_classifiers_per_level, [68, 68]) - - ran_0 = np.random.randint(0, clm2.n_classifiers_per_level[0]) - ran_1 = np.random.randint(0, clm2.n_classifiers_per_level[1]) - - assert (name_of_callable(clm2.classifiers[0][ran_0]) - == 'random_forest_predict') - assert (name_of_callable(clm2.classifiers[1][ran_1]) - == 'linear_svm_lr') - - -def test_clm_3(): - assert (clm3.n_training_images == 4) - assert (clm3.n_levels == 1) - assert (clm3.downscale == 3) - #assert (clm3.features[0] == igo and len(clm3.features) == 1) - assert_allclose(np.around(clm3.reference_shape.range()), (169., 161.)) - assert clm3.scaled_shape_models - assert clm3.pyramid_on_features - assert_allclose(clm3.patch_shape, (2, 3)) - assert (np.all([clm3.shape_models[j].n_components == 1 - for j in range(clm3.n_levels)])) - assert_allclose(clm3.n_classifiers_per_level, [68]) - ran_0 = np.random.randint(0, clm3.n_classifiers_per_level[0]) - - assert (name_of_callable(clm3.classifiers[0][ran_0]) - == 'linear_svm_lr') diff --git a/menpofit/test/clm_fitter_test.py b/menpofit/test/clm_fitter_test.py deleted file mode 100644 index c2ef74c..0000000 --- a/menpofit/test/clm_fitter_test.py +++ /dev/null @@ -1,373 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from mock import patch -import numpy as np -from numpy.testing import assert_allclose -from nose.tools import raises -from menpo.feature import sparse_hog - -import menpo.io as mio -from menpo.shape.pointcloud import PointCloud -from menpofit.clm import CLMBuilder -from menpofit.clm import GradientDescentCLMFitter -from menpofit.gradientdescent import RLMS -from menpofit.clm.classifier import linear_svm_lr -from menpofit.base import name_of_callable - - -initial_shape = [] -initial_shape.append(PointCloud(np.array([[150.9737801, 1.85331141], - [191.20452708, 1.86714624], - [237.5088486, 7.16836457], - [280.68439528, 19.1356864], - [319.00988383, 36.18921029], - [351.31395982, 61.11002727], - [375.83681819, 86.68264647], - [401.50706656, 117.12858347], - [408.46977018, 156.72258055], - [398.49810436, 197.95690492], - [375.44584527, 234.437902], - [342.35427495, 267.96920594], - [299.04149064, 309.66693535], - [250.84207113, 331.07734674], - [198.46150259, 339.47188196], - [144.62222804, 337.84178783], - [89.92321435, 327.81734317], - [101.22474793, 26.90269773], - [89.23456877, 44.52571118], - [84.04683242, 66.6369272], - [86.36993557, 88.61559027], - [94.88123162, 108.04971327], - [88.08448274, 152.88439191], - [68.71150917, 176.94681489], - [55.7165906, 204.86028035], - [53.9169657, 232.87050281], - [69.08534014, 259.8486207], - [121.82883888, 130.79001073], - [152.30894887, 128.91266055], - [183.36381228, 128.04534764], - [216.59234031, 125.86784329], - [235.18182671, 93.18819461], - [242.46006172, 117.24575711], - [246.52987701, 142.46262589], - [240.51603561, 160.38006297], - [232.61083444, 175.36132625], - [137.35714406, 56.53012228], - [124.42060774, 67.0342585], - [121.98869265, 87.71006061], - [130.4421354, 105.16741493], - [139.32511836, 89.65144616], - [144.17935107, 69.97931719], - [125.04221953, 174.72789706], - [103.0127825, 188.96555839], - [97.38196408, 210.70911033], - [107.31622619, 232.4487582], - [119.12835959, 215.57040617], - [124.80355957, 193.64317941], - [304.3174261, 101.83559243], - [293.08249678, 116.76961123], - [287.11523488, 132.55435452], - [289.39839945, 148.49971074], - [283.59574087, 162.33458018], - [286.76478391, 187.30470094], - [292.65033117, 211.98694428], - [310.75841097, 187.33036207], - [319.06250309, 165.27131484], - [321.3339324, 148.86793045], - [321.82844973, 133.03866904], - [316.60228316, 115.15885333], - [303.45716953, 109.59946563], - [301.58563675, 135.32572565], - [298.16531481, 148.240518], - [295.39615418, 162.35992687], - [293.63384823, 201.35617245], - [301.95207707, 163.05299135], - [305.27555828, 148.48478086], - [306.41382116, 133.02994058]]))) - -initial_shape.append(PointCloud(np.array([[33.08569962, 26.2373455], - [43.88613611, 26.24105964], - [56.31709803, 27.66423659], - [67.90810205, 30.87701063], - [78.19704859, 35.45523787], - [86.86947323, 42.14553624], - [93.45293474, 49.0108189], - [100.34442715, 57.18440338], - [102.21365016, 67.81389656], - [99.53663441, 78.88375569], - [93.34797327, 88.67752592], - [84.46413615, 97.67941492], - [72.83628901, 108.8736808], - [59.89656483, 114.62156782], - [45.83436002, 116.87518356], - [31.38054772, 116.43756484], - [16.69592792, 113.74637996], - [19.72996295, 32.96215989], - [16.51105259, 37.69327358], - [15.11834126, 43.62930018], - [15.74200674, 49.52974132], - [18.02696835, 54.74706954], - [16.20229791, 66.78348784], - [11.00138601, 73.24333984], - [7.51274105, 80.73705133], - [7.02960972, 88.25673842], - [11.10174551, 95.4993444], - [25.26138338, 60.85198075], - [33.44414202, 60.34798312], - [41.78120024, 60.11514235], - [50.70180534, 59.53056465], - [55.69238052, 50.75731293], - [57.6463118, 57.21586007], - [58.73890353, 63.98563718], - [57.12441419, 68.79579249], - [55.00216617, 72.817696], - [29.43014699, 40.91600468], - [25.95717546, 43.73596863], - [25.30429808, 49.2866408], - [27.57372827, 53.97328126], - [29.95847378, 49.80782952], - [31.26165197, 44.52660569], - [26.12405475, 72.64764418], - [20.20998272, 76.46991865], - [18.69832059, 82.30724133], - [21.36529486, 88.14351591], - [24.53640666, 83.6123157], - [26.05998356, 77.72568327], - [74.25267847, 53.07881273], - [71.23652416, 57.08803288], - [69.63453966, 61.32564044], - [70.24748314, 65.6063665], - [68.68968841, 69.32050656], - [69.54045681, 76.02404113], - [71.12050401, 82.6502915], - [75.9818397, 76.03093018], - [78.21117488, 70.10890893], - [78.82096788, 65.70521959], - [78.95372711, 61.4556606], - [77.55069872, 56.65560521], - [74.02173206, 55.16311953], - [73.51929617, 62.06964895], - [72.60106888, 65.53678304], - [71.85765381, 69.32731119], - [71.38454121, 79.79633067], - [73.61767156, 69.51337283], - [74.50990078, 65.60235839], - [74.81548138, 61.45331734]]))) - -initial_shape.append(PointCloud(np.array([[46.63369884, 44.08764686], - [65.31491309, 44.09407109], - [86.81640178, 46.55570064], - [106.86503868, 52.11274643], - [124.66154301, 60.0315786], - [139.66199441, 71.6036014], - [151.04922447, 83.47828965], - [162.96924699, 97.61591112], - [166.20238999, 116.0014495], - [161.57203038, 135.14867658], - [150.86767554, 152.08868824], - [135.50154984, 167.65900498], - [115.38918643, 187.02141497], - [93.00770583, 196.9633751], - [68.68470174, 200.86139148], - [43.68434508, 200.10445456], - [18.28476712, 195.44958702], - [23.53265303, 55.71937105], - [17.9649934, 63.90264665], - [15.55605939, 74.17002657], - [16.63479621, 84.37585532], - [20.58703068, 93.40012265], - [17.43094904, 114.21918023], - [8.43507654, 125.39260635], - [2.4008645, 138.35427044], - [1.56520568, 151.36086382], - [8.60866558, 163.88819772], - [33.10019692, 103.95961759], - [47.25368667, 103.08786691], - [61.67406413, 102.68512872], - [77.10378638, 101.67400095], - [85.7358453, 86.49915174], - [89.11550583, 97.67032089], - [91.00533132, 109.37981584], - [88.21279407, 117.69980754], - [84.54200076, 124.65638206], - [40.31079125, 69.47691491], - [34.3036891, 74.35452803], - [33.17442528, 83.95537112], - [37.09979548, 92.06172262], - [41.22462339, 84.85685672], - [43.47869442, 75.72207092], - [34.59233557, 124.36224816], - [24.36292985, 130.97352987], - [21.74824996, 141.07018437], - [26.36124109, 151.16502601], - [31.84622487, 143.32753518], - [34.48151342, 133.14559097], - [117.83907583, 90.5145853], - [112.62211772, 97.44922176], - [109.85120974, 104.77889356], - [110.911401, 112.18314623], - [108.21692684, 118.60739086], - [109.68847724, 130.20230795], - [112.4214409, 141.66354869], - [120.82995787, 130.21422374], - [124.68597685, 119.97106848], - [125.74071883, 112.35412967], - [125.97034877, 105.00378581], - [123.54356964, 96.70126365], - [117.43961426, 94.11975273], - [116.5705649, 106.06578435], - [114.98233273, 112.06278965], - [113.69646838, 118.61916064], - [112.87813868, 136.72713211], - [116.74072208, 118.94098628], - [118.2839861, 112.17621352], - [118.81254036, 104.99973274]]))) - -initial_shape.append(PointCloud(np.array([[109.7313602, 59.79617265], - [148.98369157, 59.80967103], - [194.16188757, 64.98196322], - [236.28740084, 76.65823864], - [273.68080984, 93.2970192], - [305.19924763, 117.61175954], - [329.12570774, 142.56245019], - [354.17165322, 172.2679391], - [360.96502322, 210.89900639], - [351.23586926, 251.13050805], - [328.74424331, 286.72428381], - [296.45746314, 319.44010324], - [254.19804988, 360.12373989], - [207.17084485, 381.01344803], - [156.06417675, 389.20382735], - [103.53427849, 387.61337726], - [50.16555004, 377.83272806], - [61.19222924, 84.23635551], - [49.49365238, 101.43077559], - [44.43208227, 123.00424473], - [46.69868733, 144.44838462], - [55.00298785, 163.40986789], - [48.37153655, 207.15416287], - [29.46971554, 230.63138542], - [16.79083463, 257.86599274], - [15.03497678, 285.19500392], - [29.83445491, 311.5170114], - [81.29522673, 185.59711915], - [111.03405753, 183.7654263], - [141.3336637, 182.91920653], - [173.75407076, 180.79465929], - [191.89145908, 148.90978278], - [198.99268671, 172.38226306], - [202.96352371, 196.98585519], - [197.0959395, 214.46753849], - [189.38299358, 229.08445602], - [96.44588204, 113.14323827], - [83.82396352, 123.3919129], - [81.45119284, 143.56487752], - [89.69904705, 160.59766732], - [98.36599502, 145.45904839], - [103.10217233, 126.26534747], - [84.43045765, 228.46643189], - [62.93677864, 242.35783193], - [57.44290226, 263.57257862], - [67.13556217, 284.78351617], - [78.66042335, 268.31564727], - [84.19760192, 246.92169276], - [259.34567409, 157.34687506], - [248.38397932, 171.9176971], - [242.5618418, 187.31855393], - [244.78947959, 202.87611756], - [239.12794222, 216.37452165], - [242.21991383, 240.73736669], - [247.96232402, 264.81933552], - [265.63001359, 240.76240374], - [273.7321494, 219.23983464], - [275.94833733, 203.23538213], - [276.43082796, 187.79108987], - [271.33176225, 170.34611298], - [258.50633904, 164.92193012], - [256.68032211, 190.02252505], - [253.34318274, 202.62322841], - [250.64136836, 216.39925191], - [248.92192186, 254.44710508], - [257.03785057, 217.075461], - [260.28050441, 202.86155077], - [261.39108462, 187.78257369]]))) - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# build clm -clm = CLMBuilder(classifier_trainers=linear_svm_lr, - patch_shape=(8, 8), - features=sparse_hog, - normalization_diagonal=100, - n_levels=2, - downscale=1.1, - scaled_shape_models=True, - max_shape_components=[2, 2], - boundary=3).build(training_images) - - -def test_clm(): - assert (clm.n_training_images == 4) - assert (clm.n_levels == 2) - assert (clm.downscale == 1.1) - #assert (clm.features[0] == sparse_hog and len(clm.features) == 1) - assert_allclose(np.around(clm.reference_shape.range()), (72., 69.)) - assert clm.scaled_shape_models - assert clm.pyramid_on_features - assert_allclose(clm.patch_shape, (8, 8)) - assert_allclose([clm.shape_models[j].n_components - for j in range(clm.n_levels)], (2, 2)) - assert_allclose(clm.n_classifiers_per_level, [68, 68]) - - ran_0 = np.random.randint(0, clm.n_classifiers_per_level[0]) - ran_1 = np.random.randint(0, clm.n_classifiers_per_level[1]) - - assert (name_of_callable(clm.classifiers[0][ran_0]) - == 'linear_svm_lr') - assert (name_of_callable(clm.classifiers[1][ran_1]) - == 'linear_svm_lr') - - -@raises(ValueError) -def test_n_shape_1_exception(): - fitter = GradientDescentCLMFitter(clm, n_shape=[3, 6, 'a']) - - -@raises(ValueError) -def test_n_shape_2_exception(): - fitter = GradientDescentCLMFitter(clm, n_shape=[10, 20, 3]) - - -def test_perturb_shape(): - fitter = GradientDescentCLMFitter(clm) - s = fitter.perturb_shape(training_images[0].landmarks[None].lms, - noise_std=0.08, rotation=False) - assert (s.n_dims == 2) - assert (s.n_landmark_groups == 0) - assert (s.n_points == 68) - - -@raises(ValueError) -def test_max_iters_exception(): - fitter = GradientDescentCLMFitter(clm) - fitter.fit(training_images[0], initial_shape[0], - max_iters=[10, 20, 30, 40]) - - -@patch('sys.stdout', new_callable=StringIO) -def test_str_mock(mock_stdout): - print(clm) - fitter = GradientDescentCLMFitter( - clm, algorithm=RLMS) - print(fitter) diff --git a/menpofit/test/fitmulitlevel_base_test.py b/menpofit/test/fitmulitlevel_base_test.py deleted file mode 100644 index 3df3c19..0000000 --- a/menpofit/test/fitmulitlevel_base_test.py +++ /dev/null @@ -1,29 +0,0 @@ -from menpo.feature import sparse_hog, igo - -from menpofit.base import (is_pyramid_on_features, - name_of_callable) - - -class Foo(): - def __call__(self): - pass - - -def test_is_pyramid_on_features_true(): - assert is_pyramid_on_features(igo) - - -def test_is_pyramid_on_features_false(): - assert not is_pyramid_on_features([igo, sparse_hog]) - - -def test_name_of_callable_partial(): - assert name_of_callable(sparse_hog) == 'sparse_hog' - - -def test_name_of_callable_function(): - assert name_of_callable(igo) == 'igo' - - -def test_name_of_callable_object_with_call(): - assert name_of_callable(Foo()) == 'Foo' diff --git a/menpofit/test/fittingresult_test.py b/menpofit/test/fittingresult_test.py deleted file mode 100644 index 311de16..0000000 --- a/menpofit/test/fittingresult_test.py +++ /dev/null @@ -1,108 +0,0 @@ -import numpy as np -from numpy.testing import assert_approx_equal -from nose.plugins.attrib import attr -from nose.tools import raises -from mock import MagicMock -from menpo.shape import PointCloud -from menpo.testing import is_same_array -from menpo.image import MaskedImage - -from menpofit.fittingresult import FittingResult, NonParametricFittingResult - - -class MockedFittingResult(FittingResult): - - def __init__(self, gt_shape=None): - FittingResult.__init__(self, MaskedImage.init_blank((10, 10)), - gt_shape=gt_shape) - @property - def n_iters(self): - return 1 - - @property - def shapes(self): - return [PointCloud(np.ones([3, 2]))] - - @property - def final_shape(self): - return PointCloud(np.ones([3, 2])) - - @property - def initial_shape(self): - return PointCloud(np.ones([3, 2])) - - -@attr('fuzzy') -def test_fittingresult_errors_me_norm(): - pcloud = PointCloud(np.array([[1., 2], [3, 4], [5, 6]])) - fr = MockedFittingResult(gt_shape=pcloud) - - assert_approx_equal(fr.errors()[0], 0.9173896) - - -@raises(ValueError) -def test_fittingresult_errors_no_gt(): - fr = MockedFittingResult() - fr.errors() - - -def test_fittingresult_gt_shape(): - pcloud = PointCloud(np.ones([3, 2])) - fr = MockedFittingResult(gt_shape=pcloud) - assert (is_same_array(fr.gt_shape.points, pcloud.points)) - - -@attr('fuzzy') -def test_fittingresult_final_error_me_norm(): - pcloud = PointCloud(np.array([[1., 2], [3, 4], [5, 6]])) - fr = MockedFittingResult(gt_shape=pcloud) - - assert_approx_equal(fr.final_error(), 0.9173896) - - -@raises(ValueError) -def test_fittingresult_final_error_no_gt(): - fr = MockedFittingResult() - fr.final_error() - - -@attr('fuzzy') -def test_fittingresult_initial_error_me_norm(): - pcloud = PointCloud(np.array([[1., 2], [3, 4], [5, 6]])) - fr = MockedFittingResult(gt_shape=pcloud) - - assert_approx_equal(fr.initial_error(), 0.9173896) - - -@raises(ValueError) -def test_fittingresult_initial_error_no_gt(): - fr = MockedFittingResult() - fr.initial_error() - - -def test_nonpara_fittingresult_as_serialized(): - image = MagicMock() - fitter = MagicMock() - parameters = [MagicMock()] - gt_shape = MagicMock() - fr = NonParametricFittingResult(image, fitter, parameters=parameters, - gt_shape=gt_shape) - s_fr = fr.as_serializable() - - image.copy.assert_called_once() - parameters[0].copy.assert_called_once() - gt_shape.copy.assert_called_once() - - -def test_nonpara_fittingresult_as_serialized(): - image = MagicMock() - fitter = MagicMock() - parameters = [MagicMock()] - gt_shape = MagicMock() - fr = NonParametricFittingResult(image, fitter, parameters=parameters, - gt_shape=gt_shape) - s_fr = fr.as_serializable() - - image.copy.assert_called_once() - parameters[0].copy.assert_called_once() - gt_shape.copy.assert_called_once() \ No newline at end of file diff --git a/menpofit/test/multilevel_fittingresult_test.py b/menpofit/test/multilevel_fittingresult_test.py deleted file mode 100644 index 5bdca7a..0000000 --- a/menpofit/test/multilevel_fittingresult_test.py +++ /dev/null @@ -1,18 +0,0 @@ -from mock import MagicMock -from menpofit.fittingresult import MultilevelFittingResult - - -def test_multilevel_fittingresult_as_serialized(): - image = MagicMock() - multiple_fitter = MagicMock() - fitting_results = [MagicMock()] - affine_correction = MagicMock() - gt_shape = MagicMock() - fr = MultilevelFittingResult(image, multiple_fitter, fitting_results, - affine_correction, gt_shape=gt_shape) - s_fr = fr.as_serializable() - - image.copy.assert_called_once() - fitting_results[0].as_serialized.assert_called_once() - affine_correction.copy.assert_called_once() - gt_shape.copy.assert_called_once() diff --git a/menpofit/test/sdm_test.py b/menpofit/test/sdm_test.py deleted file mode 100644 index e54bbf2..0000000 --- a/menpofit/test/sdm_test.py +++ /dev/null @@ -1,114 +0,0 @@ -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - -from mock import patch -from nose.tools import raises -import numpy as np -from menpo.feature import sparse_hog, igo, no_op -from menpo.transform import PiecewiseAffine - -import menpo.io as mio -from menpo.landmark import ibug_face_68_trimesh -from menpofit.sdm import SDMTrainer, SDAAMTrainer -from menpofit.clm.classifier import linear_svm_lr -from menpofit.regression.regressors import mlr_svd -from menpofit.aam import AAMBuilder -from menpofit.clm import CLMBuilder - - -# load images -filenames = ['breakingbad.jpg', 'takeo.ppm', 'lenna.png', 'einstein.jpg'] -training_images = [] -for i in range(4): - im = mio.import_builtin_asset(filenames[i]) - im.crop_to_landmarks_proportion_inplace(0.1) - if im.n_channels == 3: - im = im.as_greyscale(mode='luminosity') - training_images.append(im) - -# Seed the random number generator -np.random.seed(seed=1000) - -template_trilist_image = training_images[0].landmarks[None] -trilist = ibug_face_68_trimesh(template_trilist_image)[1].lms.trilist -aam = AAMBuilder(features=sparse_hog, - transform=PiecewiseAffine, - trilist=trilist, - normalization_diagonal=150, - n_levels=3, - downscale=1.2, - scaled_shape_models=False, - max_shape_components=None, - max_appearance_components=3, - boundary=3).build(training_images) - -clm = CLMBuilder(classifier_trainers=linear_svm_lr, - features=sparse_hog, - normalization_diagonal=100, - patch_shape=(5, 5), - n_levels=1, - downscale=1.1, - scaled_shape_models=True, - max_shape_components=25, - boundary=3).build(training_images) - -@raises(ValueError) -def test_features_exception(): - sdm = SDMTrainer(features=[igo, sparse_hog], - n_levels=3).train(training_images) - - -@raises(ValueError) -def test_regression_features_sdmtrainer_exception_1(): - sdm = SDMTrainer(n_levels=2, regression_features=[no_op, no_op, no_op]).\ - train(training_images) - - -@raises(ValueError) -def test_regression_features_sdmtrainer_exception_2(): - sdm = SDMTrainer(n_levels=3, regression_features=[no_op, sparse_hog, 1]).\ - train(training_images) - - -@raises(ValueError) -def test_regression_features_sdaamtrainer_exception_1(): - sdm = SDAAMTrainer(aam, regression_features=[no_op, sparse_hog]).\ - train(training_images) - - -@raises(ValueError) -def test_regression_features_sdaamtrainer_exception_2(): - sdm = SDAAMTrainer(aam, regression_features=7).\ - train(training_images) - - -@raises(ValueError) -def test_n_levels_exception(): - sdm = SDMTrainer(n_levels=0).train(training_images) - - -@raises(ValueError) -def test_downscale_exception(): - sdm = SDMTrainer(downscale=0).train(training_images) - - -@raises(ValueError) -def test_n_perturbations_exception(): - sdm = SDAAMTrainer(aam, n_perturbations=-10).train(training_images) - - -@patch('sys.stdout', new_callable=StringIO) -def test_verbose_mock(mock_stdout): - sdm = SDMTrainer(regression_type=mlr_svd, - regression_features=sparse_hog, - patch_shape=(16, 16), - features=no_op, - normalization_diagonal=150, - n_levels=1, - downscale=1.3, - noise_std=0.04, - rotation=False, - n_perturbations=2).train(training_images, - verbose=True) From 6b331263d499444312170c92ebbc4e9b77f822df Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 11:35:36 +0100 Subject: [PATCH 342/423] Remove spaces from docs and add super calls Just beginning to cleanup the aam code, removing some extra spaces from the docstrings (but not fixing them) and adding missing calls to super --- menpofit/aam/base.py | 67 +++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index b2b11c0..5061381 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -16,18 +16,14 @@ class AAM(object): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -43,11 +39,8 @@ class AAM(object): performing AAMs. scales : `int` or float` or list of those, optional - scale_shapes : `boolean` - scale_features : `boolean` - """ def __init__(self, shape_models, appearance_models, reference_shape, transform, features, scales, scale_shapes, scale_features): @@ -90,12 +83,10 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): Weights of the shape model that will be used to create a novel shape instance. If ``None``, the mean shape ``(shape_weights = [0, 0, ..., 0])`` is used. - appearance_weights : ``(n_weights,)`` `ndarray` or `float` list Weights of the appearance model that will be used to create a novel appearance instance. If ``None``, the mean appearance ``(appearance_weights = [0, 0, ..., 0])`` is used. - level : `int`, optional The pyramidal level to be used. @@ -383,17 +374,13 @@ class PatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -409,14 +396,16 @@ class PatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - """ + def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features): + patch_shape, features, scales, scale_shapes, scale_features, + transform): + super(PatchAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, features, + scales, scale_shapes, scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.transform = DifferentiableThinPlateSplines @@ -475,18 +464,14 @@ class LinearAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -502,15 +487,16 @@ class LinearAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - """ + def __init__(self, shape_models, appearance_models, reference_shape, transform, features, scales, scale_shapes, scale_features, n_landmarks): + super(LinearAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, features, + scales, scale_shapes, scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.transform = transform @@ -551,17 +537,13 @@ class LinearPatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -577,17 +559,18 @@ class LinearPatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - n_landmarks: `int` - """ + def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, - scale_features, n_landmarks): + patch_shape, features, scales, scale_shapes, scale_features, + n_landmarks, transform): + super(LinearPatchAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, + features, scales, scale_shapes, + scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.transform = DifferentiableThinPlateSplines @@ -629,17 +612,13 @@ class PartsAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. @@ -655,17 +634,17 @@ class PartsAAM(AAM): performing AAMs. normalize_parts: `callable` - scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - """ + def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, normalize_parts, scales, - scale_shapes, scale_features): + patch_shape, features, normalize_parts, scales, scale_shapes, + scale_features, transform): + super(PartsAAM, self).__init__(shape_models, appearance_models, + reference_shape, transform, features, + scales, scale_shapes, scale_features) self.shape_models = shape_models self.appearance_models = appearance_models self.patch_shape = patch_shape From 75d69be2d0695910b837937b6006f4e277592afa Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 19:54:30 +0100 Subject: [PATCH 343/423] Start moving the AAMBuilder into AAM This moves the build function into a train function to match the work I did on the SDM. Also, quite a big change is that the scales are no longer flipped, and are now from lowest to highest, again, to match SDM. This meant also changing some fitting code. I need to check this more thoroughly to make sure I'm doing it correctly. Similar changes need to be done in the ATM/LK code. --- menpofit/aam/__init__.py | 4 +- menpofit/aam/base.py | 208 ++++++++++++++++++++++++++++++-- menpofit/aam/builder.py | 251 +-------------------------------------- menpofit/builder.py | 22 +--- menpofit/checks.py | 13 ++ menpofit/fitter.py | 15 ++- 6 files changed, 225 insertions(+), 288 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index bb37752..e9ac26b 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,6 +1,4 @@ -from .builder import ( - AAMBuilder, PatchAAMBuilder, LinearAAMBuilder, - LinearPatchAAMBuilder, PartsAAMBuilder) +from .base import AAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 5061381..d4c9f32 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,10 +1,20 @@ from __future__ import division +from copy import deepcopy import numpy as np from menpo.shape import TriMesh -from menpofit.transform import DifferentiableThinPlateSplines -from menpofit.base import name_of_callable +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpo.model import PCAModel +from menpo.transform import Scale +from menpofit import checks +from menpofit.transform import DifferentiableThinPlateSplines, \ + DifferentiablePiecewiseAffine +from menpofit.base import name_of_callable, batch from menpofit.builder import ( - build_reference_frame, build_patch_reference_frame) + build_reference_frame, build_patch_reference_frame, + normalization_wrt_reference_shape, compute_features, scale_images, + build_shape_model, warp_images, align_shapes, + rescale_images_to_reference_shape) # TODO: document me! @@ -42,16 +52,192 @@ class AAM(object): scale_shapes : `boolean` scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform + def __init__(self, images, group=None, verbose=False, + features=no_op, transform=DifferentiablePiecewiseAffine, + diagonal=None, scales=(0.5, 1.0), scale_features=True, + max_shape_components=None, forgetting_factor=1.0, + max_appearance_components=None, batch_size=None): + # check parameters + checks.check_diagonal(diagonal) + scales, n_levels = checks.check_scales(scales) + features = checks.check_features(features, n_levels) + scale_features = checks.check_scale_features(scale_features, features) + max_shape_components = checks.check_max_components( + max_shape_components, n_levels, 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_levels, 'max_appearance_components') + # set parameters self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes + self.transform = transform self.scale_features = scale_features + self.diagonal = diagonal + self.scales = scales + self.forgetting_factor = forgetting_factor + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components + self.reference_shape = None + self.shape_models = [] + self.appearance_models = [] + + # Train AAM + self._train(images, group=group, verbose=verbose, increment=False, + batch_size=batch_size) + + def _train(self, images, group=None, verbose=False, increment=False, + batch_size=None): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest level + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + if not increment: + checks.check_trilist(image_batch[0], self.transform, + group=group) + # Normalize images and compute reference shape + self.reference_shape, image_batch = normalization_wrt_reference_shape( + image_batch, group, self.diagonal, verbose=verbose) + else: + # We are incrementing, so rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each pyramid level (low --> high) + for j in range(self.n_levels): + if verbose: + if len(self.scales) > 1: + level_str = ' - Level {}: '.format(j) + else: + level_str = ' - ' + else: + level_str = None + + # obtain image representation + if self.scale_features: + if j == 0: + # Compute features at highest level + feature_images = compute_features(image_batch, + self.features[0], + level_str=level_str, + verbose=verbose) + # Scale features at other levels + level_images = scale_images(feature_images, + self.scales[j], + level_str=level_str, + verbose=verbose) + else: + # scale images and compute features at other levels + scaled_images = scale_images(image_batch, self.scales[j], + level_str=level_str, + verbose=verbose) + level_images = compute_features(scaled_images, + self.features[j], + level_str=level_str, + verbose=verbose) + + # Extract potentially rescaled shapes + level_shapes = [i.landmarks[group].lms for i in level_images] + + # Build the shape model + if not increment: + if j == 0: + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + shape_model = self._build_shape_model( + level_shapes, self.max_shape_components[j], j) + # Store shape model + self.shape_models.append(shape_model) + else: + # Copy shape model + self.shape_models.append(deepcopy(shape_model)) + else: + # Compute aligned shapes + aligned_shapes = align_shapes(level_shapes) + # Increment shape model + self.shape_models[j].increment( + aligned_shapes, + forgetting_factor=self.forgetting_factor) + if self.max_shape_components is not None: + self.shape_models[j].trim_components( + self.max_appearance_components[j]) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(level_images, level_shapes, + scaled_reference_shape, + j, level_str, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format(level_str)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(level_str)) + + def _build_shape_model(self, shapes, max_components, level): + return build_shape_model(shapes, max_components=max_components) + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) @property def n_levels(self): diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 717a813..48c1d6d 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -35,15 +35,12 @@ class AAMBuilder(object): Note that from our experience, this approach of extracting features once and then creating a pyramid on top tends to lead to better performing AAMs. - transform : :map:`PureAlignmentTransform`, optional The :map:`PureAlignmentTransform` that will be used to warp the images. - trilist : ``(t, 3)`` `ndarray`, optional Triangle list that will be used to build the reference frame. If ``None``, defaults to performing Delaunay triangulation on the points. - diagonal : `int` >= ``20``, optional During building an AAM, all images are rescaled to ensure that the scale of their landmarks matches the scale of the mean shape. @@ -57,13 +54,9 @@ class AAMBuilder(object): landmarks, this kwarg also specifies the diagonal length of the reference frame (provided that features computation does not change the image size). - scales : `int` or float` or list of those, optional - scale_shapes : `boolean`, optional - scale_features : `boolean`, optional - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of shape components is defined per level. The first element of the list specifies the number @@ -80,7 +73,6 @@ class AAMBuilder(object): If ``None``, all the available components are kept (100% of variance). - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_levels``, then a number of appearance components is defined per level. The first element of the list specifies the number @@ -125,248 +117,7 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, trilist=None, diagonal=None, scales=(1, 0.5), scale_shapes=False, scale_features=True, max_shape_components=None, max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def build(self, images, group=None, verbose=False): - r""" - Builds an Active Appearance Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest level - """ - # normalize images and compute reference shape - reference_shape, images = normalization_wrt_reference_shape( - images, group, self.diagonal, verbose=verbose) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - shape_models = [] - appearance_models = [] - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if j == 0: - # compute features at highest level - feature_images = compute_features(images, self.features[j], - level_str=level_str, - verbose=verbose) - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) - - # extract potentially rescaled shapes - level_shapes = [i.landmarks[group].lms - for i in level_images] - - # obtain shape representation - if j == 0 or self.scale_shapes: - # obtain shape model - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) - # add shape model to the list - shape_models.append(shape_model) - else: - # copy precious shape model and add it to the list - shape_models.append(deepcopy(shape_model)) - - # obtain warped images - warped_images = self._warp_images(level_images, level_shapes, - shape_model.mean(), j, - level_str, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - appearance_models.append(appearance_model) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - appearance_models.reverse() - self.scales.reverse() - - aam = self._build_aam(shape_models, appearance_models, reference_shape) - - return aam - - def increment(self, aam, images, group=None, - forgetting_factor=1.0, verbose=False): - # normalize images with respect to reference shape of aam - images = rescale_images_to_reference_shape( - images, group, aam.reference_shape, verbose=verbose) - - # increment models at each scale - if verbose: - print_dynamic('- Incrementing models\n') - - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales[::-1]): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if j == 0: - # compute features at highest level - feature_images = compute_features(images, self.features[j], - level_str=level_str, - verbose=verbose) - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) - - # extract potentially rescaled shapes - level_shapes = [i.landmarks[group].lms - for i in level_images] - - # obtain shape representation - if j == 0 or self.scale_shapes: - if verbose: - print_dynamic('{}Incrementing shape model'.format( - level_str)) - # compute aligned shapes - aligned_shapes = align_shapes(level_shapes) - # increment shape model - aam.shape_models[j].increment( - aligned_shapes, forgetting_factor=forgetting_factor) - if self.max_shape_components is not None: - aam.shape_models[j].trim_components( - self.max_appearance_components[j]) - else: - # copy previous shape model - aam.shape_models[j] = deepcopy(aam.shape_models[j-1]) - - mean_shape = aam.appearance_models[j].mean().landmarks[ - 'source'].lms - - # obtain warped images - warped_images = self._warp_images(level_images, level_shapes, - mean_shape, j, - level_str, verbose) - - # obtain appearance representation - if verbose: - print_dynamic('{}Incrementing appearance model'.format( - level_str)) - # increment appearance model - aam.appearance_models[j].increment(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - aam.appearance_models[j].trim_components( - self.max_appearance_components[j]) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - def build_incrementally(self, images, group=None, - forgetting_factor=1.0, batch_size=100, - verbose=False): - # number of batches - n_batches = np.int(np.ceil(len(images) / batch_size)) - - # train first batch - print 'Training batch 1.' - aam = self.build(images[:batch_size], group=group, verbose=verbose) - - # train all other batches - start = batch_size - for j in range(1, n_batches): - print 'Training batch {}.'.format(j+1) - end = start + batch_size - self.increment(aam, images[start:end], group=group, - forgetting_factor=forgetting_factor, - verbose=verbose) - start = end - - return aam - - @classmethod - def _build_shape_model(cls, shapes, max_components, level): - return build_shape_model(shapes, max_components=max_components) - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return AAM(shape_models, appearance_models, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features) + pass # TODO: document me! diff --git a/menpofit/builder.py b/menpofit/builder.py index ae2fb48..fb750c9 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -174,8 +174,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images.append(Image(parts)) return parts_images -def build_reference_frame(landmarks, boundary=3, group='source', - trilist=None): +def build_reference_frame(landmarks, boundary=3, group='source'): r""" Builds a reference frame from a particular set of landmarks. @@ -183,22 +182,14 @@ def build_reference_frame(landmarks, boundary=3, group='source', ---------- landmarks : :map:`PointCloud` The landmarks that will be used to build the reference frame. - boundary : `int`, optional The number of pixels to be left as a safe margin on the boundaries of the reference frame (has potential effects on the gradient computation). - group : `string`, optional Group that will be assigned to the provided set of landmarks on the reference frame. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. - - If ``None``, defaults to performing Delaunay triangulation on the - points. - Returns ------- reference_frame : :map:`Image` @@ -206,14 +197,13 @@ def build_reference_frame(landmarks, boundary=3, group='source', """ reference_frame = _build_reference_frame(landmarks, boundary=boundary, group=group) - if trilist is not None: - reference_frame.landmarks[group] = TriMesh( - reference_frame.landmarks['source'].lms.points, trilist=trilist) + source_landmarks = reference_frame.landmarks['source'].lms + if isinstance(source_landmarks, TriMesh): + trilist = source_landmarks.trilist + else: + trilist = None - # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, - # perhaps the trilist should be directly obtained from the group landmarks reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) - return reference_frame diff --git a/menpofit/checks.py b/menpofit/checks.py index 82f4978..10a9375 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,5 +1,7 @@ import numpy as np import warnings +from menpo.shape import TriMesh +from menpo.transform import PiecewiseAffine def check_diagonal(diagonal): @@ -11,6 +13,17 @@ def check_diagonal(diagonal): raise ValueError("diagonal must be >= 20") +def check_trilist(image, transform, group=None): + trilist = image.landmarks[group].lms + + if not isinstance(trilist, TriMesh) and isinstance(transform, + PiecewiseAffine): + warnings.warn('The given images do not have an explicit triangulation ' + 'applied. A Delaunay Triangulation will be computed ' + 'and used for warping. This may be suboptimal and cause ' + 'warping artifacts.') + + # TODO: document me! def check_scales(scales): if isinstance(scales, (int, float)): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 72fd8fd..223aa58 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -149,19 +149,18 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # obtain image representation images = [] - for j, s in enumerate(self.scales[::-1]): - if j == 0: - # compute features at highest level - feature_image = self.features[j](image) - elif self.scale_features: + for j in range(self.n_levels): + if self.scale_features: + if j == 0: + # compute features at highest level + feature_image = self.features[j](image) # scale features at other levels - feature_image = images[0].rescale(s) + feature_image = feature_image.rescale(self.scales[j]) else: # scale image and compute features at other levels - scaled_image = image.rescale(s) + scaled_image = image.rescale(self.scales[j]) feature_image = self.features[j](scaled_image) images.append(feature_image) - images.reverse() # get initial shapes per level initial_shapes = [i.landmarks['initial_shape'].lms for i in images] From 5670a9a653d3341b4f21415bcb8864270cc9cd64 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 28 Jul 2015 20:28:05 +0100 Subject: [PATCH 344/423] Cleaning more code from the AAMBuider, mostly moving stuff around No real changes, just moving stuff. --- menpofit/aam/algorithm/lk.py | 3 +- menpofit/aam/base.py | 117 +++++++++++++++++++++++++---------- menpofit/aam/builder.py | 117 +---------------------------------- menpofit/fitter.py | 10 +-- menpofit/sdm/fitter.py | 1 - 5 files changed, 89 insertions(+), 159 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 756fe2a..3750cba 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -67,8 +67,7 @@ def warp_jacobian(self): dW_dp.shape[2])) def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) + return image.warp_to_mask(self.template.mask, self.transform) def gradient(self, img): nabla = fast_gradient(img) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index d4c9f32..6771faa 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -20,21 +20,11 @@ # TODO: document me! class AAM(object): r""" - Active Appearance Model class. + Active Appearance Models. Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - features : `callable` or ``[callable]``, + ---------- + features : `callable` or ``[callable]``, optional If list of length ``n_levels``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -47,10 +37,83 @@ class AAM(object): Note that from our experience, this approach of extracting features once and then creating a pyramid on top tends to lead to better performing AAMs. - + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). scales : `int` or float` or list of those, optional - scale_shapes : `boolean` - scale_features : `boolean` + scale_shapes : `boolean`, optional + scale_features : `boolean`, optional + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_levels``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ def __init__(self, images, group=None, verbose=False, features=no_op, transform=DifferentiablePiecewiseAffine, @@ -248,7 +311,6 @@ def n_levels(self): """ return len(self.scales) - # TODO: Could we directly use class names instead of this? @property def _str_title(self): r""" @@ -329,21 +391,13 @@ def _instance(self, level, shape_instance, appearance_instance): template = self.appearance_models[level].mean() landmarks = template.landmarks['source'].lms - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) + reference_frame = build_reference_frame(shape_instance) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - instance = appearance_instance.warp_to_mask( - reference_frame.mask, transform) - instance.landmarks = reference_frame.landmarks - - return instance + return appearance_instance.as_unmasked(copy=False).warp_to_mask( + reference_frame.mask, transform, warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), @@ -616,11 +670,8 @@ def _instance(self, level, shape_instance, appearance_instance): transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - instance = appearance_instance.warp_to_mask(reference_frame.mask, - transform) - instance.landmarks = reference_frame.landmarks - - return instance + return appearance_instance.as_unmasked().warp_to_mask( + reference_frame.mask, transform, warp_landmarks=True) def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py index 48c1d6d..a0d4d59 100644 --- a/menpofit/aam/builder.py +++ b/menpofit/aam/builder.py @@ -1,125 +1,14 @@ from __future__ import division -import numpy as np -from copy import deepcopy -from menpo.model import PCAModel from menpo.shape import mean_pointcloud from menpo.feature import no_op -from menpo.visualize import print_dynamic from menpofit import checks -from menpofit.builder import ( - normalization_wrt_reference_shape, compute_features, scale_images, - warp_images, extract_patches, build_shape_model, align_shapes, - build_reference_frame, build_patch_reference_frame, densify_shapes, - rescale_images_to_reference_shape) +from menpofit.builder import (warp_images, extract_patches, build_shape_model, + align_shapes, build_reference_frame, + build_patch_reference_frame, densify_shapes) from menpofit.transform import ( DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -# TODO: document me! -class AAMBuilder(object): - r""" - Class that builds Active Appearance Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - scales : `int` or float` or list of those, optional - scale_shapes : `boolean`, optional - scale_features : `boolean`, optional - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, 0.5), - scale_shapes=False, scale_features=True, - max_shape_components=None, max_appearance_components=None): - pass - - # TODO: document me! class PatchAAMBuilder(AAMBuilder): r""" diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 223aa58..330186c 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -30,20 +30,16 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, ----------- image: :map:`Image` or subclass The image to be fitted. - initial_shape: :map:`PointCloud` The initial shape estimate from which the fitting procedure will start. - max_iters: `int` or `list` of `int`, optional The maximum number of iterations. If `int`, specifies the overall maximum number of iterations. If `list` of `int`, specifies the maximum number of iterations per level. - gt_shape: :map:`PointCloud` The ground truth shape associated to the image. - crop_image: `None` or float`, optional If `float`, it specifies the proportion of the border wrt the initial shape to which the image will be internally cropped around @@ -53,7 +49,6 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, This will limit the fitting algorithm search region but is likely to speed up its running time, specially when the modeled object occupies a small portion of the image. - **kwargs: Additional keyword arguments that can be passed to specific implementations of ``_fit`` method. @@ -103,13 +98,10 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, ---------- image : :map:`Image` or subclass The image to be fitted. - initial_shape : :map:`PointCloud` The initial shape from which the fitting will start. - gt_shape : class : :map:`PointCloud`, optional The original ground truth shape associated to the image. - crop_image: `None` or float`, optional If `float`, it specifies the proportion of the border wrt the initial shape to which the image will be internally cropped around @@ -219,7 +211,7 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, shape = algorithm_result.final_shape if s != self.scales[-1]: - shape = Scale(self.scales[j+1]/s, + shape = Scale(self.scales[j + 1] / s, n_dims=shape.n_dims).apply(shape) return algorithm_results diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index ec60492..52e72cc 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -1,5 +1,4 @@ from __future__ import division -from itertools import chain import numpy as np from functools import partial import warnings From 436a2e157faf361554818821c5a5084cd3cdba48 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 09:33:56 +0100 Subject: [PATCH 345/423] Finish moving build logic into base (AAM) Now building is intimately integrated with the AAM objects. Still need to test that these things actually work though. --- menpofit/aam/__init__.py | 2 +- menpofit/aam/base.py | 185 +++++++----- menpofit/aam/builder.py | 605 --------------------------------------- 3 files changed, 121 insertions(+), 671 deletions(-) delete mode 100644 menpofit/aam/builder.py diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index e9ac26b..aca4250 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,4 +1,4 @@ -from .base import AAM +from .base import AAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 6771faa..4b961e8 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,11 +1,11 @@ from __future__ import division from copy import deepcopy import numpy as np -from menpo.shape import TriMesh from menpo.feature import no_op from menpo.visualize import print_dynamic from menpo.model import PCAModel from menpo.transform import Scale +from menpo.shape import mean_pointcloud from menpofit import checks from menpofit.transform import DifferentiableThinPlateSplines, \ DifferentiablePiecewiseAffine @@ -14,7 +14,7 @@ build_reference_frame, build_patch_reference_frame, normalization_wrt_reference_shape, compute_features, scale_images, build_shape_model, warp_images, align_shapes, - rescale_images_to_reference_shape) + rescale_images_to_reference_shape, densify_shapes, extract_patches) # TODO: document me! @@ -118,8 +118,8 @@ class AAM(object): def __init__(self, images, group=None, verbose=False, features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), scale_features=True, - max_shape_components=None, forgetting_factor=1.0, - max_appearance_components=None, batch_size=None): + max_shape_components=None, max_appearance_components=None, + batch_size=None): # check parameters checks.check_diagonal(diagonal) scales, n_levels = checks.check_scales(scales) @@ -135,7 +135,6 @@ def __init__(self, images, group=None, verbose=False, self.scale_features = scale_features self.diagonal = diagonal self.scales = scales - self.forgetting_factor = forgetting_factor self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components self.reference_shape = None @@ -147,6 +146,7 @@ def __init__(self, images, group=None, verbose=False, batch_size=batch_size) def _train(self, images, group=None, verbose=False, increment=False, + shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, batch_size=None): r""" Builds an Active Appearance Model from a list of landmarked images. @@ -255,7 +255,7 @@ def _train(self, images, group=None, verbose=False, increment=False, # Increment shape model self.shape_models[j].increment( aligned_shapes, - forgetting_factor=self.forgetting_factor) + forgetting_factor=shape_forgetting_factor) if self.max_shape_components is not None: self.shape_models[j].trim_components( self.max_appearance_components[j]) @@ -284,7 +284,9 @@ def _train(self, images, group=None, verbose=False, increment=False, self.appearance_models.append(appearance_model) else: # increment appearance model - self.appearance_models[j].increment(warped_images) + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) # trim appearance model if required if self.max_appearance_components is not None: self.appearance_models[j].trim_components( @@ -293,6 +295,18 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print_dynamic('{}Done\n'.format(level_str)) + def increment(self, images, group=None, verbose=False, + shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, + batch_size=None): + # Literally just to fit under 80 characters, but maintain the sensible + # parameter name + aff = appearance_forgetting_factor + return self._train(images, group=group, + verbose=verbose, + shape_forgetting_factor=shape_forgetting_factor, + appearance_forgetting_factor=aff, + increment=True, batch_size=batch_size) + def _build_shape_model(self, shapes, max_components, level): return build_shape_model(shapes, max_components=max_components) @@ -640,21 +654,26 @@ class PatchAAM(AAM): scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features, - transform): - super(PatchAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, features, - scales, scale_shapes, scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = DifferentiableThinPlateSplines + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + scale_features=True, max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + + super(PatchAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[level]) + return warp_images(images, shapes, reference_frame, self.transform, + level_str=level_str, verbose=verbose) @property def _str_title(self): @@ -728,21 +747,36 @@ class LinearAAM(AAM): scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features, - n_landmarks): - super(LinearAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, features, - scales, scale_shapes, scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + def __init__(self, images, group=None, verbose=False, features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), scale_features=True, + max_shape_components=None, max_appearance_components=None, + batch_size=None): + + super(LinearAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=transform, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model( + dense_shapes, max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) # TODO: implement me! def _instance(self, level, shape_instance, appearance_instance): @@ -801,23 +835,37 @@ class LinearPatchAAM(AAM): n_landmarks: `int` """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features, - n_landmarks, transform): - super(LinearPatchAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, - features, scales, scale_shapes, - scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = DifferentiableThinPlateSplines + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + scale_features=True, max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + + super(LinearPatchAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _build_shape_model(self, shapes, max_components, level): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[level]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes, + max_components=max_components) + return shape_model + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, level_str=level_str, + verbose=verbose) # TODO: implement me! def _instance(self, level, shape_instance, appearance_instance): @@ -841,6 +889,7 @@ def __str__(self): # TODO: document me! +# TODO: implement offsets support? class PartsAAM(AAM): r""" Parts based Active Appearance Model class. @@ -876,21 +925,27 @@ class PartsAAM(AAM): scale_features : `boolean` """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, normalize_parts, scales, scale_shapes, - scale_features, transform): - super(PartsAAM, self).__init__(shape_models, appearance_models, - reference_shape, transform, features, - scales, scale_shapes, scale_features) - self.shape_models = shape_models - self.appearance_models = appearance_models + def __init__(self, images, group=None, verbose=False, features=no_op, + normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), scale_features=True, + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_shape = patch_shape - self.features = features self.normalize_parts = normalize_parts - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + + super(PartsAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, scale_features=scale_features, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _warp_images(self, images, shapes, reference_shape, level, level_str, + verbose): + return extract_patches(images, shapes, self.patch_shape[level], + normalize_function=self.normalize_parts, + level_str=level_str, verbose=verbose) # TODO: implement me! def _instance(self, level, shape_instance, appearance_instance): diff --git a/menpofit/aam/builder.py b/menpofit/aam/builder.py deleted file mode 100644 index a0d4d59..0000000 --- a/menpofit/aam/builder.py +++ /dev/null @@ -1,605 +0,0 @@ -from __future__ import division -from menpo.shape import mean_pointcloud -from menpo.feature import no_op -from menpofit import checks -from menpofit.builder import (warp_images, extract_patches, build_shape_model, - align_shapes, build_reference_frame, - build_patch_reference_frame, densify_shapes) -from menpofit.transform import ( - DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) - - -# TODO: document me! -class PatchAAMBuilder(AAMBuilder): - r""" - Class that builds Patch based Active Appearance Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=True, - scale_features=True, max_shape_components=None, - max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[level]) - return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return PatchAAM(shape_models, appearance_models, reference_shape, - self.patch_shape, self.features, self.scales, - self.scale_shapes, self.scale_features) - - -# TODO: document me! -class LinearAAMBuilder(AAMBuilder): - r""" - Class that builds Linear Active Appearance Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None, max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_reference_frame(mean_aligned_shape) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model( - dense_shapes, max_components=max_components) - return shape_model - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return LinearAAM(shape_models, appearance_models, - reference_shape, self.transform, - self.features, self.scales, - self.scale_shapes, self.scale_features, - self.n_landmarks) - - -# TODO: document me! -class LinearPatchAAMBuilder(AAMBuilder): - r""" - Class that builds Linear Patch based Active Appearance Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=False, - scale_features=True, max_shape_components=None, - max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[level]) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model(dense_shapes, - max_components=max_components) - return shape_model - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return LinearPatchAAM(shape_models, appearance_models, - reference_shape, self.patch_shape, - self.features, self.scales, self.scale_shapes, - self.scale_features, self.n_landmarks) - - -# TODO: document me! -# TODO: implement offsets support? -class PartsAAMBuilder(AAMBuilder): - r""" - Class that builds Parts based Active Appearance Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - normalize_parts : `callable`, optional - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - normalize_parts=no_op, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None, max_appearance_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return extract_patches(images, shapes, self.patch_shape[level], - normalize_function=self.normalize_parts, - level_str=level_str, verbose=verbose) - - def _build_aam(self, shape_models, appearance_models, reference_shape): - return PartsAAM(shape_models, appearance_models, reference_shape, - self.patch_shape, self.features, - self.normalize_parts, self.scales, - self.scale_shapes, self.scale_features) - - -from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM From e942f6ccae41b63bfbd5cc2d0b69f50dafdca47e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 11:17:02 +0100 Subject: [PATCH 346/423] Add _increment_shape_model This works the same as _build_shape_model and allows densify to be called first for Linear AAMs. --- menpofit/aam/base.py | 50 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 4b961e8..ad7cf61 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -238,10 +238,11 @@ def _train(self, images, group=None, verbose=False, increment=False, level_shapes = [i.landmarks[group].lms for i in level_images] # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(level_str)) + if not increment: if j == 0: - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) shape_model = self._build_shape_model( level_shapes, self.max_shape_components[j], j) # Store shape model @@ -250,15 +251,10 @@ def _train(self, images, group=None, verbose=False, increment=False, # Copy shape model self.shape_models.append(deepcopy(shape_model)) else: - # Compute aligned shapes - aligned_shapes = align_shapes(level_shapes) - # Increment shape model - self.shape_models[j].increment( - aligned_shapes, - forgetting_factor=shape_forgetting_factor) - if self.max_shape_components is not None: - self.shape_models[j].trim_components( - self.max_appearance_components[j]) + self._increment_shape_model( + level_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor, + max_components=self.max_shape_components[j]) # Obtain warped images - we use a scaled version of the # reference shape, computed here. This is because the mean @@ -310,6 +306,16 @@ def increment(self, images, group=None, verbose=False, def _build_shape_model(self, shapes, max_components, level): return build_shape_model(shapes, max_components=max_components) + def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, + max_components=None): + # Compute aligned shapes + aligned_shapes = align_shapes(shapes) + # Increment shape model + shape_model.increment(aligned_shapes, + forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): reference_frame = build_reference_frame(reference_shape) @@ -772,6 +778,17 @@ def _build_shape_model(self, shapes, max_components, level): dense_shapes, max_components=max_components) return shape_model + def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, + max_components=None): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return warp_images(images, shapes, self.reference_frame, @@ -861,6 +878,17 @@ def _build_shape_model(self, shapes, max_components, level): max_components=max_components) return shape_model + def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, + max_components=None): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + def _warp_images(self, images, shapes, reference_shape, level, level_str, verbose): return warp_images(images, shapes, self.reference_frame, From 4cacf9a7ef1a47e4c4e02a219e0b2e89ff98b9be Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 11:17:29 +0100 Subject: [PATCH 347/423] Missing check for patch features --- menpofit/aam/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index ad7cf61..4292197 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -664,7 +664,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), scale_features=True, max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = patch_shape + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -856,7 +856,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), scale_features=True, max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = patch_shape + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(LinearPatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -958,7 +958,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, patch_shape=(17, 17), scale_features=True, max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = patch_shape + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts super(PartsAAM, self).__init__( From ae43fc36f74b20522b695963e18ca4efba4dddfb Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 11:17:49 +0100 Subject: [PATCH 348/423] Only raise warning for scale_features it True --- menpofit/checks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/menpofit/checks.py b/menpofit/checks.py index 10a9375..e248ef4 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -73,11 +73,14 @@ def check_scale_features(scale_features, features): """ if np.alltrue([f == features[0] for f in features]): return scale_features - else: + elif scale_features: + # Only raise warning if True was passed. warnings.warn('scale_features has been automatically set to False ' 'because different types of features are used at each ' 'level.') return False + else: + return scale_features # TODO: document me! From 5d6584e7c55820e80146f6191aba56bae3a83f62 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:49:42 +0100 Subject: [PATCH 349/423] Accidentally worked on too many things... This contains four changes: 1. Rename iterations to n_iterations in both SDM fitter and algorithm 2. Fix a bug with first_image, use image_batch[0] to make sure that generating bounding boxes works properly. 3. Provide the ability to pass a reference shape, otherwise we calculate it from the first batch. Raise a new warning if we do use the first batch. 4. Refactor the SDM algorithms so that they share code more, since the only difference between increment train was the call to increment, we can just have a flag about whether to increment or not. --- menpofit/builder.py | 7 ++ menpofit/sdm/algorithm.py | 136 ++++++++++++++++---------------------- menpofit/sdm/fitter.py | 83 +++++++++++++---------- 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index fb750c9..a55f832 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -310,3 +310,10 @@ def build_shape_model(shapes, max_components=None): shape_model.trim_components(max_components) return shape_model + + +class MenpoFitBuilderWarning(Warning): + r""" + A warning that some part of building the model may cause issues. + """ + pass diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index b4fc7e7..fe713c7 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -13,8 +13,26 @@ class SupervisedDescentAlgorithm(object): r""" """ + + def __init__(self): + self.regressors = [] + def train(self, images, gt_shapes, current_shapes, level_str='', verbose=False): + return self._train(images, gt_shapes, current_shapes, increment=False, + level_str=level_str, verbose=verbose) + + def increment(self, images, gt_shapes, current_shapes, level_str='', + verbose=False): + return self._train(images, gt_shapes, current_shapes, increment=True, + level_str=level_str, verbose=verbose) + + def _train(self, images, gt_shapes, current_shapes, increment=False, + level_str='', verbose=False): + + if not increment: + # Reset the regressors + self.regressors = [] n_perturbations = len(current_shapes[0]) template_shape = gt_shapes[0] @@ -22,93 +40,32 @@ def train(self, images, gt_shapes, current_shapes, level_str='', # obtain delta_x and gt_x delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) - # initialize iteration counter and list of regressors - self.regressors = [] - # Cascaded Regression loop - for k in range(self.iterations): + for k in range(self.n_iterations): # generate regression data features = features_per_image( images, current_shapes, self.patch_shape, self.features, level_str='{}(Iteration {}) - '.format(level_str, k), verbose=verbose) - # Perform regression if verbose: print_dynamic('{}(Iteration {}) - Performing regression'.format( level_str, k)) - r = self._regressor_cls() - r.train(features, delta_x) - # add regressor to list - self.regressors.append(r) - # Estimate delta_points - estimated_delta_x = r.predict(features) - - if verbose: - print_dynamic('{}(Iteration {}) - Calculating errors'.format( - level_str, k)) - errors = [] - for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): - s1 = template_shape.from_vector(dx) - s2 = template_shape.from_vector(edx) - gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] - errors.append(self._compute_error(s1, s2, gt_s)) - mean = np.mean(errors) - std = np.std(errors) - median = np.median(errors) - print_dynamic('{}(Iteration {}) - Training error -> ' - 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. - format(level_str, k, mean, std, median)) + if not increment: + r = self._regressor_cls() + r.train(features, delta_x) + self.regressors.append(r) + else: + self.regressors[k].increment(features, delta_x) - j = 0 - for shapes in current_shapes: - for s in shapes: - # update current x - current_x = s.as_vector() + estimated_delta_x[j] - # update current shape inplace - s.from_vector_inplace(current_x) - # update delta_x - delta_x[j] = gt_x[j] - current_x - # increase index - j += 1 - - return current_shapes - - def increment(self, images, gt_shapes, current_shapes, verbose=False): - - n_perturbations = len(current_shapes[0]) - template_shape = gt_shapes[0] - - # obtain delta_x and gt_x - delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes) - - # Cascaded Regression loop - for r in self.regressors: - # generate regression data - features = features_per_image(images, current_shapes, - self.patch_shape, self.features) - - # update regression - if verbose: - print_dynamic('- Updating regression') - r.increment(features, delta_x) - - # estimate delta_points - estimated_delta_x = r.predict(features) + # Estimate delta_points + estimated_delta_x = self.regressors[k].predict(features) if verbose: - errors = [] - for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): - s1 = template_shape.from_vector(dx) - s2 = template_shape.from_vector(edx) - gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] - errors.append(self._compute_error(s1, s2, gt_s)) - mean = np.mean(errors) - std = np.std(errors) - median = np.median(errors) - print_dynamic('- Training error -> mean: {0:.4f}, ' - 'std: {1:.4f}, median: {2:.4f}.\n'. - format(mean, std, median)) + self._print_regression_info(template_shape, gt_shapes, + n_perturbations, delta_x, + estimated_delta_x, k, + level_str=level_str) j = 0 for shapes in current_shapes: @@ -122,7 +79,6 @@ def increment(self, images, gt_shapes, current_shapes, verbose=False): # increase index j += 1 - # rearrange current shapes into their original list of list form return current_shapes def run(self, image, initial_shape, gt_shape=None, **kwargs): @@ -148,19 +104,39 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): return NonParametricAlgorithmResult(image, self, shapes, gt_shape=gt_shape) + def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, + delta_x, estimated_delta_x, level_index, + level_str=''): + print_dynamic('{}(Iteration {}) - Calculating errors'.format( + level_str, level_index)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + s1 = template_shape.from_vector(dx) + s2 = template_shape.from_vector(edx) + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('{}(Iteration {}) - Training error -> ' + 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. + format(level_str, level_index, mean, std, median)) + # TODO: document me! class Newton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True): + super(Newton, self).__init__() + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape - self.iterations = iterations + self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps @@ -169,15 +145,17 @@ def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, class GaussNewton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), iterations=3, + def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True, alpha2=0): + super(GaussNewton, self).__init__() + self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, alpha2=alpha2) self.patch_shape = patch_shape self.features = features self.patch_shape = patch_shape - self.iterations = iterations + self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 52e72cc..23b5237 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -6,8 +6,8 @@ from menpo.feature import no_op from menpofit.visualize import print_progress from menpofit.base import batch, name_of_callable -from menpofit.builder import (normalization_wrt_reference_shape, scale_images, - rescale_images_to_reference_shape) +from menpofit.builder import (scale_images, rescale_images_to_reference_shape, + compute_reference_shape, MenpoFitBuilderWarning) from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, align_shape_with_bounding_box) from menpofit.result import MultiFitterResult @@ -20,9 +20,10 @@ class SupervisedDescentFitter(MultiFitter): r""" """ def __init__(self, images, group=None, bounding_box_group=None, - sd_algorithm_cls=Newton, holistic_feature=no_op, - patch_features=no_op, patch_shape=(17, 17), diagonal=None, - scales=(0.5, 1.0), iterations=6, n_perturbations=30, + reference_shape=None, sd_algorithm_cls=Newton, + holistic_feature=no_op, patch_features=no_op, patch_shape=(17, + 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, + n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): # check parameters @@ -32,7 +33,7 @@ def __init__(self, images, group=None, bounding_box_group=None, patch_shape = checks.check_patch_shape(patch_shape, n_levels) # set parameters self.algorithms = [] - self.reference_shape = None + self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls self._holistic_feature = holistic_feature self._patch_features = patch_features @@ -40,7 +41,7 @@ def __init__(self, images, group=None, bounding_box_group=None, self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations - self.iterations = checks.check_max_iters(iterations, n_levels) + self.n_iterations = checks.check_max_iters(n_iterations, n_levels) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._setup_algorithms() @@ -54,7 +55,7 @@ def _setup_algorithms(self): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], patch_shape=self._patch_shape[j], - iterations=self.iterations[j])) + n_iterations=self.n_iterations[j])) def perturb_from_bounding_box(self, bounding_box): return self._perturb_from_bounding_box(self.reference_shape, @@ -78,37 +79,49 @@ def _train(self, images, group=None, bounding_box_group=None, increment = True if verbose: - print('Computing batch {}'.format(k)) + print('Computing batch {} - ({})'.format(k, len(image_batch))) # In the case where group is None, we need to get the only key so # that we can attach landmarks below and not get a complaint about # using None - first_image = image_batch[0] if group is None: - group = first_image.landmarks.group_labels[0] - - if not increment: - # Normalize images and compute reference shape - self.reference_shape, image_batch = normalization_wrt_reference_shape( - image_batch, group, self.diagonal, verbose=verbose) - else: - # We are incrementing, so rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) + group = image_batch[0].landmarks.group_labels[0] + + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) # No bounding box is given, so we will use the ground truth box if bounding_box_group is None: - bounding_box_group = '__gt_bb_' + # It's important to use bb_group for batching, so that we + # generate ground truth bounding boxes for each batch, every + # time + bb_group = '__gt_bb_' for i in image_batch: gt_s = i.landmarks[group].lms - perturb_bbox_group = bounding_box_group + '0' + perturb_bbox_group = bb_group + '0' i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + else: + bb_group = bounding_box_group # Find all bounding boxes on the images with the given bounding # box key - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) n_perturbations = len(all_bb_keys) # If there is only one example bounding box, then we will generate @@ -128,20 +141,20 @@ def _train(self, images, group=None, bounding_box_group=None, # This is customizable by passing in the correct method p_s = self._perturb_from_bounding_box(gt_s, bb) - perturb_bbox_group = '{}_{}'.format(bounding_box_group, - j) + perturb_bbox_group = '{}_{}'.format(bb_group, j) i.landmarks[perturb_bbox_group] = p_s elif n_perturbations != self.n_perturbations: warnings.warn('The original value of n_perturbation {} ' 'will be reset to {} in order to agree with ' 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations)) + format(self.n_perturbations, n_perturbations), + MenpoFitBuilderWarning) self.n_perturbations = n_perturbations # Re-grab all the bounding box keys for iterating over when # calculating perturbations - all_bb_keys = list(first_image.landmarks.keys_matching( - '*{}*'.format(bounding_box_group))) + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) # Before scaling, we compute the holistic feature on the whole image msg = '- Computing holistic features ({})'.format( @@ -187,12 +200,12 @@ def _train(self, images, group=None, bounding_box_group=None, current_shapes.append(c_shapes) # train supervised descent algorithm - if increment: - current_shapes = self.algorithms[j].increment( + if not increment: + current_shapes = self.algorithms[j].train( level_images, level_gt_shapes, current_shapes, - verbose=verbose) + level_str=level_str, verbose=verbose) else: - current_shapes = self.algorithms[j].train( + current_shapes = self.algorithms[j].increment( level_images, level_gt_shapes, current_shapes, level_str=level_str, verbose=verbose) @@ -310,7 +323,7 @@ def __str__(self): - Patch shape: {}""" for k, s in enumerate(self.scales): level_info.append(lvl_str_tmplt.format(k, s, - self.iterations[k], + self.n_iterations[k], self._patch_shape[k])) level_info = '\n'.join(level_info) From 94bda5259ba19f6b9388f021cde912de81901bd0 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:52:29 +0100 Subject: [PATCH 350/423] Add a RegularizedSDM alias Just allow passing alpha (the regularization parameter) explicitly. --- menpofit/sdm/fitter.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 23b5237..fd110eb 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -346,4 +346,24 @@ def __str__(self): return cls_str +# Aliases for common combinations of supervised descent fitting SDM = partial(SupervisedDescentFitter, sd_algorithm_cls=Newton) + +class RegularizedSDM(SupervisedDescentFitter): + + def __init__(self, images, group=None, bounding_box_group=None, + alpha=1.0, reference_shape=None, + holistic_feature=no_op, patch_features=no_op, + patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + n_iterations=6, n_perturbations=30, + perturb_from_bounding_box=noisy_shape_from_bounding_box, + batch_size=None, verbose=False): + super(RegularizedSDM, self).__init__( + images, group=group, bounding_box_group=bounding_box_group, + reference_shape=reference_shape, + sd_algorithm_cls=partial(Newton, alpha=alpha), + holistic_feature=holistic_feature, patch_features=patch_features, + patch_shape=patch_shape, diagonal=diagonal, scales=scales, + n_iterations=n_iterations, n_perturbations=n_perturbations, + perturb_from_bounding_box=perturb_from_bounding_box, + batch_size=batch_size, verbose=verbose) From ab5cc3394bf9b58ec1e7206c38ac6135fe70e15b Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:53:39 +0100 Subject: [PATCH 351/423] Don't split patch parameter over two lines --- menpofit/sdm/fitter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index fd110eb..e15bedf 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -21,9 +21,9 @@ class SupervisedDescentFitter(MultiFitter): """ def __init__(self, images, group=None, bounding_box_group=None, reference_shape=None, sd_algorithm_cls=Newton, - holistic_feature=no_op, patch_features=no_op, patch_shape=(17, - 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, - n_perturbations=30, + holistic_feature=no_op, patch_features=no_op, + patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): # check parameters From 69f76135978e450703b5f08c79319da6d0145104 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 16:55:40 +0100 Subject: [PATCH 352/423] Allow passing an explicit reference_shape to AAM --- menpofit/aam/base.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 4292197..5ac9f2d 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,5 +1,6 @@ from __future__ import division from copy import deepcopy +import warnings import numpy as np from menpo.feature import no_op from menpo.visualize import print_dynamic @@ -7,14 +8,14 @@ from menpo.transform import Scale from menpo.shape import mean_pointcloud from menpofit import checks -from menpofit.transform import DifferentiableThinPlateSplines, \ - DifferentiablePiecewiseAffine +from menpofit.transform import (DifferentiableThinPlateSplines, + DifferentiablePiecewiseAffine) from menpofit.base import name_of_callable, batch from menpofit.builder import ( build_reference_frame, build_patch_reference_frame, - normalization_wrt_reference_shape, compute_features, scale_images, - build_shape_model, warp_images, align_shapes, - rescale_images_to_reference_shape, densify_shapes, extract_patches) + compute_features, scale_images, build_shape_model, warp_images, + align_shapes, rescale_images_to_reference_shape, densify_shapes, + extract_patches, MenpoFitBuilderWarning, compute_reference_shape) # TODO: document me! @@ -115,7 +116,7 @@ class AAM(object): ``0`` <= `float` <= ``1`` or a list of those containing 1 or ``len(scales)`` elements """ - def __init__(self, images, group=None, verbose=False, + def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), scale_features=True, max_shape_components=None, max_appearance_components=None, @@ -137,7 +138,7 @@ def __init__(self, images, group=None, verbose=False, self.scales = scales self.max_shape_components = max_shape_components self.max_appearance_components = max_appearance_components - self.reference_shape = None + self.reference_shape = reference_shape self.shape_models = [] self.appearance_models = [] @@ -184,17 +185,25 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print('Computing batch {}'.format(k)) - if not increment: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) checks.check_trilist(image_batch[0], self.transform, group=group) - # Normalize images and compute reference shape - self.reference_shape, image_batch = normalization_wrt_reference_shape( - image_batch, group, self.diagonal, verbose=verbose) - else: - # We are incrementing, so rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) # build models at each scale if verbose: From 034893a0594cc6298741a937b675468acdb1edc9 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 17:02:55 +0100 Subject: [PATCH 353/423] Rename n_levels to n_scales This is a much more descriptive name. n_levels should be scrapped for n_scales --- menpofit/aam/base.py | 14 +++++++------- menpofit/aam/fitter.py | 12 ++++++------ menpofit/atm/base.py | 10 +++++----- menpofit/atm/builder.py | 20 ++++++++++---------- menpofit/checks.py | 4 ++-- menpofit/fitter.py | 12 ++++++------ menpofit/fittingresult.py | 4 ++-- menpofit/result.py | 4 ++-- menpofit/sdm/fitter.py | 4 ++-- menpofit/visualize/widgets/base.py | 26 +++++++++++++------------- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 5ac9f2d..76cf156 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -26,7 +26,7 @@ class AAM(object): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -61,7 +61,7 @@ class AAM(object): scale_shapes : `boolean`, optional scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -77,7 +77,7 @@ class AAM(object): If ``None``, all the available components are kept (100% of variance). max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of appearance components + If list of length ``n_scales``, then a number of appearance components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -651,7 +651,7 @@ class PatchAAM(AAM): patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -744,7 +744,7 @@ class LinearAAM(AAM): The transform used to warp the images from which the AAM was constructed. features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -842,7 +842,7 @@ class LinearPatchAAM(AAM): patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -943,7 +943,7 @@ class PartsAAM(AAM): patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 24fe85c..8fe78c2 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -32,16 +32,16 @@ def _check_n_appearance(self, n_appearance): if type(n_appearance) is int or type(n_appearance) is float: for am in self.aam.appearance_models: am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_levels > 1: + elif len(n_appearance) == 1 and self.aam.n_scales > 1: for am in self.aam.appearance_models: am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_levels: + elif len(n_appearance) == self.aam.n_scales: for am, n in zip(self.aam.appearance_models, n_appearance): am.n_active_components = n else: raise ValueError('n_appearance can be an integer or a float ' 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_levels)) + 'those'.format(self.aam.n_scales)) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): @@ -58,7 +58,7 @@ def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_levels) + sampling = checks.check_sampling(sampling, self.n_scales) self._set_up(lk_algorithm_cls, sampling, **kwargs) def _set_up(self, lk_algorithm_cls, sampling, **kwargs): @@ -115,10 +115,10 @@ def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_levels) + sampling = checks.check_sampling(sampling, self.n_scales) self.n_perturbations = n_perturbations self.noise_std = noise_std - self.max_iters = checks.check_max_iters(max_iters, self.n_levels) + self.max_iters = checks.check_max_iters(max_iters, self.n_scales) self._set_up(sd_algorithm_cls, sampling, **kwargs) def _set_up(self, sd_algorithm_cls, sampling, **kwargs): diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 10c513a..63ccbc4 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -28,7 +28,7 @@ class ATM(object): constructed. features : `callable` or ``[callable]``, - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -333,7 +333,7 @@ class PatchATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -414,7 +414,7 @@ class LinearATM(ATM): constructed. features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -483,7 +483,7 @@ class LinearPatchATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -555,7 +555,7 @@ class PartsATM(ATM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 0f44545..3f0c8c4 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -22,7 +22,7 @@ class ATMBuilder(object): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -64,7 +64,7 @@ class ATMBuilder(object): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -258,7 +258,7 @@ class PatchATMBuilder(ATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -292,7 +292,7 @@ class PatchATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -383,7 +383,7 @@ class LinearATMBuilder(ATMBuilder): Parameters ---------- features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -425,7 +425,7 @@ class LinearATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -524,7 +524,7 @@ class LinearPatchATMBuilder(LinearATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -558,7 +558,7 @@ class LinearPatchATMBuilder(LinearATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. @@ -651,7 +651,7 @@ class PartsATMBuilder(ATMBuilder): patch_shape: (`int`, `int`) or list or list of (`int`, `int`) features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at + If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at the lowest pyramidal level and so on. @@ -687,7 +687,7 @@ class PartsATMBuilder(ATMBuilder): scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is + If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number of components of the lowest pyramidal level and so on. diff --git a/menpofit/checks.py b/menpofit/checks.py index e248ef4..16a7188 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -47,7 +47,7 @@ def check_features(features, n_levels): ---------- features : callable or list of callables The features to apply to the images. - n_levels : int + n_scales : int The number of pyramid levels. Returns @@ -103,7 +103,7 @@ def check_max_components(max_components, n_levels, var_name): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be None or int or float or a list of - those containing 1 or {n_levels} elements. + those containing 1 or {n_scales} elements. """ str_error = ("{} must be None or an int > 0 or a 0 <= float <= 1 or " "a list of those containing 1 or {} elements").format( diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 330186c..22cee87 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -13,7 +13,7 @@ class MultiFitter(object): r""" """ @property - def n_levels(self): + def n_scales(self): r""" The number of pyramidal levels used during alignment. @@ -141,7 +141,7 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # obtain image representation images = [] - for j in range(self.n_levels): + for j in range(self.n_scales): if self.scale_features: if j == 0: # compute features at highest level @@ -196,7 +196,7 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, The fitting object containing the state of the whole fitting procedure. """ - max_iters = checks.check_max_iters(max_iters, self.n_levels) + max_iters = checks.check_max_iters(max_iters, self.n_scales) shape = initial_shape gt_shape = None algorithm_results = [] @@ -267,16 +267,16 @@ def _check_n_shape(self, n_shape): if type(n_shape) is int or type(n_shape) is float: for sm in self._model.shape_models: sm.n_active_components = n_shape - elif len(n_shape) == 1 and self._model.n_levels > 1: + elif len(n_shape) == 1 and self._model.n_scales > 1: for sm in self._model.shape_models: sm.n_active_components = n_shape[0] - elif len(n_shape) == self._model.n_levels: + elif len(n_shape) == self._model.n_scales: for sm, n in zip(self._model.shape_models, n_shape): sm.n_active_components = n else: raise ValueError('n_shape can be an integer or a float or None' 'or a list containing 1 or {} of ' - 'those'.format(self._model.n_levels)) + 'those'.format(self._model.n_scales)) def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): transform = noisy_alignment_similarity_transform( diff --git a/menpofit/fittingresult.py b/menpofit/fittingresult.py index 7fdedef..3c88302 100644 --- a/menpofit/fittingresult.py +++ b/menpofit/fittingresult.py @@ -719,7 +719,7 @@ def n_levels(self): :type: `int` """ - return self.fitter.n_levels + return self.fitter.n_scales @property def downscale(self): @@ -904,7 +904,7 @@ class SerializableMultilevelFittingResult(FittingResult): The list of fitted shapes per iteration of the fitting procedure. gt_shape : :map:`PointCloud` The ground truth shape associated to the image. - n_levels : `int` + n_scales : `int` Number of levels within the multilevel fitter. downscale : `int` Scale of downscaling applied to the image. diff --git a/menpofit/result.py b/menpofit/result.py index f19c516..a8e7364 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -508,13 +508,13 @@ def __init__(self, image, fitter, algorithm_results, affine_correction, self._gt_shape = gt_shape @property - def n_levels(self): + def n_scales(self): r""" The number of levels of the fitter object. :type: `int` """ - return self.fitter.n_levels + return self.fitter.n_scales @property def scales(self): diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index e15bedf..12f9bec 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -51,7 +51,7 @@ def __init__(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=False, batch_size=batch_size) def _setup_algorithms(self): - for j in range(self.n_levels): + for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], patch_shape=self._patch_shape[j], @@ -165,7 +165,7 @@ def _train(self, images, group=None, bounding_box_group=None, # for each pyramid level (low --> high) current_shapes = [] - for j in range(self.n_levels): + for j in range(self.n_scales): if verbose: if len(self.scales) > 1: level_str = ' - Level {}: '.format(j) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 302a5e2..01dbb30 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -31,7 +31,7 @@ def _check_n_parameters(n_params, n_levels, max_n_params): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be ``None`` or `int` or `float` or a `list` - of those containing ``1`` or ``n_levels`` elements. + of those containing ``1`` or ``n_scales`` elements. """ str_error = ("n_params must be None or 1 <= int <= max_n_params or " "a list of those containing 1 or {} elements").format(n_levels) @@ -128,7 +128,7 @@ def visualize_shape_model(shape_model, n_parameters=5, mode='multiple', max_n_params = [sp.n_active_components for sp in shape_model] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_parameters = _check_n_parameters(n_parameters, n_levels, max_n_params) # Initial options dictionaries @@ -487,7 +487,7 @@ def visualize_appearance_model(appearance_model, n_parameters=5, max_n_params = [ap.n_active_components for ap in appearance_model] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_parameters = _check_n_parameters(n_parameters, n_levels, max_n_params) # Find initial groups and labels that will be passed to the landmark options @@ -790,7 +790,7 @@ def visualize_aam(aam, n_shape_parameters=5, n_appearance_parameters=5, print('Initializing...') # Get the number of levels - n_levels = aam.n_levels + n_levels = aam.n_scales # Define the styling options if style == 'coloured': @@ -829,7 +829,7 @@ def visualize_aam(aam, n_shape_parameters=5, n_appearance_parameters=5, max_n_appearance = [ap.n_active_components for ap in aam.appearance_models] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_shape_parameters = _check_n_parameters(n_shape_parameters, n_levels, max_n_shape) n_appearance_parameters = _check_n_parameters(n_appearance_parameters, @@ -972,7 +972,7 @@ def update_info(aam, instance, level, group): if n_levels == 1: tmp_shape_models = '' tmp_pyramid = '' - else: # n_levels > 1 + else: # n_scales > 1 # shape models info if aam.scaled_shape_models: tmp_shape_models = "Each level has a scaled shape model " \ @@ -993,7 +993,7 @@ def update_info(aam, instance, level, group): "> Warp using {} transform".format(aam.transform.__name__), "> {}".format(tmp_pyramid), "> Level {}/{} (downscale={:.1f})".format( - level + 1, aam.n_levels, aam.downscale), + level + 1, aam.n_scales, aam.downscale), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -1216,7 +1216,7 @@ def visualize_atm(atm, n_shape_parameters=5, mode='multiple', print('Initializing...') # Get the number of levels - n_levels = atm.n_levels + n_levels = atm.n_scales # Define the styling options if style == 'coloured': @@ -1252,7 +1252,7 @@ def visualize_atm(atm, n_shape_parameters=5, mode='multiple', max_n_shape = [sp.n_active_components for sp in atm.shape_models] # Check the given number of parameters (the returned n_parameters is a list - # of len n_levels) + # of len n_scales) n_shape_parameters = _check_n_parameters(n_shape_parameters, n_levels, max_n_shape) @@ -1388,7 +1388,7 @@ def update_info(atm, instance, level, group): if n_levels == 1: tmp_shape_models = '' tmp_pyramid = '' - else: # n_levels > 1 + else: # n_scales > 1 # shape models info if atm.scaled_shape_models: tmp_shape_models = "Each level has a scaled shape model " \ @@ -1409,7 +1409,7 @@ def update_info(atm, instance, level, group): "> Warp using {} transform".format(atm.transform.__name__), "> {}".format(tmp_pyramid), "> Level {}/{} (downscale={:.1f})".format( - level + 1, atm.n_levels, atm.downscale), + level + 1, atm.n_scales, atm.downscale), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -2466,9 +2466,9 @@ def update_info(name, value): else: text_per_line = [ "> {} iterations".format(fitting_results[im].n_iters)] - if hasattr(fitting_results[im], 'n_levels'): # Multilevel result + if hasattr(fitting_results[im], 'n_scales'): # Multilevel result text_per_line.append("> {} levels with downscale of {:.1f}".format( - fitting_results[im].n_levels, fitting_results[im].downscale)) + fitting_results[im].n_scales, fitting_results[im].downscale)) info_wid.set_widget_state(n_lines=len(text_per_line), text_per_line=text_per_line) From 6739040d4cf9c35f305e5d8cae59760f55228717 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 29 Jul 2015 17:11:37 +0100 Subject: [PATCH 354/423] Take @jalabort _prepare_image method About to copy the flow into the SDM. Will add a commit after that renames to holistic_features --- menpofit/fitter.py | 56 +++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 22cee87..ec3dcf7 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -107,59 +107,59 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, initial shape to which the image will be internally cropped around the initial shape range. If `None`, no cropping is performed. - This will limit the fitting algorithm search region but is likely to speed up its running time, specially when the modeled object occupies a small portion of the image. - Returns ------- images : `list` of :map:`Image` or subclass The list of images that will be fitted by the fitters. - initial_shapes : `list` of :map:`PointCloud` The initial shape for each one of the previous images. - gt_shapes : `list` of :map:`PointCloud` The ground truth shape for each one of the previous images. """ - - # attach landmarks to the image - image.landmarks['initial_shape'] = initial_shape + # Attach landmarks to the image + image.landmarks['__initial_shape'] = initial_shape if gt_shape: - image.landmarks['gt_shape'] = gt_shape + image.landmarks['__gt_shape'] = gt_shape - # if specified, crop the image if crop_image: + # If specified, crop the image image = image.crop_to_landmarks_proportion(crop_image, - group='initial_shape') + group='__initial_shape') - # rescale image wrt the scale factor between reference_shape and + # Rescale image wrt the scale factor between reference_shape and # initial_shape image = image.rescale_to_reference_shape(self.reference_shape, - group='initial_shape') + group='__initial_shape') - # obtain image representation + # Compute image representation images = [] - for j in range(self.n_scales): - if self.scale_features: - if j == 0: - # compute features at highest level - feature_image = self.features[j](image) - # scale features at other levels - feature_image = feature_image.rescale(self.scales[j]) + for i in range(self.n_scales): + # Handle features + if i == 0 or self.features[i] is not self.features[i - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_image = self.features[i](image) + + # Handle scales + if self.scales[i] != 1: + # Scale feature images only if scale is different than 1 + scaled_image = feature_image.rescale(self.scales[i]) else: - # scale image and compute features at other levels - scaled_image = image.rescale(self.scales[j]) - feature_image = self.features[j](scaled_image) - images.append(feature_image) + scaled_image = feature_image + + # Add scaled image to list + images.append(scaled_image) - # get initial shapes per level - initial_shapes = [i.landmarks['initial_shape'].lms for i in images] + # Get initial shapes per level + initial_shapes = [i.landmarks['__initial_shape'].lms for i in images] - # get ground truth shapes per level + # Get ground truth shapes per level if gt_shape: - gt_shapes = [i.landmarks['gt_shape'].lms for i in images] + gt_shapes = [i.landmarks['__gt_shape'].lms for i in images] else: gt_shapes = None From 9d1460f124d10b92cef56285bb1ba5ab757665b5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 11:36:57 +0100 Subject: [PATCH 355/423] Move AAM to scales away from 'levels' Mostly simple refactoring. Also, added the correct feature scaling logic from @jalabort CLM branch --- menpofit/aam/base.py | 280 +++++++++-------------------- menpofit/base.py | 2 +- menpofit/fitter.py | 36 ++-- menpofit/visualize/widgets/base.py | 4 +- 4 files changed, 113 insertions(+), 209 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 76cf156..acb847e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -123,13 +123,13 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, batch_size=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales, n_scales = checks.check_scales(scales) + features = checks.check_features(features, n_scales) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') + max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( - max_appearance_components, n_levels, 'max_appearance_components') + max_appearance_components, n_scales, 'max_appearance_components') # set parameters self.features = features self.transform = transform @@ -166,7 +166,7 @@ def _train(self, images, group=None, verbose=False, increment=False, ------- aam : :map:`AAM` The AAM object. Shape and appearance models are stored from - lowest to highest level + lowest to highest scale """ # If batch_size is not None, then we may have a generator, else we # assume we have a list. @@ -210,50 +210,45 @@ def _train(self, images, group=None, verbose=False, increment=False, print_dynamic('- Building models\n') feature_images = [] - # for each pyramid level (low --> high) - for j in range(self.n_levels): + # for each scale (low --> high) + for j in range(self.n_scales): if verbose: if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) + scale_prefix = ' - Scale {}: '.format(j) else: - level_str = ' - ' + scale_prefix = ' - ' else: - level_str = None - - # obtain image representation - if self.scale_features: - if j == 0: - # Compute features at highest level - feature_images = compute_features(image_batch, - self.features[0], - level_str=level_str, - verbose=verbose) - # Scale features at other levels - level_images = scale_images(feature_images, - self.scales[j], - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(image_batch, self.scales[j], - level_str=level_str, + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + level_str=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[k] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + level_str=scale_prefix, verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) + else: + scaled_images = feature_images # Extract potentially rescaled shapes - level_shapes = [i.landmarks[group].lms for i in level_images] + scale_shapes = [i.landmarks[group].lms for i in scaled_images] # Build the shape model if verbose: - print_dynamic('{}Building shape model'.format(level_str)) + print_dynamic('{}Building shape model'.format(scale_prefix)) if not increment: if j == 0: shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) + scale_shapes, self.max_shape_components[j], j) # Store shape model self.shape_models.append(shape_model) else: @@ -261,7 +256,7 @@ def _train(self, images, group=None, verbose=False, increment=False, self.shape_models.append(deepcopy(shape_model)) else: self._increment_shape_model( - level_shapes, self.shape_models[j], + scale_shapes, self.shape_models[j], forgetting_factor=shape_forgetting_factor, max_components=self.max_shape_components[j]) @@ -271,13 +266,14 @@ def _train(self, images, group=None, verbose=False, increment=False, # reference frame. scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( self.reference_shape) - warped_images = self._warp_images(level_images, level_shapes, + warped_images = self._warp_images(scaled_images, scale_shapes, scaled_reference_shape, - j, level_str, verbose) + j, scale_prefix, verbose) # obtain appearance model if verbose: - print_dynamic('{}Building appearance model'.format(level_str)) + print_dynamic('{}Building appearance model'.format( + scale_prefix)) if not increment: appearance_model = PCAModel(warped_images) @@ -298,7 +294,7 @@ def _train(self, images, group=None, verbose=False, increment=False, self.max_appearance_components[j]) if verbose: - print_dynamic('{}Done\n'.format(level_str)) + print_dynamic('{}Done\n'.format(scale_prefix)) def increment(self, images, group=None, verbose=False, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, @@ -312,7 +308,7 @@ def increment(self, images, group=None, verbose=False, appearance_forgetting_factor=aff, increment=True, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, level): + def _build_shape_model(self, shapes, max_components, scale_index): return build_shape_model(shapes, max_components=max_components) def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, @@ -325,16 +321,16 @@ def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, if max_components is not None: shape_model.trim_components(max_components) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): reference_frame = build_reference_frame(reference_shape) return warp_images(images, shapes, reference_frame, self.transform, level_str=level_str, verbose=verbose) @property - def n_levels(self): + def n_scales(self): """ - The number of scale levels of the AAM. + The number of scales of the AAM. :type: `int` """ @@ -348,7 +344,8 @@ def _str_title(self): """ return 'Active Appearance Model' - def instance(self, shape_weights=None, appearance_weights=None, level=-1): + def instance(self, shape_weights=None, appearance_weights=None, + scale_index=-1): r""" Generates a novel AAM instance given a set of shape and appearance weights. If no weights are provided, the mean AAM instance is @@ -364,16 +361,16 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): Weights of the appearance model that will be used to create a novel appearance instance. If ``None``, the mean appearance ``(appearance_weights = [0, 0, ..., 0])`` is used. - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[level] - am = self.appearance_models[level] + sm = self.shape_models[scale_index] + am = self.appearance_models[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel if shape_weights is None: @@ -387,24 +384,24 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): appearance_weights *= am.eigenvalues[:n_appearance_weights] ** 0.5 appearance_instance = am.instance(appearance_weights) - return self._instance(level, shape_instance, appearance_instance) + return self._instance(scale_index, shape_instance, appearance_instance) - def random_instance(self, level=-1): + def random_instance(self, scale_index=-1): r""" Generates a novel random instance of the AAM. Parameters ----------- - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[level] - am = self.appearance_models[level] + sm = self.shape_models[scale_index] + am = self.appearance_models[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel shape_weights = (np.random.randn(sm.n_active_components) * @@ -414,10 +411,10 @@ def random_instance(self, level=-1): am.eigenvalues[:am.n_active_components]**0.5) appearance_instance = am.instance(appearance_weights) - return self._instance(level, shape_instance, appearance_instance) + return self._instance(scale_index, shape_instance, appearance_instance) - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() + def _instance(self, scale_index, shape_instance, appearance_instance): + template = self.appearance_models[scale_index].mean() landmarks = template.landmarks['source'].lms reference_frame = build_reference_frame(shape_instance) @@ -470,11 +467,11 @@ def view_appearance_models_widget(self, n_parameters=5, n_parameters : `int` or `list` of `int` or ``None``, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -502,19 +499,19 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, n_shape_parameters : `int` or `list` of `int` or None, optional The number of shape principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. n_appearance_parameters : `int` or `list` of `int` or None, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -532,106 +529,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: fix me! def __str__(self): - out = "{}\n - {} training images.\n".format(self._str_title, - self.n_training_images) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_levels): - n_channels.append( - self.appearance_models[j].template_instance.n_channels) - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_levels > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.appearance_models[0].n_features, - self.appearance_models[0].template_instance.n_true_pixels(), - n_channels[0], - self.appearance_models[0].template_instance._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self.appearance_models[i].n_features, - self.appearance_models[i].template_instance.n_true_pixels(), - n_channels[i], - self.appearance_models[i].template_instance._str_shape, - n_channels[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n - {3} appearance components " \ - "({4:.2f}% of variance)\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100, - self.appearance_models[i].n_components, - self.appearance_models[i].variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} shape components ({10:.2f}% of " \ - "variance)\n - {11} appearance components ({12:.2f}% of " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.appearance_models[0].n_features, - self.appearance_models[0].template_instance.n_true_pixels(), - n_channels[0], - self.appearance_models[0].template_instance._str_shape, - n_channels[0], self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100, - self.appearance_models[0].n_components, - self.appearance_models[0].variance_ratio() * 100) - return out + return '' # TODO: document me! @@ -652,9 +550,9 @@ class PatchAAM(AAM): The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. + each scale after downscaling of the image. The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. + the lowest scale and so on. If ``callable`` the specified feature will be applied to the original image and pyramid generation will be performed on top of the feature @@ -683,10 +581,10 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[level]) + reference_shape, patch_shape=self.patch_shape[scale_index]) return warp_images(images, shapes, reference_frame, self.transform, level_str=level_str, verbose=verbose) @@ -694,8 +592,8 @@ def _warp_images(self, images, shapes, reference_shape, level, level_str, def _str_title(self): return 'Patch-Based Active Appearance Model' - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean + def _instance(self, scale_index, shape_instance, appearance_instance): + template = self.appearance_models[scale_index].mean landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( @@ -776,7 +674,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, level): + def _build_shape_model(self, shapes, max_components, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_reference_frame(mean_aligned_shape) @@ -798,14 +696,14 @@ def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, if max_components is not None: shape_model.trim_components(max_components) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): return warp_images(images, shapes, self.reference_frame, self.transform, level_str=level_str, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -875,11 +773,11 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, level): + def _build_shape_model(self, shapes, max_components, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[level]) + mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model @@ -898,14 +796,14 @@ def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, if max_components is not None: shape_model.trim_components(max_components) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): return warp_images(images, shapes, self.reference_frame, self.transform, level_str=level_str, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -978,14 +876,14 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _warp_images(self, images, shapes, reference_shape, level, level_str, - verbose): - return extract_patches(images, shapes, self.patch_shape[level], + def _warp_images(self, images, shapes, reference_shape, scale_index, + level_str, verbose): + return extract_patches(images, shapes, self.patch_shape[scale_index], normalize_function=self.normalize_parts, level_str=level_str, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! diff --git a/menpofit/base.py b/menpofit/base.py index 8d84311..863328a 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -46,7 +46,7 @@ def create_pyramid(images, n_levels, downscale, features, verbose=False): images: list of :map:`Image` The set of landmarked images from which to build the AAM. - n_levels: int + n_scales: int The number of multi-resolution pyramidal levels to be used. downscale: float diff --git a/menpofit/fitter.py b/menpofit/fitter.py index ec3dcf7..af4e093 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -63,11 +63,6 @@ def fit(self, image, initial_shape, max_iters=50, gt_shape=None, images, initial_shapes, gt_shapes = self._prepare_image( image, initial_shape, gt_shape=gt_shape, crop_image=crop_image) - # detach added landmarks from image - del image.landmarks['initial_shape'] - if gt_shape: - del image.landmarks['gt_shape'] - # work out the affine transform between the initial shape of the # highest pyramidal level and the initial shape of the original image affine_correction = AlignmentAffine(initial_shapes[-1], initial_shape) @@ -192,28 +187,39 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, Returns ------- - algorithm_results: :class:`menpo.fg2015.fittingresult.FittingResult` list + algorithm_results: :class:`FittingResult` list The fitting object containing the state of the whole fitting procedure. """ + # Perform check max_iters = checks.check_max_iters(max_iters, self.n_scales) + + # Set initial and ground truth shapes shape = initial_shape gt_shape = None - algorithm_results = [] - for j, (i, alg, it, s) in enumerate(zip(images, self.algorithms, - max_iters, self.scales)): - if gt_shapes: - gt_shape = gt_shapes[j] - algorithm_result = alg.run(i, shape, gt_shape=gt_shape, - max_iters=it, **kwargs) + # Initialize list of algorithm results + algorithm_results = [] + for i in range(self.n_scales): + # Handle ground truth shape + if gt_shapes is not None: + gt_shape = gt_shapes[i] + + # Run algorithm + algorithm_result = self.algorithms[i].run(images[i], shape, + gt_shape=gt_shape, + max_iters=max_iters[i], + **kwargs) + # Add algorithm result to the list algorithm_results.append(algorithm_result) + # Prepare this scale's final shape for the next scale shape = algorithm_result.final_shape - if s != self.scales[-1]: - shape = Scale(self.scales[j + 1] / s, + if self.scales[i] != self.scales[-1]: + shape = Scale(self.scales[i + 1] / self.scales[i], n_dims=shape.n_dims).apply(shape) + # Return list of algorithm results return algorithm_results diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 01dbb30..396e7c4 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -897,7 +897,7 @@ def render_function(name, value): # Compute weights and instance shape_weights = shape_model_parameters_wid.parameters appearance_weights = appearance_model_parameters_wid.parameters - instance = aam.instance(level=level, shape_weights=shape_weights, + instance = aam.instance(scale_index=level, shape_weights=shape_weights, appearance_weights=appearance_weights) # Update info @@ -1317,7 +1317,7 @@ def render_function(name, value): # Compute weights and instance shape_weights = shape_model_parameters_wid.parameters - instance = atm.instance(level=level, shape_weights=shape_weights) + instance = atm.instance(scale_index=level, shape_weights=shape_weights) # Update info update_info(atm, instance, level, From 644c4226e046c3df19f4599cb495fa06107b1e96 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 11:38:08 +0100 Subject: [PATCH 356/423] Move SDM to scales away from levels Use correct features scaling logic and also get rid of the custom _prepare_image method. --- menpofit/sdm/fitter.py | 171 ++++++++++++----------------------------- 1 file changed, 50 insertions(+), 121 deletions(-) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 12f9bec..e74a90e 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -7,7 +7,8 @@ from menpofit.visualize import print_progress from menpofit.base import batch, name_of_callable from menpofit.builder import (scale_images, rescale_images_to_reference_shape, - compute_reference_shape, MenpoFitBuilderWarning) + compute_reference_shape, MenpoFitBuilderWarning, + compute_features) from menpofit.fitter import (MultiFitter, noisy_shape_from_bounding_box, align_shape_with_bounding_box) from menpofit.result import MultiFitterResult @@ -28,20 +29,21 @@ def __init__(self, images, group=None, bounding_box_group=None, batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_features = checks.check_features(patch_features, n_levels) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) + scales, n_scales = checks.check_scales(scales) + patch_features = checks.check_features(patch_features, n_scales) + holistic_features = checks.check_features(holistic_feature, n_scales) + patch_shape = checks.check_patch_shape(patch_shape, n_scales) # set parameters self.algorithms = [] self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls - self._holistic_feature = holistic_feature + self.features = holistic_features self._patch_features = patch_features self._patch_shape = patch_shape self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations - self.n_iterations = checks.check_max_iters(n_iterations, n_levels) + self.n_iterations = checks.check_max_iters(n_iterations, n_scales) self._perturb_from_bounding_box = perturb_from_bounding_box # set up algorithms self._setup_algorithms() @@ -156,41 +158,46 @@ def _train(self, images, group=None, bounding_box_group=None, all_bb_keys = list(image_batch[0].landmarks.keys_matching( '*{}*'.format(bb_group))) - # Before scaling, we compute the holistic feature on the whole image - msg = '- Computing holistic features ({})'.format( - name_of_callable(self._holistic_feature)) - wrap = partial(print_progress, prefix=msg, verbose=verbose) - image_batch = [self._holistic_feature(im) - for im in wrap(image_batch)] - - # for each pyramid level (low --> high) + # for each scale (low --> high) current_shapes = [] for j in range(self.n_scales): if verbose: if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) + scale_prefix = ' - Scale {}: '.format(j) else: - level_str = ' - ' + scale_prefix = ' - ' else: - level_str = None - - # Scale images - level_images = scale_images(image_batch, self.scales[j], - level_str=level_str, - verbose=verbose) + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + level_str=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[k] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + level_str=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images - # Extract scaled ground truth shapes for current level - level_gt_shapes = [i.landmarks[group].lms - for i in level_images] + # Extract scaled ground truth shapes for current scale + scaled_shapes = [i.landmarks[group].lms for i in scaled_images] if j == 0: msg = '{}Generating {} perturbations per image'.format( - level_str, self.n_perturbations) + scale_prefix, self.n_perturbations) wrap = partial(print_progress, prefix=msg, end_with_newline=False, verbose=verbose) # Extract perturbations at the very bottom level - for i in wrap(level_images): + for i in wrap(scaled_images): c_shapes = [] for perturb_bbox_group in all_bb_keys: bbox = i.landmarks[perturb_bbox_group].lms @@ -202,14 +209,14 @@ def _train(self, images, group=None, bounding_box_group=None, # train supervised descent algorithm if not increment: current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose) + scaled_images, scaled_shapes, current_shapes, + level_str=scale_prefix, verbose=verbose) else: current_shapes = self.algorithms[j].increment( - level_images, level_gt_shapes, current_shapes, - level_str=level_str, verbose=verbose) + scaled_images, scaled_shapes, current_shapes, + level_str=scale_prefix, verbose=verbose) - # Scale current shapes to next level resolution + # Scale current shapes to next resolution if self.scales[j] != (1 or self.scales[-1]): transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) @@ -224,83 +231,6 @@ def increment(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=True, batch_size=batch_size) - def _prepare_image(self, image, initial_shape, gt_shape=None, - crop_image=0.5): - r""" - Prepares the image to be fitted. - - The image is first rescaled wrt the ``reference_landmarks`` and then - a gaussian pyramid is applied. Depending on the - ``pyramid_on_features`` flag, the pyramid is either applied to the - features image computed from the rescaled imaged or applied to the - rescaled image and features extracted at each pyramidal level. - - Parameters - ---------- - image : :map:`Image` or subclass - The image to be fitted. - initial_shape : :map:`PointCloud` - The initial shape from which the fitting will start. - gt_shape : :map:`PointCloud`, optional - The original ground truth shape associated to the image. - crop_image: `None` or float`, optional - If `float`, it specifies the proportion of the border wrt the - initial shape to which the image will be internally cropped around - the initial shape range. - If `None`, no cropping is performed. - - This will limit the fitting algorithm search region but is - likely to speed up its running time, specially when the - modeled object occupies a small portion of the image. - - Returns - ------- - images : `list` of :map:`Image` or subclass - The list of images that will be fitted by the fitters. - initial_shapes : `list` of :map:`PointCloud` - The initial shape for each one of the previous images. - gt_shapes : `list` of :map:`PointCloud` - The ground truth shape for each one of the previous images. - """ - # Attach landmarks to the image - image.landmarks['initial_shape'] = initial_shape - if gt_shape: - image.landmarks['gt_shape'] = gt_shape - - # If specified, crop the image - if crop_image: - image = image.crop_to_landmarks_proportion(crop_image, - group='initial_shape') - - # Rescale image w.r.t the scale factor between reference_shape and - # initial_shape - image = image.rescale_to_reference_shape(self.reference_shape, - group='initial_shape') - - # Compute the holistic feature on the normalized image - image = self._holistic_feature(image) - - # Obtain image representation - images = [] - for s in self.scales: - if s != 1: - # scale image - scaled_image = image.rescale(s) - else: - scaled_image = image - images.append(scaled_image) - - # Get initial shapes per level - initial_shapes = [i.landmarks['initial_shape'].lms for i in images] - - # Get ground truth shapes per level - if gt_shape: - gt_shapes = [i.landmarks['gt_shape'].lms for i in images] - else: - gt_shapes = None - - return images, initial_shapes, gt_shapes - def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): return MultiFitterResult(image, self, algorithm_results, @@ -316,30 +246,29 @@ def __str__(self): noisy_shape_from_bounding_box) regressor_cls = self.algorithms[0]._regressor_cls - # Compute level info strings - level_info = [] - lvl_str_tmplt = r""" - Level {} (Scale {}) + # Compute scale info strings + scales_info = [] + lvl_str_tmplt = r""" - Scale {} - {} iterations - Patch shape: {}""" for k, s in enumerate(self.scales): - level_info.append(lvl_str_tmplt.format(k, s, - self.n_iterations[k], - self._patch_shape[k])) - level_info = '\n'.join(level_info) + scales_info.append(lvl_str_tmplt.format(s, + self.n_iterations[k], + self._patch_shape[k])) + scales_info = '\n'.join(scales_info) cls_str = r"""Supervised Descent Method - Regression performed using the {reg_alg} algorithm - Regression class: {reg_cls} - - Levels: {levels} -{level_info} + - Scales: {scales} +{scales_info} - Perturbations generated per shape: {n_perturbations} - Images scaled to diagonal: {diagonal:.2f} - Custom perturbation scheme used: {is_custom_perturb_func}""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), reg_cls=name_of_callable(regressor_cls), - n_levels=len(self.scales), - levels=self.scales, - level_info=level_info, + scales=self.scales, + scales_info=scales_info, n_perturbations=self.n_perturbations, diagonal=diagonal, is_custom_perturb_func=is_custom_perturb_func) From dd311e13a63c2268e4ea69b64b40382425f3348c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 11:48:22 +0100 Subject: [PATCH 357/423] Remove scale_features --- menpofit/aam/base.py | 47 +++++++++++++------------------------------- menpofit/checks.py | 16 --------------- menpofit/fitter.py | 14 ------------- 3 files changed, 14 insertions(+), 63 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index acb847e..2f6cfd2 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -58,8 +58,6 @@ class AAM(object): reference frame (provided that features computation does not change the image size). scales : `int` or float` or list of those, optional - scale_shapes : `boolean`, optional - scale_features : `boolean`, optional max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional If list of length ``n_scales``, then a number of shape components is defined per level. The first element of the list specifies the number @@ -118,14 +116,12 @@ class AAM(object): """ def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), scale_features=True, - max_shape_components=None, max_appearance_components=None, - batch_size=None): + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + max_appearance_components=None, batch_size=None): # check parameters checks.check_diagonal(diagonal) scales, n_scales = checks.check_scales(scales) features = checks.check_features(features, n_scales) - scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( @@ -133,7 +129,6 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, # set parameters self.features = features self.transform = transform - self.scale_features = scale_features self.diagonal = diagonal self.scales = scales self.max_shape_components = max_shape_components @@ -564,20 +559,18 @@ class PatchAAM(AAM): scales : `int` or float` or list of those scale_shapes : `boolean` - scale_features : `boolean` """ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - scale_features=True, max_shape_components=None, - max_appearance_components=None, batch_size=None): + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, scale_features=scale_features, - max_shape_components=max_shape_components, + scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -656,20 +649,16 @@ class LinearAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` """ def __init__(self, images, group=None, verbose=False, features=no_op, transform=DifferentiableThinPlateSplines, diagonal=None, - scales=(0.5, 1.0), scale_features=True, - max_shape_components=None, max_appearance_components=None, - batch_size=None): + scales=(0.5, 1.0), max_shape_components=None, + max_appearance_components=None, batch_size=None): super(LinearAAM, self).__init__( images, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, - scales=scales, scale_features=scale_features, + transform=transform, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -754,22 +743,18 @@ class LinearPatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` - n_landmarks: `int` """ def __init__(self, images, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - scale_features=True, max_shape_components=None, - max_appearance_components=None, batch_size=None): + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(LinearPatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, scale_features=scale_features, - max_shape_components=max_shape_components, + scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -856,23 +841,19 @@ class PartsAAM(AAM): normalize_parts: `callable` scales : `int` or float` or list of those - scale_shapes : `boolean` - scale_features : `boolean` """ def __init__(self, images, group=None, verbose=False, features=no_op, normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), scale_features=True, - max_shape_components=None, max_appearance_components=None, - batch_size=None): + patch_shape=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts super(PartsAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, scale_features=scale_features, - max_shape_components=max_shape_components, + scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) diff --git a/menpofit/checks.py b/menpofit/checks.py index 16a7188..0547791 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -67,22 +67,6 @@ def check_features(features, n_levels): "callables with the same length as scales") -# TODO: document me! -def check_scale_features(scale_features, features): - r""" - """ - if np.alltrue([f == features[0] for f in features]): - return scale_features - elif scale_features: - # Only raise warning if True was passed. - warnings.warn('scale_features has been automatically set to False ' - 'because different types of features are used at each ' - 'level.') - return False - else: - return scale_features - - # TODO: document me! def check_patch_shape(patch_shape, n_levels): if len(patch_shape) == 2 and isinstance(patch_shape[0], int): diff --git a/menpofit/fitter.py b/menpofit/fitter.py index af4e093..d963fda 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -254,20 +254,6 @@ def features(self): def scales(self): return self._model.scales - @property - def scale_features(self): - r""" - Flag that defined the nature of Gaussian pyramid used to build the - AAM. - If ``True``, the feature space is computed once at the highest scale - and the Gaussian pyramid is applied to the feature images. - If ``False``, the Gaussian pyramid is applied to the original images - and features are extracted at each level. - - :type: `boolean` - """ - return self._model.scale_features - def _check_n_shape(self, n_shape): if n_shape is not None: if type(n_shape) is int or type(n_shape) is float: From 58a506f3a2de65f7eb98a27327a855bfc3567d66 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 30 Jul 2015 16:56:37 +0100 Subject: [PATCH 358/423] Bug using wrong index Fixed SDM no scaling logic --- menpofit/aam/base.py | 2 +- menpofit/sdm/fitter.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 2f6cfd2..d3a1746 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -225,7 +225,7 @@ def _train(self, images, group=None, verbose=False, increment=False, level_str=scale_prefix, verbose=verbose) # handle scales - if self.scales[k] != 1: + if self.scales[j] != 1: # Scale feature images only if scale is different than 1 scaled_images = scale_images(feature_images, self.scales[j], level_str=scale_prefix, diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index e74a90e..741e1dc 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -179,7 +179,7 @@ def _train(self, images, group=None, bounding_box_group=None, level_str=scale_prefix, verbose=verbose) # handle scales - if self.scales[k] != 1: + if self.scales[j] != 1: # Scale feature images only if scale is different than 1 scaled_images = scale_images(feature_images, self.scales[j], level_str=scale_prefix, @@ -216,8 +216,9 @@ def _train(self, images, group=None, bounding_box_group=None, scaled_images, scaled_shapes, current_shapes, level_str=scale_prefix, verbose=verbose) - # Scale current shapes to next resolution - if self.scales[j] != (1 or self.scales[-1]): + # Scale current shapes to next resolution, don't bother + # scaling final level + if j != (self.n_scales - 1): transform = Scale(self.scales[j + 1] / self.scales[j], n_dims=2) for image_shapes in current_shapes: From ebbdb4212bc795b39cfaf1fae8b59e1ba16e65e4 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 11:21:54 +0100 Subject: [PATCH 359/423] Add menpofit.feature package - We may consider moving these feature to menpocore in the long run. --- menpofit/feature/features.py | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 menpofit/feature/features.py diff --git a/menpofit/feature/features.py b/menpofit/feature/features.py new file mode 100644 index 0000000..8344b35 --- /dev/null +++ b/menpofit/feature/features.py @@ -0,0 +1,74 @@ +from __future__ import division +import numpy as np +import warnings +from menpo.feature import ndfeature + + +# TODO: Document me! +@ndfeature +def centralize(x, axes=(-2, -1)): + r""" + """ + mean = np.mean(x, axis=axes, keepdims=True) + return x - mean + + +# TODO: Document me! +@ndfeature +def normalize_norm(x, axes=(-2, -1)): + r""" + """ + x = centralize(x, axes=axes) + norm = np.asarray(np.linalg.norm(x, axis=axes)) + positions = np.asarray(axes) + len(x.shape) + for axis in positions: + norm = np.expand_dims(norm, axis=axis) + return handle_div_by_zero(x, norm) + + +# TODO: document me! +@ndfeature +def normalize_std(x, axes=(-2, -1)): + r""" + """ + x = centralize(x, axes=axes) + std = np.std(x, axis=axes, keepdims=True) + return handle_div_by_zero(x, std) + + +# TODO: document me! +@ndfeature +def normalize_var(x, axes=(-2, -1)): + r""" + """ + x = centralize(x, axes=axes) + var = np.var(x, axis=axes, keepdims=True) + return handle_div_by_zero(x, var) + + +# TODO: document me! +@ndfeature +def probability_map(x, axes=(-2, -1)): + r""" + """ + x = x - np.min(x, axis=axes, keepdims=True) + total = np.sum(x, axis=axes, keepdims=True) + nonzero = total > 0 + if np.any(~nonzero): + warnings.warn("some of x axes have 0 variance - uniform probability " + "maps are used them.") + x[nonzero] /= total[nonzero] + x[~nonzero] = 1 / np.prod(axes) + return x + + +# TODO: document me! +def handle_div_by_zero(x, normalizer): + r""" + """ + nonzero = normalizer > 0 + if np.any(~nonzero): + warnings.warn("some of x axes have 0 variance - they cannot be " + "normalized.") + x[nonzero] /= normalizer[nonzero] + return x From e7fc36f2d10ac89513b74605a0e010eb93b1e6f6 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 11:23:09 +0100 Subject: [PATCH 360/423] Add __init__.py --- menpofit/feature/__init__.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 menpofit/feature/__init__.py diff --git a/menpofit/feature/__init__.py b/menpofit/feature/__init__.py new file mode 100644 index 0000000..03a2607 --- /dev/null +++ b/menpofit/feature/__init__.py @@ -0,0 +1,2 @@ +from features import ( + centralize, normalize_norm, normalize_std, normalize_var, probability_map) From c7fb14549de880d715bed2cdf671b8d699edb00d Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:19:57 +0100 Subject: [PATCH 361/423] Add ftt_utils.py to menpofit.math --- menpofit/math/fft_utils.py | 244 +++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 menpofit/math/fft_utils.py diff --git a/menpofit/math/fft_utils.py b/menpofit/math/fft_utils.py new file mode 100644 index 0000000..a3e9a85 --- /dev/null +++ b/menpofit/math/fft_utils.py @@ -0,0 +1,244 @@ +from __future__ import division +import warnings +import numpy as np +from functools import wraps +from menpo.feature.base import rebuild_feature_image +try: + # try importing pyfftw + from pyfftw.interfaces.numpy_fft import fft2, ifft2, fftshift, ifftshift + + try: + # try calling fft2 on a 4-dimensional array (this is known to have + # problem in some linux distributions) + fft2(np.zeros((1, 1, 1, 1))) + except RuntimeError: + warnings.warn("pyfftw is known to be buggy on your system, numpy.fft " + "will be used instead. Consequently, all algorithms " + "using ffts will be running at a slower speed.", + RuntimeWarning) + from numpy.fft import fft2, ifft2, fftshift, ifftshift +except ImportError: + warnings.warn("pyfftw is not installed on your system, numpy.fft will be " + "used instead. Consequently, all algorithms using ffts " + "will be running at a slower speed. Consider installing " + "pyfftw (pip install pyfftw) to speed up your ffts.", + ImportWarning) + from numpy.fft import fft2, ifft2, fftshift, ifftshift + + +# TODO: Document me! +def pad(pixels, ext_shape, boundary='constant'): + r""" + """ + h, w = pixels.shape[-2:] + + h_margin = (ext_shape[0] - h) // 2 + w_margin = (ext_shape[1] - w) // 2 + + h_margin2 = h_margin + if h + 2 * h_margin < ext_shape[0]: + h_margin += 1 + + w_margin2 = w_margin + if w + 2 * w_margin < ext_shape[1]: + w_margin += 1 + + pad_width = [] + for _ in pixels.shape[:-2]: + pad_width.append((0, 0)) + pad_width += [(h_margin, h_margin2), (w_margin, w_margin2)] + pad_width = tuple(pad_width) + + return np.lib.pad(pixels, pad_width, mode=boundary) + + +# TODO: Document me! +def crop(pixels, shape): + r""" + """ + h, w = pixels.shape[-2:] + + h_margin = (h - shape[0]) // 2 + w_margin = (w - shape[1]) // 2 + + h_corrector = 1 if np.remainder(h - shape[0], 2) != 0 else 0 + w_corrector = 1 if np.remainder(w - shape[1], 2) != 0 else 0 + + return pixels[..., + h_margin + h_corrector:-h_margin, + w_margin + w_corrector:-w_margin] + + +# TODO: Document me! +def ndconvolution(wrapped): + r""" + """ + @wraps(wrapped) + def wrapper(image, filter, *args, **kwargs): + if not isinstance(image, np.ndarray) and not isinstance(filter, np.ndarray): + # Both image and filter are menpo images + feature = wrapped(image.pixels, filter.pixels, *args, **kwargs) + return rebuild_feature_image(image, feature) + elif not isinstance(image, np.ndarray): + # Image is menpo image + feature = wrapped(image.pixels, filter, *args, **kwargs) + return rebuild_feature_image(image, feature) + elif not isinstance(filter, np.ndarray): + # filter is menpo image + return wrapped(image, filter, *args, **kwargs) + else: + return wrapped(image, filter, *args, **kwargs) + return wrapper + + +# TODO: Document me! +@ndconvolution +def fft_convolve2d(x, f, mode='same', boundary='constant', fft_filter=False): + r""" + Performs fast 2d convolution in the frequency domain convolving each image + channel with its corresponding filter channel. + + Parameters + ---------- + x : ``(channels, height, width)`` `ndarray` + Image. + f : ``(channels, height, width)`` `ndarray` + Filter. + mode : str {`full`, `same`, `valid`}, optional + Determines the shape of the resulting convolution. + boundary: str {`constant`, `symmetric`}, optional + Determines how the image is padded. + fft_filter: `bool`, optional + If `True`, the filter is assumed to be defined on the frequency + domain. If `False` the filter is assumed to be defined on the + spatial domain. + + Returns + ------- + c: ``(channels, height, width)`` `ndarray` + Result of convolving each image channel with its corresponding + filter channel. + """ + if fft_filter: + # extended shape is filter shape + ext_shape = np.asarray(f.shape[-2:]) + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + + # compute ffts of extended image + fft_ext_x = fft2(ext_x) + fft_ext_f = f + else: + # extended shape + x_shape = np.asarray(x.shape[-2:]) + f_shape = np.asarray(f.shape[-2:]) + f_half_shape = (f_shape / 2).astype(int) + ext_shape = x_shape + f_half_shape - 1 + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + ext_f = pad(f, ext_shape) + + # compute ffts of extended image and extended filter + fft_ext_x = fft2(ext_x) + fft_ext_f = fft2(ext_f) + + # compute extended convolution in Fourier domain + fft_ext_c = fft_ext_f * fft_ext_x + + # compute ifft of extended convolution + ext_c = np.real(ifftshift(ifft2(fft_ext_c), axes=(-2, -1))) + + if mode is 'full': + return ext_c + elif mode is 'same': + return crop(ext_c, x_shape) + elif mode is 'valid': + return crop(ext_c, x_shape - f_half_shape + 1) + else: + raise ValueError( + "mode={}, is not supported. The only supported " + "modes are: 'full', 'same' and 'valid'.".format(mode)) + + +# TODO: Document me! +@ndconvolution +def fft_convolve2d_sum(x, f, mode='same', boundary='constant', + fft_filter=False, axis=0, keepdims=True): + r""" + Performs fast 2d convolution in the frequency domain convolving each image + channel with its corresponding filter channel and summing across the + channel axis. + + Parameters + ---------- + x : ``(channels, height, width)`` `ndarray` + Image. + f : ``(channels, height, width)`` `ndarray` + Filter. + mode : str {`full`, `same`, `valid`}, optional + Determines the shape of the resulting convolution. + boundary: str {`constant`, `symmetric`}, optional + Determines how the image is padded. + fft_filter: `bool`, optional + If `True`, the filter is assumed to be defined on the frequency + domain. If `False` the filter is assumed to be defined on the + spatial domain. + axis : `int`, optional + The axis across to which the summation is performed. + keepdims: `boolean`, optional + If `True` the number of dimensions of the result is the same as the + number of dimensions of the filter. If `False` the channel dimension + is lost in the result. + Returns + ------- + c: ``(1, height, width)`` `ndarray` + Result of convolving each image channel with its corresponding + filter channel and summing across the channel axis. + """ + if fft_filter: + fft_ext_f = f + + # extended shape is fft_ext_filter shape + x_shape = np.asarray(x.shape[-2:]) + f_shape = ((np.asarray(fft_ext_f.shape[-2:]) + 1) / 1.5).astype(int) + f_half_shape = (f_shape / 2).astype(int) + ext_shape = np.asarray(f.shape[-2:]) + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + + # compute ffts of extended image + fft_ext_x = fft2(ext_x) + else: + # extended shape + x_shape = np.asarray(x.shape[-2:]) + f_shape = np.asarray(f.shape[-2:]) + f_half_shape = (f_shape / 2).astype(int) + ext_shape = x_shape + f_half_shape - 1 + + # extend image and filter + ext_x = pad(x, ext_shape, boundary=boundary) + ext_f = pad(f, ext_shape) + + # compute ffts of extended image and extended filter + fft_ext_x = fft2(ext_x) + fft_ext_f = fft2(ext_f) + + # compute extended convolution in Fourier domain + fft_ext_c = np.sum(fft_ext_f * fft_ext_x, axis=axis, keepdims=keepdims) + + # compute ifft of extended convolution + ext_c = np.real(ifftshift(ifft2(fft_ext_c), axes=(-2, -1))) + + if mode is 'full': + return ext_c + elif mode is 'same': + return crop(ext_c, x_shape) + elif mode is 'valid': + return crop(ext_c, x_shape - f_half_shape + 1) + else: + raise ValueError( + "mode={}, is not supported. The only supported " + "modes are: 'full', 'same' and 'valid'.".format(mode)) From bb230f50709789bc691f15ccad7d45558cd16f3a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:20:38 +0100 Subject: [PATCH 362/423] Add correlationfilters.py to menpofit.math - Add MOSSE, MCCF and their respective incremental versions --- menpofit/math/correlationfilter.py | 373 +++++++++++++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100644 menpofit/math/correlationfilter.py diff --git a/menpofit/math/correlationfilter.py b/menpofit/math/correlationfilter.py new file mode 100644 index 0000000..753a55e --- /dev/null +++ b/menpofit/math/correlationfilter.py @@ -0,0 +1,373 @@ +import numpy as np +from numpy.fft import fft2, ifft2, ifftshift +from scipy.sparse import spdiags, eye as speye +from scipy.sparse.linalg import spsolve +from menpofit.math.fft_utils import pad, crop + + +# TODO: Document me! +def mosse(X, y, l=0.01, boundary='constant', crop_filter=True): + r""" + Minimum Output Sum of Squared Errors (MOSSE) filter. + + Parameters + ---------- + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l: `float`, optional + Regularization parameter. + boundary: str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter: `bool`, optional + If ``True``, the shape of the MOSSE filter is the same as the shape + of the desired response. If ``False``, the filter's shape is equal to: + ``X[0].shape + y.shape - 1`` + + Returns + ------- + mosse: ``(1, height, width)`` `ndarray` + Minimum Output Sum od Squared Errors (MOSSE) filter associated to + the training images. + + References + ---------- + .. [1] David S. Bolme, J. Ross Beveridge, Bruce A. Draper and Yui Man Lui. + "Visual Object Tracking using Adaptive Correlation Filters". CVPR, 2010. + """ + # number of images, number of channels, height and width + n, k, hx, wx = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # extended shape + ext_h = hx + hy - 1 + ext_w = wx + wy - 1 + ext_shape = (ext_h, ext_w) + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for x in X: + # extend image + ext_x = pad(x, ext_shape, boundary=boundary) + # fft of extended image + fft_ext_x = fft2(ext_x) + + # update auto and cross spectral energy matrices + sXX += fft_ext_x.conj() * fft_ext_x + sXY += fft_ext_x.conj() * fft_ext_y + + # compute desired correlation filter + fft_ext_f = sXY / (sXX + l) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute extended filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX + + +def imosse(A, B, n_ab, X, y, l=0.01, boundary='constant', + crop_filter=True, f=1.0): + r""" + Incremental Minimum Output Sum od Squared Errors (iMOSSE) filter + + Parameters + ---------- + A : + B : + n_ab : `int` + Total number of samples used to produce A and B. + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l : `float`, optional + Regularization parameter. + boundary : str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter : `bool`, optional + f : ``[0, 1]`` `float`, optional + Forgetting factor that weights the relative contribution of new + samples vs old samples. If 1.0, all samples are weighted equally. + If <1.0, more emphasis is put on the new samples. + + Returns + ------- + mccf : ``(1, height, width)`` `ndarray` + Multi-Channel Correlation Filter (MCCF) filter associated to the + training images. + sXY : + sXX : + + References + ---------- + .. [1] David S. Bolme, J. Ross Beveridge, Bruce A. Draper and Yui Man Lui. + "Visual Object Tracking using Adaptive Correlation Filters". CVPR, 2010. + """ + # number of images; number of channels, height and width + n_x, k, hz, wz = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # multiply the number of samples used to produce the auto and cross + # spectral energy matrices A and B by forgetting factor + n_ab *= f + # total number of samples + n = n_ab + n_x + # compute weighting factors + nu_ab = n_ab / n + nu_x = n_x / n + + # extended shape + ext_h = hz + hy - 1 + ext_w = wz + wy - 1 + ext_shape = (ext_h, ext_w) + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # extend images + ext_X = pad(X, ext_shape, boundary=boundary) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for ext_x in ext_X: + # fft of extended image + fft_ext_x = fft2(ext_x) + + # update auto and cross spectral energy matrices + sXX += fft_ext_x.conj() * fft_ext_x + sXY += fft_ext_x.conj() * fft_ext_y + + # combine old and new auto and cross spectral energy matrices + sXY = nu_ab * A + nu_x * sXY + sXX = nu_ab * B + nu_x * sXX + # compute desired correlation filter + fft_ext_f = sXY / (sXX + l) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX + + +# TODO: Document me! +def mccf(X, y, l=0.01, boundary='constant', crop_filter=True): + r""" + Multi-Channel Correlation Filter (MCCF). + + Parameters + ---------- + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l : `float`, optional + Regularization parameter. + boundary : str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter : `bool`, optional + + Returns + ------- + mccf: ``(1, height, width)`` `ndarray` + Multi-Channel Correlation Filter (MCCF) filter associated to the + training images. + sXY : + sXX : + + References + ---------- + .. [1] Hamed Kiani Galoogahi, Terence Sim, Simon Lucey. "Multi-Channel + Correlation Filters". ICCV, 2013. + """ + # number of images; number of channels, height and width + n, k, hx, wx = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # extended shape + ext_h = hx + hy - 1 + ext_w = wx + wy - 1 + ext_shape = (ext_h, ext_w) + # extended dimensionality + ext_d = ext_h * ext_w + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # extend images + ext_X = pad(X, ext_shape, boundary=boundary) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for ext_x in ext_X: + # fft of extended image + fft_ext_x = fft2(ext_x) + + # store extended image fft as sparse diagonal matrix + diag_fft_x = spdiags(fft_ext_x.reshape((k, -1)), + -np.arange(0, k) * ext_d, ext_d * k, ext_d).T + # vectorize extended desired response fft + diag_fft_y = fft_ext_y.ravel() + + # update auto and cross spectral energy matrices + sXX += diag_fft_x.conj().T.dot(diag_fft_x) + sXY += diag_fft_x.conj().T.dot(diag_fft_y) + + # solve ext_d independent k x k linear systems (with regularization) + # to obtain desired extended multi-channel correlation filter + fft_ext_f = spsolve(sXX + l * speye(sXX.shape[-1]), sXY) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX + + +# TODO: Document me! +def imccf(A, B, n_ab, X, y, l=0.01, boundary='constant', crop_filter=True, + f=1.0): + r""" + Incremental Multi-Channel Correlation Filter (MCCF) + + Parameters + ---------- + A : + B : + n_ab : `int` + Total number of samples used to produce A and B. + X : ``(n_images, n_channels, height, width)`` `ndarray` + Training images. + y : ``(1, height, width)`` `ndarray` + Desired response. + l : `float`, optional + Regularization parameter. + boundary : str {`constant`, `symmetric`}, optional + Determines how the image is padded. + crop_filter : `bool`, optional + f : ``[0, 1]`` `float`, optional + Forgetting factor that weights the relative contribution of new + samples vs old samples. If 1.0, all samples are weighted equally. + If <1.0, more emphasis is put on the new samples. + + Returns + ------- + mccf : ``(1, height, width)`` `ndarray` + Multi-Channel Correlation Filter (MCCF) filter associated to the + training images. + sXY : + sXX : + + References + ---------- + .. [1] David S. Bolme, J. Ross Beveridge, Bruce A. Draper and Yui Man Lui. + "Visual Object Tracking using Adaptive Correlation Filters". CVPR, 2010. + .. [2] Hamed Kiani Galoogahi, Terence Sim, Simon Lucey. "Multi-Channel + Correlation Filters". ICCV, 2013. + """ + # number of images; number of channels, height and width + n_x, k, hz, wz = X.shape + + # height and width of desired responses + _, hy, wy = y.shape + y_shape = (hy, wy) + + # multiply the number of samples used to produce the auto and cross + # spectral energy matrices A and B by forgetting factor + n_ab *= f + # total number of samples + n = n_ab + n_x + # compute weighting factors + nu_ab = n_ab / n + nu_x = n_x / n + + # extended shape + ext_h = hz + hy - 1 + ext_w = wz + wy - 1 + ext_shape = (ext_h, ext_w) + # extended dimensionality + ext_d = ext_h * ext_w + + # extend desired response + ext_y = pad(y, ext_shape) + # fft of extended desired response + fft_ext_y = fft2(ext_y) + + # extend images + ext_X = pad(X, ext_shape, boundary=boundary) + + # auto and cross spectral energy matrices + sXX = 0 + sXY = 0 + # for each training image and desired response + for ext_x in ext_X: + # fft of extended image + fft_ext_x = fft2(ext_x) + + # store extended image fft as sparse diagonal matrix + diag_fft_x = spdiags(fft_ext_x.reshape((k, -1)), + -np.arange(0, k) * ext_d, ext_d * k, ext_d).T + # vectorize extended desired response fft + diag_fft_y = fft_ext_y.ravel() + + # update auto and cross spectral energy matrices + sXX += diag_fft_x.conj().T.dot(diag_fft_x) + sXY += diag_fft_x.conj().T.dot(diag_fft_y) + + # combine old and new auto and cross spectral energy matrices + sXY = nu_ab * A + nu_x * sXY + sXX = nu_ab * B + nu_x * sXX + # solve ext_d independent k x k linear systems (with regularization) + # to obtain desired extended multi-channel correlation filter + fft_ext_f = spsolve(sXX + l * speye(sXX.shape[-1]), sXY) + # reshape extended filter to extended image shape + fft_ext_f = fft_ext_f.reshape((k, ext_h, ext_w)) + + # compute filter inverse fft + f = np.real(ifftshift(ifft2(fft_ext_f), axes=(-2, -1))) + if crop_filter: + # crop extended filter to match desired response shape + f = crop(f, y_shape) + + return f, sXY, sXX From e1f4ef8c12ba2a2e7b4224beb593d36304ca3e4e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:22:58 +0100 Subject: [PATCH 363/423] Update __init__.py from menpofit.math - The math package is an advanced package and nothing should be importable at the first level --- menpofit/math/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py index d916940..aa9091f 100644 --- a/menpofit/math/__init__.py +++ b/menpofit/math/__init__.py @@ -1 +1 @@ -from regression import IRLRegression, IIRLRegression \ No newline at end of file +from .regression import IRLRegression, IIRLRegression \ No newline at end of file From 6cac7de95dc0081bbaee3cba28483aeb0d116cb8 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:24:36 +0100 Subject: [PATCH 364/423] Add clm results --- menpofit/clm/result.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 menpofit/clm/result.py diff --git a/menpofit/clm/result.py b/menpofit/clm/result.py new file mode 100644 index 0000000..09e532b --- /dev/null +++ b/menpofit/clm/result.py @@ -0,0 +1,14 @@ +from menpofit.result import ParametricAlgorithmResult, MultiFitterResult + + +# TODO: document me! +class CLMAlgorithmResult(ParametricAlgorithmResult): + r""" + """ + + +# TODO: document me! +class CLMFitterResult(MultiFitterResult): + r""" + """ + From bd2ceb003e5b1b536c635d3fe453c5dc6997c249 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:28:01 +0100 Subject: [PATCH 365/423] Add CLM class - Some things could still be improved; mainly how to properly deal with shape models. --- menpofit/clm/base.py | 465 +++++++++++++++++-------------------------- 1 file changed, 187 insertions(+), 278 deletions(-) diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 3711a5d..7feabf3 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -1,321 +1,230 @@ -import numpy as np -from menpo.image import Image - -from menpofit.base import DeformableModel - - -class CLM(DeformableModel): +from __future__ import division +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpofit import checks +from menpofit.base import batch +from menpofit.builder import ( + normalization_wrt_reference_shape, compute_features, scale_images, + build_shape_model, increment_shape_model) +from expert import ExpertEnsemble, CorrelationFilterExpertEnsemble + + +# TODO: Document me! +# TODO: Introduce shape_model_cls +# TODO: Get rid of max_shape_components and shape_forgetting_factor +class CLM(object): r""" - Constrained Local Model class. + Constrained Local Model (CLM) class. Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the CLM. - - classifiers : ``[[callable]]`` - A list containing the list of classifier callables per each pyramidal - level of the CLM. - - n_training_images : `int` - The number of training images used to build the AAM. - - patch_shape : tuple of `int` - The shape of the patches used to train the classifiers. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - downscale : `float` - The downscale factor that was used to create the different pyramidal - levels. - - scaled_shape_models : `boolean`, Optional - If ``True``, the reference frames are the mean shapes of each pyramid - level, so the shape models are scaled. - - If ``False``, the reference frames of all levels are the mean shape of - the highest level, so the shape models are not scaled; they have the - same size. + ---------- + Returns + ------- + clm : :map:`CLM` + The CLM object """ - def __init__(self, shape_models, classifiers, n_training_images, - patch_shape, features, reference_shape, downscale, - scaled_shape_models): - DeformableModel.__init__(self, features) - self.shape_models = shape_models - self.classifiers = classifiers - self.n_training_images = n_training_images - self.patch_shape = patch_shape - self.reference_shape = reference_shape - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models + def __init__(self, images, group=None, verbose=False, batch_size=None, + diagonal=None, scales=(0.5, 1), features=no_op, + # shape_model_cls=build_normalised_pca_shape_model, + expert_ensemble_cls=CorrelationFilterExpertEnsemble, + max_shape_components=None, + shape_forgetting_factor=1.0): + self.diagonal = checks.check_diagonal(diagonal) + self.scales = checks.check_scales(scales) + self.features = checks.check_features(features, self.n_scales) + # self.shape_model_cls = checks.check_algorithm_cls( + # shape_model_cls, self.n_scales, ShapeModel) + self.expert_ensemble_cls = checks.check_algorithm_cls( + expert_ensemble_cls, self.n_scales, ExpertEnsemble) + + self.max_shape_components = checks.check_max_components( + max_shape_components, self.n_scales, 'max_shape_components') + self.shape_forgetting_factor = shape_forgetting_factor + + # Train CLM + self.train(images, group=group, verbose=verbose, batch_size=batch_size) @property - def n_levels(self): - """ - The number of multi-resolution pyramidal levels of the CLM. - - :type: `int` - """ - return len(self.shape_models) - - @property - def n_classifiers_per_level(self): - """ - The number of classifiers per pyramidal level of the CLM. + def n_scales(self): + r""" + The number of scales of the CLM. :type: `int` """ - return [len(clf) for clf in self.classifiers] + return len(self.scales) - def instance(self, shape_weights=None, level=-1): + def _train_batch(self, image_batch, increment, group=None, verbose=False): r""" - Generates a novel CLM instance given a set of shape weights. If no - weights are provided, the mean CLM instance is returned. - - Parameters - ----------- - shape_weights : ``(n_weights,)`` `ndarray` or `float` list - Weights of the shape model that will be used to create - a novel shape instance. If `None`, the mean shape - ``(shape_weights = [0, 0, ..., 0])`` is used. - - level : `int`, optional - The pyramidal level to be used. - - Returns - ------- - shape_instance : :map:`PointCloud` - The novel CLM instance. """ - sm = self.shape_models[level] - # TODO: this bit of logic should to be transferred down to PCAModel - if shape_weights is None: - shape_weights = [0] - n_shape_weights = len(shape_weights) - shape_weights *= sm.eigenvalues[:n_shape_weights] ** 0.5 - shape_instance = sm.instance(shape_weights) - return shape_instance + # If increment is False, we need to initialise/reset both shape models + # and ensembles of experts + if not increment: + self.shape_models = [] + self.expert_ensembles = [] + + # normalize images and compute reference shape + self.reference_shape, image_batch = normalization_wrt_reference_shape( + image_batch, group, self.diagonal, verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Training models\n') + + # for each level (low --> high) + for i in range(self.n_scales): + if verbose: + if self.n_scales > 1: + prefix = ' - Scale {}: '.format(i) + else: + prefix = ' - ' + + # handle features + if i == 0 or self.features[i] is not self.features[i-1]: + # compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[i], + prefix=prefix, + verbose=verbose) + # handle scales + if self.scales[i] != 1: + # scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, + self.scales[i], + prefix=prefix, + verbose=verbose) + else: + scaled_images = feature_images - def random_instance(self, level=-1): - r""" - Generates a novel random CLM instance. + # extract scaled shapes + scaled_shapes = [image.landmarks[group].lms + for image in scaled_images] - Parameters - ----------- - level : `int`, optional - The pyramidal level to be used. + # train shape model + if verbose: + print_dynamic('{}Training shape model'.format(prefix)) - Returns - ------- - shape_instance : :map:`PointCloud` - The novel CLM instance. - """ - sm = self.shape_models[level] - # TODO: this bit of logic should to be transferred down to PCAModel - shape_weights = (np.random.randn(sm.n_active_components) * - sm.eigenvalues[:sm.n_active_components]**0.5) - shape_instance = sm.instance(shape_weights) - return shape_instance + # TODO: This should be cleaned up by defining shape model classes + if increment: + increment_shape_model( + self.shape_models[i], scaled_shapes, + max_components=self.max_shape_components[i], + forgetting_factor=self.shape_forgetting_factor, + prefix=prefix, verbose=verbose) - def response_image(self, image, group=None, label=None, level=-1): - r""" - Generates a response image result of applying the classifiers of a - particular pyramidal level of the CLM to an image. + else: + shape_model = build_shape_model( + scaled_shapes, max_components=self.max_shape_components[i], + prefix=prefix, verbose=verbose) + self.shape_models.append(shape_model) + + # train expert ensemble + if verbose: + print_dynamic('{}Training expert ensemble'.format(prefix)) + + if increment: + self.expert_ensembles[i].increment(scaled_images, + scaled_shapes, + prefix=prefix, + verbose=verbose) + else: + expert_ensemble = self.expert_ensemble_cls[i](scaled_images, + scaled_shapes, + prefix=prefix, + verbose=verbose) + self.expert_ensembles.append(expert_ensemble) - Parameters - ----------- - image: :map:`Image` - The image. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label : `string`, optional - The label of of the landmark manager that you wish to use. If no - label is passed, the convex hull of all landmarks is used. - level: `int`, optional - The pyramidal level to be used. + if verbose: + print_dynamic('{}Done\n'.format(prefix)) - Returns - ------- - image : :map:`Image` - The response image. + def _train(self, images, increment, group=None, verbose=False, + batch_size=None): + r""" """ - # rescale image - image = image.rescale_to_reference_shape(self.reference_shape, - group=group, label=label) - - # apply pyramid - if self.n_levels > 1: - if self.pyramid_on_features: - # compute features at highest level - feature_image = self.features(image) - - # apply pyramid on feature image - pyramid = feature_image.gaussian_pyramid( - n_levels=self.n_levels, downscale=self.downscale) + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] - # get rescaled feature images - images = list(pyramid) - else: - # create pyramid on intensities image - pyramid = image.gaussian_pyramid( - n_levels=self.n_levels, downscale=self.downscale) + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True - # compute features at each level - images = [self.features[self.n_levels - j - 1](i) - for j, i in enumerate(pyramid)] - images.reverse() - else: - images = [self.features(image)] + if verbose: + print('Batch {}'.format(k)) - # initialize responses - image = images[level] - image_pixels = np.reshape(image.pixels, (-1, image.n_channels)) - response_data = np.zeros((image.shape[0], image.shape[1], - self.n_classifiers_per_level[level])) - # Compute responses - for j, clf in enumerate(self.classifiers[level]): - response_data[:, :, j] = np.reshape(clf(image_pixels), - image.shape) - return Image(image_data=response_data) + # Train each batch + self._train_batch(image_batch, increment, group=group, + verbose=verbose) - @property - def _str_title(self): + def train(self, images, group=None, verbose=False, batch_size=None): r""" - Returns a string containing name of the model. + """ + return self._train(images, False, group=group, verbose=verbose, + batch_size=batch_size) - : str + def increment(self, images, group=None, verbose=False, batch_size=None): + r""" """ - return 'Constrained Local Model' + return self._train(images, True, group=group, verbose=verbose, + batch_size=batch_size) - def view_shape_models_widget(self, n_parameters=5, mode='multiple', + def view_shape_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), - figure_size=(10, 8), style='coloured'): + mode='multiple', figure_size=(10, 8)): r""" - Visualizes the shape models of the CLM object using the + Visualizes the shape models of the AAM object using the `menpo.visualize.widgets.visualize_shape_model` widget. Parameters ----------- n_parameters : `int` or `list` of `int` or ``None``, optional - The number of principal components to be used for the parameters - sliders. If `int`, then the number of sliders per level is the - minimum between `n_parameters` and the number of active components - per level. If `list` of `int`, then a number of sliders is defined - per level. If ``None``, all the active components per level will - have a slider. - mode : {``'single'``, ``'multiple'``}, optional - If ``'single'``, then only a single slider is constructed along with - a drop down menu. If ``'multiple'``, then a slider is constructed - for each parameter. + The number of shape principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per level is the minimum + between `n_parameters` and the number of active components per + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. + mode : {``single``, ``multiple``}, optional + If ``'single'``, only a single slider is constructed along with a + drop down menu. + If ``'multiple'``, a slider is constructed for each parameter. + popup : `bool`, optional + If ``True``, the widget will appear as a popup window. figure_size : (`int`, `int`), optional The size of the plotted figures. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. """ from menpofit.visualize import visualize_shape_model - visualize_shape_model( - self.shape_models, n_parameters=n_parameters, - parameters_bounds=parameters_bounds, figure_size=figure_size, - mode=mode, style=style) + visualize_shape_model(self.shape_models, n_parameters=n_parameters, + parameters_bounds=parameters_bounds, + figure_size=figure_size, mode=mode,) - def __str__(self): - from menpofit.base import name_of_callable - out = "{}\n - {} training images.\n".format(self._str_title, - self.n_training_images) - # small strings about number of channels, channels string and downscale - down_str = [] - for j in range(self.n_levels): - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - temp_img = Image(image_data=np.random.rand(50, 50)) - if self.pyramid_on_features: - temp = self.features(temp_img) - n_channels = [temp.n_channels] * self.n_levels - else: - n_channels = [] - for j in range(self.n_levels): - temp = self.features[j](temp_img) - n_channels.append(temp.n_channels) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.patch_shape[1], self.patch_shape[0]) + # TODO: Implement me! + def view_expert_ensemble_widget(self): + r""" + """ + raise NotImplementedError - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.patch_shape[1], self.patch_shape[0]) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n - {3} {4} classifiers.\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100, - self.n_classifiers_per_level[i], - name_of_callable(self.classifiers[i][0])) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - {4} shape components ({5:.2f}% of " \ - "variance)\n - {6} {7} classifiers.".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100, - self.n_classifiers_per_level[0], - name_of_callable(self.classifiers[0][0])) - return out + # TODO: Implement me! + def view_clm_widget(self): + r""" + """ + raise NotImplementedError + + # TODO: Implement me! + def __str__(self): + r""" + """ + raise NotImplementedError From ac9d87bf486f8062d1c2d8122e2dc5660546a22f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:31:55 +0100 Subject: [PATCH 366/423] Add dummy wrapper for correlation filters --- menpofit/clm/expert/base.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 menpofit/clm/expert/base.py diff --git a/menpofit/clm/expert/base.py b/menpofit/clm/expert/base.py new file mode 100644 index 0000000..c2f0d38 --- /dev/null +++ b/menpofit/clm/expert/base.py @@ -0,0 +1,28 @@ +import numpy as np +from menpofit.math.correlationfilter import mccf, imccf + + +# TODO: document me! +class IncrementalCorrelationFilterThinWrapper(object): + r""" + """ + def __init__(self, cf_callable=mccf, icf_callable=imccf): + self.cf_callable = cf_callable + self.icf_callable = icf_callable + + def increment(self, A, B, n_x, Z, t): + r""" + """ + # Turn list of X into ndarray + if isinstance(Z, list): + Z = np.asarray(Z) + return self.icf_callable(A, B, n_x, Z, t) + + def train(self, X, t): + r""" + """ + # Turn list of X into ndarray + if isinstance(X, list): + X = np.asarray(X) + # Return linear svm filter and bias + return self.cf_callable(X, t) From 477ea0315d519a54166c9a4f6390ebe02e78f10f Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:32:45 +0100 Subject: [PATCH 367/423] Add CorrelationFilterExpertEnsamble - Supports multichannel features. - Supports incremental training. --- menpofit/clm/expert/ensemble.py | 262 ++++++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 menpofit/clm/expert/ensemble.py diff --git a/menpofit/clm/expert/ensemble.py b/menpofit/clm/expert/ensemble.py new file mode 100644 index 0000000..a120345 --- /dev/null +++ b/menpofit/clm/expert/ensemble.py @@ -0,0 +1,262 @@ +from __future__ import division +from functools import partial +import numpy as np +from scipy.stats import multivariate_normal +from menpo.shape import PointCloud +from menpo.image import Image +from menpofit.base import build_grid +from menpofit.feature import normalize_norm, probability_map +from menpofit.math.fft_utils import ( + fft2, ifft2, fftshift, pad, crop, fft_convolve2d_sum) +from menpofit.visualize import print_progress +from .base import IncrementalCorrelationFilterThinWrapper + + +# TODO: Document me! +class ExpertEnsemble(object): + r""" + """ + + +# TODO: Document me! +# TODO: Should convolutional experts of ensembles support patch features? +class ConvolutionBasedExpertEnsemble(ExpertEnsemble): + r""" + """ + @property + def n_experts(self): + r""" + """ + return self.fft_padded_filters.shape[0] + + @property + def n_sample_offsets(self): + r""" + """ + if self.sample_offsets: + return self.sample_offsets.n_points + else: + return 1 + + @property + def padded_size(self): + r""" + """ + pad_size = np.floor(1.5 * np.asarray(self.patch_size) - 1).astype(int) + return tuple(pad_size) + + @property + def search_size(self): + r""" + """ + return self.patch_size + + def increment(self, images, shapes, prefix='', verbose=False): + r""" + """ + self._train(images, shapes, prefix=prefix, verbose=verbose, + increment=True) + + @property + def spatial_filter_images(self): + r""" + """ + filter_images = [] + for fft_padded_filter in self.fft_padded_filters: + spatial_filter = np.real(ifft2(fft_padded_filter)) + spatial_filter = crop(spatial_filter, + self.patch_size)[:, ::-1, ::-1] + filter_images.append(Image(spatial_filter)) + return filter_images + + @property + def frequency_filter_images(self): + r""" + """ + filter_images = [] + for fft_padded_filter in self.fft_padded_filters: + spatial_filter = np.real(ifft2(fft_padded_filter)) + spatial_filter = crop(spatial_filter, + self.patch_size)[:, ::-1, ::-1] + frequency_filter = np.abs(fftshift(fft2(spatial_filter))) + filter_images.append(Image(frequency_filter)) + return filter_images + + def _extract_patch(self, image, landmark): + r""" + """ + # Extract patch from image + patch = image.extract_patches( + landmark, patch_size=self.patch_size, + sample_offsets=self.sample_offsets, as_single_array=True) + # Reshape patch + # patch: (offsets x ch) x h x w + patch = patch.reshape((-1,) + patch.shape[-2:]) + # Normalise patch + return self.normalise_callable(patch) + + def _extract_patches(self, image, shape): + r""" + """ + # Obtain patch ensemble, the whole shape is used to extract patches + # from all landmarks at once + patches = image.extract_patches(shape, patch_size=self.patch_size, + sample_offsets=self.sample_offsets, + as_single_array=True) + # Reshape patches + # patches: n_patches x (n_offsets x n_channels) x height x width + patches = patches.reshape((patches.shape[0], -1) + patches.shape[-2:]) + # Normalise patches + return self.normalise_callable(patches) + + def predict_response(self, image, shape): + r""" + """ + # Extract patches + patches = self._extract_patches(image, shape) + # Predict responses + return fft_convolve2d_sum(patches, self.fft_padded_filters, + fft_filter=True, axis=1) + + def predict_probability(self, image, shape): + r""" + """ + # Predict responses + responses = self.predict_response(image, shape) + # Turn them into proper probability maps + return probability_map(responses) + + +# TODO: Document me! +class CorrelationFilterExpertEnsemble(ConvolutionBasedExpertEnsemble): + r""" + """ + def __init__(self, images, shapes, verbose=False, prefix='', + icf_cls=IncrementalCorrelationFilterThinWrapper, + patch_size=(17, 17), context_size=(34, 34), + response_covariance=3, normalise_callable=normalize_norm, + cosine_mask=True, sample_offsets=None): + # TODO: check parameters? + # Set parameters + self._icf = icf_cls() + self.patch_size = patch_size + self.context_size = context_size + self.response_covariance = response_covariance + self.normalise_callable = normalise_callable + self.cosine_mask = cosine_mask + self.sample_offsets = sample_offsets + + # Generate cosine mask + self._cosine_mask = generate_cosine_mask(self.context_size) + + # Generate desired response, i.e. a Gaussian response with the + # specified covariance centred at the middle of the patch + self.response = generate_gaussian_response( + self.patch_size, self.response_covariance)[None, ...] + + # Train ensemble of correlation filter experts + self._train(images, shapes, verbose=verbose, prefix=prefix) + + def _extract_patch(self, image, landmark): + r""" + """ + # Extract patch from image + patch = image.extract_patches( + landmark, patch_size=self.context_size, + sample_offsets=self.sample_offsets, as_single_array=True) + # Reshape patch + # patch: (offsets x ch) x h x w + patch = patch.reshape((-1,) + patch.shape[-2:]) + # Normalise patch + patch = self.normalise_callable(patch) + if self.cosine_mask: + # Apply cosine mask if require + patch = self._cosine_mask * patch + return patch + + def _train(self, images, shapes, prefix='', verbose=False, + increment=False): + r""" + """ + # Define print_progress partial + wrap = partial(print_progress, + prefix='{}Training experts' + .format(prefix), + end_with_newline=not prefix, + verbose=verbose) + + # If increment is False, we need to initialise/reset the ensemble of + # experts + if not increment: + self.fft_padded_filters = [] + self.auto_correlations = [] + self.cross_correlations = [] + # Set number of images + self.n_images = len(images) + else: + # Update number of images + self.n_images += len(images) + + # Obtain total number of experts + n_experts = shapes[0].n_points + + # Train ensemble of correlation filter experts + fft_padded_filters = [] + auto_correlations = [] + cross_correlations = [] + for i in wrap(range(n_experts)): + patches = [] + for image, shape in zip(images, shapes): + # Select the appropriate landmark + landmark = PointCloud([shape.points[i]]) + # Extract patch + patch = self._extract_patch(image, landmark) + # Add patch to the list + patches.append(patch) + + if increment: + # Increment correlation filter + correlation_filter, auto_correlation, cross_correlation = ( + self._icf.increment(self.auto_correlations[i], + self.cross_correlations[i], + self.n_images, + patches, + self.response)) + else: + # Train correlation filter + correlation_filter, auto_correlation, cross_correlation = ( + self._icf.train(patches, self.response)) + + # Pad filter with zeros + padded_filter = pad(correlation_filter, self.padded_size) + # Compute fft of padded filter + fft_padded_filter = fft2(padded_filter) + # Add fft padded filter to list + fft_padded_filters.append(fft_padded_filter) + auto_correlations.append(auto_correlation) + cross_correlations.append(cross_correlation) + + # Turn list into ndarray + self.fft_padded_filters = np.asarray(fft_padded_filters) + self.auto_correlations = np.asarray(auto_correlations) + self.cross_correlations = np.asarray(cross_correlations) + + +# TODO: Document me! +def generate_gaussian_response(patch_size, response_covariance): + r""" + """ + grid = build_grid(patch_size) + mvn = multivariate_normal(mean=np.zeros(2), cov=response_covariance) + return mvn.pdf(grid) + + +# TODO: Document me! +def generate_cosine_mask(patch_size): + r""" + """ + cy = np.hanning(patch_size[0]) + cx = np.hanning(patch_size[1]) + return cy[..., None].dot(cx[None, ...]) + + From bd3427e61a5d72bdf06ec4e9d907173b0d925932 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:34:09 +0100 Subject: [PATCH 368/423] Add __init__.py --- menpofit/clm/expert/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 menpofit/clm/expert/__init__.py diff --git a/menpofit/clm/expert/__init__.py b/menpofit/clm/expert/__init__.py new file mode 100644 index 0000000..393b44d --- /dev/null +++ b/menpofit/clm/expert/__init__.py @@ -0,0 +1 @@ +from ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble From d4c7e05dea481c5a9ca8d157e95aa47407af73aa Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:36:03 +0100 Subject: [PATCH 369/423] Add CLMFitters --- menpofit/clm/fitter.py | 297 +++++------------------------------------ 1 file changed, 36 insertions(+), 261 deletions(-) diff --git a/menpofit/clm/fitter.py b/menpofit/clm/fitter.py index 4ad4a56..bf6e499 100644 --- a/menpofit/clm/fitter.py +++ b/menpofit/clm/fitter.py @@ -1,272 +1,47 @@ -from __future__ import division -import numpy as np -from menpo.image import Image +from menpofit import checks +from menpofit.fitter import ModelFitter +from menpofit.modelinstance import OrthoPDM +from .algorithm import CLMAlgorithm, RegularisedLandmarkMeanShift +from .result import CLMFitterResult -from menpofit.transform import DifferentiableAlignmentSimilarity -from menpofit.modelinstance import PDM, OrthoPDM -from menpofit.fitter import MultilevelFitter -from menpofit.gradientdescent import RLMS - -class CLMFitter(MultilevelFitter): +# TODO: Document me! +class CLMFitter(ModelFitter): r""" - Abstract Interface for defining Constrained Local Models Fitters. - - Parameters - ----------- - clm : :map:`CLM` - The Constrained Local Model to be used. """ - def __init__(self, clm): - self.clm = clm - - @property - def reference_shape(self): - r""" - The reference shape of the CLM. - - :type: :map:`PointCloud` - """ - return self.clm.reference_shape - - @property - def features(self): - r""" - The feature extracted at each pyramidal level during CLM building. - Stored in ascending pyramidal order. - - :type: `list` - """ - return self.clm.features - @property - def n_levels(self): - r""" - The number of pyramidal levels used during CLM building. + def clm(self): + return self._model - :type: `int` - """ - return self.clm.n_levels - - @property - def downscale(self): - r""" - The downscale used to generate the final scale factor applied at - each pyramidal level during CLM building. - The scale factor is computed as: - - ``(downscale ** k) for k in range(n_levels)`` - - :type: `float` - """ - return self.clm.downscale + def _fitter_result(self, image, algorithm_results, affine_correction, + gt_shape=None): + return CLMFitterResult(image, self, algorithm_results, + affine_correction, gt_shape=gt_shape) +# TODO: Document me! +# TODO: Rethink shape model and OrthoPDM relation class GradientDescentCLMFitter(CLMFitter): r""" - Gradient Descent based :map:`Fitter` for Constrained Local Models. - - Parameters - ----------- - clm : :map:`CLM` - The Constrained Local Model to be used. - - algorithm : subclass :map:`GradientDescent`, optional - The :map:`GradientDescent` class to be used. - - pdm_transform : :map:`GlobalPDM` or subclass, optional - The point distribution class to be used. - - .. note:: - - Only :map:`GlobalPDM` and its subclasses are supported. - :map:`PDM` is not supported at the moment. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per pyramidal level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of shape components - capturing the specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. """ - def __init__(self, clm, algorithm=RLMS, - pdm_transform=OrthoPDM, n_shape=None, **kwargs): - super(GradientDescentCLMFitter, self).__init__(clm) - self._set_up(algorithm=algorithm, pdm_transform=pdm_transform, - n_shape=n_shape, **kwargs) - - @property - def algorithm(self): - r""" - Returns a string containing the name of fitting algorithm. - - :type: `string` - """ - return 'GD-CLM-' + self._fitters[0].algorithm - - def _set_up(self, algorithm=RLMS, - pdm_transform=OrthoPDM, - global_transform=DifferentiableAlignmentSimilarity, - n_shape=None, **kwargs): - r""" - Sets up the Gradient Descent Fitter object. - - Parameters - ----------- - algorithm : :map:`GradientDescent`, optional - The Gradient Descent class to be used. - - pdm_transform : :map:`GlobalPDM` or subclass, optional - The point distribution class to be used. - - n_shape : `int` ``> 1``, ``0. <=`` `float` ``<= 1.``, `list` of the - previous or ``None``, optional - The number of shape components or amount of shape variance to be - used per fitting level. - - If `None`, all available shape components ``(n_active_components)`` - will be used. - If `int` ``> 1``, the specified number of shape components will be - used. - If ``0. <=`` `float` ``<= 1.``, the number of components capturing the - specified variance ratio will be computed and used. - - If `list` of length ``n_levels``, then the number of components is - defined per level. The first element of the list corresponds to the - lowest pyramidal level and so on. - If not a `list` or a `list` of length 1, then the specified number of - components will be used for all levels. - """ - # check n_shape parameter - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self.clm.n_levels > 1: - for sm in self.clm.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self.clm.n_levels: - for sm, n in zip(self.clm.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None' - 'or a list containing 1 or {} of ' - 'those'.format(self.clm.n_levels)) - - self._fitters = [] - for j, (sm, clf) in enumerate(zip(self.clm.shape_models, - self.clm.classifiers)): - - if pdm_transform is not PDM: - pdm_trans = pdm_transform(sm, global_transform) - else: - pdm_trans = pdm_transform(sm) - self._fitters.append(algorithm(clf, self.clm.patch_shape, - pdm_trans, **kwargs)) - - def __str__(self): - from menpofit.base import name_of_callable - out = "{0} Fitter\n" \ - " - Gradient-Descent {1}\n" \ - " - Transform is {2}.\n" \ - " - {3} training images.\n".format( - self.clm._str_title, self._fitters[0].algorithm, - self._fitters[0].transform.__class__.__name__, - self.clm.n_training_images) - # small strings about number of channels, channels string and downscale - down_str = [] - for j in range(self.n_levels): - if j == self.n_levels - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) - temp_img = Image(image_data=np.random.rand(50, 50)) - if self.pyramid_on_features: - temp = self.features(temp_img) - n_channels = [temp.n_channels] * self.n_levels - else: - n_channels = [] - for j in range(self.n_levels): - temp = self.features[j](temp_img) - n_channels.append(temp.n_channels) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_levels): - if isinstance(self.features[j], str): - feat_str.append("- Feature is {} with ".format( - self.features[j])) - elif self.features[j] is None: - feat_str.append("- No features extracted. ") - else: - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - if self.n_levels > 1: - if self.clm.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.clm.patch_shape[1], - self.clm.patch_shape[0]) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n - Patch size is {}W x " \ - "{}H.\n".format(out, self.n_levels, self.downscale, - self.clm.patch_shape[1], - self.clm.patch_shape[0]) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - out = "{0} - {1} motion components\n - {2} {3} " \ - "classifiers.\n".format( - out, self._fitters[i].transform.n_parameters, - len(self._fitters[i].classifiers), - name_of_callable(self._fitters[i].classifiers[0])) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - {4} motion components\n - {5} {6} " \ - "classifiers.".format( - out, feat_str[0], n_channels[0], ch_str[0], - out, self._fitters[0].transform.n_parameters, - len(self._fitters[0].classifiers), - name_of_callable(self._fitters[0].classifiers[0])) - return out + def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, + n_shape=None): + self._model = clm + self._gd_algorithms_cls = checks.check_algorithm_cls( + gd_algorithm_cls, self.n_scales, CLMAlgorithm) + self._check_n_shape(n_shape) + + # Construct algorithms + self.algorithms = [] + for i in range(self.clm.n_scales): + pdm = OrthoPDM(self.clm.shape_models[i]) + algorithm = self._gd_algorithms_cls[i]( + self.clm.expert_ensembles[i], pdm) + self.algorithms.append(algorithm) + + +# TODO: Implement me! +# TODO: Document me! +class SupervisedDescentCLMFitter(CLMFitter): + r""" + """ From 65b69787d99ed21327f9aff2c1e13cddfbc15578 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Fri, 31 Jul 2015 13:38:03 +0100 Subject: [PATCH 370/423] Add __init__.py to menpofit.clm - Also modified initi from expert --- menpofit/clm/__init__.py | 4 +++- menpofit/clm/expert/__init__.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) mode change 100755 => 100644 menpofit/clm/__init__.py diff --git a/menpofit/clm/__init__.py b/menpofit/clm/__init__.py old mode 100755 new mode 100644 index 18186e9..83b81e9 --- a/menpofit/clm/__init__.py +++ b/menpofit/clm/__init__.py @@ -1,3 +1,5 @@ from .base import CLM -from .builder import CLMBuilder from .fitter import GradientDescentCLMFitter +from .algorithm import RegularisedLandmarkMeanShift +from .expert import ( + CorrelationFilterExpertEnsemble, IncrementalCorrelationFilterThinWrapper) diff --git a/menpofit/clm/expert/__init__.py b/menpofit/clm/expert/__init__.py index 393b44d..673e02b 100644 --- a/menpofit/clm/expert/__init__.py +++ b/menpofit/clm/expert/__init__.py @@ -1 +1,2 @@ from ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble +from base import IncrementalCorrelationFilterThinWrapper From 08d3c40305887717a77351cecbd779f89c614095 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 31 Jul 2015 16:29:39 +0100 Subject: [PATCH 371/423] Refactoring SD-AAM Derive directly from the SDM - abstract out the check appearance model and shape model methods so that we don't need to derive from the AAMFitter as well. Some more, heavier refactoring is about to happen to pull the SD-AAM in line with the new and improved SDM --- menpofit/aam/fitter.py | 140 ++++++++++++----------------------------- menpofit/checks.py | 16 +++++ menpofit/fitter.py | 25 +++----- 3 files changed, 63 insertions(+), 118 deletions(-) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 8fe78c2..58d3404 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -1,12 +1,11 @@ from __future__ import division import numpy as np from copy import deepcopy -from menpo.transform import Scale, AlignmentUniformScale +from menpo.transform import AlignmentUniformScale from menpo.image import BooleanImage -from menpofit.builder import ( - rescale_images_to_reference_shape, compute_features, scale_images) -from menpofit.fitter import ModelFitter +from menpofit.fitter import ModelFitter, noisy_shape_from_bounding_box from menpofit.modelinstance import OrthoPDM +from menpofit.sdm import SupervisedDescentFitter from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform import menpofit.checks as checks from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM @@ -28,20 +27,7 @@ def aam(self): return self._model def _check_n_appearance(self, n_appearance): - if n_appearance is not None: - if type(n_appearance) is int or type(n_appearance) is float: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance - elif len(n_appearance) == 1 and self.aam.n_scales > 1: - for am in self.aam.appearance_models: - am.n_active_components = n_appearance[0] - elif len(n_appearance) == self.aam.n_scales: - for am, n in zip(self.aam.appearance_models, n_appearance): - am.n_active_components = n - else: - raise ValueError('n_appearance can be an integer or a float ' - 'or None or a list containing 1 or {} of ' - 'those'.format(self.aam.n_scales)) + checks.set_models_components(self.aam.appearance_models, n_appearance) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): @@ -106,25 +92,39 @@ def _set_up(self, lk_algorithm_cls, sampling, **kwargs): # TODO: document me! -class SupervisedDescentAAMFitter(AAMFitter): +class SupervisedDescentAAMFitter(SupervisedDescentFitter): r""" """ - def __init__(self, aam, sd_algorithm_cls=ProjectOutNewton, + def __init__(self, images, aam, group=None, bounding_box_group=None, n_shape=None, n_appearance=None, sampling=None, - n_perturbations=10, noise_std=0.05, max_iters=6, **kwargs): - self._model = aam - self._check_n_shape(n_shape) - self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_scales) - self.n_perturbations = n_perturbations - self.noise_std = noise_std - self.max_iters = checks.check_max_iters(max_iters, self.n_scales) - self._set_up(sd_algorithm_cls, sampling, **kwargs) - - def _set_up(self, sd_algorithm_cls, sampling, **kwargs): + sd_algorithm_cls=ProjectOutNewton, + n_iterations=6, n_perturbations=30, + perturb_from_bounding_box=noisy_shape_from_bounding_box, + batch_size=None, verbose=False): + self.aam = aam + checks.set_models_components(aam.appearance_models, n_appearance) + checks.set_models_components(aam.shape_models, n_shape) + self._sampling = checks.check_sampling(sampling, aam.n_scales) + + # patch_feature and patch_shape are not actually + # used because they are fully defined by the AAM already. Therefore, + # we just leave them as their 'defaults' because they won't be used. + super(SupervisedDescentAAMFitter, self).__init__( + images, group=group, bounding_box_group=bounding_box_group, + reference_shape=self.aam.reference_shape, + sd_algorithm_cls=sd_algorithm_cls, + holistic_feature=self.aam.features, + diagonal=self.aam.diagonal, + scales=self.aam.scales, n_iterations=n_iterations, + n_perturbations=n_perturbations, + perturb_from_bounding_box=perturb_from_bounding_box, + batch_size=batch_size, verbose=verbose) + + def _setup_algorithms(self): self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models, sampling)): + self.aam.shape_models, + self._sampling)): if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform @@ -132,9 +132,9 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): sm, self.aam.transform, source=am.mean().landmarks['source'].lms) # set up algorithm using standard aam interface - algorithm = sd_algorithm_cls( + algorithm = self._sd_algorithm_cls( SupervisedDescentStandardInterface, am, md_transform, - sampling=s, max_iters=self.max_iters[j], **kwargs) + sampling=s, max_iters=self.n_iterations[j]) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): @@ -142,19 +142,19 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) # set up algorithm using linear aam interface - algorithm = sd_algorithm_cls( + algorithm = self._sd_algorithm_cls( SupervisedDescentLinearInterface, am, md_transform, - sampling=s, max_iters=self.max_iters[j], **kwargs) + sampling=s, max_iters=self.n_iterations[j]) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) # set up algorithm using parts aam interface - algorithm = sd_algorithm_cls( + algorithm = self._sd_algorithm_cls( SupervisedDescentPartsInterface, am, pdm, - sampling=s, max_iters=self.max_iters[j], + sampling=s, max_iters=self.n_iterations[j], patch_shape=self.aam.patch_shape[j], - normalize_parts=self.aam.normalize_parts, **kwargs) + normalize_parts=self.aam.normalize_parts) else: raise ValueError("AAM object must be of one of the " @@ -165,67 +165,6 @@ def _set_up(self, sd_algorithm_cls, sampling, **kwargs): # append algorithms to list self.algorithms.append(algorithm) - # TODO: Allow training from bounding boxes - def train(self, images, group=None, verbose=False, **kwargs): - # normalize images with respect to reference shape of aam - images = rescale_images_to_reference_shape( - images, group, self.reference_shape, verbose=verbose) - - if self.scale_features: - # compute features at highest level - feature_images = compute_features(images, self.features[0], - verbose=verbose) - - # for each pyramid level (low --> high) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain image representation - if s == self.scales[-1]: - level_images = feature_images - elif self.scale_features: - # scale features at other levels - level_images = scale_images(feature_images, s, - level_str=level_str, - verbose=verbose) - else: - # scale images and compute features at other levels - scaled_images = scale_images(images, s, level_str=level_str, - verbose=verbose) - level_images = compute_features(scaled_images, - self.features[j], - level_str=level_str, - verbose=verbose) - - # extract ground truth shapes for current level - level_gt_shapes = [i.landmarks[group].lms for i in level_images] - - if j == 0: - # generate perturbed shapes - current_shapes = [] - for gt_s in level_gt_shapes: - perturbed_shapes = [] - for _ in range(self.n_perturbations): - p_s = self.noisy_shape_from_shape(gt_s, self.noise_std) - perturbed_shapes.append(p_s) - current_shapes.append(perturbed_shapes) - - # train cascaded regression algorithm - current_shapes = self.algorithms[j].train( - level_images, level_gt_shapes, current_shapes, - verbose=verbose, **kwargs) - - # scale current shapes to next level resolution - if s != self.scales[-1]: - transform = Scale(self.scales[j+1]/s, n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) - # TODO: Document me! def holistic_sampling_from_scale(aam, scale=0.35): @@ -247,6 +186,7 @@ def holistic_sampling_from_scale(aam, scale=0.35): return true_positions, BooleanImage(modified_mask[0]) +# TODO: Document me! def holistic_sampling_from_step(aam, step=8): reference = aam.appearance_models[0].mean() diff --git a/menpofit/checks.py b/menpofit/checks.py index 0547791..3c9a516 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -145,3 +145,19 @@ def check_sampling(sampling, n_levels): 'None'.format(n_levels)) +def set_models_components(models, n_components): + if n_components is not None: + n_scales = len(models) + if type(n_components) is int or type(n_components) is float: + for am in models: + am.n_active_components = n_components + elif len(n_components) == 1 and n_scales > 1: + for am in models: + am.n_active_components = n_components[0] + elif len(n_components) == n_scales: + for am, n in zip(models, n_components): + am.n_active_components = n + else: + raise ValueError('n_components can be an integer or a float ' + 'or None or a list containing 1 or {} of ' + 'those'.format(n_scales)) diff --git a/menpofit/fitter.py b/menpofit/fitter.py index d963fda..07f0ec8 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -255,29 +255,18 @@ def scales(self): return self._model.scales def _check_n_shape(self, n_shape): - if n_shape is not None: - if type(n_shape) is int or type(n_shape) is float: - for sm in self._model.shape_models: - sm.n_active_components = n_shape - elif len(n_shape) == 1 and self._model.n_scales > 1: - for sm in self._model.shape_models: - sm.n_active_components = n_shape[0] - elif len(n_shape) == self._model.n_scales: - for sm, n in zip(self._model.shape_models, n_shape): - sm.n_active_components = n - else: - raise ValueError('n_shape can be an integer or a float or None' - 'or a list containing 1 or {} of ' - 'those'.format(self._model.n_scales)) + checks.set_models_components(self._model.shape_models, n_shape) - def noisy_shape_from_bounding_box(self, bounding_box, noise_std=0.05): + def noisy_shape_from_bounding_box(self, bounding_box, + noise_percentage=0.05): transform = noisy_alignment_similarity_transform( - self.reference_bounding_box, bounding_box, noise_std=noise_std) + self.reference_bounding_box, bounding_box, + noise_percentage=noise_percentage) return transform.apply(self.reference_shape) - def noisy_shape_from_shape(self, shape, noise_std=0.05): + def noisy_shape_from_shape(self, shape, noise_percentage=0.05): return self.noisy_shape_from_bounding_box( - shape.bounding_box(), noise_std=noise_std) + shape.bounding_box(), noise_percentage=noise_percentage) def noisy_alignment_similarity_transform(source, target, noise_type='uniform', From 9335a9635dedb5c8f73d302911288ea851c52f2e Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:31:10 +0100 Subject: [PATCH 372/423] Add clm base algorithm --- menpofit/clm/algorithm/base.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 menpofit/clm/algorithm/base.py diff --git a/menpofit/clm/algorithm/base.py b/menpofit/clm/algorithm/base.py new file mode 100644 index 0000000..7966c2a --- /dev/null +++ b/menpofit/clm/algorithm/base.py @@ -0,0 +1,6 @@ + + +# TODO: document me! +class CLMAlgorithm(object): + r""" + """ From c33f88fba54460f975eea0a3f90373d419d2a57a Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:37:04 +0100 Subject: [PATCH 373/423] Add clm algorithms - ASM and RLMS --- menpofit/clm/algorithm/base.py | 6 - menpofit/clm/algorithm/gd.py | 237 +++++++++++++++++++++++++++++++++ menpofit/clm/fitter.py | 7 +- 3 files changed, 241 insertions(+), 9 deletions(-) delete mode 100644 menpofit/clm/algorithm/base.py create mode 100644 menpofit/clm/algorithm/gd.py diff --git a/menpofit/clm/algorithm/base.py b/menpofit/clm/algorithm/base.py deleted file mode 100644 index 7966c2a..0000000 --- a/menpofit/clm/algorithm/base.py +++ /dev/null @@ -1,6 +0,0 @@ - - -# TODO: document me! -class CLMAlgorithm(object): - r""" - """ diff --git a/menpofit/clm/algorithm/gd.py b/menpofit/clm/algorithm/gd.py new file mode 100644 index 0000000..6a7539b --- /dev/null +++ b/menpofit/clm/algorithm/gd.py @@ -0,0 +1,237 @@ +from __future__ import division +import numpy as np +from menpofit.base import build_grid +from menpofit.clm.result import CLMAlgorithmResult + +multivariate_normal = None # expensive, from scipy.stats + + +# TODO: document me! +class GradientDescentCLMAlgorithm(object): + r""" + """ + + +# TODO: Document me! +class ActiveShapeModel(GradientDescentCLMAlgorithm): + r""" + Active Shape Model (ASM) algorithm + """ + def __init__(self, expert_ensemble, shape_model, gaussian_covariance=10, + eps=10**-5): + # Set parameters + self.expert_ensemble, = expert_ensemble, + self.transform = shape_model + self.gaussian_covariance = gaussian_covariance + self.eps = eps + # Perform pre-computations + self._precompute() + + def _precompute(self): + r""" + """ + # Import multivariate normal distribution from scipy + global multivariate_normal + if multivariate_normal is None: + from scipy.stats import multivariate_normal # expensive + + # Build grid associated to size of the search space + search_size = self.expert_ensemble.search_size + self.half_search_size = np.round( + np.asarray(self.expert_ensemble.search_size) / 2) + self.search_grid = build_grid(search_size)[None, None] + + # set rho2 + self.rho2 = self.transform.model.noise_variance() + + # Compute Gaussian-KDE grid + self.mvn = multivariate_normal(mean=np.zeros(2), + cov=self.gaussian_covariance) + + # Compute shape model prior + sim_prior = np.zeros((4,)) + pdm_prior = self.rho2 / self.transform.model.eigenvalues + self.rho2_inv_L = np.hstack((sim_prior, pdm_prior)) + + # Compute Jacobian + J = np.rollaxis(self.transform.d_dp(None), -1, 1) + self.J = J.reshape((-1, J.shape[-1])) + # Compute inverse Hessian + self.JJ = self.J.T.dot(self.J) + # Compute Jacobian pseudo-inverse + self.pinv_J = np.linalg.solve(self.JJ, self.J.T) + self.inv_JJ_prior = np.linalg.inv(self.JJ + np.diag(self.rho2_inv_L)) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + r""" + """ + # Initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # Initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Expectation-Maximisation loop + while k < max_iters and eps > self.eps: + + target = self.transform.target + # Obtain all landmark positions l_i = (x_i, y_i) being considered + # ie all pixel positions in each landmark's search space + candidate_landmarks = (target.points[:, None, None, None, :] + + self.search_grid) + + # Compute responses + responses = self.expert_ensemble.predict_probability(image, target) + + # Approximate responses using isotropic Gaussian + max_indices = np.argmax( + responses.reshape(responses.shape[:2] + (-1,)), axis=-1) + max_indices = np.unravel_index(max_indices, responses.shape)[-2:] + max_indices = np.hstack((max_indices[0], max_indices[1])) + max_indices = max_indices[:, None, None, None, ...] + max_indices -= self.half_search_size + gaussian_responses = self.mvn.pdf(max_indices + self.search_grid) + # Normalise smoothed responses + gaussian_responses /= np.sum(gaussian_responses, + axis=(-2, -1))[..., None, None] + + # Compute new target + new_target = np.sum(gaussian_responses[:, None, ..., None] * + candidate_landmarks, axis=(-3, -2)) + + # Compute shape error term + error = target.as_vector() - new_target.ravel() + + # Solve for increments on the shape parameters + if map_inference: + Je = (self.rho2_inv_L * self.transform.as_vector() - + self.J.T.dot(error)) + dp = -self.inv_JJ_prior.dot(Je) + else: + dp = self.pinv_J.dot(error) + + # Update pdm + s_k = self.transform.target.points + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # Test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # Increase iteration counter + k += 1 + + # Return algorithm result + return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) + + +# TODO: Document me! +class RegularisedLandmarkMeanShift(GradientDescentCLMAlgorithm): + r""" + Regularized Landmark Mean-Shift (RLMS) algorithm + """ + def __init__(self, expert_ensemble, shape_model, kernel_covariance=10, + eps=10**-5): + # Set parameters + self.expert_ensemble, = expert_ensemble, + self.transform = shape_model + self.kernel_covariance = kernel_covariance + self.eps = eps + # Perform pre-computations + self._precompute() + + def _precompute(self): + r""" + """ + # Import multivariate normal distribution from scipy + global multivariate_normal + if multivariate_normal is None: + from scipy.stats import multivariate_normal # expensive + + # Build grid associated to size of the search space + search_size = self.expert_ensemble.search_size + self.search_grid = build_grid(search_size) + + # set rho2 + self.rho2 = self.transform.model.noise_variance() + + # Compute Gaussian-KDE grid + mvn = multivariate_normal(mean=np.zeros(2), cov=self.kernel_covariance) + self.kernel_grid = mvn.pdf(self.search_grid)[None, None] + + # Compute shape model prior + sim_prior = np.zeros((4,)) + pdm_prior = self.rho2 / self.transform.model.eigenvalues + self.rho2_inv_L = np.hstack((sim_prior, pdm_prior)) + + # Compute Jacobian + J = np.rollaxis(self.transform.d_dp(None), -1, 1) + self.J = J.reshape((-1, J.shape[-1])) + # Compute inverse Hessian + self.JJ = self.J.T.dot(self.J) + # Compute Jacobian pseudo-inverse + self.pinv_J = np.linalg.solve(self.JJ, self.J.T) + self.inv_JJ_prior = np.linalg.inv(self.JJ + np.diag(self.rho2_inv_L)) + + def run(self, image, initial_shape, max_iters=20, gt_shape=None, + map_inference=False): + r""" + """ + # Initialize transform + self.transform.set_target(initial_shape) + p_list = [self.transform.as_vector()] + + # Initialize iteration counter and epsilon + k = 0 + eps = np.Inf + + # Expectation-Maximisation loop + while k < max_iters and eps > self.eps: + + target = self.transform.target + # Obtain all landmark positions l_i = (x_i, y_i) being considered + # ie all pixel positions in each landmark's search space + candidate_landmarks = (target.points[:, None, None, None, :] + + self.search_grid) + + # Compute patch responses + patch_responses = self.expert_ensemble.predict_probability(image, + target) + + # Smooth responses using the Gaussian-KDE grid + patch_kernels = patch_responses * self.kernel_grid + # Normalise smoothed responses + patch_kernels /= np.sum(patch_kernels, + axis=(-2, -1))[..., None, None] + + # Compute mean shift target + mean_shift_target = np.sum(patch_kernels[..., None] * + candidate_landmarks, axis=(-3, -2)) + + # Compute shape error term + error = mean_shift_target.ravel() - target.as_vector() + + # Solve for increments on the shape parameters + if map_inference: + Je = (self.rho2_inv_L * self.transform.as_vector() - + self.J.T.dot(error)) + dp = -self.inv_JJ_prior.dot(Je) + else: + dp = self.pinv_J.dot(error) + + # Update pdm + s_k = self.transform.target.points + self.transform.from_vector_inplace(self.transform.as_vector() + dp) + p_list.append(self.transform.as_vector()) + + # Test convergence + eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) + + # Increase iteration counter + k += 1 + + # Return algorithm result + return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) \ No newline at end of file diff --git a/menpofit/clm/fitter.py b/menpofit/clm/fitter.py index bf6e499..c83c223 100644 --- a/menpofit/clm/fitter.py +++ b/menpofit/clm/fitter.py @@ -1,7 +1,8 @@ from menpofit import checks from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM -from .algorithm import CLMAlgorithm, RegularisedLandmarkMeanShift +from .algorithm import ( + GradientDescentCLMAlgorithm, RegularisedLandmarkMeanShift) from .result import CLMFitterResult @@ -28,10 +29,9 @@ def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, n_shape=None): self._model = clm self._gd_algorithms_cls = checks.check_algorithm_cls( - gd_algorithm_cls, self.n_scales, CLMAlgorithm) + gd_algorithm_cls, self.n_scales, GradientDescentCLMAlgorithm) self._check_n_shape(n_shape) - # Construct algorithms self.algorithms = [] for i in range(self.clm.n_scales): pdm = OrthoPDM(self.clm.shape_models[i]) @@ -45,3 +45,4 @@ def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, class SupervisedDescentCLMFitter(CLMFitter): r""" """ + raise NotImplementedError From 656ee2f69c6a246fb5b8122cf5eb7ffa5f67bb31 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:37:40 +0100 Subject: [PATCH 374/423] Add not implemented supervised descent algorthm for clms - Future template for methods like drmf-clm --- menpofit/clm/algorithm/sd.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 menpofit/clm/algorithm/sd.py diff --git a/menpofit/clm/algorithm/sd.py b/menpofit/clm/algorithm/sd.py new file mode 100644 index 0000000..a77b245 --- /dev/null +++ b/menpofit/clm/algorithm/sd.py @@ -0,0 +1,8 @@ + + +# TODO: implement me! +# TODO: document me! +class SupervisedDescentCLMAlgorithm(object): + r""" + """ + raise NotImplementedError From 7886a9bcf7b97d55086abf395e21c4751e61fada Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:39:01 +0100 Subject: [PATCH 375/423] Add __init__.py --- menpofit/clm/algorithm/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 menpofit/clm/algorithm/__init__.py diff --git a/menpofit/clm/algorithm/__init__.py b/menpofit/clm/algorithm/__init__.py new file mode 100644 index 0000000..1df3fab --- /dev/null +++ b/menpofit/clm/algorithm/__init__.py @@ -0,0 +1,3 @@ +from gd import ( + GradientDescentCLMAlgorithm, ActiveShapeModel, + RegularisedLandmarkMeanShift) From 242813c86a4266b9cf86f9c0d2c60a30d6b8ff21 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 10:48:10 +0100 Subject: [PATCH 376/423] Update __init__.py --- menpofit/clm/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/clm/__init__.py b/menpofit/clm/__init__.py index 83b81e9..9406f54 100644 --- a/menpofit/clm/__init__.py +++ b/menpofit/clm/__init__.py @@ -1,5 +1,5 @@ from .base import CLM from .fitter import GradientDescentCLMFitter -from .algorithm import RegularisedLandmarkMeanShift +from .algorithm import ActiveShapeModel, RegularisedLandmarkMeanShift from .expert import ( CorrelationFilterExpertEnsemble, IncrementalCorrelationFilterThinWrapper) From c4679c9e58370ab1df902614d7efab4afc393b78 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:10:31 +0100 Subject: [PATCH 377/423] Change n_levels for n_scales in aam --- menpofit/aam/base.py | 719 ++++++++++++++++--------------------------- 1 file changed, 263 insertions(+), 456 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index d3a1746..3fe1c63 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,31 +1,34 @@ from __future__ import division -from copy import deepcopy -import warnings import numpy as np -from menpo.feature import no_op -from menpo.visualize import print_dynamic -from menpo.model import PCAModel -from menpo.transform import Scale -from menpo.shape import mean_pointcloud -from menpofit import checks -from menpofit.transform import (DifferentiableThinPlateSplines, - DifferentiablePiecewiseAffine) -from menpofit.base import name_of_callable, batch +from menpo.shape import TriMesh +from menpofit.transform import DifferentiableThinPlateSplines +from menpofit.base import name_of_callable from menpofit.builder import ( - build_reference_frame, build_patch_reference_frame, - compute_features, scale_images, build_shape_model, warp_images, - align_shapes, rescale_images_to_reference_shape, densify_shapes, - extract_patches, MenpoFitBuilderWarning, compute_reference_shape) + build_reference_frame, build_patch_reference_frame) # TODO: document me! class AAM(object): r""" - Active Appearance Models. + Active Appearance Model class. Parameters - ---------- - features : `callable` or ``[callable]``, optional + ----------- + shape_models : :map:`PCAModel` list + A list containing the shape models of the AAM. + + appearance_models : :map:`PCAModel` list + A list containing the appearance models of the AAM. + + reference_shape : :map:`PointCloud` + The reference shape that was used to resize all training images to a + consistent object size. + + transform : :map:`PureAlignmentTransform` + The transform used to warp the images from which the AAM was + constructed. + + features : `callable` or ``[callable]``, If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -38,299 +41,35 @@ class AAM(object): Note that from our experience, this approach of extracting features once and then creating a pyramid on top tends to lead to better performing AAMs. - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - scales : `int` or float` or list of those, optional - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of appearance components - is defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - appearance components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - aam : :map:`AAMBuilder` - The AAM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, images, group=None, verbose=False, reference_shape=None, - features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, - max_appearance_components=None, batch_size=None): - # check parameters - checks.check_diagonal(diagonal) - scales, n_scales = checks.check_scales(scales) - features = checks.check_features(features, n_scales) - max_shape_components = checks.check_max_components( - max_shape_components, n_scales, 'max_shape_components') - max_appearance_components = checks.check_max_components( - max_appearance_components, n_scales, 'max_appearance_components') - # set parameters - self.features = features - self.transform = transform - self.diagonal = diagonal - self.scales = scales - self.max_shape_components = max_shape_components - self.max_appearance_components = max_appearance_components - self.reference_shape = reference_shape - self.shape_models = [] - self.appearance_models = [] - # Train AAM - self._train(images, group=group, verbose=verbose, increment=False, - batch_size=batch_size) + scales : `int` or float` or list of those, optional - def _train(self, images, group=None, verbose=False, increment=False, - shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, - batch_size=None): - r""" - Builds an Active Appearance Model from a list of landmarked images. + scale_shapes : `boolean` - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. + scale_features : `boolean` - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest scale - """ - # If batch_size is not None, then we may have a generator, else we - # assume we have a list. - if batch_size is not None: - # Create a generator of fixed sized batches. Will still work even - # on an infinite list. - image_batches = batch(images, batch_size) - else: - image_batches = [list(images)] - - for k, image_batch in enumerate(image_batches): - # After the first batch, we are incrementing the model - if k > 0: - increment = True - - if verbose: - print('Computing batch {}'.format(k)) - - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(image_batch[0], self.transform, - group=group) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - - feature_images = [] - # for each scale (low --> high) - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' - else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features(image_batch, - self.features[j], - level_str=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - level_str=scale_prefix, - verbose=verbose) - else: - scaled_images = feature_images - - # Extract potentially rescaled shapes - scale_shapes = [i.landmarks[group].lms for i in scaled_images] - - # Build the shape model - if verbose: - print_dynamic('{}Building shape model'.format(scale_prefix)) - - if not increment: - if j == 0: - shape_model = self._build_shape_model( - scale_shapes, self.max_shape_components[j], j) - # Store shape model - self.shape_models.append(shape_model) - else: - # Copy shape model - self.shape_models.append(deepcopy(shape_model)) - else: - self._increment_shape_model( - scale_shapes, self.shape_models[j], - forgetting_factor=shape_forgetting_factor, - max_components=self.max_shape_components[j]) - - # Obtain warped images - we use a scaled version of the - # reference shape, computed here. This is because the mean - # moves when we are incrementing, and we need a consistent - # reference frame. - scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( - self.reference_shape) - warped_images = self._warp_images(scaled_images, scale_shapes, - scaled_reference_shape, - j, scale_prefix, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format( - scale_prefix)) - - if not increment: - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - self.appearance_models.append(appearance_model) - else: - # increment appearance model - self.appearance_models[j].increment( - warped_images, - forgetting_factor=appearance_forgetting_factor) - # trim appearance model if required - if self.max_appearance_components is not None: - self.appearance_models[j].trim_components( - self.max_appearance_components[j]) - - if verbose: - print_dynamic('{}Done\n'.format(scale_prefix)) - - def increment(self, images, group=None, verbose=False, - shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, - batch_size=None): - # Literally just to fit under 80 characters, but maintain the sensible - # parameter name - aff = appearance_forgetting_factor - return self._train(images, group=group, - verbose=verbose, - shape_forgetting_factor=shape_forgetting_factor, - appearance_forgetting_factor=aff, - increment=True, batch_size=batch_size) - - def _build_shape_model(self, shapes, max_components, scale_index): - return build_shape_model(shapes, max_components=max_components) - - def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, - max_components=None): - # Compute aligned shapes - aligned_shapes = align_shapes(shapes) - # Increment shape model - shape_model.increment(aligned_shapes, - forgetting_factor=forgetting_factor) - if max_components is not None: - shape_model.trim_components(max_components) - - def _warp_images(self, images, shapes, reference_shape, scale_index, - level_str, verbose): - reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features @property def n_scales(self): """ - The number of scales of the AAM. + The number of scale levels of the AAM. :type: `int` """ return len(self.scales) + # TODO: Could we directly use class names instead of this? @property def _str_title(self): r""" @@ -339,8 +78,7 @@ def _str_title(self): """ return 'Active Appearance Model' - def instance(self, shape_weights=None, appearance_weights=None, - scale_index=-1): + def instance(self, shape_weights=None, appearance_weights=None, level=-1): r""" Generates a novel AAM instance given a set of shape and appearance weights. If no weights are provided, the mean AAM instance is @@ -352,20 +90,22 @@ def instance(self, shape_weights=None, appearance_weights=None, Weights of the shape model that will be used to create a novel shape instance. If ``None``, the mean shape ``(shape_weights = [0, 0, ..., 0])`` is used. + appearance_weights : ``(n_weights,)`` `ndarray` or `float` list Weights of the appearance model that will be used to create a novel appearance instance. If ``None``, the mean appearance ``(appearance_weights = [0, 0, ..., 0])`` is used. - scale_index : `int`, optional - The scale to be used. + + level : `int`, optional + The pyramidal level to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[scale_index] - am = self.appearance_models[scale_index] + sm = self.shape_models[level] + am = self.appearance_models[level] # TODO: this bit of logic should to be transferred down to PCAModel if shape_weights is None: @@ -379,24 +119,24 @@ def instance(self, shape_weights=None, appearance_weights=None, appearance_weights *= am.eigenvalues[:n_appearance_weights] ** 0.5 appearance_instance = am.instance(appearance_weights) - return self._instance(scale_index, shape_instance, appearance_instance) + return self._instance(level, shape_instance, appearance_instance) - def random_instance(self, scale_index=-1): + def random_instance(self, level=-1): r""" Generates a novel random instance of the AAM. Parameters ----------- - scale_index : `int`, optional - The scale to be used. + level : `int`, optional + The pyramidal level to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[scale_index] - am = self.appearance_models[scale_index] + sm = self.shape_models[level] + am = self.appearance_models[level] # TODO: this bit of logic should to be transferred down to PCAModel shape_weights = (np.random.randn(sm.n_active_components) * @@ -406,18 +146,23 @@ def random_instance(self, scale_index=-1): am.eigenvalues[:am.n_active_components]**0.5) appearance_instance = am.instance(appearance_weights) - return self._instance(scale_index, shape_instance, appearance_instance) + return self._instance(level, shape_instance, appearance_instance) - def _instance(self, scale_index, shape_instance, appearance_instance): - template = self.appearance_models[scale_index].mean() + def _instance(self, level, shape_instance, appearance_instance): + template = self.appearance_models[level].mean() landmarks = template.landmarks['source'].lms - reference_frame = build_reference_frame(shape_instance) + if type(landmarks) == TriMesh: + trilist = landmarks.trilist + else: + trilist = None + reference_frame = build_reference_frame(shape_instance, + trilist=trilist) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - return appearance_instance.as_unmasked(copy=False).warp_to_mask( + return appearance_instance.as_unmasked().warp_to_mask( reference_frame.mask, transform, warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, @@ -462,11 +207,11 @@ def view_appearance_models_widget(self, n_parameters=5, n_parameters : `int` or `list` of `int` or ``None``, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per scale is the minimum + If `int`, then the number of sliders per level is the minimum between `n_parameters` and the number of active components per - scale. - If `list` of `int`, then a number of sliders is defined per scale. - If ``None``, all the active components per scale will have a slider. + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -494,19 +239,19 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, n_shape_parameters : `int` or `list` of `int` or None, optional The number of shape principal components to be used for the parameters sliders. - If `int`, then the number of sliders per scale is the minimum + If `int`, then the number of sliders per level is the minimum between `n_parameters` and the number of active components per - scale. - If `list` of `int`, then a number of sliders is defined per scale. - If ``None``, all the active components per scale will have a slider. + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. n_appearance_parameters : `int` or `list` of `int` or None, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per scale is the minimum + If `int`, then the number of sliders per level is the minimum between `n_parameters` and the number of active components per - scale. - If `list` of `int`, then a number of sliders is defined per scale. - If ``None``, all the active components per scale will have a slider. + level. + If `list` of `int`, then a number of sliders is defined per level. + If ``None``, all the active components per level will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -524,7 +269,106 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: fix me! def __str__(self): - return '' + out = "{}\n - {} training images.\n".format(self._str_title, + self.n_training_images) + # small strings about number of channels, channels string and downscale + n_channels = [] + down_str = [] + for j in range(self.n_scales): + n_channels.append( + self.appearance_models[j].template_instance.n_channels) + if j == self.n_scales - 1: + down_str.append('(no downscale)') + else: + down_str.append('(downscale by {})'.format( + self.downscale**(self.n_scales - j - 1))) + # string about features and channels + if self.pyramid_on_features: + feat_str = "- Feature is {} with ".format( + name_of_callable(self.features)) + if n_channels[0] == 1: + ch_str = ["channel"] + else: + ch_str = ["channels"] + else: + feat_str = [] + ch_str = [] + for j in range(self.n_scales): + feat_str.append("- Feature is {} with ".format( + name_of_callable(self.features[j]))) + if n_channels[j] == 1: + ch_str.append("channel") + else: + ch_str.append("channels") + out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) + if self.n_scales > 1: + if self.scaled_shape_models: + out = "{} - Gaussian pyramid with {} levels and downscale " \ + "factor of {}.\n - Each level has a scaled shape " \ + "model (reference frame).\n".format(out, self.n_scales, + self.downscale) + + else: + out = "{} - Gaussian pyramid with {} levels and downscale " \ + "factor of {}:\n - Shape models (reference frames) " \ + "are not scaled.\n".format(out, self.n_scales, + self.downscale) + if self.pyramid_on_features: + out = "{} - Pyramid was applied on feature space.\n " \ + "{}{} {} per image.\n".format(out, feat_str, + n_channels[0], ch_str[0]) + if not self.scaled_shape_models: + out = "{} - Reference frames of length {} " \ + "({} x {}C, {} x {}C)\n".format( + out, + self.appearance_models[0].n_features, + self.appearance_models[0].template_instance.n_true_pixels(), + n_channels[0], + self.appearance_models[0].template_instance._str_shape, + n_channels[0]) + else: + out = "{} - Features were extracted at each pyramid " \ + "level.\n".format(out) + for i in range(self.n_scales - 1, -1, -1): + out = "{} - Level {} {}: \n".format(out, self.n_scales - i, + down_str[i]) + if not self.pyramid_on_features: + out = "{} {}{} {} per image.\n".format( + out, feat_str[i], n_channels[i], ch_str[i]) + if (self.scaled_shape_models or + (not self.pyramid_on_features)): + out = "{} - Reference frame of length {} " \ + "({} x {}C, {} x {}C)\n".format( + out, self.appearance_models[i].n_features, + self.appearance_models[i].template_instance.n_true_pixels(), + n_channels[i], + self.appearance_models[i].template_instance._str_shape, + n_channels[i]) + out = "{0} - {1} shape components ({2:.2f}% of " \ + "variance)\n - {3} appearance components " \ + "({4:.2f}% of variance)\n".format( + out, self.shape_models[i].n_components, + self.shape_models[i].variance_ratio() * 100, + self.appearance_models[i].n_components, + self.appearance_models[i].variance_ratio() * 100) + else: + if self.pyramid_on_features: + feat_str = [feat_str] + out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ + " - Reference frame of length {4} ({5} x {6}C, " \ + "{7} x {8}C)\n - {9} shape components ({10:.2f}% of " \ + "variance)\n - {11} appearance components ({12:.2f}% of " \ + "variance)\n".format( + out, feat_str[0], n_channels[0], ch_str[0], + self.appearance_models[0].n_features, + self.appearance_models[0].template_instance.n_true_pixels(), + n_channels[0], + self.appearance_models[0].template_instance._str_shape, + n_channels[0], self.shape_models[0].n_components, + self.shape_models[0].variance_ratio() * 100, + self.appearance_models[0].n_components, + self.appearance_models[0].variance_ratio() * 100) + return out # TODO: document me! @@ -536,18 +380,22 @@ class PatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. + appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. + reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. + patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. + features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at - each scale after downscaling of the image. + each level after downscaling of the image. The first element of the list specifies the features to be extracted at - the lowest scale and so on. + the lowest pyramidal level and so on. If ``callable`` the specified feature will be applied to the original image and pyramid generation will be performed on top of the feature @@ -558,35 +406,30 @@ class PatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those + scale_shapes : `boolean` - """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - max_shape_components=None, max_appearance_components=None, - batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - - super(PatchAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, max_shape_components=max_shape_components, - max_appearance_components=max_appearance_components, - batch_size=batch_size) - - def _warp_images(self, images, shapes, reference_shape, scale_index, - level_str, verbose): - reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[scale_index]) - return warp_images(images, shapes, reference_frame, self.transform, - level_str=level_str, verbose=verbose) + scale_features : `boolean` + + """ + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, scales, scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = DifferentiableThinPlateSplines + self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features @property def _str_title(self): return 'Patch-Based Active Appearance Model' - def _instance(self, scale_index, shape_instance, appearance_instance): - template = self.appearance_models[scale_index].mean + def _instance(self, level, shape_instance, appearance_instance): + template = self.appearance_models[level].mean landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( @@ -626,14 +469,18 @@ class LinearAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. + appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. + reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. + transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. + features : `callable` or ``[callable]``, optional If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. @@ -649,50 +496,27 @@ class LinearAAM(AAM): performing AAMs. scales : `int` or float` or list of those - """ - def __init__(self, images, group=None, verbose=False, features=no_op, - transform=DifferentiableThinPlateSplines, diagonal=None, - scales=(0.5, 1.0), max_shape_components=None, - max_appearance_components=None, batch_size=None): - - super(LinearAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, scales=scales, - max_shape_components=max_shape_components, - max_appearance_components=max_appearance_components, - batch_size=batch_size) - - def _build_shape_model(self, shapes, max_components, scale_index): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_reference_frame(mean_aligned_shape) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model( - dense_shapes, max_components=max_components) - return shape_model - - def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, - max_components=None): - aligned_shapes = align_shapes(shapes) - dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, - self.transform) - # Increment shape model - shape_model.increment(dense_shapes, - forgetting_factor=forgetting_factor) - if max_components is not None: - shape_model.trim_components(max_components) - - def _warp_images(self, images, shapes, reference_shape, scale_index, - level_str, verbose): - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) + scale_shapes : `boolean` + + scale_features : `boolean` + + """ + def __init__(self, shape_models, appearance_models, reference_shape, + transform, features, scales, scale_shapes, scale_features, + n_landmarks): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = transform + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks # TODO: implement me! - def _instance(self, scale_index, shape_instance, appearance_instance): + def _instance(self, level, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -721,13 +545,17 @@ class LinearPatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. + appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. + reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. + patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. + features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. @@ -743,52 +571,30 @@ class LinearPatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - max_shape_components=None, max_appearance_components=None, - batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - - super(LinearPatchAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, max_shape_components=max_shape_components, - max_appearance_components=max_appearance_components, - batch_size=batch_size) - - def _build_shape_model(self, shapes, max_components, scale_index): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model(dense_shapes, - max_components=max_components) - return shape_model - - def _increment_shape_model(self, shapes, shape_model, forgetting_factor=1.0, - max_components=None): - aligned_shapes = align_shapes(shapes) - dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, - self.transform) - # Increment shape model - shape_model.increment(dense_shapes, - forgetting_factor=forgetting_factor) - if max_components is not None: - shape_model.trim_components(max_components) - - def _warp_images(self, images, shapes, reference_shape, scale_index, - level_str, verbose): - return warp_images(images, shapes, self.reference_frame, - self.transform, level_str=level_str, - verbose=verbose) + scale_shapes : `boolean` + + scale_features : `boolean` + + n_landmarks: `int` + + """ + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, scales, scale_shapes, + scale_features, n_landmarks): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.transform = DifferentiableThinPlateSplines + self.patch_shape = patch_shape + self.features = features + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features + self.n_landmarks = n_landmarks # TODO: implement me! - def _instance(self, scale_index, shape_instance, appearance_instance): + def _instance(self, level, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -809,7 +615,6 @@ def __str__(self): # TODO: document me! -# TODO: implement offsets support? class PartsAAM(AAM): r""" Parts based Active Appearance Model class. @@ -818,13 +623,17 @@ class PartsAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. + appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. + reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. + patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. + features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. @@ -840,31 +649,29 @@ class PartsAAM(AAM): performing AAMs. normalize_parts: `callable` + scales : `int` or float` or list of those - """ - def __init__(self, images, group=None, verbose=False, features=no_op, - normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, - max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - self.normalize_parts = normalize_parts + scale_shapes : `boolean` - super(PartsAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=DifferentiableThinPlateSplines, diagonal=diagonal, - scales=scales, max_shape_components=max_shape_components, - max_appearance_components=max_appearance_components, - batch_size=batch_size) + scale_features : `boolean` - def _warp_images(self, images, shapes, reference_shape, scale_index, - level_str, verbose): - return extract_patches(images, shapes, self.patch_shape[scale_index], - normalize_function=self.normalize_parts, - level_str=level_str, verbose=verbose) + """ + def __init__(self, shape_models, appearance_models, reference_shape, + patch_shape, features, normalize_parts, scales, + scale_shapes, scale_features): + self.shape_models = shape_models + self.appearance_models = appearance_models + self.patch_shape = patch_shape + self.features = features + self.normalize_parts = normalize_parts + self.reference_shape = reference_shape + self.scales = scales + self.scale_shapes = scale_shapes + self.scale_features = scale_features # TODO: implement me! - def _instance(self, scale_index, shape_instance, appearance_instance): + def _instance(self, level, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! From 7785306fd2473a217c4e8498480f63776a176237 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:26:19 +0100 Subject: [PATCH 378/423] Change n_levels for n_scales in atm --- menpofit/atm/algorithm.py | 4 ++-- menpofit/atm/base.py | 20 ++++++++++---------- menpofit/atm/builder.py | 28 ++++++++++++++-------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 43e27a1..d92df72 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -220,8 +220,8 @@ class Compositional(LucasKanade): def run(self, image, initial_shape, gt_shape=None, max_iters=20, map_inference=False): # define cost closure - def cost_closure(x, f): - return lambda: x.T.dot(f(x)) + def cost_closure(x): + return lambda: x.T.dot(x) # initialize transform self.transform.set_target(initial_shape) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 63ccbc4..c11455f 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -60,7 +60,7 @@ def __init__(self, shape_models, warped_templates, reference_shape, self.scale_features = scale_features @property - def n_levels(self): + def n_scales(self): """ The number of scale level of the ATM. @@ -219,14 +219,14 @@ def __str__(self): # small strings about number of channels, channels string and downscale n_channels = [] down_str = [] - for j in range(self.n_levels): + for j in range(self.n_scales): n_channels.append( self.warped_templates[j].n_channels) - if j == self.n_levels - 1: + if j == self.n_scales - 1: down_str.append('(no downscale)') else: down_str.append('(downscale by {})'.format( - self.downscale**(self.n_levels - j - 1))) + self.downscale**(self.n_scales - j - 1))) # string about features and channels if self.pyramid_on_features: feat_str = "- Feature is {} with ".format( @@ -238,7 +238,7 @@ def __str__(self): else: feat_str = [] ch_str = [] - for j in range(self.n_levels): + for j in range(self.n_scales): feat_str.append("- Feature is {} with ".format( name_of_callable(self.features[j]))) if n_channels[j] == 1: @@ -246,17 +246,17 @@ def __str__(self): else: ch_str.append("channels") out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_levels > 1: + if self.n_scales > 1: if self.scaled_shape_models: out = "{} - Gaussian pyramid with {} levels and downscale " \ "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_levels, + "model (reference frame).\n".format(out, self.n_scales, self.downscale) else: out = "{} - Gaussian pyramid with {} levels and downscale " \ "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_levels, + "are not scaled.\n".format(out, self.n_scales, self.downscale) if self.pyramid_on_features: out = "{} - Pyramid was applied on feature space.\n " \ @@ -275,8 +275,8 @@ def __str__(self): else: out = "{} - Features were extracted at each pyramid " \ "level.\n".format(out) - for i in range(self.n_levels - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_levels - i, + for i in range(self.n_scales - 1, -1, -1): + out = "{} - Level {} {}: \n".format(out, self.n_scales - i, down_str[i]) if not self.pyramid_on_features: out = "{} {}{} {} per image.\n".format( diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py index 3f0c8c4..da049f0 100644 --- a/menpofit/atm/builder.py +++ b/menpofit/atm/builder.py @@ -109,8 +109,8 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -153,7 +153,7 @@ def build(self, shapes, template, group=None, verbose=False): verbose=verbose) # normalize the template size using the reference_shape scaling - template = template.rescale_to_reference_shape( + template = template.rescale_to_pointcloud( reference_shape, group=group) # build models at each scale @@ -339,9 +339,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scale_features=True, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -470,8 +470,8 @@ def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -605,9 +605,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, scale_features=True, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') @@ -735,9 +735,9 @@ def __init__(self, patch_shape=(17, 17), features=no_op, max_shape_components=None): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, n_levels) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) max_shape_components = checks.check_max_components( max_shape_components, len(scales), 'max_shape_components') From 23f617aed2233ab5883f4f6573a0384ec4c465ec Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:30:08 +0100 Subject: [PATCH 379/423] Change n_levels for n_scales in lk --- menpofit/lk/fitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index e00501c..11130c2 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -19,8 +19,8 @@ def __init__(self, template, group=None, features=no_op, **kwargs): # check parameters checks.check_diagonal(diagonal) - scales, n_levels = checks.check_scales(scales) - features = checks.check_features(features, n_levels) + scales = checks.check_scales(scales) + features = checks.check_features(features, len(scales)) scale_features = checks.check_scale_features(scale_features, features) # set parameters self.features = features From ca68468a775c24607ea52dd7fce305b25c781780 Mon Sep 17 00:00:00 2001 From: Joan Alabort Date: Mon, 3 Aug 2015 11:37:58 +0100 Subject: [PATCH 380/423] Final changes - Update builder.py - Update checks.py - Other minor changes --- menpofit/base.py | 14 +- menpofit/builder.py | 72 +- menpofit/checks.py | 107 +- menpofit/clm/algorithm/sd.py | 3 +- menpofit/clm/builder.py | 361 ----- menpofit/clm/classifier.py | 19 - menpofit/clm/fitter.py | 3 +- menpofit/fitter.py | 21 +- menpofit/fittingresult.py | 1258 ----------------- menpofit/gradientdescent/__init__.py | 1 - menpofit/gradientdescent/base.py | 201 --- menpofit/gradientdescent/residual.py | 104 -- menpofit/lucaskanade/__init__.py | 3 - menpofit/lucaskanade/appearance/__init__.py | 3 - .../lucaskanade/appearance/alternating.py | 174 --- menpofit/lucaskanade/appearance/base.py | 21 - menpofit/lucaskanade/appearance/projectout.py | 59 - .../lucaskanade/appearance/simultaneous.py | 184 --- menpofit/lucaskanade/base.py | 91 -- menpofit/lucaskanade/image.py | 184 --- menpofit/lucaskanade/residual.py | 573 -------- menpofit/regression/__init__.py | 0 menpofit/regression/base.py | 295 ---- menpofit/regression/parametricfeatures.py | 176 --- menpofit/regression/regressors.py | 152 -- menpofit/regression/trainer.py | 649 --------- menpofit/transform/modeldriven.py | 1 + menpofit/visualize/widgets/base.py | 2 +- 28 files changed, 140 insertions(+), 4591 deletions(-) delete mode 100644 menpofit/clm/builder.py delete mode 100644 menpofit/clm/classifier.py delete mode 100644 menpofit/fittingresult.py delete mode 100755 menpofit/gradientdescent/__init__.py delete mode 100644 menpofit/gradientdescent/base.py delete mode 100755 menpofit/gradientdescent/residual.py delete mode 100755 menpofit/lucaskanade/__init__.py delete mode 100644 menpofit/lucaskanade/appearance/__init__.py delete mode 100644 menpofit/lucaskanade/appearance/alternating.py delete mode 100644 menpofit/lucaskanade/appearance/base.py delete mode 100644 menpofit/lucaskanade/appearance/projectout.py delete mode 100644 menpofit/lucaskanade/appearance/simultaneous.py delete mode 100644 menpofit/lucaskanade/base.py delete mode 100644 menpofit/lucaskanade/image.py delete mode 100755 menpofit/lucaskanade/residual.py delete mode 100644 menpofit/regression/__init__.py delete mode 100644 menpofit/regression/base.py delete mode 100644 menpofit/regression/parametricfeatures.py delete mode 100644 menpofit/regression/regressors.py delete mode 100644 menpofit/regression/trainer.py diff --git a/menpofit/base.py b/menpofit/base.py index 863328a..cdedcec 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -111,13 +111,13 @@ def pyramid_on_features(self): return is_pyramid_on_features(self.features) - -def build_sampling_grid(patch_shape): +def build_grid(shape): r""" """ - patch_shape = np.array(patch_shape) - patch_half_shape = np.require(np.floor(patch_shape / 2), dtype=int) - start = -patch_half_shape - end = patch_half_shape + 1 + shape = np.asarray(shape) + half_shape = np.floor(shape / 2) + half_shape = np.require(half_shape, dtype=int) + start = -half_shape + end = half_shape + shape % 2 sampling_grid = np.mgrid[start[0]:end[0], start[1]:end[1]] - return sampling_grid.swapaxes(0, 2).swapaxes(0, 1) + return np.rollaxis(sampling_grid, 0, 3) diff --git a/menpofit/builder.py b/menpofit/builder.py index a55f832..b148916 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -54,11 +54,10 @@ def rescale_images_to_reference_shape(images, group, reference_shape, r""" """ wrap = partial(print_progress, prefix='- Normalizing images size', - verbose=verbose) + end_with_newline=False, verbose=verbose) # Normalize the scaling of all images wrt the reference_shape size - normalized_images = [i.rescale_to_reference_shape(reference_shape, - group=group) + normalized_images = [i.rescale_to_pointcloud(reference_shape, group=group) for i in wrap(images)] return normalized_images @@ -117,19 +116,19 @@ def normalization_wrt_reference_shape(images, group, diagonal, verbose=False): # TODO: document me! -def compute_features(images, features, level_str='', verbose=False): +def compute_features(images, features, prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Computing feature space'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Computing feature space'.format(prefix), + end_with_newline=not prefix, verbose=verbose) return [features(i) for i in wrap(images)] # TODO: document me! -def scale_images(images, scale, level_str='', verbose=False): +def scale_images(images, scale, prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Scaling images'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Scaling images'.format(prefix), + end_with_newline=not prefix, verbose=verbose) if not np.allclose(scale, 1): return [i.rescale(scale) for i in wrap(images)] @@ -138,11 +137,11 @@ def scale_images(images, scale, level_str='', verbose=False): # TODO: document me! -def warp_images(images, shapes, reference_frame, transform, level_str='', +def warp_images(images, shapes, reference_frame, transform, prefix='', verbose=None): wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Warping images'.format(prefix), + end_with_newline=not prefix, verbose=verbose) warped_images = [] # Build a dummy transform, use set_target for efficiency @@ -161,10 +160,10 @@ def warp_images(images, shapes, reference_frame, transform, level_str='', # TODO: document me! def extract_patches(images, shapes, patch_shape, normalize_function=no_op, - level_str='', verbose=False): + prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Warping images'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Warping images'.format(prefix), + end_with_newline=not prefix, verbose=verbose) parts_images = [] for i, s in wrap(zip(images, shapes)): @@ -174,7 +173,9 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images.append(Image(parts)) return parts_images -def build_reference_frame(landmarks, boundary=3, group='source'): + +def build_reference_frame(landmarks, boundary=3, group='source', + trilist=None): r""" Builds a reference frame from a particular set of landmarks. @@ -182,14 +183,22 @@ def build_reference_frame(landmarks, boundary=3, group='source'): ---------- landmarks : :map:`PointCloud` The landmarks that will be used to build the reference frame. + boundary : `int`, optional The number of pixels to be left as a safe margin on the boundaries of the reference frame (has potential effects on the gradient computation). + group : `string`, optional Group that will be assigned to the provided set of landmarks on the reference frame. + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. + + If ``None``, defaults to performing Delaunay triangulation on the + points. + Returns ------- reference_frame : :map:`Image` @@ -197,13 +206,14 @@ def build_reference_frame(landmarks, boundary=3, group='source'): """ reference_frame = _build_reference_frame(landmarks, boundary=boundary, group=group) - source_landmarks = reference_frame.landmarks['source'].lms - if isinstance(source_landmarks, TriMesh): - trilist = source_landmarks.trilist - else: - trilist = None + if trilist is not None: + reference_frame.landmarks[group] = TriMesh( + reference_frame.landmarks['source'].lms.points, trilist=trilist) + # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, + # perhaps the trilist should be directly obtained from the group landmarks reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) + return reference_frame @@ -282,7 +292,8 @@ def align_shapes(shapes): return [s.aligned_source() for s in gpa.transforms] -def build_shape_model(shapes, max_components=None): +# TODO: rethink OrthoPDM, should this function be its constructor? +def build_shape_model(shapes, max_components=None, prefix='', verbose=False): r""" Builds a shape model given a set of shapes. @@ -301,6 +312,8 @@ def build_shape_model(shapes, max_components=None): shape_model: :class:`menpo.model.pca` The PCA shape model. """ + if verbose: + print_dynamic('{}Building shape model'.format(prefix)) # compute aligned shapes aligned_shapes = align_shapes(shapes) # build shape model @@ -308,12 +321,19 @@ def build_shape_model(shapes, max_components=None): if max_components is not None: # trim shape model if required shape_model.trim_components(max_components) - return shape_model -class MenpoFitBuilderWarning(Warning): +def increment_shape_model(shape_model, shapes, forgetting_factor=None, + max_components=None, prefix='', verbose=False): r""" - A warning that some part of building the model may cause issues. """ - pass + if verbose: + print_dynamic('{}Incrementing shape model'.format(prefix)) + # compute aligned shapes + aligned_shapes = align_shapes(shapes) + # increment shape model + shape_model.increment(aligned_shapes, forgetting_factor=forgetting_factor) + if max_components is not None: + shape_model.trim_components(max_components) + return shape_model diff --git a/menpofit/checks.py b/menpofit/checks.py index 3c9a516..5c954c7 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -1,3 +1,5 @@ +import warnings +from functools import partial import numpy as np import warnings from menpo.shape import TriMesh @@ -11,6 +13,7 @@ def check_diagonal(diagonal): """ if diagonal is not None and diagonal < 20: raise ValueError("diagonal must be >= 20") + return diagonal def check_trilist(image, transform, group=None): @@ -27,19 +30,17 @@ def check_trilist(image, transform, group=None): # TODO: document me! def check_scales(scales): if isinstance(scales, (int, float)): - return [scales], 1 + return [scales] elif len(scales) == 1 and isinstance(scales[0], (int, float)): - return list(scales), 1 + return list(scales) elif len(scales) > 1: - l1, n1 = check_scales(scales[0]) - l2, n2 = check_scales(scales[1:]) - return l1 + l2, n1 + n2 + return check_scales(scales[0]) + check_scales(scales[1:]) else: raise ValueError("scales must be an int/float or a list/tuple of " "int/float") -def check_features(features, n_levels): +def check_features(features, n_scales): r""" Checks the feature type per level. @@ -56,34 +57,48 @@ def check_features(features, n_levels): A list of feature function. """ if callable(features): - return [features] * n_levels + return [features] * n_scales elif len(features) == 1 and np.alltrue([callable(f) for f in features]): - return list(features) * n_levels - elif len(features) == n_levels and np.alltrue([callable(f) + return list(features) * n_scales + elif len(features) == n_scales and np.alltrue([callable(f) for f in features]): return list(features) else: raise ValueError("features must be a callable or a list/tuple of " - "callables with the same length as scales") + "callables with the same length as the number " + "of scales") + + +# TODO: document me! +def check_scale_features(scale_features, features): + r""" + """ + if np.alltrue([f == features[0] for f in features]): + return scale_features + else: + warnings.warn('scale_features has been automatically set to False ' + 'because different types of features are used at each ' + 'level.') + return False # TODO: document me! -def check_patch_shape(patch_shape, n_levels): +def check_patch_shape(patch_shape, n_scales): if len(patch_shape) == 2 and isinstance(patch_shape[0], int): - return [patch_shape] * n_levels + return [patch_shape] * n_scales elif len(patch_shape) == 1: return check_patch_shape(patch_shape[0], 1) - elif len(patch_shape) == n_levels: + elif len(patch_shape) == n_scales: l1 = check_patch_shape(patch_shape[0], 1) - l2 = check_patch_shape(patch_shape[1:], n_levels-1) + l2 = check_patch_shape(patch_shape[1:], n_scales-1) return l1 + l2 else: raise ValueError("patch_shape must be a list/tuple of int or a " "list/tuple of lit/tuple of int/float with the " - "same length as scales") + "same length as the number of scales") -def check_max_components(max_components, n_levels, var_name): +def check_max_components(max_components, n_scales, var_name): r""" Checks the maximum number of components per level either of the shape or the appearance model. It must be None or int or float or a list of @@ -91,12 +106,12 @@ def check_max_components(max_components, n_levels, var_name): """ str_error = ("{} must be None or an int > 0 or a 0 <= float <= 1 or " "a list of those containing 1 or {} elements").format( - var_name, n_levels) + var_name, n_scales) if not isinstance(max_components, (list, tuple)): - max_components_list = [max_components] * n_levels + max_components_list = [max_components] * n_scales elif len(max_components) == 1: - max_components_list = [max_components[0]] * n_levels - elif len(max_components) == n_levels: + max_components_list = [max_components[0]] * n_scales + elif len(max_components) == n_scales: max_components_list = max_components else: raise ValueError(str_error) @@ -109,41 +124,40 @@ def check_max_components(max_components, n_levels, var_name): # TODO: document me! -def check_max_iters(max_iters, n_levels): +def check_max_iters(max_iters, n_scales): if type(max_iters) is int: - max_iters = [np.round(max_iters/n_levels) - for _ in range(n_levels)] - elif len(max_iters) == 1 and n_levels > 1: - max_iters = [np.round(max_iters[0]/n_levels) - for _ in range(n_levels)] - elif len(max_iters) != n_levels: + max_iters = [np.round(max_iters/n_scales) + for _ in range(n_scales)] + elif len(max_iters) == 1 and n_scales > 1: + max_iters = [np.round(max_iters[0]/n_scales) + for _ in range(n_scales)] + elif len(max_iters) != n_scales: raise ValueError('max_iters can be integer, integer list ' 'containing 1 or {} elements or ' - 'None'.format(n_levels)) + 'None'.format(n_scales)) return np.require(max_iters, dtype=np.int) # TODO: document me! -def check_sampling(sampling, n_levels): +def check_sampling(sampling, n_scales): if (isinstance(sampling, (list, tuple)) and np.alltrue([isinstance(s, (np.ndarray, np.int)) or sampling is None for s in sampling])): if len(sampling) == 1: - return sampling * n_levels - elif len(sampling) == n_levels: + return sampling * n_scales + elif len(sampling) == n_scales: return sampling else: raise ValueError('A sampling list can only ' 'contain 1 element or {} ' - 'elements'.format(n_levels)) + 'elements'.format(n_scales)) elif isinstance(sampling, (np.ndarray, np.int)) or sampling is None: - return [sampling] * n_levels + return [sampling] * n_scales else: raise ValueError('sampling can be an integer or ndarray, ' 'a integer or ndarray list ' 'containing 1 or {} elements or ' - 'None'.format(n_levels)) - + 'None'.format(n_scales)) def set_models_components(models, n_components): if n_components is not None: @@ -161,3 +175,26 @@ def set_models_components(models, n_components): raise ValueError('n_components can be an integer or a float ' 'or None or a list containing 1 or {} of ' 'those'.format(n_scales)) + + +def check_algorithm_cls(algorithm_cls, n_scales, base_algorithm_cls): + r""" + """ + if (isinstance(algorithm_cls, partial) and + base_algorithm_cls in algorithm_cls.func.mro()): + return [algorithm_cls] * n_scales + elif (isinstance(algorithm_cls, type) and + base_algorithm_cls in algorithm_cls.mro()): + return [algorithm_cls] * n_scales + elif len(algorithm_cls) == 1: + return check_algorithm_cls(algorithm_cls[0], n_scales, + base_algorithm_cls) + elif len(algorithm_cls) == n_scales: + return [check_algorithm_cls(a, 1, base_algorithm_cls)[0] + for a in algorithm_cls] + else: + raise ValueError("algorithm_cls must be a subclass of {} or a " + "list/tuple of {} subclasses with the same length " + "as the number of scales {}" + .format(base_algorithm_cls, base_algorithm_cls, + n_scales)) diff --git a/menpofit/clm/algorithm/sd.py b/menpofit/clm/algorithm/sd.py index a77b245..29d9bbc 100644 --- a/menpofit/clm/algorithm/sd.py +++ b/menpofit/clm/algorithm/sd.py @@ -5,4 +5,5 @@ class SupervisedDescentCLMAlgorithm(object): r""" """ - raise NotImplementedError + def __init__(self): + raise NotImplementedError diff --git a/menpofit/clm/builder.py b/menpofit/clm/builder.py deleted file mode 100644 index 4f05930..0000000 --- a/menpofit/clm/builder.py +++ /dev/null @@ -1,361 +0,0 @@ -from __future__ import division, print_function -import numpy as np -from menpo.feature import sparse_hog -from menpo.visualize import print_dynamic, progress_bar_str - -from menpofit import checks -from menpofit.base import create_pyramid, build_sampling_grid -from menpofit.builder import (DeformableModelBuilder, build_shape_model, - normalization_wrt_reference_shape) -from .classifier import linear_svm_lr - - -class CLMBuilder(DeformableModelBuilder): - r""" - Class that builds Multilevel Constrained Local Models. - - Parameters - ---------- - classifier_trainers : ``callable -> callable`` or ``[callable -> callable]`` - - Each ``classifier_trainers`` is a callable that will be invoked as: - - classifer = classifier_trainer(X, t) - - where X is a matrix of samples and t is a matrix of classifications - for each sample. `classifier` is then itself a callable, - which will be used to classify novel instance by the CLM. - - If list of length ``n_levels``, then a classifier_trainer callable is - defined per level. The first element of the list specifies the - classifier_trainer to be used at the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified - classifier_trainer will be used for all levels. - - - Examples of such classifier trainers can be found in - `menpo.fitmultilevel.clm.classifier` - - patch_shape : tuple of `int` - The shape of the patches used by the classifier trainers. - - features : `callable` or ``[callable]``, optional - If list of length ``n_levels``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - - normalization_diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the ``normalization_diagonal`` - value. - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - n_levels : `int` > ``0``, optional - The number of multi-resolution pyramidal levels to be used. - - downscale : `float` >= ``1``, optional - The downscale factor that will be used to create the different - pyramidal levels. The scale factor will be:: - - (downscale ** k) for k in range(n_levels) - - scaled_shape_models : `boolean`, optional - If ``True``, the reference frames will be the mean shapes of each - pyramid level, so the shape models will be scaled. - - If ``False``, the reference frames of all levels will be the mean shape - of the highest level, so the shape models will not be scaled; they will - have the same size. - - max_shape_components : ``None`` or `int` > ``0`` or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_levels``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - boundary : `int` >= ``0``, optional - The number of pixels to be left as a safe margin on the boundaries - of the reference frame (has potential effects on the gradient - computation). - - Returns - ------- - clm : :map:`CLMBuilder` - The CLM Builder object - """ - def __init__(self, classifier_trainers=linear_svm_lr, patch_shape=(5, 5), - features=sparse_hog, normalization_diagonal=None, - n_levels=3, downscale=1.1, scaled_shape_models=True, - max_shape_components=None, boundary=3): - - # general deformable model checks - checks.check_n_levels(n_levels) - checks.check_downscale(downscale) - checks.check_normalization_diagonal(normalization_diagonal) - checks.check_boundary(boundary) - max_shape_components = checks.check_max_components( - max_shape_components, n_levels, 'max_shape_components') - features = checks.check_features(features, n_levels) - - # CLM specific checks - classifier_trainers = check_classifier_trainers(classifier_trainers, n_levels) - patch_shape = check_patch_shape(patch_shape) - - # store parameters - self.classifier_trainers = classifier_trainers - self.patch_shape = patch_shape - self.features = features - self.normalization_diagonal = normalization_diagonal - self.n_levels = n_levels - self.downscale = downscale - self.scaled_shape_models = scaled_shape_models - self.max_shape_components = max_shape_components - self.boundary = boundary - - def build(self, images, group=None, label=None, verbose=False): - r""" - Builds a Multilevel Constrained Local Model from a list of - landmarked images. - - Parameters - ---------- - images : list of :map:`Image` - The set of landmarked images from which to build the AAM. - group : string, Optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - label : `string`, optional - The label of of the landmark manager that you wish to use. If - ``None``, the convex hull of all landmarks is used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - clm : :map:`CLM` - The CLM object - """ - # compute reference_shape and normalize images size - self.reference_shape, normalized_images = \ - normalization_wrt_reference_shape( - images, group, label, self.normalization_diagonal, - verbose=verbose) - - # create pyramid - generators = create_pyramid(normalized_images, self.n_levels, - self.downscale, self.features, - verbose=verbose) - - # build the model at each pyramid level - if verbose: - if self.n_levels > 1: - print_dynamic('- Building model for each of the {} pyramid ' - 'levels\n'.format(self.n_levels)) - else: - print_dynamic('- Building model\n') - - shape_models = [] - classifiers = [] - # for each pyramid level (high --> low) - for j in range(self.n_levels): - # since models are built from highest to lowest level, the - # parameters of type list need to use a reversed index - rj = self.n_levels - j - 1 - - if verbose: - level_str = ' - ' - if self.n_levels > 1: - level_str = ' - Level {}: '.format(j + 1) - - # get images of current level - feature_images = [] - for c, g in enumerate(generators): - if verbose: - print_dynamic( - '{}Computing feature space/rescaling - {}'.format( - level_str, - progress_bar_str((c + 1.) / len(generators), - show_bar=False))) - feature_images.append(next(g)) - - # extract potentially rescaled shapes - shapes = [i.landmarks[group][label] for i in feature_images] - - # define shapes that will be used for training - if j == 0: - original_shapes = shapes - train_shapes = shapes - else: - if self.scaled_shape_models: - train_shapes = shapes - else: - train_shapes = original_shapes - - # train shape model and find reference frame - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = build_shape_model( - train_shapes, self.max_shape_components[rj]) - - # add shape model to the list - shape_models.append(shape_model) - - # build classifiers - sampling_grid = build_sampling_grid(self.patch_shape) - n_points = shapes[0].n_points - level_classifiers = [] - for k in range(n_points): - if verbose: - print_dynamic('{}Building classifiers - {}'.format( - level_str, - progress_bar_str((k + 1.) / n_points, - show_bar=False))) - - positive_labels = [] - negative_labels = [] - positive_samples = [] - negative_samples = [] - - for i, s in zip(feature_images, shapes): - - max_x = i.shape[0] - 1 - max_y = i.shape[1] - 1 - - point = (np.round(s.points[k, :])).astype(int) - patch_grid = sampling_grid + point[None, None, ...] - positive, negative = get_pos_neg_grid_positions( - patch_grid, positive_grid_size=(1, 1)) - - x = positive[:, 0] - y = positive[:, 1] - x[x > max_x] = max_x - y[y > max_y] = max_y - x[x < 0] = 0 - y[y < 0] = 0 - - positive_sample = i.pixels[:, x, y].T - positive_samples.append(positive_sample) - positive_labels.append(np.ones(positive_sample.shape[0])) - - x = negative[:, 0] - y = negative[:, 1] - x[x > max_x] = max_x - y[y > max_y] = max_y - x[x < 0] = 0 - y[y < 0] = 0 - - negative_sample = i.pixels[:, x, y].T - negative_samples.append(negative_sample) - negative_labels.append(-np.ones(negative_sample.shape[0])) - - positive_samples = np.asanyarray(positive_samples) - positive_samples = np.reshape(positive_samples, - (-1, positive_samples.shape[-1])) - positive_labels = np.asanyarray(positive_labels).flatten() - - negative_samples = np.asanyarray(negative_samples) - negative_samples = np.reshape(negative_samples, - (-1, negative_samples.shape[-1])) - negative_labels = np.asanyarray(negative_labels).flatten() - - X = np.vstack((positive_samples, negative_samples)) - t = np.hstack((positive_labels, negative_labels)) - - clf = self.classifier_trainers[rj](X, t) - level_classifiers.append(clf) - - # add level classifiers to the list - classifiers.append(level_classifiers) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and appearance models so that they are - # ordered from lower to higher resolution - shape_models.reverse() - classifiers.reverse() - n_training_images = len(images) - - from .base import CLM - return CLM(shape_models, classifiers, n_training_images, - self.patch_shape, self.features, self.reference_shape, - self.downscale, self.scaled_shape_models) - - -def get_pos_neg_grid_positions(sampling_grid, positive_grid_size=(1, 1)): - r""" - Divides a sampling grid in positive and negative pixel positions. By - default only the centre of the grid is considered to be positive. - """ - positive_grid_size = np.array(positive_grid_size) - mask = np.zeros(sampling_grid.shape[:-1], dtype=np.bool) - centre = np.round(np.array(mask.shape) / 2).astype(int) - positive_grid_size -= [1, 1] - start = centre - positive_grid_size - end = centre + positive_grid_size + 1 - mask[start[0]:end[0], start[1]:end[1]] = True - positive = sampling_grid[mask] - negative = sampling_grid[~mask] - return positive, negative - - -def check_classifier_trainers(classifier_trainers, n_levels): - r""" - Checks the classifier_trainers. Must be a ``callable`` -> - ``callable`` or - or a list containing 1 or {n_levels} callables each of which returns a - callable. - """ - str_error = ("classifier must be a callable " - "of a list containing 1 or {} callables").format(n_levels) - if not isinstance(classifier_trainers, list): - classifier_list = [classifier_trainers] * n_levels - elif len(classifier_trainers) == 1: - classifier_list = [classifier_trainers[0]] * n_levels - elif len(classifier_trainers) == n_levels: - classifier_list = classifier_trainers - else: - raise ValueError(str_error) - for classifier in classifier_list: - if not callable(classifier): - raise ValueError(str_error) - return classifier_list - - -def check_patch_shape(patch_shape): - r""" - Checks the patch shape. It must be a tuple with `int` > ``1``. - """ - str_error = "patch_size mast be a tuple with two integers" - if not isinstance(patch_shape, tuple) or len(patch_shape) != 2: - raise ValueError(str_error) - for sh in patch_shape: - if not isinstance(sh, int) or sh < 2: - raise ValueError(str_error) - return patch_shape diff --git a/menpofit/clm/classifier.py b/menpofit/clm/classifier.py deleted file mode 100644 index 377b4cd..0000000 --- a/menpofit/clm/classifier.py +++ /dev/null @@ -1,19 +0,0 @@ -from sklearn import svm -from sklearn import linear_model - - -class linear_svm_lr(object): - r""" - Binary classifier that combines Linear Support Vector Machines and - Logistic Regression. - """ - def __init__(self, X, t): - self.clf1 = svm.LinearSVC(class_weight='auto') - self.clf1.fit(X, t) - t1 = self.clf1.decision_function(X) - self.clf2 = linear_model.LogisticRegression(class_weight='auto') - self.clf2.fit(t1[..., None], t) - - def __call__(self, x): - t1_pred = self.clf1.decision_function(x) - return self.clf2.predict_proba(t1_pred[..., None])[:, 1] diff --git a/menpofit/clm/fitter.py b/menpofit/clm/fitter.py index c83c223..26fe326 100644 --- a/menpofit/clm/fitter.py +++ b/menpofit/clm/fitter.py @@ -45,4 +45,5 @@ def __init__(self, clm, gd_algorithm_cls=RegularisedLandmarkMeanShift, class SupervisedDescentCLMFitter(CLMFitter): r""" """ - raise NotImplementedError + def __init__(self): + raise NotImplementedError diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 07f0ec8..7f1992d 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -21,7 +21,7 @@ def n_scales(self): """ return len(self.scales) - def fit(self, image, initial_shape, max_iters=50, gt_shape=None, + def fit(self, image, initial_shape, max_iters=20, gt_shape=None, crop_image=0.5, **kwargs): r""" Fits the multilevel fitter to an image. @@ -126,8 +126,8 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, # Rescale image wrt the scale factor between reference_shape and # initial_shape - image = image.rescale_to_reference_shape(self.reference_shape, - group='__initial_shape') + image = image.rescale_to_pointcloud(self.reference_shape, + group='__initial_shape') # Compute image representation images = [] @@ -158,9 +158,14 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, else: gt_shapes = None + # detach added landmarks from image + del image.landmarks['__initial_shape'] + if gt_shape: + del image.landmarks['__gt_shape'] + return images, initial_shapes, gt_shapes - def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, + def _fit(self, images, initial_shape, gt_shapes=None, max_iters=20, **kwargs): r""" Fits the fitter to the multilevel pyramidal images. @@ -174,8 +179,6 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, gt_shapes: :class:`menpo.shape.PointCloud` list, optional The original ground truth shapes associated to the multilevel images. - - Default: None max_iters: int or list, optional The maximum number of iterations. If int, then this will be the overall maximum number of iterations @@ -183,8 +186,6 @@ def _fit(self, images, initial_shape, gt_shapes=None, max_iters=50, If list, then a maximum number of iterations is specified for each pyramidal level. - Default: 50 - Returns ------- algorithm_results: :class:`FittingResult` list @@ -236,10 +237,6 @@ def reference_shape(self): """ return self._model.reference_shape - @property - def reference_bounding_box(self): - return self.reference_shape.bounding_box() - @property def features(self): r""" diff --git a/menpofit/fittingresult.py b/menpofit/fittingresult.py deleted file mode 100644 index 3c88302..0000000 --- a/menpofit/fittingresult.py +++ /dev/null @@ -1,1258 +0,0 @@ -from __future__ import division - -import abc -from itertools import chain -import numpy as np - -from menpo.shape.pointcloud import PointCloud -from menpo.image import Image -from menpo.transform import Scale -from menpofit.base import name_of_callable - - -class FittingResult(object): - r""" - Object that holds the state of a single fitting object, during and after it - has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` or subclass - The fitted image. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - - def __init__(self, image, gt_shape=None): - self.image = image - self._gt_shape = gt_shape - - @property - def n_iters(self): - return len(self.shapes) - 1 - - @abc.abstractproperty - def shapes(self): - r""" - A list containing the shapes obtained at each fitting iteration. - - :type: `list` of :map:`PointCloud` - """ - - def displacements(self): - r""" - A list containing the displacement between the shape of each iteration - and the shape of the previous one. - - :type: `list` of ndarray - """ - return [np.linalg.norm(s1.points - s2.points, axis=1) - for s1, s2 in zip(self.shapes, self.shapes[1:])] - - def displacements_stats(self, stat_type='mean'): - r""" - A list containing the a statistical metric on the displacement between - the shape of each iteration and the shape of the previous one. - - Parameters - ----------- - stat_type : `str` ``{'mean', 'median', 'min', 'max'}``, optional - Specifies a statistic metric to be extracted from the displacements. - - Returns - ------- - :type: `list` of `float` - The statistical metric on the points displacements for each - iteration. - """ - if stat_type == 'mean': - return [np.mean(d) for d in self.displacements()] - elif stat_type == 'median': - return [np.median(d) for d in self.displacements()] - elif stat_type == 'max': - return [np.max(d) for d in self.displacements()] - elif stat_type == 'min': - return [np.min(d) for d in self.displacements()] - else: - raise ValueError("type must be 'mean', 'median', 'min' or 'max'") - - @abc.abstractproperty - def final_shape(self): - r""" - Returns the final fitted shape. - """ - - @abc.abstractproperty - def initial_shape(self): - r""" - Returns the initial shape from which the fitting started. - """ - - @property - def gt_shape(self): - r""" - Returns the original ground truth shape associated to the image. - """ - return self._gt_shape - - @property - def fitted_image(self): - r""" - Returns a copy of the fitted image with the following landmark - groups attached to it: - - ``initial``, containing the initial fitted shape . - - ``final``, containing the final shape. - - ``ground``, containing the ground truth shape. Only returned if - the ground truth shape was provided. - - :type: :map:`Image` - """ - image = Image(self.image.pixels) - - image.landmarks['initial'] = self.initial_shape - image.landmarks['final'] = self.final_shape - if self.gt_shape is not None: - image.landmarks['ground'] = self.gt_shape - return image - - @property - def iter_image(self): - r""" - Returns a copy of the fitted image with as many landmark groups as - iteration run by fitting procedure: - - ``iter_0``, containing the initial shape. - - ``iter_1``, containing the the fitted shape at the first - iteration. - - ``...`` - - ``iter_n``, containing the final fitted shape. - - :type: :map:`Image` - """ - image = Image(self.image.pixels) - for j, s in enumerate(self.shapes): - key = 'iter_{}'.format(j) - image.landmarks[key] = s - return image - - def errors(self, error_type='me_norm'): - r""" - Returns a list containing the error at each fitting iteration. - - Parameters - ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Returns - ------- - errors : `list` of `float` - The errors at each iteration of the fitting process. - """ - if self.gt_shape is not None: - return [compute_error(t, self.gt_shape, error_type) - for t in self.shapes] - else: - raise ValueError('Ground truth has not been set, errors cannot ' - 'be computed') - - def final_error(self, error_type='me_norm'): - r""" - Returns the final fitting error. - - Parameters - ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Returns - ------- - final_error : `float` - The final error at the end of the fitting procedure. - """ - if self.gt_shape is not None: - return compute_error(self.final_shape, self.gt_shape, error_type) - else: - raise ValueError('Ground truth has not been set, final error ' - 'cannot be computed') - - def initial_error(self, error_type='me_norm'): - r""" - Returns the initial fitting error. - - Parameters - ----------- - error_type : `str` ``{'me_norm', 'me', 'rmse'}``, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - - Returns - ------- - initial_error : `float` - The initial error at the start of the fitting procedure. - """ - if self.gt_shape is not None: - return compute_error(self.initial_shape, self.gt_shape, error_type) - else: - raise ValueError('Ground truth has not been set, final error ' - 'cannot be computed') - - def view_widget(self, browser_style='buttons', figure_size=(10, 8), - style='coloured'): - r""" - Visualizes the multilevel fitting result object using the - `menpo.visualize.widgets.visualize_fitting_result` widget. - - Parameters - ----------- - browser_style : {``'buttons'``, ``'slider'``}, optional - It defines whether the selector of the fitting results will have the - form of plus/minus buttons or a slider. - figure_size : (`int`, `int`), optional - The initial size of the rendered figure. - style : {``'coloured'``, ``'minimal'``}, optional - If ``'coloured'``, then the style of the widget will be coloured. If - ``minimal``, then the style is simple using black and white colours. - """ - from menpofit.visualize import visualize_fitting_result - visualize_fitting_result(self, figure_size=figure_size, - browser_style=browser_style, style=style) - - def plot_errors(self, error_type='me_norm', figure_id=None, - new_figure=False, render_lines=True, line_colour='b', - line_style='-', line_width=2, render_markers=True, - marker_style='o', marker_size=4, marker_face_colour='b', - marker_edge_colour='k', marker_edge_width=1., - render_axes=True, axes_font_name='sans-serif', - axes_font_size=10, axes_font_style='normal', - axes_font_weight='normal', figure_size=(10, 6), - render_grid=True, grid_line_style='--', - grid_line_width=0.5): - r""" - Plot of the error evolution at each fitting iteration. - - Parameters - ---------- - error_type : {``me_norm``, ``me``, ``rmse``}, optional - Specifies the way in which the error between the fitted and - ground truth shapes is to be computed. - figure_id : `object`, optional - The id of the figure to be used. - new_figure : `bool`, optional - If ``True``, a new figure is created. - render_lines : `bool`, optional - If ``True``, the line will be rendered. - line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or - ``(3, )`` `ndarray`, optional - The colour of the lines. - line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the lines. - line_width : `float`, optional - The width of the lines. - render_markers : `bool`, optional - If ``True``, the markers will be rendered. - marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, - ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, - ``1``, ``2``, ``3``, ``4``, ``8``}, optional - The style of the markers. - marker_size : `int`, optional - The size of the markers in points^2. - marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The face (filling) colour of the markers. - marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The edge colour of the markers. - marker_edge_width : `float`, optional - The width of the markers' edge. - render_axes : `bool`, optional - If ``True``, the axes will be rendered. - axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the axes. - axes_font_size : `int`, optional - The font size of the axes. - axes_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the axes. - axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, - ``book``, ``medium``, ``roman``, ``semibold``, - ``demibold``, ``demi``, ``bold``, ``heavy``, - ``extra bold``, ``black``}, optional - The font weight of the axes. - figure_size : (`float`, `float`) or `None`, optional - The size of the figure in inches. - render_grid : `bool`, optional - If ``True``, the grid will be rendered. - grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the grid lines. - grid_line_width : `float`, optional - The width of the grid lines. - - Returns - ------- - viewer : :map:`GraphPlotter` - The viewer object. - """ - from menpo.visualize import GraphPlotter - errors_list = self.errors(error_type=error_type) - return GraphPlotter(figure_id=figure_id, new_figure=new_figure, - x_axis=range(len(errors_list)), - y_axis=[errors_list], - title='Fitting Errors per Iteration', - x_label='Iteration', y_label='Fitting Error', - x_axis_limits=(0, len(errors_list)-1), - y_axis_limits=None).render( - render_lines=render_lines, line_colour=line_colour, - line_style=line_style, line_width=line_width, - render_markers=render_markers, marker_style=marker_style, - marker_size=marker_size, marker_face_colour=marker_face_colour, - marker_edge_colour=marker_edge_colour, - marker_edge_width=marker_edge_width, render_legend=False, - render_axes=render_axes, axes_font_name=axes_font_name, - axes_font_size=axes_font_size, axes_font_style=axes_font_style, - axes_font_weight=axes_font_weight, render_grid=render_grid, - grid_line_style=grid_line_style, grid_line_width=grid_line_width, - figure_size=figure_size) - - def plot_displacements(self, stat_type='mean', figure_id=None, - new_figure=False, render_lines=True, line_colour='b', - line_style='-', line_width=2, render_markers=True, - marker_style='o', marker_size=4, - marker_face_colour='b', marker_edge_colour='k', - marker_edge_width=1., render_axes=True, - axes_font_name='sans-serif', axes_font_size=10, - axes_font_style='normal', axes_font_weight='normal', - figure_size=(10, 6), render_grid=True, - grid_line_style='--', grid_line_width=0.5): - r""" - Plot of a statistical metric of the displacement between the shape of - each iteration and the shape of the previous one. - - Parameters - ---------- - stat_type : {``mean``, ``median``, ``min``, ``max``}, optional - Specifies a statistic metric to be extracted from the displacements - (see also `displacements_stats()` method). - figure_id : `object`, optional - The id of the figure to be used. - new_figure : `bool`, optional - If ``True``, a new figure is created. - render_lines : `bool`, optional - If ``True``, the line will be rendered. - line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or - ``(3, )`` `ndarray`, optional - The colour of the lines. - line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the lines. - line_width : `float`, optional - The width of the lines. - render_markers : `bool`, optional - If ``True``, the markers will be rendered. - marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, - ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, - ``1``, ``2``, ``3``, ``4``, ``8``}, optional - The style of the markers. - marker_size : `int`, optional - The size of the markers in points^2. - marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The face (filling) colour of the markers. - marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray`, optional - The edge colour of the markers. - marker_edge_width : `float`, optional - The width of the markers' edge. - render_axes : `bool`, optional - If ``True``, the axes will be rendered. - axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the axes. - axes_font_size : `int`, optional - The font size of the axes. - axes_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the axes. - axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, - ``book``, ``medium``, ``roman``, ``semibold``, - ``demibold``, ``demi``, ``bold``, ``heavy``, - ``extra bold``, ``black``}, optional - The font weight of the axes. - figure_size : (`float`, `float`) or `None`, optional - The size of the figure in inches. - render_grid : `bool`, optional - If ``True``, the grid will be rendered. - grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the grid lines. - grid_line_width : `float`, optional - The width of the grid lines. - - Returns - ------- - viewer : :map:`GraphPlotter` - The viewer object. - """ - from menpo.visualize import GraphPlotter - # set labels - if stat_type == 'max': - ylabel = 'Maximum Displacement' - title = 'Maximum displacement per Iteration' - elif stat_type == 'min': - ylabel = 'Minimum Displacement' - title = 'Minimum displacement per Iteration' - elif stat_type == 'mean': - ylabel = 'Mean Displacement' - title = 'Mean displacement per Iteration' - elif stat_type == 'median': - ylabel = 'Median Displacement' - title = 'Median displacement per Iteration' - else: - raise ValueError('stat_type must be one of {max, min, mean, ' - 'median}.') - # plot - displacements_list = self.displacements_stats(stat_type=stat_type) - return GraphPlotter(figure_id=figure_id, new_figure=new_figure, - x_axis=range(len(displacements_list)), - y_axis=[displacements_list], - title=title, - x_label='Iteration', y_label=ylabel, - x_axis_limits=(0, len(displacements_list)-1), - y_axis_limits=None).render( - render_lines=render_lines, line_colour=line_colour, - line_style=line_style, line_width=line_width, - render_markers=render_markers, marker_style=marker_style, - marker_size=marker_size, marker_face_colour=marker_face_colour, - marker_edge_colour=marker_edge_colour, - marker_edge_width=marker_edge_width, render_legend=False, - render_axes=render_axes, axes_font_name=axes_font_name, - axes_font_size=axes_font_size, axes_font_style=axes_font_style, - axes_font_weight=axes_font_weight, render_grid=render_grid, - grid_line_style=grid_line_style, grid_line_width=grid_line_width, - figure_size=figure_size) - - def as_serializable(self): - r"""" - Returns a serializable version of the fitting result. This is a much - lighter weight object than the initial fitting result. For example, - it won't contain the original fitting object. - - Returns - ------- - serializable_fitting_result : :map:`SerializableFittingResult` - The lightweight serializable version of this fitting result. - """ - if self.parameters is not None: - parameters = [p.copy() for p in self.parameters] - else: - parameters = [] - gt_shape = self.gt_shape.copy() if self.gt_shape else None - return SerializableFittingResult(self.image.copy(), - parameters, - [s.copy() for s in self.shapes], - gt_shape) - - -class NonParametricFittingResult(FittingResult): - r""" - Object that holds the state of a Non Parametric :map:`Fitter` object - before, during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - fitter : :map:`Fitter` - The Fitter object used to fitter the image. - shapes : `list` of :map:`PointCloud` - The list of fitted shapes per iteration of the fitting procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - - def __init__(self, image, fitter, parameters=None, gt_shape=None): - super(NonParametricFittingResult, self).__init__(image, - gt_shape=gt_shape) - self.fitter = fitter - # The parameters are the shapes for Non-Parametric algorithms - self.parameters = parameters - - @property - def shapes(self): - return self.parameters - - @property - def final_shape(self): - return self.parameters[-1].copy() - - @property - def initial_shape(self): - return self.parameters[0].copy() - - @FittingResult.gt_shape.setter - def gt_shape(self, value): - r""" - Setter for the ground truth shape associated to the image. - """ - if isinstance(value, PointCloud): - self._gt_shape = value - else: - raise ValueError("Accepted values for gt_shape setter are " - "PointClouds.") - - -class SemiParametricFittingResult(FittingResult): - r""" - Object that holds the state of a Semi Parametric :map:`Fitter` object - before, during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - fitter : :map:`Fitter` - The Fitter object used to fitter the image. - parameters : `list` of `ndarray` - The list of optimal transform parameters per iteration of the fitting - procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - - def __init__(self, image, fitter, parameters=None, gt_shape=None): - FittingResult.__init__(self, image, gt_shape=gt_shape) - self.fitter = fitter - self.parameters = parameters - - @property - def transforms(self): - r""" - Generates a list containing the transforms obtained at each fitting - iteration. - """ - return [self.fitter.transform.from_vector(p) for p in self.parameters] - - @property - def final_transform(self): - r""" - Returns the final transform. - """ - return self.fitter.transform.from_vector(self.parameters[-1]) - - @property - def initial_transform(self): - r""" - Returns the initial transform from which the fitting started. - """ - return self.fitter.transform.from_vector(self.parameters[0]) - - @property - def shapes(self): - return [self.fitter.transform.from_vector(p).target - for p in self.parameters] - - @property - def final_shape(self): - return self.final_transform.target - - @property - def initial_shape(self): - return self.initial_transform.target - - @FittingResult.gt_shape.setter - def gt_shape(self, value): - r""" - Setter for the ground truth shape associated to the image. - """ - if type(value) is PointCloud: - self._gt_shape = value - elif type(value) is list and value[0] is float: - transform = self.fitter.transform.from_vector(value) - self._gt_shape = transform.target - else: - raise ValueError("Accepted values for gt_shape setter are " - "PointClouds or float lists " - "specifying transform shapes.") - - -class ParametricFittingResult(SemiParametricFittingResult): - r""" - Object that holds the state of a Fully Parametric :map:`Fitter` object - before, during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - fitter : :map:`Fitter` - The Fitter object used to fitter the image. - parameters : `list` of `ndarray` - The list of optimal transform parameters per iteration of the fitting - procedure. - weights : `list` of `ndarray` - The list of optimal appearance parameters per iteration of the fitting - procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - def __init__(self, image, fitter, parameters=None, weights=None, - gt_shape=None): - SemiParametricFittingResult.__init__(self, image, fitter, parameters, - gt_shape=gt_shape) - self.weights = weights - - @property - def warped_images(self): - r""" - The list containing the warped images obtained at each fitting - iteration. - - :type: `list` of :map:`Image` or subclass - """ - mask = self.fitter.template.mask - transform = self.fitter.transform - return [self.image.warp_to_mask(mask, transform.from_vector(p)) - for p in self.parameters] - - @property - def appearance_reconstructions(self): - r""" - The list containing the appearance reconstruction obtained at - each fitting iteration. - - :type: list` of :map:`Image` or subclass - """ - if self.weights: - return [self.fitter.appearance_model.instance(w) - for w in self.weights] - else: - return [self.fitter.template for _ in self.shapes] - - @property - def error_images(self): - r""" - The list containing the error images obtained at - each fitting iteration. - - :type: list` of :map:`Image` or subclass - """ - template = self.fitter.template - warped_images = self.warped_images - appearances = self.appearance_reconstructions - - error_images = [] - for a, i in zip(appearances, warped_images): - error = a.as_vector() - i.as_vector() - error_image = template.from_vector(error) - error_images.append(error_image) - - return error_images - - -class SerializableFittingResult(FittingResult): - r""" - Designed to allow the fitting results to be easily serializable. In - comparison to the other fitting result objects, the serializable fitting - results contain a much stricter set of data. For example, the major data - components of a serializable fitting result are the fitted shapes, the - parameters and the fitted image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - parameters : `list` of `ndarray` - The list of optimal transform parameters per iteration of the fitting - procedure. - shapes : `list` of :map:`PointCloud` - The list of fitted shapes per iteration of the fitting procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - """ - def __init__(self, image, parameters, shapes, gt_shape): - FittingResult.__init__(self, image, gt_shape=gt_shape) - - self.parameters = parameters - self._shapes = shapes - - @property - def shapes(self): - return self._shapes - - @property - def initial_shape(self): - return self._shapes[0] - - @property - def final_shape(self): - return self._shapes[-1] - - -class MultilevelFittingResult(FittingResult): - r""" - Class that holds the state of a :map:`MultilevelFitter` object before, - during and after it has fitted a particular image. - - Parameters - ----------- - image : :map:`Image` or subclass - The fitted image. - multilevel_fitter : :map:`MultilevelFitter` - The multilevel fitter object used to fit the image. - fitting_results : `list` of :map:`FittingResult` - The list of fitting results. - affine_correction : :map:`Affine` - The affine transform between the initial shape of the highest - pyramidal level and the initial shape of the original image - gt_shape : class:`PointCloud`, optional - The ground truth shape associated to the image. - """ - def __init__(self, image, multiple_fitter, fitting_results, - affine_correction, gt_shape=None): - super(MultilevelFittingResult, self).__init__(image, gt_shape=gt_shape) - self.fitter = multiple_fitter - self.fitting_results = fitting_results - self._affine_correction = affine_correction - - @property - def n_levels(self): - r""" - The number of levels of the fitter object. - - :type: `int` - """ - return self.fitter.n_scales - - @property - def downscale(self): - r""" - The downscale factor used by the multiple fitter. - - :type: `float` - """ - return self.fitter.downscale - - @property - def n_iters(self): - r""" - The total number of iterations used to fitter the image. - - :type: `int` - """ - n_iters = 0 - for f in self.fitting_results: - n_iters += f.n_iters - return n_iters - - @property - def shapes(self): - r""" - A list containing the shapes obtained at each fitting iteration. - - :type: `list` of :map:`PointCloud` - """ - return _rescale_shapes_to_reference(self.fitting_results, self.n_levels, - self.downscale, - self._affine_correction) - - @property - def final_shape(self): - r""" - The final fitted shape. - - :type: :map:`PointCloud` - """ - return self._affine_correction.apply( - self.fitting_results[-1].final_shape) - - @property - def initial_shape(self): - r""" - The initial shape from which the fitting started. - - :type: :map:`PointCloud` - """ - n = self.n_levels - 1 - initial_shape = self.fitting_results[0].initial_shape - Scale(self.downscale ** n, initial_shape.n_dims).apply_inplace( - initial_shape) - - return self._affine_correction.apply(initial_shape) - - @FittingResult.gt_shape.setter - def gt_shape(self, value): - r""" - Setter for the ground truth shape associated to the image. - - type: :map:`PointCloud` - """ - self._gt_shape = value - - def __str__(self): - if self.fitter.pyramid_on_features: - feat_str = name_of_callable(self.fitter.features) - else: - feat_str = [] - for j in range(self.n_levels): - if isinstance(self.fitter.features[j], str): - feat_str.append(self.fitter.features[j]) - elif self.fitter.features[j] is None: - feat_str.append("none") - else: - feat_str.append(name_of_callable(self.fitter.features[j])) - out = "Fitting Result\n" \ - " - Initial error: {0:.4f}\n" \ - " - Final error: {1:.4f}\n" \ - " - {2} method with {3} pyramid levels, {4} iterations " \ - "and using {5} features.".format( - self.initial_error(), self.final_error(), self.fitter.algorithm, - self.n_levels, self.n_iters, feat_str) - return out - - def as_serializable(self): - r"""" - Returns a serializable version of the fitting result. This is a much - lighter weight object than the initial fitting result. For example, - it won't contain the original fitting object. - - Returns - ------- - serializable_fitting_result : :map:`SerializableFittingResult` - The lightweight serializable version of this fitting result. - """ - gt_shape = self.gt_shape.copy() if self.gt_shape else None - fr_copies = [fr.as_serializable() for fr in self.fitting_results] - - return SerializableMultilevelFittingResult( - self.image.copy(), fr_copies, - gt_shape, self.n_levels, self.downscale, self.n_iters, - self._affine_correction.copy()) - - -class AMMultilevelFittingResult(MultilevelFittingResult): - r""" - Class that holds the state of an Active Model (either AAM or ATM). - """ - @property - def costs(self): - r""" - Returns a list containing the cost at each fitting iteration. - - :type: `list` of `float` - """ - raise ValueError('costs not implemented yet.') - - @property - def final_cost(self): - r""" - Returns the final fitting cost. - - :type: `float` - """ - raise ValueError('costs not implemented yet.') - - @property - def initial_cost(self): - r""" - Returns the initial fitting cost. - - :type: `float` - """ - raise ValueError('costs not implemented yet.') - - @property - def warped_images(self): - r""" - The list containing the warped images obtained at each fitting - iteration. - - :type: `list` of :map:`Image` or subclass - """ - mask = self.fitting_results[-1].fitter.template.mask - transform = self.fitting_results[-1].fitter.transform - warped_images = [] - for s in self.shapes(): - transform.set_target(s) - image = self.image.warp_to_mask(mask, transform) - warped_images.append(image) - - return warped_images - - @property - def error_images(self): - r""" - The list containing the error images obtained at each fitting - iteration. - - :type: `list` of :map:`Image` or subclass - """ - return list(chain( - *[f.error_images for f in self.fitting_results])) - - -class SerializableMultilevelFittingResult(FittingResult): - r""" - Designed to allow the fitting results to be easily serializable. In - comparison to the other fitting result objects, the serializable fitting - results contain a much stricter set of data. For example, the major data - components of a serializable fitting result are the fitted shapes, the - parameters and the fitted image. - - Parameters - ----------- - image : :map:`Image` - The fitted image. - shapes : `list` of :map:`PointCloud` - The list of fitted shapes per iteration of the fitting procedure. - gt_shape : :map:`PointCloud` - The ground truth shape associated to the image. - n_scales : `int` - Number of levels within the multilevel fitter. - downscale : `int` - Scale of downscaling applied to the image. - n_iters : `int` - Number of iterations the fitter performed. - """ - def __init__(self, image, fitting_results, gt_shape, n_levels, - downscale, n_iters, affine_correction): - FittingResult.__init__(self, image, gt_shape=gt_shape) - self.fitting_results = fitting_results - self.n_levels = n_levels - self._n_iters = n_iters - self.downscale = downscale - self.affine_correction = affine_correction - - @property - def n_iters(self): - return self._n_iters - - @property - def final_shape(self): - return self.shapes[-1] - - @property - def initial_shape(self): - return self.shapes[0] - - @property - def shapes(self): - return _rescale_shapes_to_reference(self.fitting_results, self.n_levels, - self.downscale, - self.affine_correction) - - -def _rescale_shapes_to_reference(fitting_results, n_levels, downscale, - affine_correction): - n = n_levels - 1 - shapes = [] - for j, f in enumerate(fitting_results): - transform = Scale(downscale ** (n - j), f.final_shape.n_dims) - for t in f.shapes: - t = transform.apply(t) - shapes.append(affine_correction.apply(t)) - return shapes - - -def compute_error(target, ground_truth, error_type='me_norm'): - r""" - """ - gt_points = ground_truth.points - target_points = target.points - - if error_type == 'me_norm': - return _compute_me_norm(target_points, gt_points) - elif error_type == 'me': - return _compute_me(target_points, gt_points) - elif error_type == 'rmse': - return _compute_rmse(target_points, gt_points) - else: - raise ValueError("Unknown error_type string selected. Valid options " - "are: me_norm, me, rmse'") - - -def _compute_me(target, ground_truth): - r""" - """ - return np.mean(np.sqrt(np.sum((target - ground_truth) ** 2, axis=-1))) - - -def _compute_rmse(target, ground_truth): - r""" - """ - return np.sqrt(np.mean((target.flatten() - ground_truth.flatten()) ** 2)) - - -def _compute_me_norm(target, ground_truth): - r""" - """ - normalizer = np.mean(np.max(ground_truth, axis=0) - - np.min(ground_truth, axis=0)) - return _compute_me(target, ground_truth) / normalizer - - -def compute_cumulative_error(errors, x_axis): - r""" - """ - n_errors = len(errors) - return [np.count_nonzero([errors <= x]) / n_errors for x in x_axis] - - -def plot_cumulative_error_distribution(errors, error_range=None, figure_id=None, - new_figure=False, - title='Cumulative Error Distribution', - x_label='Normalized Point-to-Point Error', - y_label='Images Proportion', - legend_entries=None, render_lines=True, - line_colour=None, line_style='-', - line_width=2, render_markers=True, - marker_style='s', marker_size=10, - marker_face_colour='w', - marker_edge_colour=None, - marker_edge_width=2, render_legend=True, - legend_title=None, - legend_font_name='sans-serif', - legend_font_style='normal', - legend_font_size=10, - legend_font_weight='normal', - legend_marker_scale=1., - legend_location=2, - legend_bbox_to_anchor=(1.05, 1.), - legend_border_axes_pad=1., - legend_n_columns=1, - legend_horizontal_spacing=1., - legend_vertical_spacing=1., - legend_border=True, - legend_border_padding=0.5, - legend_shadow=False, - legend_rounded_corners=False, - render_axes=True, - axes_font_name='sans-serif', - axes_font_size=10, - axes_font_style='normal', - axes_font_weight='normal', - axes_x_limits=None, axes_y_limits=None, - figure_size=(10, 8), render_grid=True, - grid_line_style='--', - grid_line_width=0.5): - r""" - Plot the cumulative error distribution (CED) of the provided fitting errors. - - Parameters - ---------- - errors : `list` of `lists` - A `list` with `lists` of fitting errors. A separate CED curve will be - rendered for each errors `list`. - error_range : `list` of `float` with length 3, optional - Specifies the horizontal axis range, i.e. - - :: - - error_range[0] = min_error - error_range[1] = max_error - error_range[2] = error_step - - If ``None``, then ``'error_range = [0., 0.101, 0.005]'``. - figure_id : `object`, optional - The id of the figure to be used. - new_figure : `bool`, optional - If ``True``, a new figure is created. - title : `str`, optional - The figure's title. - x_label : `str`, optional - The label of the horizontal axis. - y_label : `str`, optional - The label of the vertical axis. - legend_entries : `list of `str` or ``None``, optional - If `list` of `str`, it must have the same length as `errors` `list` and - each `str` will be used to name each curve. If ``None``, the CED curves - will be named as `'Curve %d'`. - render_lines : `bool` or `list` of `bool`, optional - If ``True``, the line will be rendered. If `bool`, this value will be - used for all curves. If `list`, a value must be specified for each - fitting errors curve, thus it must have the same length as `errors`. - line_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} or - ``(3, )`` `ndarray` or `list` of those or ``None``, optional - The colour of the lines. If not a `list`, this value will be - used for all curves. If `list`, a value must be specified for each - fitting errors curve, thus it must have the same length as `errors`. If - ``None``, the colours will be linearly sampled from jet colormap. - line_style : {``-``, ``--``, ``-.``, ``:``} or `list` of those, optional - The style of the lines. If not a `list`, this value will be used for all - curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - line_width : `float` or `list` of `float`, optional - The width of the lines. If `float`, this value will be used for all - curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - render_markers : `bool` or `list` of `bool`, optional - If ``True``, the markers will be rendered. If `bool`, this value will be - used for all curves. If `list`, a value must be specified for each - fitting errors curve, thus it must have the same length as `errors`. - marker_style : {``.``, ``,``, ``o``, ``v``, ``^``, ``<``, ``>``, ``+``, - ``x``, ``D``, ``d``, ``s``, ``p``, ``*``, ``h``, ``H``, - ``1``, ``2``, ``3``, ``4``, ``8``} or `list` of those, optional - The style of the markers. If not a `list`, this value will be used for - all curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - marker_size : `int` or `list` of `int`, optional - The size of the markers in points^2. If `int`, this value will be used - for all curves. If `list`, a value must be specified for each fitting - errors curve, thus it must have the same length as `errors`. - marker_face_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray` or `list` of those or ``None``, optional - The face (filling) colour of the markers. If not a `list`, this value - will be used for all curves. If `list`, a value must be specified for - each fitting errors curve, thus it must have the same length as - `errors`. If ``None``, the colours will be linearly sampled from jet - colormap. - marker_edge_colour : {``r``, ``g``, ``b``, ``c``, ``m``, ``k``, ``w``} - or ``(3, )`` `ndarray` or `list` of those or ``None``, optional - The edge colour of the markers. If not a `list`, this value will be used - for all curves. If `list`, a value must be specified for each fitting - errors curve, thus it must have the same length as `errors`. If - ``None``, the colours will be linearly sampled from jet colormap. - marker_edge_width : `float` or `list` of `float`, optional - The width of the markers' edge. If `float`, this value will be used for - all curves. If `list`, a value must be specified for each fitting errors - curve, thus it must have the same length as `errors`. - render_legend : `bool`, optional - If ``True``, the legend will be rendered. - legend_title : `str`, optional - The title of the legend. - legend_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the legend. - legend_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the legend. - legend_font_size : `int`, optional - The font size of the legend. - legend_font_weight : {``ultralight``, ``light``, ``normal``, - ``regular``, ``book``, ``medium``, ``roman``, - ``semibold``, ``demibold``, ``demi``, ``bold``, - ``heavy``, ``extra bold``, ``black``}, optional - The font weight of the legend. - legend_marker_scale : `float`, optional - The relative size of the legend markers with respect to the original - legend_location : `int`, optional - The location of the legend. The predefined values are: - - =============== === - 'best' 0 - 'upper right' 1 - 'upper left' 2 - 'lower left' 3 - 'lower right' 4 - 'right' 5 - 'center left' 6 - 'center right' 7 - 'lower center' 8 - 'upper center' 9 - 'center' 10 - =============== === - - legend_bbox_to_anchor : (`float`, `float`), optional - The bbox that the legend will be anchored. - legend_border_axes_pad : `float`, optional - The pad between the axes and legend border. - legend_n_columns : `int`, optional - The number of the legend's columns. - legend_horizontal_spacing : `float`, optional - The spacing between the columns. - legend_vertical_spacing : `float`, optional - The vertical space between the legend entries. - legend_border : `bool`, optional - If ``True``, a frame will be drawn around the legend. - legend_border_padding : `float`, optional - The fractional whitespace inside the legend border. - legend_shadow : `bool`, optional - If ``True``, a shadow will be drawn behind legend. - legend_rounded_corners : `bool`, optional - If ``True``, the frame's corners will be rounded (fancybox). - render_axes : `bool`, optional - If ``True``, the axes will be rendered. - axes_font_name : {``serif``, ``sans-serif``, ``cursive``, ``fantasy``, - ``monospace``}, optional - The font of the axes. - axes_font_size : `int`, optional - The font size of the axes. - axes_font_style : {``normal``, ``italic``, ``oblique``}, optional - The font style of the axes. - axes_font_weight : {``ultralight``, ``light``, ``normal``, ``regular``, - ``book``, ``medium``, ``roman``, ``semibold``, - ``demibold``, ``demi``, ``bold``, ``heavy``, - ``extra bold``, ``black``}, optional - The font weight of the axes. - axes_x_limits : (`float`, `float`) or ``None``, optional - The limits of the x axis. If ``None``, it is set to - ``(0., 'errors_max')``. - axes_y_limits : (`float`, `float`) or ``None``, optional - The limits of the y axis. If ``None``, it is set to ``(0., 1.)``. - figure_size : (`float`, `float`) or ``None``, optional - The size of the figure in inches. - render_grid : `bool`, optional - If ``True``, the grid will be rendered. - grid_line_style : {``-``, ``--``, ``-.``, ``:``}, optional - The style of the grid lines. - grid_line_width : `float`, optional - The width of the grid lines. - - Raises - ------ - ValueError - legend_entries list has different length than errors list - - Returns - ------- - viewer : :map:`GraphPlotter` - The viewer object. - """ - from menpo.visualize import GraphPlotter - - # make sure that errors is a list even with one list member - if not isinstance(errors[0], list): - errors = [errors] - - # create x and y axes lists - x_axis = list(np.arange(error_range[0], error_range[1], error_range[2])) - ceds = [compute_cumulative_error(e, x_axis) for e in errors] - - # parse legend_entries, axes_x_limits and axes_y_limits - if legend_entries is None: - legend_entries = ["Curve {}".format(k) for k in range(len(ceds))] - if len(legend_entries) != len(ceds): - raise ValueError('legend_entries list has different length than errors ' - 'list') - if axes_x_limits is None: - axes_x_limits = (0., x_axis[-1]) - if axes_y_limits is None: - axes_y_limits = (0., 1.) - - # render - return GraphPlotter(figure_id=figure_id, new_figure=new_figure, - x_axis=x_axis, y_axis=ceds, title=title, - legend_entries=legend_entries, x_label=x_label, - y_label=y_label, x_axis_limits=axes_x_limits, - y_axis_limits=axes_y_limits).render( - render_lines=render_lines, line_colour=line_colour, - line_style=line_style, line_width=line_width, - render_markers=render_markers, marker_style=marker_style, - marker_size=marker_size, marker_face_colour=marker_face_colour, - marker_edge_colour=marker_edge_colour, - marker_edge_width=marker_edge_width, render_legend=render_legend, - legend_title=legend_title, legend_font_name=legend_font_name, - legend_font_style=legend_font_style, legend_font_size=legend_font_size, - legend_font_weight=legend_font_weight, - legend_marker_scale=legend_marker_scale, - legend_location=legend_location, - legend_bbox_to_anchor=legend_bbox_to_anchor, - legend_border_axes_pad=legend_border_axes_pad, - legend_n_columns=legend_n_columns, - legend_horizontal_spacing=legend_horizontal_spacing, - legend_vertical_spacing=legend_vertical_spacing, - legend_border=legend_border, - legend_border_padding=legend_border_padding, - legend_shadow=legend_shadow, - legend_rounded_corners=legend_rounded_corners, render_axes=render_axes, - axes_font_name=axes_font_name, axes_font_size=axes_font_size, - axes_font_style=axes_font_style, axes_font_weight=axes_font_weight, - figure_size=figure_size, render_grid=render_grid, - grid_line_style=grid_line_style, grid_line_width=grid_line_width) - diff --git a/menpofit/gradientdescent/__init__.py b/menpofit/gradientdescent/__init__.py deleted file mode 100755 index 8d1122e..0000000 --- a/menpofit/gradientdescent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import RLMS diff --git a/menpofit/gradientdescent/base.py b/menpofit/gradientdescent/base.py deleted file mode 100644 index c9546ee..0000000 --- a/menpofit/gradientdescent/base.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import division -import numpy as np -from menpofit.base import build_sampling_grid - -multivariate_normal = None # expensive, from scipy.stats - -from menpofit.fitter import Fitter -from menpofit.fittingresult import SemiParametricFittingResult - - -# TODO: incorporate different residuals -# TODO: generalize transform prior, and map the changes to LK methods -class GradientDescent(Fitter): - r""" - Abstract Interface for defining Gradient Descent based fitting algorithms - for Constrained Local Models [1]_. - - Parameters - ---------- - classifiers : `list` of ``classifier_callable`` - The list containing the classifier that will produce the response - maps for each landmark point. - patch_shape : `tuple` of `int` - The shape of the patches used to train the classifiers. - transform : :map:`GlobalPDM` or subclass - The global point distribution model to be used. - - .. note:: - - Only :map:`GlobalPDM` and its subclasses are supported. - :map:`PDM` is not supported at the moment. - eps : `float`, optional - The convergence value. When calculating the level of convergence, if - the norm of the delta parameter updates is less than ``eps``, the - algorithm is considered to have converged. - - References - ---------- - .. [1] J. Saragih, S. Lucey and J. Cohn, ''Deformable Model Fitting by - Regularized Landmark Mean-Shifts", International Journal of Computer - Vision (IJCV), 2010. - """ - def __init__(self, classifiers, patch_shape, pdm, eps=10**-10): - self.classifiers = classifiers - self.patch_shape = patch_shape - self.transform = pdm - self.eps = eps - # pre-computations - self._set_up() - - def _create_fitting_result(self, image, parameters, gt_shape=None): - return SemiParametricFittingResult( - image, self, parameters=[parameters], gt_shape=gt_shape) - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - self.transform.from_vector_inplace(initial_parameters) - return Fitter.fit(self, image, initial_parameters, gt_shape=gt_shape, - **kwargs) - - def get_parameters(self, shape): - self.transform.set_target(shape) - return self.transform.as_vector() - - -class RLMS(GradientDescent): - r""" - Implementation of the Regularized Landmark Mean-Shifts algorithm for - fitting Constrained Local Models described in [1]_. - - Parameters - ---------- - classifiers : `list` of ``classifier_callable`` - The list containing the classifier that will produce the response - maps for each landmark point. - patch_shape : `tuple` of `int` - The shape of the patches used to train the classifiers. - transform : :map:`GlobalPDM` or subclass - The global point distribution model to be used. - - .. note:: - - Only :map:`GlobalPDM` and its subclasses are supported. - :map:`PDM` is not supported at the moment. - eps : `float`, optional - The convergence value. When calculating the level of convergence, if - the norm of the delta parameter updates is less than ``eps``, the - algorithm is considered to have converged. - scale: `float`, optional - Constant value that will be multiplied to the `noise_variance` of - the pdm in order to compute the covariance of the KDE - approximation. - - References - ---------- - .. [1] J. Saragih, S. Lucey and J. Cohn, ''Deformable Model Fitting by - Regularized Landmark Mean-Shifts", International Journal of Computer - Vision (IJCV), 2010. - """ - def __init__(self, classifiers, patch_shape, pdm, eps=10**-10, scale=10): - self.scale = scale - super(RLMS, self).__init__( - classifiers, patch_shape, pdm, eps=eps) - - @property - def algorithm(self): - return 'RLMS' - - def _set_up(self): - global multivariate_normal - if multivariate_normal is None: - from scipy.stats import multivariate_normal # expensive - # Build the sampling grid associated to the patch shape - self._sampling_grid = build_sampling_grid(self.patch_shape) - # Define the 2-dimensional gaussian distribution - mean = np.zeros(self.transform.n_dims) - covariance = self.scale * self.transform.model.noise_variance() - mvn = multivariate_normal(mean=mean, cov=covariance) - # Compute Gaussian-KDE grid - self._kernel_grid = mvn.pdf(self._sampling_grid) - - # Jacobian - self._J = self.transform.d_dp([]) - - # Prior - sim_prior = np.zeros((4,)) - pdm_prior = 1 / self.transform.model.eigenvalues - self._J_prior = np.hstack((sim_prior, pdm_prior)) - - # Inverse Hessian - H = np.einsum('ijk, ilk -> jl', self._J, self._J) - self._inv_H = np.linalg.inv(np.diag(self._J_prior) + H) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - target = self.transform.target - n_iters = 0 - - max_h = image.shape[-2] - 1 - max_w = image.shape[-1] - 1 - - image_pixels = np.reshape(image.pixels, (image.n_channels, -1)).T - response_image = np.zeros((target.n_points, image.shape[-2], - image.shape[-1])) - - # Compute response maps - for j, clf in enumerate(self.classifiers): - response_image[j, :, :] = np.reshape(clf(image_pixels), - image.shape) - - while n_iters < max_iters and error > self.eps: - - mean_shift_target = np.zeros_like(target.points) - - # Compute mean-shift vectors - for j, point in enumerate(target.points): - - patch_grid = (self._sampling_grid + - np.round(point[None, None, ...]).astype(int)) - - x = patch_grid[:, :, 0] - y = patch_grid[:, :, 1] - - # deal with boundaries - x[x > max_h] = max_h - y[y > max_w] = max_w - x[x < 0] = 0 - y[y < 0] = 0 - - kernel_response = response_image[j, x, y] * self._kernel_grid - normalizer = np.sum(kernel_response) - normalized_kernel_response = kernel_response / normalizer - - mean_shift_target[j, :] = np.sum( - normalized_kernel_response * (x, y), axis=(1, 2)) - - # Compute (shape) error term - error = mean_shift_target - target.points - - # Compute steepest descent parameter updates - sd_delta_p = np.einsum('ijk, ik -> j', self._J, error) - - # TODO: a similar approach could be implemented in LK - # Deal with prior - prior = self._J_prior * self.transform.as_vector() - - # Compute parameter updates - delta_p = -np.dot(self._inv_H, prior - sd_delta_p) - - # Update transform weights - parameters = self.transform.as_vector() + delta_p - fitting_result.parameters.append(parameters) - self.transform.from_vector_inplace(parameters) - target = self.transform.target - - # Test convergence - error = np.abs(np.linalg.norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/gradientdescent/residual.py b/menpofit/gradientdescent/residual.py deleted file mode 100755 index 658e0ca..0000000 --- a/menpofit/gradientdescent/residual.py +++ /dev/null @@ -1,104 +0,0 @@ -import abc - - -class Residual(object): - r""" - """ - __metaclass__ = abc.ABCMeta - - @abc.abstractproperty - def error(self): - pass - - @abc.abstractproperty - def error_derivative(self): - pass - - @abc.abstractproperty - def d_dp(self): - pass - - @abc.abstractproperty - def hessian(self): - pass - - -class SSD(Residual): - - type = 'SSD' - - def error(self): - raise ValueError("Not implemented") - - def error_derivative(self): - raise ValueError("Not implemented") - - def d_dp(self): - raise ValueError("Not implemented") - - def hessian(self): - raise ValueError("Not implemented") - - -class Robust(Residual): - - def __init__(self): - raise ValueError("Not implemented") - - def error(self): - raise ValueError("Not implemented") - - def error_derivative(self): - raise ValueError("Not implemented") - - def d_dp(self): - raise ValueError("Not implemented") - - def hessian(self): - raise ValueError("Not implemented") - - @abc.abstractmethod - def _weights(self): - pass - - -class Fair(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class L1L2(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class GemanMcClure(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Cauchy(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Welsch(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Huber(Robust): - - def _weights(self): - raise ValueError("Not implemented") - - -class Turkey(Robust): - - def _weights(self): - raise ValueError("Not implemented") diff --git a/menpofit/lucaskanade/__init__.py b/menpofit/lucaskanade/__init__.py deleted file mode 100755 index a01f4c8..0000000 --- a/menpofit/lucaskanade/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .appearance import SFA, SFC, SIC, AFA, AFC, AIC, PIC - -from .image import FA, FC, IC diff --git a/menpofit/lucaskanade/appearance/__init__.py b/menpofit/lucaskanade/appearance/__init__.py deleted file mode 100644 index 46657ca..0000000 --- a/menpofit/lucaskanade/appearance/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .simultaneous import SFA, SFC, SIC -from .alternating import AFA, AFC, AIC -from .projectout import PIC diff --git a/menpofit/lucaskanade/appearance/alternating.py b/menpofit/lucaskanade/appearance/alternating.py deleted file mode 100644 index b80955d..0000000 --- a/menpofit/lucaskanade/appearance/alternating.py +++ /dev/null @@ -1,174 +0,0 @@ -from scipy.linalg import norm -import numpy as np - -from .base import AppearanceLucasKanade - - -class AFA(AppearanceLucasKanade): - r""" - Alternating Forward Additive algorithm - """ - @property - def algorithm(self): - return 'Alternating-FA' - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [[0]] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute appearance - weights = self.appearance_model.project(IWxp) - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute warp Jacobian - dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images( - image, dW_dp, forward=(self.template, self.transform)) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - fitting_result.parameters.append(parameters) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class AFC(AppearanceLucasKanade): - r""" - Alternating Forward Compositional algorithm - """ - @property - def algorithm(self): - return 'Alternating-FC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [[0]] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute template by projection - weights = self.appearance_model.project(IWxp) - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images(IWxp, self._dW_dp) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class AIC(AppearanceLucasKanade): - r""" - Alternating Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'Alternating-IC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [[0]] - n_iters = 0 - - # Baker-Matthews, Inverse Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute appearance - weights = self.appearance_model.project(IWxp) - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute steepest descent images, VT_dW_dp - self._J = self.residual.steepest_descent_images(self.template, - self._dW_dp) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, self.template) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/lucaskanade/appearance/base.py b/menpofit/lucaskanade/appearance/base.py deleted file mode 100644 index 6cf28d6..0000000 --- a/menpofit/lucaskanade/appearance/base.py +++ /dev/null @@ -1,21 +0,0 @@ -from menpofit.lucaskanade.residual import SSD -from menpofit.lucaskanade.base import LucasKanade - - -class AppearanceLucasKanade(LucasKanade): - - def __init__(self, model, transform, eps=10**-6): - # Note that the only supported residual for Appearance LK is SSD. - # This is because, in general, we don't know how to take the appropriate - # derivatives for arbitrary residuals with (for instance) a project out - # AAM. - # See https://github.com/menpo/menpo/issues/130 for details. - super(AppearanceLucasKanade, self).__init__(SSD(), - transform, eps=eps) - - # in appearance alignment, target image is aligned to appearance model - self.appearance_model = model - # by default, template is assigned to mean appearance - self.template = model.mean() - # pre-compute - self._set_up() diff --git a/menpofit/lucaskanade/appearance/projectout.py b/menpofit/lucaskanade/appearance/projectout.py deleted file mode 100644 index 0d1cd76..0000000 --- a/menpofit/lucaskanade/appearance/projectout.py +++ /dev/null @@ -1,59 +0,0 @@ -import numpy as np -from scipy.linalg import norm - -from .base import AppearanceLucasKanade - - -class PIC(AppearanceLucasKanade): - r""" - Project-Out Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'ProjectOut-IC' - - def _set_up(self): - # Compute warp Jacobian - dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VT_dW_dp - J = self.residual.steepest_descent_images( - self.template, dW_dp) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Baker-Matthews, Inverse Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, self.template) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/lucaskanade/appearance/simultaneous.py b/menpofit/lucaskanade/appearance/simultaneous.py deleted file mode 100644 index a29d35f..0000000 --- a/menpofit/lucaskanade/appearance/simultaneous.py +++ /dev/null @@ -1,184 +0,0 @@ -from scipy.linalg import norm -import numpy as np - -from .base import AppearanceLucasKanade - - -class SFA(AppearanceLucasKanade): - r""" - Simultaneous Forward Additive algorithm - """ - @property - def algorithm(self): - return 'Simultaneous-FA' - - def _fit(self, fitting_result, max_iters=20, project=True): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute warp Jacobian - dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VI_dW_dp - J_aux = self.residual.steepest_descent_images( - image, dW_dp, forward=(self.template, self.transform)) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J_aux.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - fitting_result.parameters.append(parameters) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class SFC(AppearanceLucasKanade): - r""" - Simultaneous Forward Compositional algorithm - """ - @property - def algorithm(self): - return 'Simultaneous-FC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20, project=True): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [] - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute steepest descent images, VI_dW_dp - J_aux = self.residual.steepest_descent_images(IWxp, self._dW_dp) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J_aux.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result - - -class SIC(AppearanceLucasKanade): - r""" - Simultaneous Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'Simultaneous-IC' - - def _set_up(self): - # Compute warp Jacobian - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20, project=True): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - fitting_result.weights = [] - n_iters = 0 - - mean = self.appearance_model.mean() - - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - if n_iters == 0: - # Project image onto the model bases - weights = self.appearance_model.project(IWxp) - else: - # Compute Gauss-Newton appearance parameters updates - diff = (self.template.as_vector() - mean.as_vector()) - self.template.from_vector_inplace(IWxp.as_vector() - diff - - np.dot(J_aux, delta_p)) - delta_weights = self.appearance_model.project(self.template) - weights += delta_weights - - # Reconstruct appearance - self.template = self.appearance_model.instance(weights) - fitting_result.weights.append(weights) - - # Compute steepest descent images, VT_dW_dp - J_aux = self.residual.steepest_descent_images(self.template, - self._dW_dp) - - # Project out appearance model from VT_dW_dp - self._J = self.appearance_model.project_out_vectors(J_aux.T).T - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, mean) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - return fitting_result diff --git a/menpofit/lucaskanade/base.py b/menpofit/lucaskanade/base.py deleted file mode 100644 index 9184805..0000000 --- a/menpofit/lucaskanade/base.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import division -import numpy as np - -from menpofit.fitter import Fitter -from menpofit.fittingresult import ParametricFittingResult - - -class LucasKanade(Fitter): - r""" - An abstract base class for implementations of Lucas-Kanade [1]_ - type algorithms. - - This is to abstract away optimisation specific functionality such as the - calculation of the Hessian (which could be derived using a number of - techniques, currently only Gauss-Newton). - - Parameters - ---------- - image : :map:`Image` - The image to perform the alignment upon. - - .. note:: Only the image is expected within the base class because - different algorithms expect different kinds of template - (image/model) - - residual : :map:`Residual` - The kind of residual to be calculated. This is used to quantify the - error between the input image and the reference object. - - transform : :map:`Alignment` - The transformation type used to warp the image in to the appropriate - reference frame. This is used by the warping function to calculate - sub-pixel coordinates of the input image in the reference frame. - - eps : float, optional - The convergence value. When calculating the level of convergence, if - the norm of the delta parameter updates is less than `eps`, the - algorithm is considered to have converged. - - Default: 1**-10 - - Notes - ----- - The type of optimisation technique chosen will determine properties such - as the convergence rate of the algorithm. The supported optimisation - techniques are detailed below: - - ===== ==================== =============================================== - type full name hessian approximation - ===== ==================== =============================================== - 'GN' Gauss-Newton :math:`\mathbf{J^T J}` - ===== ==================== =============================================== - - Attributes - ---------- - transform - weights - n_iters - - References - ---------- - .. [1] Lucas, Bruce D., and Takeo Kanade. - "An iterative image registration technique with an application to - stereo vision." IJCAI. Vol. 81. 1981. - """ - def __init__(self, residual, transform, eps=10**-10): - # set basic state for all Lucas Kanade algorithms - self.transform = transform - self.residual = residual - self.eps = eps - # setup the optimisation approach - self._calculate_delta_p = self._gauss_newton_update - - def _gauss_newton_update(self, sd_delta_p): - return np.linalg.solve(self._H, sd_delta_p) - - def _set_up(self, **kwargs): - pass - - def _create_fitting_result(self, image, parameters, gt_shape=None): - return ParametricFittingResult(image, self, parameters=[parameters], - gt_shape=gt_shape) - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - self.transform.from_vector_inplace(initial_parameters) - return Fitter.fit(self, image, initial_parameters, gt_shape=gt_shape, - **kwargs) - - def get_parameters(self, shape): - self.transform.set_target(shape) - return self.transform.as_vector() diff --git a/menpofit/lucaskanade/image.py b/menpofit/lucaskanade/image.py deleted file mode 100644 index 76d68e9..0000000 --- a/menpofit/lucaskanade/image.py +++ /dev/null @@ -1,184 +0,0 @@ -from scipy.linalg import norm -import numpy as np - -from .base import LucasKanade - - -class ImageLucasKanade(LucasKanade): - - def __init__(self, template, residual, transform, eps=10 ** -6): - super(ImageLucasKanade, self).__init__(residual, transform, eps=eps) - # in image alignment, we align a template image to the target image - self.template = template - # pre-compute - self._set_up() - - -class FA(ImageLucasKanade): - r""" - Forward Additive algorithm - """ - @property - def algorithm(self): - return 'Image-FA' - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Forward Additive Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute the Jacobian of the warp - dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - # TODO: rename kwarg "forward" to "forward_additive" - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images( - image, dW_dp, forward=(self.template, self.transform)) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - fitting_result.parameters.append(parameters) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - fitting_result.fitted = True - return fitting_result - - -class FC(ImageLucasKanade): - r""" - Forward Compositional algorithm - """ - @property - def algorithm(self): - return 'Image-FC' - - def _set_up(self): - r""" - The forward compositional algorithm pre-computes the Jacobian of the - warp. This is set as an attribute on the class. - """ - # Compute the Jacobian of the warp - self._dW_dp = np.rollaxis( - self.transform.d_dp(self.template.indices()), -1) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Forward Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # TODO: add "forward_compositional" kwarg with options - # In the forward compositional algorithm there are two different - # ways of computing the steepest descent images: - # 1. V[I(x)](W(x,p)) * dW/dx * dW/dp - # 2. V[I(W(x,p))] * dW/dp -> this is what is currently used - # Compute steepest descent images, VI_dW_dp - self._J = self.residual.steepest_descent_images(IWxp, self._dW_dp) - - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - # Compute steepest descent parameter updates - sd_delta_p = self.residual.steepest_descent_update( - self._J, self.template, IWxp) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - fitting_result.fitted = True - return fitting_result - - -class IC(ImageLucasKanade): - r""" - Inverse Compositional algorithm - """ - @property - def algorithm(self): - return 'Image-IC' - - def _set_up(self): - r""" - The Inverse Compositional algorithm pre-computes the Jacobian of the - warp, the steepest descent images and the Hessian. These are all - stored as attributes on the class. - """ - # Compute the Jacobian of the warp - dW_dp = np.rollaxis(self.transform.d_dp(self.template.indices()), -1) - - # Compute steepest descent images, VT_dW_dp - self._J = self.residual.steepest_descent_images( - self.template, dW_dp) - - # TODO: Pre-compute the inverse - # Compute Hessian and inverse - self._H = self.residual.calculate_hessian(self._J) - - def _fit(self, fitting_result, max_iters=20): - # Initial error > eps - error = self.eps + 1 - image = fitting_result.image - n_iters = 0 - - # Baker-Matthews, Inverse Compositional Algorithm - while n_iters < max_iters and error > self.eps: - # Compute warped image with current weights - IWxp = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - - # Compute steepest descent parameter updates. - sd_delta_p = self.residual.steepest_descent_update( - self._J, IWxp, self.template) - - # Compute gradient descent parameter updates - delta_p = np.real(self._calculate_delta_p(sd_delta_p)) - - # Request the pesudoinverse vector from the transform - inv_delta_p = self.transform.pseudoinverse_vector(delta_p) - - # Update warp weights - self.transform.compose_after_from_vector_inplace(inv_delta_p) - fitting_result.parameters.append(self.transform.as_vector()) - - # Test convergence - error = np.abs(norm(delta_p)) - n_iters += 1 - - fitting_result.fitted = True - return fitting_result diff --git a/menpofit/lucaskanade/residual.py b/menpofit/lucaskanade/residual.py deleted file mode 100755 index f97d06f..0000000 --- a/menpofit/lucaskanade/residual.py +++ /dev/null @@ -1,573 +0,0 @@ -""" -This module contains a set of similarity measures that was designed for use -within the Lucas-Kanade framework. They therefore expose a number of methods -that make them useful for inverse compositional and forward additive -Lucas-Kanade. - -These similarity measures are designed to be dimension independent where -possible. For this reason, some methods look more complicated than would be -normally the case. For example, calculating the Hessian involves summing -a multi-dimensional array, so we dynamically calculate the list of axes -to sum over. However, the basics of the logic, other than dimension -reduction, should be similar to the original algorithms. - -References ----------- - -.. [1] Lucas, Bruce D., and Takeo Kanade. - "An iterative image registration technique with an application to stereo - vision." - IJCAI. Vol. 81. 1981. -""" -import abc -import numpy as np -from numpy.fft import fftshift, fft2 -import scipy.linalg - -from menpo.math import log_gabor -from menpo.image import MaskedImage -from menpo.feature import gradient - - -class Residual(object): - """ - An abstract base class for calculating the residual between two images - within the Lucas-Kanade algorithm. The classes were designed - specifically to work within the Lucas-Kanade framework and so no - guarantee is made that calling methods on these subclasses will generate - correct results. - """ - __metaclass__ = abc.ABCMeta - - def __init__(self): - pass - - @property - def error(self): - r""" - The RMS of the error image. - - :type: float - - Notes - ----- - Will only generate a result if the - :func:`steepest_descent_update` function has been calculated prior. - - .. math:: - error = \sqrt{\sum_x E(x)^2} - - where :math:`E(x) = T(x) - I(W(x;p))` within the forward additive - framework. - """ - return np.sqrt(np.mean(self._error_img ** 2)) - - @abc.abstractmethod - def steepest_descent_images(self, image, dW_dp, **kwargs): - r""" - Calculates the standard steepest descent images. - - Within the forward additive framework this is defined as - - .. math:: - \nabla I \frac{\partial W}{\partial p} - - The input image is vectorised (`N`-pixels) so that masked images can - be handled. - - Parameters - ---------- - image : :class:`menpo.image.base.Image` - The image to calculate the steepest descent images from, could be - either the template or input image depending on which framework is - used. - dW_dp : ndarray - The Jacobian of the warp. - - Returns - ------- - VT_dW_dp : (N, n_params) ndarray - The steepest descent images - """ - pass - - @abc.abstractmethod - def calculate_hessian(self, VT_dW_dp): - r""" - Calculates the Gauss-Newton approximation to the Hessian. - - This is abstracted because some residuals expect the Hessian to be - pre-processed. The Gauss-Newton approximation to the Hessian is - defined as: - - .. math:: - \mathbf{J J^T} - - Parameters - ---------- - VT_dW_dp : (N, n_params) ndarray - The steepest descent images. - - Returns - ------- - H : (n_params, n_params) ndarray - The approximation to the Hessian - """ - pass - - @abc.abstractmethod - def steepest_descent_update(self, VT_dW_dp, IWxp, template): - r""" - Calculates the steepest descent parameter updates. - - These are defined, for the forward additive algorithm, as: - - .. math:: - \sum_x [ \nabla I \frac{\partial W}{\partial p} ]^T [ T(x) - I(W(x;p)) ] - - Parameters - ---------- - VT_dW_dp : (N, n_params) ndarray - The steepest descent images. - IWxp : :class:`menpo.image.base.Image` - Either the warped image or the template - (depending on the framework) - template : :class:`menpo.image.base.Image` - Either the warped image or the template - (depending on the framework) - - Returns - ------- - sd_delta_p : (n_params,) ndarray - The steepest descent parameter updates. - """ - pass - - def _calculate_gradients(self, image, forward=None): - r""" - Calculates the gradients of the given method. - - If `forward` is provided, then the gradients are warped - (as required in the forward additive algorithm) - - Parameters - ---------- - image : :class:`menpo.image.base.Image` - The image to calculate the gradients for - forward : (:map:`Image`, :map:`AlignableTransform>`), optional - A tuple containing the extra weights required for the function - `warp` (which should be passed as a function handle). - - Default: `None` - """ - if forward: - # Calculate the gradient over the image - # grad: (dims x ch) x H x W - grad = gradient(image) - # Warp gradient for forward additive using the given transform - # grad: (dims x ch) x h x w - template, transform = forward - grad = grad.warp_to_mask(template.mask, transform, - warp_landmarks=False) - else: - # Calculate the gradient over the image and set one pixels along - # the boundary of the image mask to zero (no reliable gradient - # can be computed there!) - # grad: (dims x ch) x h x w - grad = gradient(image) - grad.set_boundary_pixels() - return grad - - -class SSD(Residual): - - type = 'SSD' - - def steepest_descent_images(self, image, dW_dp, forward=None): - # compute gradient - # grad: dims x ch x pixels - grad = self._calculate_gradients(image, forward=forward) - grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) - - # compute steepest descent images - # gradient: dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - a = grad[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi, sdi2=None): - # compute hessian - # sdi.T: params x (ch x pixels) - # sdi: (ch x pixels) x params - # hessian: params x x params - if sdi2 is None: - H = sdi.T.dot(sdi) - else: - H = sdi.T.dot(sdi2) - return H - - def steepest_descent_update(self, sdi, IWxp, template): - self._error_img = IWxp.as_vector() - template.as_vector() - return sdi.T.dot(self._error_img) - - -class GaborFourier(Residual): - - type = 'GaborFourier' - - def __init__(self, image_shape, **kwargs): - super(GaborFourier, self).__init__() - - if 'filter_bank' in kwargs: - self._filter_bank = kwargs.get('filter_bank') - if self._filter_bank.shape != image_shape: - raise ValueError('Filter bank shape must match the shape ' - 'of the image') - else: - gabor = log_gabor(np.ones(image_shape), **kwargs) - # Get filter bank matrix - self._filter_bank = gabor[2] - - # Flatten the filter bank for vectorized calculations - self._filter_bank = self._filter_bank.ravel() - - def steepest_descent_images(self, image, dW_dp, forward=None): - n_dims = image.n_dims - n_channels = image.n_channels - n_params = dW_dp.shape[-1] - - # compute gradient - # grad: dims x ch x pixels - grad_img = self._calculate_gradients(image, forward=forward) - grad = grad_img.as_vector().reshape((n_dims, n_channels, -1)) - - # compute steepest descent images - # gradient: dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - a = grad[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - - # make sdi images - # sdi_img: ch x h x w x params - sdi_mask = np.tile(grad_img.mask.pixels[0, ..., None], - (1, 1, n_params)) - sdi_img = MaskedImage.blank(grad_img.shape + (n_params,), - n_channels=n_channels, - mask=sdi_mask) - sdi_img.from_vector_inplace(sdi.ravel()) - - # compute FFT over each channel, parameter and dimension - # fft_sdi: ch x h x w x params - fft_sdi = fftshift(fft2(sdi_img.pixels, axes=(-3, -2)), axes=(-3, -2)) - # Note that, fft_sdi is rectangular, i.e. is not define in - # terms of the mask pixels, but in terms of the whole image. - # Selecting mask pixels once the fft has been computed makes no - # sense because they have lost their original spatial meaning. - - # reshape steepest descent images - # sdi: (ch x h x w) x params - return fft_sdi.reshape((-1, fft_sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # reshape steepest descent images - # sdi: ch x (h x w) x params - sdi = sdi.reshape((-1, self._filter_bank.shape[0], sdi.shape[-1])) - - # compute filtered steepest descent images - # filter_bank: (h x w) - # sdi: ch x (h x w) x params - # filtered_sdi: ch x (h x w) x params - filtered_sdi = (self._filter_bank[None, ..., None] ** 0.5) * sdi - - # reshape filtered steepest descent images - # filtered_sdi: (ch x h x w) x params - filtered_sdi = filtered_sdi.reshape((-1, sdi.shape[-1])) - - # compute filtered hessian - # filtered_sdi.T: params x (ch x h x w) - # filtered_sdi: (ch x h x w) x params - # hessian: params x x n_param - return np.conjugate(filtered_sdi).T.dot(filtered_sdi) - - def steepest_descent_update(self, sdi, IWxp, template): - # compute error image - # error_img: ch x h x w - error_img = IWxp.pixels - template.pixels - - # compute FFT error image - # fft_error_img: ch x (h x w) - fft_error_img = fftshift(fft2(error_img)) - fft_error_img = fft_error_img.reshape((IWxp.n_channels, -1)) - - # compute filtered steepest descent images - # filter_bank: (h x w) - # fft_error_img: ch x (h x w) - # filtered_error_img: ch x (h x w) - filtered_error_img = self._filter_bank * fft_error_img - - # reshape _error_img - # error_img: (ch x h x w) - self._error_img = filtered_error_img.ravel() - - # compute steepest descent update - # sdi: params x (ch x h x w) - # error_img: (ch x h x w) - # sdu: params - return sdi.T.dot(np.conjugate(self._error_img)) - - -class ECC(Residual): - - type = 'ECC' - - def _normalise_images(self, image): - # TODO: do we need to copy the image? - # TODO: is this supposed to be per channel normalization? - norm_image = image.copy() - norm_image.normalize_norm_inplace() - return norm_image - - def steepest_descent_images(self, image, dW_dp, forward=None): - # normalize image - norm_image = self._normalise_images(image) - - # compute gradient - # gradient: dims x ch x pixels - grad = self._calculate_gradients(norm_image, forward=forward) - grad = grad.as_vector().reshape((image.n_dims, image.n_channels, -1)) - - # compute steepest descent images - # gradient: dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - a = grad[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # compute hessian - # sdi.T: params x (ch x pixels) - # sdi: (ch x pixels) x params - # hessian: params x x params - H = sdi.T.dot(sdi) - self._H_inv = scipy.linalg.inv(H) - return H - - def steepest_descent_update(self, sdi, IWxp, template): - normalised_IWxp = self._normalise_images(IWxp).as_vector() - normalised_template = self._normalise_images(template).as_vector() - - Gt = sdi.T.dot(normalised_template) - Gw = sdi.T.dot(normalised_IWxp) - - # Calculate the numerator - IWxp_norm = scipy.linalg.norm(normalised_IWxp) - num1 = IWxp_norm ** 2 - num2 = np.dot(Gw.T, np.dot(self._H_inv, Gw)) - num = num1 - num2 - - # Calculate the denominator - den1 = np.dot(normalised_template, normalised_IWxp) - den2 = np.dot(Gt.T, np.dot(self._H_inv, Gw)) - den = den1 - den2 - - # Calculate lambda to choose the step size - # Avoid division by zero - if den > 0: - l = num / den - else: - den3 = np.dot(Gt.T, np.dot(self._H_inv, Gt)) - l1 = np.sqrt(num2 / den3) - l2 = - den / den3 - l = np.maximum(l1, l2) - - self._error_img = l * normalised_IWxp - normalised_template - - return sdi.T.dot(self._error_img) - - -class GradientImages(Residual): - - type = 'GradientImages' - - def _regularise_gradients(self, grad): - pixels = grad.pixels - ab = np.sqrt(np.sum(pixels**2, axis=0)) - m_ab = np.median(ab) - ab = ab + m_ab - grad.pixels = pixels / ab - return grad - - def steepest_descent_images(self, image, dW_dp, forward=None): - n_dims = image.n_dims - n_channels = image.n_channels - - # compute gradient - first_grad = self._calculate_gradients(image, forward=forward) - self._template_grad = self._regularise_gradients(first_grad) - - # compute gradient - # second_grad: dims x dims x ch x pixels - second_grad = self._calculate_gradients(self._template_grad) - second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels, -1)) - - # Fix crossed derivatives: dydx = dxdy - second_grad[1, 0, ...] = second_grad[0, 1, ...] - - # compute steepest descent images - # gradient: dims x dims x ch x (h x w) - # dw_dp: dims x x (h x w) x params - # sdi: dims x ch x (h x w) x params - sdi = 0 - a = second_grad[..., None] * dW_dp[:, None, None, ...] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (dims x ch x h x w) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # compute hessian - # sdi.T: params x (dims x ch x pixels) - # sdi: (dims x ch x pixels) x params - # hessian: params x x params - return sdi.T.dot(sdi) - - def steepest_descent_update(self, sdi, IWxp, template): - # compute IWxp regularized gradient - IWxp_grad = self._calculate_gradients(IWxp) - IWxp_grad = self._regularise_gradients(IWxp_grad) - - # compute vectorized error_image - # error_img: (dims x ch x pixels) - self._error_img = (IWxp_grad.as_vector() - - self._template_grad.as_vector()) - - # compute steepest descent update - # sdi.T: params x (dims x ch x pixels) - # error_img: (dims x ch x pixels) - # sdu: params - return sdi.T.dot(self._error_img) - - -class GradientCorrelation(Residual): - - type = 'GradientCorrelation' - - def steepest_descent_images(self, image, dW_dp, forward=None): - n_dims = image.n_dims - n_channels = image.n_channels - - # compute gradient - # grad: dims x ch x pixels - grad = self._calculate_gradients(image, forward=forward) - grad2 = grad.as_vector().reshape((n_dims, n_channels, -1)) - - # compute IGOs (remember axis 0 is y, axis 1 is x) - # grad: dims x ch x pixels - # phi: ch x pixels - # cos_phi: ch x pixels - # sin_phi: ch x pixels - phi = np.angle(grad2[1, ...] + 1j * grad2[0, ...]) - self._cos_phi = np.cos(phi) - self._sin_phi = np.sin(phi) - - # concatenate sin and cos terms so that we can take the second - # derivatives correctly. sin(phi) = y and cos(phi) = x which is the - # correct ordering when multiplying against the warp Jacobian - # cos_phi: ch x pixels - # sin_phi: ch x pixels - # grad: (dims x ch) x pixels - grad.from_vector_inplace( - np.concatenate((self._sin_phi[None, ...], - self._cos_phi[None, ...]), axis=0).ravel()) - - # compute IGOs gradient - # second_grad: dims x dims x ch x pixels - second_grad = self._calculate_gradients(grad) - second_grad = second_grad.masked_pixels().flatten().reshape( - (n_dims, n_dims, n_channels, -1)) - - # Fix crossed derivatives: dydx = dxdy - second_grad[1, 0, ...] = second_grad[0, 1, ...] - - # complete full IGOs gradient computation - # second_grad: dims x dims x ch x pixels - second_grad[1, ...] = (-self._sin_phi[None, ...] * second_grad[1, ...]) - second_grad[0, ...] = (self._cos_phi[None, ...] * second_grad[0, ...]) - - # compute steepest descent images - # gradient: dims x dims x ch x pixels - # dw_dp: dims x x pixels x params - # sdi: ch x pixels x params - sdi = 0 - aux = second_grad[..., None] * dW_dp[None, :, None, ...] - for a in aux.reshape(((-1,) + aux.shape[2:])): - sdi += a - - # compute constant N - # N: 1 - self._N = grad.n_parameters / 2 - - # reshape steepest descent images - # sdi: (ch x pixels) x params - return sdi.reshape((-1, sdi.shape[-1])) - - def calculate_hessian(self, sdi): - # compute hessian - # sdi.T: params x (dims x ch x pixels) - # sdi: (dims x ch x pixels) x params - # hessian: params x x params - return sdi.T.dot(sdi) - - def steepest_descent_update(self, sdi, IWxp, template): - n_dims = IWxp.n_dims - n_channels = IWxp.n_channels - - # compute IWxp gradient - IWxp_grad = self._calculate_gradients(IWxp) - IWxp_grad = IWxp_grad.as_vector().reshape( - (n_dims, n_channels, -1)) - - # compute IGOs (remember axis 0 is y, axis 1 is x) - # IWxp_grad: dims x ch x pixels - # phi: ch x pixels - # IWxp_cos_phi: ch x pixels - # IWxp_sin_phi: ch x pixels - phi = np.angle(IWxp_grad[1, ...] + 1j * IWxp_grad[0, ...]) - IWxp_cos_phi = np.cos(phi) - IWxp_sin_phi = np.sin(phi) - - # compute error image - # error_img: (ch x h x w) - self._error_img = (self._cos_phi * IWxp_sin_phi - - self._sin_phi * IWxp_cos_phi).ravel() - - # compute steepest descent update - # sdi: (ch x pixels) x params - # error_img: (ch x pixels) - # sdu: params - sdu = sdi.T.dot(self._error_img) - - # compute step size - qp = np.sum(self._cos_phi * IWxp_cos_phi + - self._sin_phi * IWxp_sin_phi) - l = self._N / qp - return l * sdu diff --git a/menpofit/regression/__init__.py b/menpofit/regression/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/menpofit/regression/base.py b/menpofit/regression/base.py deleted file mode 100644 index edd2dcf..0000000 --- a/menpofit/regression/base.py +++ /dev/null @@ -1,295 +0,0 @@ -import abc - -from menpofit.fitter import Fitter -from menpofit.fittingresult import (NonParametricFittingResult, - SemiParametricFittingResult, - ParametricFittingResult) - - -class Regressor(Fitter): - r""" - An abstract base class for fitting Regressors. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features): - self.regressor = regressor - self.features = features - - def _set_up(self): - r""" - Abstract method that sets up the fitter object. - """ - pass - - def _fit(self, fitting_result, max_iters=1): - r""" - Abstract method to fit an image. - - Parameters - ---------- - fitting_result : `menpo.fit.fittingresult` - The fitting result object. - max_iters : int - The maximum number of iterations. - """ - image = fitting_result.image - initial_shape = fitting_result.initial_shape - n_iters = 0 - - while n_iters < max_iters: - features = self.features(image, initial_shape) - delta_p = self.regressor(features) - - fitted_shape, parameters = self.update(delta_p, initial_shape) - fitting_result.parameters.append(parameters) - n_iters += 1 - - return fitting_result - - @abc.abstractmethod - def update(self, delta_p, initial_shape): - r""" - Abstract method to update the parameters. - """ - pass - - -class NonParametricRegressor(Regressor): - r""" - Fitter of Non-Parametric Regressor. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features): - super(NonParametricRegressor, self).__init__( - regressor, features) - - @property - def algorithm(self): - r""" - Returns the regression type. - """ - return "Non-Parametric" - - def _create_fitting_result(self, image, shapes, gt_shape=None): - r""" - Creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image.. - shape : :map:`PointCloud` - The current shape. - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return NonParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def update(self, delta_shape, initial_shape): - r""" - Updates the shape. - - Parameters - ---------- - delta_shape : :map:`PointCloud` - The shape increment. - initial_shape : :map:`PointCloud` - The current shape. - """ - fitted_shape = initial_shape.from_vector( - initial_shape.as_vector() + delta_shape) - return fitted_shape, fitted_shape - - def get_parameters(self, shape): - r""" - Method that makes sure that the parameter passed to the fit method is - the shape. - - Parameters - ---------- - shape: :map:`PointCloud` - The current shape. - """ - return shape - - -class SemiParametricRegressor(Regressor): - r""" - Fitter of Semi-Parametric Regressor. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features, transform, update='composition'): - super(SemiParametricRegressor, self).__init__( - regressor, features) - self.transform = transform - self._update = self._select_update(update) - - @property - def algorithm(self): - r""" - Returns the regression type. - """ - return "Semi-Parametric" - - def _create_fitting_result(self, image, parameters, gt_shape=None): - r""" - Creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image.. - shape : :map:`PointCloud` - The current shape. - gt_shape : :map:`PointCloud`, optional - The ground truth shape. - """ - self.transform.from_vector_inplace(parameters) - return SemiParametricFittingResult( - image, self, parameters=[self.transform.as_vector()], - gt_shape=gt_shape) - - def fit(self, image, initial_parameters, gt_shape=None, **kwargs): - self.transform.from_vector_inplace(initial_parameters) - return Fitter.fit(self, image, initial_parameters, gt_shape=gt_shape, - **kwargs) - - def _select_update(self, update): - r""" - Select the way to update the parameters. - - Parameters - ---------- - update : {'compositional', 'additive'} - The update method. - - Returns - ------- - update : `function` - The correct function to apply the update chosen. - """ - if update == 'additive': - return self._additive - elif update == 'compositional': - return self._compositional - else: - raise ValueError('Unknown update string selected. Valid' - 'options are: additive, compositional') - - def _additive(self, delta_p): - r""" - Updates the parameters in the additive way. - - Parameters - ---------- - delta_p : `ndarray` - The parameters increment - """ - parameters = self.transform.as_vector() + delta_p - self.transform.from_vector_inplace(parameters) - - def _compositional(self, delta_p): - r""" - Updates the parameters in the compositional way. - - Parameters - ---------- - delta_p : `ndarray` - The parameters increment - """ - self.transform.compose_after_from_vector_inplace(delta_p) - - def update(self, delta_p, initial_shape): - r""" - Updates the parameters of the shape model. - - Parameters - ---------- - delta_p : `ndarray` - The parameters increment. - - initial_shape : :map:`PointCloud` - The current shape. - """ - self._update(delta_p) - return self.transform.target, self.transform.as_vector() - - def get_parameters(self, shape): - r""" - Method that makes sure that the parameter passed to the fit method is - the model parameters. - - Parameters - ---------- - shape : :map:`PointCloud` - The current shape. - """ - self.transform.set_target(shape) - return self.transform.as_vector() - - -class ParametricRegressor(SemiParametricRegressor): - r""" - Fitter of Parametric Regressor. - - Parameters - ---------- - regressor : callable - The regressor to be used from - `menpo.fit.regression.regressioncallables`. - features : function - The feature function used to regress. - """ - def __init__(self, regressor, features, appearance_model, transform, - update='composition'): - super(ParametricRegressor, self).__init__( - regressor, features, transform, update=update) - self.appearance_model = appearance_model - self.template = appearance_model.mean() - - @property - def algorithm(self): - r""" - Returns the regression type. - """ - return "Parametric" - - def _create_fitting_result(self, image, parameters, gt_shape=None): - r""" - Creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image.. - shape : :map:`PointCloud` - The current shape. - gt_shape : :map:`PointCloud`, optional - The ground truth shape. - """ - self.transform.from_vector_inplace(parameters) - return ParametricFittingResult( - image, self, parameters=[self.transform.as_vector()], - gt_shape=gt_shape) diff --git a/menpofit/regression/parametricfeatures.py b/menpofit/regression/parametricfeatures.py deleted file mode 100644 index eedbe89..0000000 --- a/menpofit/regression/parametricfeatures.py +++ /dev/null @@ -1,176 +0,0 @@ -import numpy as np - - -def extract_parametric_features(appearance_model, warped_image, - rergession_features): - r""" - Extracts a particular parametric feature given an appearance model and - a warped image. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - rergession_features : callable - Defines the function from which the parametric features will be - extracted. - - Non-default regression feature options and new experimental features - can be used by defining a callable. In this case, the callable must - define a constructor that receives as an input an appearance model and - a warped masked image and on calling returns a particular parametric - feature representation. - - Returns - ------- - features : `ndarray` - The resulting parametric features. - """ - if rergession_features is None: - features = weights(appearance_model, warped_image) - elif hasattr(rergession_features, '__call__'): - features = rergession_features(appearance_model, warped_image) - else: - raise ValueError("regression_features can only be: (1) None " - "or (2) a callable defining a non-standard " - "feature computation (see `menpo.fit.regression." - "parametricfeatures`") - return features - - -def weights(appearance_model, warped_image): - r""" - Returns the resulting weights after projecting the warped image to the - appearance PCA model. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return appearance_model.project(warped_image) - - -def whitened_weights(appearance_model, warped_image): - r""" - Returns the sheared weights after projecting the warped image to the - appearance PCA model. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return appearance_model.project_whitened(warped_image) - - -def appearance(appearance_model, warped_image): - r""" - Projects the warped image onto the appearance model and rebuilds from the - weights found. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return appearance_model.reconstruct(warped_image).as_vector() - - -def difference(appearance_model, warped_image): - r""" - Returns the difference between the warped image and the image constructed - by projecting the warped image onto the appearance model and rebuilding it - from the weights found. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model based on which the parametric features will be - computed. - warped_image : :map:`MaskedImage` - The warped image. - """ - return (warped_image.as_vector() - - appearance(appearance_model, warped_image)) - - -def project_out(appearance_model, warped_image): - r""" - Returns a version of the whitened warped image where all the basis of the - model have been projected out and which has been scaled by the inverse of - the appearance model's noise_variance. - - Parameters - ---------- - appearance_model: :class:`menpo.model.pca` - The appearance model based on which the parametric features will be - computed. - warped_image: :class:`menpo.image.masked` - The warped image. - """ - diff = warped_image.as_vector() - appearance_model.mean().as_vector() - return appearance_model.distance_to_subspace_vector(diff).ravel() - - -class nonparametric_regression_features(object): - - def __init__(self, patch_shape, feature_patch_length, regression_features): - self.patch_shape = patch_shape - self.feature_patch_length = feature_patch_length - self.regression_features = regression_features - - def __call__(self, image, shape): - # extract patches - patches = image.extract_patches(shape, patch_size=self.patch_shape) - - features = np.zeros((shape.n_points, self.feature_patch_length)) - for j, patch in enumerate(patches): - # compute features - features[j, ...] = self.regression_features(patch).as_vector() - - return np.hstack((features.ravel(), 1)) - - -class parametric_regression_features(object): - - def __init__(self, transform, template, appearance_model, - regression_features): - self.transform = transform - self.template = template - self.appearance_model = appearance_model - self.regression_features = regression_features - - def __call__(self, image, shape): - self.transform.set_target(shape) - # TODO should the template be a mask or a shape? warp_to_shape here - warped_image = image.warp_to_mask(self.template.mask, self.transform, - warp_landmarks=False) - features = extract_parametric_features( - self.appearance_model, warped_image, self.regression_features) - return np.hstack((features, 1)) - - -class semiparametric_classifier_regression_features(object): - - def __init__(self, patch_shape, classifiers): - self.patch_shape = patch_shape - self.classifiers = classifiers - - def __call__(self, image, shape): - patches = image.extract_patches(shape, patch_size=self.patch_shape) - features = [clf(patch.as_vector(keep_channels=True)) - for (clf, patch) in zip(self.classifiers, patches)] - return np.hstack((np.asarray(features).ravel(), 1)) diff --git a/menpofit/regression/regressors.py b/menpofit/regression/regressors.py deleted file mode 100644 index 5f1b534..0000000 --- a/menpofit/regression/regressors.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import division -import numpy as np - - -class mlr(object): - r""" - Multivariate Linear Regression - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - """ - def __init__(self, X, T, lmda=0): - XX = np.dot(X.T, X) - if lmda > 0: - np.fill_diagonal(XX, lmda + np.diag(XX)) - XT = np.dot(X.T, T) - self.R = np.linalg.solve(XX, XT) - - def __call__(self, x): - return np.dot(x, self.R) - - -class mlr_svd(object): - r""" - Multivariate Linear Regression using SVD decomposition - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - def __init__(self, X, T, variance=None): - self.R, _, _, _ = _svd_regression(X, T, variance=variance) - - def __call__(self, x): - return np.dot(x, self.R) - - -class mlr_pca(object): - r""" - Multivariate Linear Regression using PCA reconstructions - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - def __init__(self, X, T, variance=None): - self.R, _, _, self.V = _svd_regression(X, T, variance=variance) - - def _call__(self, x): - x = np.dot(np.dot(x, self.V.T), self.V) - return np.dot(x, self.R) - - -class mlr_pca_weights(object): - r""" - Multivariate Linear Regression using PCA weights - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - def __init__(self, X, T, variance=None): - _, _, _, self.V = _svd_regression(X, T, variance=variance) - W = np.dot(X, self.V.T) - self.R, _, _, _ = _svd_regression(W, T) - - def __call__(self, x): - w = np.dot(x, self.V.T) - return np.dot(w, self.R) - - -def _svd_regression(X, T, variance=None): - r""" - SVD decomposition for regression. - - Parameters - ---------- - X: numpy.array - The regression features used to create the coefficient matrix. - T: numpy.array - The shapes differential that denote the dependent variable. - variance: float or None, Optional - The SVD variance. - - Default: None - - Raises - ------ - ValueError - variance must be set to a number between 0 and 1 - """ - if variance is not None and not (0 < variance <= 1): - raise ValueError("variance must be set to a number between 0 and 1.") - - U, s, V = np.linalg.svd(X) - if variance: - total = sum(s) - acc = 0 - for j, y in enumerate(s): - acc += y - if acc / total >= variance: - r = j+1 - break - else: - tol = np.max(X.shape) * np.spacing(np.max(s)) - r = np.sum(s > tol) - U = U[:, :r] - s = 1 / s[:r] - V = V[:r, :] - R = np.dot(np.dot(V.T * s, U.T), T) - - return R, U, s, V diff --git a/menpofit/regression/trainer.py b/menpofit/regression/trainer.py deleted file mode 100644 index c1ea8cc..0000000 --- a/menpofit/regression/trainer.py +++ /dev/null @@ -1,649 +0,0 @@ -from __future__ import division, print_function -import abc -import numpy as np -from menpo.image import Image -from menpo.feature import sparse_hog -from menpo.visualize import print_dynamic, progress_bar_str - -from menpofit.base import noisy_align, build_sampling_grid -from menpofit.fittingresult import (NonParametricFittingResult, - SemiParametricFittingResult, - ParametricFittingResult) -from .base import (NonParametricRegressor, SemiParametricRegressor, - ParametricRegressor) -from .parametricfeatures import extract_parametric_features, weights, \ - nonparametric_regression_features, parametric_regression_features, \ - semiparametric_classifier_regression_features -from .regressors import mlr - - -class RegressorTrainer(object): - r""" - An abstract base class for training regressors. - - Parameters - ---------- - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : ``None`` or `string` or `function`, optional - The features that are used during the regression. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : boolean, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - """ - __metaclass__ = abc.ABCMeta - - def __init__(self, reference_shape, regression_type=mlr, - regression_features=None, noise_std=0.04, rotation=False, - n_perturbations=10): - self.reference_shape = reference_shape - self.regression_type = regression_type - self.regression_features = regression_features - self.rotation = rotation - self.noise_std = noise_std - self.n_perturbations = n_perturbations - - def _regression_data(self, images, gt_shapes, perturbed_shapes, - verbose=False): - r""" - Method that generates the regression data : features and delta_ps. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images. - - gt_shapes : :map:`PointCloud` list - List of the ground truth shapes that correspond to the images. - - perturbed_shapes : :map:`PointCloud` list - List of the perturbed shapes in order to regress. - - verbose : `boolean`, optional - If ``True``, the progress is printed. - """ - if verbose: - print_dynamic('- Generating regression data') - - n_images = len(images) - features = [] - delta_ps = [] - for j, (i, s, p_shape) in enumerate(zip(images, gt_shapes, - perturbed_shapes)): - if verbose: - print_dynamic('- Generating regression data - {}'.format( - progress_bar_str((j + 1.) / n_images, show_bar=False))) - for ps in p_shape: - features.append(self.features(i, ps)) - delta_ps.append(self.delta_ps(s, ps)) - return np.asarray(features), np.asarray(delta_ps) - - @abc.abstractmethod - def features(self, image, shape): - r""" - Abstract method to generate the features for the regression. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - pass - - @abc.abstractmethod - def get_features_function(self): - r""" - Abstract method to return the function that computes the features for - the regression. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - pass - - @abc.abstractmethod - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Abstract method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - pass - - def train(self, images, shapes, perturbed_shapes=None, verbose=False, - **kwargs): - r""" - Trains a Regressor given a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to train the regressor. - - shapes : :map:`PointCloud` list - List of the shapes that correspond to the images. - - perturbed_shapes : :map:`PointCloud` list, optional - List of the perturbed shapes used for the regressor training. - - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - regressor : :map:`Regressor` - A regressor object. - - Raises - ------ - ValueError - The number of shapes must be equal to the number of images. - ValueError - The number of perturbed shapes must be equal or multiple to - the number of images. - """ - n_images = len(images) - n_shapes = len(shapes) - - # generate regression data - if n_images != n_shapes: - raise ValueError("The number of shapes must be equal to " - "the number of images.") - elif not perturbed_shapes: - perturbed_shapes = self.perturb_shapes(shapes) - features, delta_ps = self._regression_data( - images, shapes, perturbed_shapes, verbose=verbose) - elif n_images == len(perturbed_shapes): - features, delta_ps = self._regression_data( - images, shapes, perturbed_shapes, verbose=verbose) - else: - raise ValueError("The number of perturbed shapes must be " - "equal or multiple to the number of images.") - - # perform regression - if verbose: - print_dynamic('- Performing regression...') - # Expected to be a callable - regressor = self.regression_type(features, delta_ps, **kwargs) - - # compute regressor RMSE - estimated_delta_ps = regressor(features) - error = np.sqrt(np.mean(np.sum((delta_ps - estimated_delta_ps) ** 2, - axis=1))) - if verbose: - print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(error)) - return self._build_regressor(regressor, self.get_features_function()) - - def perturb_shapes(self, gt_shape): - r""" - Perturbs the given shapes. The number of perturbations is defined by - ``n_perturbations``. - - Parameters - ---------- - gt_shape : :map:`PointCloud` list - List of the shapes that correspond to the images. - will be perturbed. - - Returns - ------- - perturbed_shapes : :map:`PointCloud` list - List of the perturbed shapes. - """ - return [[self._perturb_shape(s) for _ in range(self.n_perturbations)] - for s in gt_shape] - - def _perturb_shape(self, gt_shape): - r""" - Method that performs noisy alignment between the given ground truth - shape and the reference shape. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return noisy_align(self.reference_shape, gt_shape, - noise_std=self.noise_std - ).apply(self.reference_shape) - - @abc.abstractmethod - def _build_regressor(self, regressor, features): - r""" - Abstract method to build a regressor model. - """ - pass - - -class NonParametricRegressorTrainer(RegressorTrainer): - r""" - Class for training a Non-Parametric Regressor. - - Parameters - ---------- - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : `function`, optional - The features that are used during the regression. - - See `menpo.features` for details more details on - Menpo's standard image features and feature options. - See :ref:`feature_functions` for non standard - features definitions. - patch_shape : tuple, optional - The shape of the patches that will be extracted. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - - """ - def __init__(self, reference_shape, regression_type=mlr, - regression_features=sparse_hog, patch_shape=(16, 16), - noise_std=0.04, rotation=False, n_perturbations=10): - super(NonParametricRegressorTrainer, self).__init__( - reference_shape, regression_type=regression_type, - regression_features=regression_features, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.patch_shape = patch_shape - self._set_up() - - def _set_up(self): - # work out feature length per patch - patch_img = Image.init_blank(self.patch_shape, fill=0) - self._feature_patch_length = self.regression_features(patch_img).n_parameters - - @property - def algorithm(self): - r""" - Returns the algorithm name. - """ - return "Non-Parametric" - - def _create_fitting(self, image, shapes, gt_shape=None): - r""" - Method that creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The image object. - - shapes : :map:`PointCloud` list - The shapes. - - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return NonParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def get_features_function(self): - return nonparametric_regression_features(self.patch_shape, - self._feature_patch_length, - self.regression_features) - - def features(self, image, shape): - r""" - Method that extracts the features for the regression, which in this - case are patch based. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - return self.get_features_function()(image, shape) - - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - return (gt_shape.as_vector() - - perturbed_shape.as_vector()) - - def _build_regressor(self, regressor, features): - r""" - Method to build the NonParametricRegressor regressor object. - """ - return NonParametricRegressor(regressor, features) - - -class SemiParametricRegressorTrainer(NonParametricRegressorTrainer): - r""" - Class for training a Semi-Parametric Regressor. - - This means that a parametric shape model and a non-parametric appearance - representation are employed. - - Parameters - ---------- - reference_shape : PointCloud - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : `function`, optional - The features that are used during the regression. - - See :ref:`menpo.features` for details more details on - Menpos standard image features and feature options. - patch_shape : tuple, optional - The shape of the patches that will be extracted. - update : 'compositional' or 'additive' - Defines the way to update the warp. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - - """ - def __init__(self, transform, reference_shape, regression_type=mlr, - regression_features=sparse_hog, patch_shape=(16, 16), - update='compositional', noise_std=0.04, rotation=False, - n_perturbations=10): - super(SemiParametricRegressorTrainer, self).__init__( - reference_shape, regression_type=regression_type, - regression_features=regression_features, patch_shape=patch_shape, - noise_std=noise_std, rotation=rotation, - n_perturbations=n_perturbations) - self.transform = transform - self.update = update - - @property - def algorithm(self): - r""" - Returns the algorithm name. - """ - return "Semi-Parametric" - - def _create_fitting(self, image, shapes, gt_shape=None): - r""" - Method that creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The image object. - - shapes : :map:`PointCloud` list - The shapes. - - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return SemiParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - self.transform.set_target(gt_shape) - gt_ps = self.transform.as_vector() - self.transform.set_target(perturbed_shape) - perturbed_ps = self.transform.as_vector() - return gt_ps - perturbed_ps - - def _build_regressor(self, regressor, features): - r""" - Method to build the NonParametricRegressor regressor object. - """ - return SemiParametricRegressor(regressor, features, self.transform, - self.update) - - -class ParametricRegressorTrainer(RegressorTrainer): - r""" - Class for training a Parametric Regressor. - - Parameters - ---------- - appearance_model : :map:`PCAModel` - The appearance model to be used. - transform : :map:`Affine` - The transform used for warping. - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - regression_features : ``None`` or `function`, optional - The parametric features that are used during the regression. - - If ``None``, the reconstruction appearance weights will be used as - feature. - - If `string` or `function`, the feature representation will be - computed using one of the function in: - - If `string`, the feature representation will be extracted by - executing a parametric feature function. - - Note that this feature type can only be one of the parametric - feature functions defined :ref:`parametric_features`. - patch_shape : tuple, optional - The shape of the patches that will be extracted. - update : 'compositional' or 'additive' - Defines the way to update the warp. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - - """ - def __init__(self, appearance_model, transform, reference_shape, - regression_type=mlr, regression_features=weights, - update='compositional', noise_std=0.04, rotation=False, - n_perturbations=10): - super(ParametricRegressorTrainer, self).__init__( - reference_shape, regression_type=regression_type, - regression_features=regression_features, noise_std=noise_std, - rotation=rotation, n_perturbations=n_perturbations) - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.regression_features = regression_features - self.transform = transform - self.update = update - - @property - def algorithm(self): - r""" - Returns the algorithm name. - """ - return "Parametric" - - def _create_fitting(self, image, shapes, gt_shape=None): - r""" - Method that creates the fitting result object. - - Parameters - ---------- - image : :map:`MaskedImage` - The image object. - - shapes : :map:`PointCloud` list - The shapes. - - gt_shape : :map:`PointCloud` - The ground truth shape. - """ - return ParametricFittingResult(image, self, parameters=[shapes], - gt_shape=gt_shape) - - def get_features_function(self): - return parametric_regression_features(self.transform, self.template, - self.appearance_model, - self.regression_features) - - def features(self, image, shape): - r""" - Method that extracts the features for the regression, which in this - case are patch based. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - return self.get_features_function()(image, shape) - - def delta_ps(self, gt_shape, perturbed_shape): - r""" - Method to generate the delta_ps for the regression. - - Parameters - ---------- - gt_shape : :map:`PointCloud` - The ground truth shape. - - perturbed_shape : :map:`PointCloud` - The perturbed shape. - """ - self.transform.set_target(gt_shape) - gt_ps = self.transform.as_vector() - self.transform.set_target(perturbed_shape) - perturbed_ps = self.transform.as_vector() - return gt_ps - perturbed_ps - - def _build_regressor(self, regressor, features): - r""" - Method to build the NonParametricRegressor regressor object. - """ - return ParametricRegressor( - regressor, features, self.appearance_model, self.transform, - self.update) - - -class SemiParametricClassifierBasedRegressorTrainer( - SemiParametricRegressorTrainer): - r""" - Class for training a Semi-Parametric Classifier-Based Regressor. This means - that the classifiers are used instead of features. - - Parameters - ---------- - classifiers : list of :map:`classifiers` - List of classifiers. - transform : :map:`Affine` - The transform used for warping. - reference_shape : :map:`PointCloud` - The reference shape that will be used. - regression_type : `callable`, optional - A `callable` that defines the regression technique to be used. - Examples of such callables can be found in - :ref:`regression_callables` - patch_shape : tuple, optional - The shape of the patches that will be extracted. - noise_std : `float`, optional - The standard deviation of the gaussian noise used to produce the - training shapes. - rotation : `boolean`, optional - Specifies whether ground truth in-plane rotation is to be used - to produce the training shapes. - n_perturbations : `int`, optional - Defines the number of perturbations that will be applied to the - training shapes. - """ - def __init__(self, classifiers, transform, reference_shape, - regression_type=mlr, patch_shape=(16, 16), - update='compositional', noise_std=0.04, rotation=False, - n_perturbations=10): - super(SemiParametricClassifierBasedRegressorTrainer, self).__init__( - transform, reference_shape, regression_type=regression_type, - patch_shape=patch_shape, update=update, - noise_std=noise_std, rotation=rotation, - n_perturbations=n_perturbations) - self.classifiers = classifiers - - def _set_up(self): - # TODO: CLMs should use slices instead of sampling grid, and the - # need of the _set_up method will probably disappear - # set up sampling grid - self.sampling_grid = build_sampling_grid(self.patch_shape) - - def get_features_function(self): - return semiparametric_classifier_regression_features(self.patch_shape, - self.classifiers) - - def features(self, image, shape): - r""" - Method that extracts the features for the regression, which in this - case are patch based. - - Parameters - ---------- - image : :map:`MaskedImage` - The current image. - - shape : :map:`PointCloud` - The current shape. - """ - return self.get_features_function()(image, shape) diff --git a/menpofit/transform/modeldriven.py b/menpofit/transform/modeldriven.py index 0238db5..42c1993 100644 --- a/menpofit/transform/modeldriven.py +++ b/menpofit/transform/modeldriven.py @@ -5,6 +5,7 @@ from menpo.transform.base import Transform, VComposable, VInvertible from menpofit.differentiable import DP + # TODO: Should MDT implement VComposable and VInvertible? class ModelDrivenTransform(Transform, Targetable, Vectorizable, VComposable, VInvertible, DP): diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 396e7c4..05739ee 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -1590,7 +1590,7 @@ def plot_ced(errors, legend_entries=None, error_range=None, as part of a parent widget. If ``False``, the widget object is not returned, it is just visualized. """ - from menpofit.fittingresult import plot_cumulative_error_distribution + from menpofit.result import plot_cumulative_error_distribution print('Initializing...') # Make sure that errors is a list even with one list member From 2946407799031fc06fdd4bfcdcc8cbc1b81f1104 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:15:52 +0100 Subject: [PATCH 381/423] GaussNewton regression needed transposing Seems to be correct as it doesn't cause an exception - but it doesn't work particularly well. --- menpofit/math/regression.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/menpofit/math/regression.py b/menpofit/math/regression.py index 82dcce0..9b9c65d 100644 --- a/menpofit/math/regression.py +++ b/menpofit/math/regression.py @@ -50,8 +50,9 @@ class IIRLRegression(IRLRegression): r""" Indirect Incremental Regularized Linear Regression """ - def __init__(self, alpha=0, bias=True, alpha2=0): - super(IIRLRegression, self).__init__(alpha=alpha, bias=bias) + def __init__(self, alpha=0, bias=False, alpha2=0): + # TODO: Can we model the bias? May need to slice off of prediction? + super(IIRLRegression, self).__init__(alpha=alpha, bias=False) self.alpha2 = alpha2 def train(self, X, Y): @@ -60,9 +61,10 @@ def train(self, X, Y): J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution - H = J.T.dot(J) + # Note that everything is transposed from the above exchanging of roles + H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J.T) + self.W = np.linalg.solve(H, J).T def increment(self, X, Y): # incremental least squares exchanging the roles of X and Y @@ -70,6 +72,7 @@ def increment(self, X, Y): J = self.W # solve the original problem by computing the pseudo-inverse of the # previous solution - H = J.T.dot(J) + # Note that everything is transposed from the above exchanging of roles + H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J.T) + self.W = np.linalg.solve(H, J).T From e0080f3e31d167cd6a9f7b7e32a64106e09aef21 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:16:45 +0100 Subject: [PATCH 382/423] Remove extra space --- menpofit/modelinstance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/menpofit/modelinstance.py b/menpofit/modelinstance.py index 7810e72..e18c540 100644 --- a/menpofit/modelinstance.py +++ b/menpofit/modelinstance.py @@ -246,7 +246,6 @@ def _weights_for_target(self, target): Weights of the statistical model that generate the closest PointCloud to the requested target """ - self._update_global_transform(target) projected_target = self.global_transform.pseudoinverse().apply(target) # now we have the target in model space, project it to recover the From 67559341ee0b086d1629b5b3377b76c74388cbad Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:17:13 +0100 Subject: [PATCH 383/423] Remove unused methods and correct spelling from SDM package --- menpofit/sdm/algorithm.py | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index fe713c7..b8affbb 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -160,7 +160,7 @@ def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, self.eps = eps -# TODO: docment me! +# TODO: document me! def features_per_patch(image, shape, patch_shape, features_callable): """r """ @@ -171,7 +171,7 @@ def features_per_patch(image, shape, patch_shape, features_callable): return np.asarray(patch_features).ravel() -# TODO: docment me! +# TODO: document me! def features_per_shape(image, shapes, patch_shape, features_callable): """r """ @@ -182,7 +182,7 @@ def features_per_shape(image, shapes, patch_shape, features_callable): return np.asarray(patch_features) -# TODO: docment me! +# TODO: document me! def features_per_image(images, shapes, patch_shape, features_callable, level_str='', verbose=False): """r @@ -250,25 +250,3 @@ def compute_features_info(image, shape, features_callable, return (features_patch_shape, features_patch_length, features_shape, features_length) - - -# def initialize_sampling(self, image, group=None, label=None): -# if self._sampling is None: -# sampling = np.ones(self.patch_shape, dtype=np.bool) -# else: -# sampling = self._sampling -# -# # TODO: include offsets support? -# patches = image.extract_patches_around_landmarks( -# group=group, label=label, patch_size=self.patch_shape, -# as_single_array=True) -# -# # TODO: include offsets support? -# features_patch_shape = self.features(patches[0, 0]).shape -# self._features_patch_length = np.prod(features_patch_shape) -# self._features_shape = (patches.shape[0], features_patch_shape) -# self._features_length = np.prod(self._features_shape) -# -# feature_mask = np.tile(sampling[None, None, None, ...], -# self._feature_shape[:3] + (1, 1)) -# self._feature_mask = np.nonzero(feature_mask.flatten())[0] From 11f69b73991b3f5311bfe5f858247c891f59f9c8 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 13:20:57 +0100 Subject: [PATCH 384/423] Attempting to refactor SD-AAM This attempts to refactor the Supervised Descent AAM code - including attempting to get PO-CR implemented. It hasn't been exhaustively tested, but I need to make a number of other changes and I wanted a discrete commit for this refactoring work. Changes: The SupervisedDescentAAMFitter is a subclass of the SupervisedDescentFitter. Therefore, it acts more like an SDM than an AAM. This is great because it borrows a lot of code from the SDFitter - which we are pretty happy with. As an extension to this, the SDAAM algorithms have been totally refactored. Now, they act exactly like SupervisedDescentAlgorithm, but parametric. The major change is that the take an interface directly rather than an interface class. Thus, the interface properly encapsulates the AAM state rather than the algorithm. Finally, accepting the refactoring of the train method to match the SDAlgorithm class format (but with parameters), the interfaces now don't have a circular dependancy on the SDAlgorithm. The SDInterfaces now encapsulate warping and the transform. The subclasses of SDAlgorithm, like ProjectOut are thus fairly simple. The finaly thing is that results need to be refactored to now have this dependancy on the whole algorithm. Therefore, I've added a horrible shim that just properly returns the transform to allow the algorithms to keep working for the time being. --- menpofit/aam/algorithm/sd.py | 500 +++++++++++++++++------------------ menpofit/aam/fitter.py | 31 +-- 2 files changed, 251 insertions(+), 280 deletions(-) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index 903f4bf..bb3242e 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -1,8 +1,13 @@ from __future__ import division +from functools import partial import numpy as np from menpo.image import Image from menpo.feature import no_op -from menpo.visualize import print_dynamic, progress_bar_str +from menpo.visualize import print_dynamic +from menpofit.math import IRLRegression, IIRLRegression +from menpofit.result import compute_normalise_point_to_point_error +from menpofit.sdm.algorithm import SupervisedDescentAlgorithm +from menpofit.visualize import print_progress from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult @@ -10,9 +15,14 @@ class SupervisedDescentStandardInterface(object): r""" """ - def __init__(self, cr_aam_algorithm, sampling=None): - self.algorithm = cr_aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None): + self.appearance_model = appearance_model + self.transform = transform + self.template = template + + self._build_sampling_mask(sampling) + def _build_sampling_mask(self, sampling): n_true_pixels = self.template.n_true_pixels() n_channels = self.template.n_channels sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) @@ -29,18 +39,6 @@ def __init__(self, cr_aam_algorithm, sampling=None): def shape_model(self): return self.transform.pdm.model - @property - def appearance_model(self): - return self.algorithm.appearance_model - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - @property def n(self): return self.transform.n_parameters @@ -55,8 +53,11 @@ def warp(self, image): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): + # TODO: Faking an 'algorithm' + algorithm = lambda x: x + algorithm.transform = self.transform return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, algorithm, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -70,8 +71,11 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): + # TODO: Faking an 'algorithm' + algorithm = lambda x: x + algorithm.transform = self.transform return LinearAAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, algorithm, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -79,16 +83,20 @@ def algorithm_result(self, image, shape_parameters, class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): r""" """ - def __init__(self, cr_aam_algorithm, sampling=None, patch_shape=(17, 17), - normalize_parts=no_op): - self.algorithm = cr_aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None, + patch_shape=(17, 17), normalize_parts=no_op): self.patch_shape = patch_shape + # TODO: Refactor to patch_features self.normalize_parts = normalize_parts + super(SupervisedDescentPartsInterface, self).__init__( + appearance_model, transform, template, sampling=sampling) + + def _build_sampling_mask(self, sampling): if sampling is None: sampling = np.ones(self.patch_shape, dtype=np.bool) - image_shape = self.algorithm.template.pixels.shape + image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], image_shape[:3] + (1, 1)) self.i_mask = np.nonzero(image_mask.flatten())[0] @@ -102,137 +110,143 @@ def warp(self, image): patch_size=self.patch_shape, as_single_array=True) parts = self.normalize_parts(parts) - return Image(parts) + return Image(parts, copy=False) + + +def _weights_for_target(transform, target): + transform.set_target(target) + return transform.as_vector() # TODO document me! -class SupervisedDescent(object): +def obtain_parametric_delta_x(gt_shapes, current_shapes, transform): + # initialize current and delta parameters arrays + n_samples = len(gt_shapes) * len(current_shapes[0]) + gt_params = np.empty((n_samples, transform.n_parameters)) + delta_params = np.empty_like(gt_params) + + k = 0 + for gt_s, c_s in zip(gt_shapes, current_shapes): + # Compute and cache ground truth parameters + c_gt_params = _weights_for_target(transform, gt_s) + for s in c_s: + gt_params[k] = c_gt_params + + current_params = _weights_for_target(transform, s) + delta_params[k] = c_gt_params - current_params + + k += 1 + + return delta_params, gt_params + + +class ParametricSupervisedDescentAlgorithm(SupervisedDescentAlgorithm): r""" """ - def __init__(self, aam_interface, appearance_model, transform, max_iters=3, - eps=10**-5, **kwargs): - # set common state for all AAM algorithms - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.transform = transform - self.max_iters = max_iters - # TODO: Make use of eps in self.train? + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5): + super(ParametricSupervisedDescentAlgorithm, self).__init__() + + self.interface = aam_interface + self.n_iterations = n_iterations self.eps = eps - # set interface - self.interface = aam_interface(self, **kwargs) - # perform pre-computations + + self._compute_error = compute_error self._precompute() + @property + def appearance_model(self): + return self.interface.appearance_model + + @property + def transform(self): + return self.interface.transform + def _precompute(self): - # grab appearance model mean + # Grab appearance model mean a_bar = self.appearance_model.mean() - # vectorize it and mask it + # Vectorise it and mask it self.a_bar_m = a_bar.as_vector()[self.interface.i_mask] - def train(self, images, gt_shapes, current_shapes, verbose=False, - **kwargs): - n_images = len(images) - n_samples_image = len(current_shapes[0]) + def _train(self, images, gt_shapes, current_shapes, increment=False, + level_str='', verbose=False): - # set number of iterations and initialize list of regressors - self.regressors = [] + if not increment: + # Reset the regressors + self.regressors = [] - # compute current and delta parameters from current and ground truth - # shapes - delta_params, current_params, gt_params = self._generate_params( - gt_shapes, current_shapes) - # initialize iteration counter - k = 0 + n_perturbations = len(current_shapes[0]) + template_shape = gt_shapes[0] + + # obtain delta_x and gt_x (parameters rather than shapes) + delta_x, gt_x = obtain_parametric_delta_x(gt_shapes, current_shapes, + self.transform) # Cascaded Regression loop - while k < self.max_iters: + for k in range(self.n_iterations): # generate regression data - features = self._generate_features(images, current_params, - verbose=verbose) + features = self._generate_features( + images, current_shapes, + level_str='{}(Iteration {}) - '.format(level_str, k), + verbose=verbose) - # perform regression if verbose: - print_dynamic('- Performing regression...') - regressor = self._perform_regression(features, delta_params, - **kwargs) - # add regressor to list - self.regressors.append(regressor) - - # compute regression rmse - estimated_delta_params = regressor(features) - # TODO: Should print a more informative error here? - rmse = _compute_rmse(delta_params, estimated_delta_params) + print_dynamic('{}(Iteration {}) - Performing regression'.format( + level_str, k)) + + if not increment: + r = self._regressor_cls() + r.train(features, delta_x) + self.regressors.append(r) + else: + self.regressors[k].increment(features, delta_x) + + # Estimate delta_points + estimated_delta_x = self.regressors[k].predict(features) if verbose: - print_dynamic('- Regression RMSE is {0:.5f}.\n'.format(rmse)) - - current_params += estimated_delta_params + self._print_regression_info(template_shape, gt_shapes, + n_perturbations, delta_x, + estimated_delta_x, k, + level_str=level_str) + + j = 0 + for shapes in current_shapes: + for s in shapes: + # Estimate parameters + edx = estimated_delta_x[j] + # Current parameters + cx = _weights_for_target(self.transform, s) + edx + + # Uses less memory to find updated target shape + self.transform.from_vector_inplace(cx) + # Update current shape inplace + s.from_vector_inplace(self.transform.target.as_vector()) + + delta_x[j] = gt_x[j] - cx + j += 1 + + return current_shapes + + def _generate_features(self, images, current_shapes, level_str='', + verbose=False): + # Initialize features array - since current_shapes is a list of lists + # we need to know the total size + n_samples = len(images) * len(current_shapes[0]) + features = np.empty((n_samples,) + self.a_bar_m.shape) + + wrap = partial(print_progress, + prefix='{}Computing features'.format(level_str), + end_with_newline=not level_str, verbose=verbose) - delta_params = gt_params - current_params - # increase iteration counter - k += 1 - - # obtain current shapes from current parameters - current_shapes = [] - for p in current_params: - current_shapes.append(self.transform.from_vector(p).target) - - # convert current shapes into a list of list and return - final_shapes = [] - for j in range(n_images): - k = j * n_samples_image - l = k + n_samples_image - final_shapes.append(current_shapes[k:l]) - return final_shapes - - def _generate_params(self, gt_shapes, current_shapes): - # initialize current and delta parameters arrays - n_samples = len(gt_shapes) * len(current_shapes[0]) - current_params = np.empty((n_samples, self.transform.n_parameters)) - gt_params = np.empty((n_samples, self.transform.n_parameters)) - delta_params = np.empty((n_samples, self.transform.n_parameters)) # initialize sample counter k = 0 - # compute ground truth and current shape parameters - for gt_s, c_s in zip(gt_shapes, current_shapes): - for s in c_s: - # compute current parameters - current_params[k] = self._compute_params(s) - # compute ground truth parameters - gt_params[k] = self._compute_params(gt_s) - # compute delta parameters - delta_params[k] = gt_params[k] - current_params[k] - # increment counter - k += 1 + for img, img_shapes in wrap(zip(images, current_shapes)): + for s in img_shapes: + self.transform.set_target(s) + # Assumes that the transform is correctly set + features[k] = self._compute_features(img) - return delta_params, current_params, gt_params - - def _compute_params(self, shape): - self.transform.set_target(shape) - return self.transform.as_vector() - - def _generate_features(self, images, current_params, verbose=False): - # initialize features array - n_images = len(images) - n_samples = len(current_params) - n_samples_image = int(n_samples / n_images) - features = np.zeros((n_samples,) + self.a_bar_m.shape) - - # initialize sample counter - k = 0 - for i in images: - for _ in range(n_samples_image): - if verbose: - print_dynamic('- Generating regression features - {' - '}'.format( - progress_bar_str((k + 1.) / n_samples, - show_bar=False))) - # set transform - self.transform.from_vector_inplace(current_params[k]) - # compute regression features - f = self._compute_train_features(i) - # add to features array - features[k] = f - # increment counter k += 1 return features @@ -242,47 +256,53 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] - # initialize iteration counter - k = 0 - # Cascaded Regression loop - while k < self.max_iters: - # compute regression features - features = self._compute_test_features(image) + for r in self.regressors: + # Assumes that the transform is correctly set + features = self._compute_features(image) # solve for increments on the shape parameters - dp = self.regressors[k](features) + dx = r.predict(features) - # update warp - self.transform.from_vector_inplace(self.transform.as_vector() + dp) - p_list.append(self.transform.as_vector()) - - # increase iteration counter - k += 1 + # We need to update the transform to set the state for the warping + # of the image above. + new_x = p_list[-1] + dx + self.transform.from_vector_inplace(new_x) + p_list.append(new_x) # return algorithm result return self.interface.algorithm_result( image, p_list, gt_shape=gt_shape) + def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, + delta_x, estimated_delta_x, level_index, + level_str=''): + print_dynamic('{}(Iteration {}) - Calculating errors'.format( + level_str, level_index)) + errors = [] + for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): + self.transform.from_vector_inplace(dx) + s1 = self.transform.target + self.transform.from_vector_inplace(edx) + s2 = self.transform.target + + gt_s = gt_shapes[np.floor_divide(j, n_perturbations)] + errors.append(self._compute_error(s1, s2, gt_s)) + mean = np.mean(errors) + std = np.std(errors) + median = np.median(errors) + print_dynamic('{}(Iteration {}) - Training error -> ' + 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. + format(level_str, level_index, mean, std, median)) + # TODO: document me! -class MeanTemplate(SupervisedDescent): +class MeanTemplate(ParametricSupervisedDescentAlgorithm): r""" """ - def _compute_train_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - return i_m - self.a_bar_m - - def _compute_test_features(self, image): - # warp image + def _compute_features(self, image): i = self.interface.warp(image) - # vectorize it and mask it i_m = i.as_vector()[self.interface.i_mask] - # compute masked error return i_m - self.a_bar_m @@ -290,111 +310,103 @@ def _compute_test_features(self, image): class MeanTemplateNewton(MeanTemplate): r""" """ - def _perform_regression(self, features, deltas, gamma=None, - dtype=np.float64): - return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, bias=True): + super(MeanTemplateNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) # TODO: document me! class MeanTemplateGaussNewton(MeanTemplate): r""" """ - def _perform_regression(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - return _supervised_gauss_newton(features, deltas, gamma=gamma, - psi=psi, dtype=dtype) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, alpha2=0, bias=True): + super(MeanTemplateGaussNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IIRLRegression, alpha=alpha, + alpha2=alpha2, bias=bias) # TODO: document me! -class ProjectOut(SupervisedDescent): +class ProjectOut(ParametricSupervisedDescentAlgorithm): r""" """ def _precompute(self): - # call super method super(ProjectOut, self)._precompute() - # grab appearance model components A = self.appearance_model.components - # mask them self.A_m = A.T[self.interface.i_mask, :] - # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(self.A_m) def project_out(self, J): - # project-out appearance bases from a particular vector or matrix + # Project-out appearance bases from a particular vector or matrix return J - self.A_m.dot(self.pinv_A_m.dot(J)) - def _compute_train_features(self, image): - # warp image + def _compute_features(self, image): i = self.interface.warp(image) - # vectorize it and mask it i_m = i.as_vector()[self.interface.i_mask] - # compute masked error + # TODO: This project out could actually be cached at test time - + # but we need to think about the best way to implement this and still + # allow incrementing e_m = i_m - self.a_bar_m return self.project_out(e_m) - def _compute_test_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # compute masked error - return i_m - self.a_bar_m - # TODO: document me! class ProjectOutNewton(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None, - dtype=np.float64): - regressor = _supervised_newton(features, deltas, gamma=gamma, - dtype=dtype) - regressor.R = self.project_out(regressor.R) - return regressor + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, bias=True): + super(ProjectOutNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) # TODO: document me! class ProjectOutGaussNewton(ProjectOut): r""" """ - def _perform_regression(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - return _supervised_gauss_newton(features, deltas, gamma=gamma, - psi=psi, dtype=dtype) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, alpha2=0, bias=True): + super(ProjectOutGaussNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + self._regressor_cls = partial(IIRLRegression, alpha=alpha, + alpha2=alpha2, bias=bias) # TODO: document me! -class AppearanceWeights(SupervisedDescent): +class AppearanceWeights(ParametricSupervisedDescentAlgorithm): r""" """ def _precompute(self): - # call super method super(AppearanceWeights, self)._precompute() - # grab appearance model components A = self.appearance_model.components - # mask them A_m = A.T[self.interface.i_mask, :] - # compute their pseudoinverse + self.pinv_A_m = np.linalg.pinv(A_m) def project(self, J): - # project a particular vector or matrix onto the appearance bases + # Project a particular vector or matrix onto the appearance bases return self.pinv_A_m.dot(J - self.a_bar_m) - def _compute_train_features(self, image): - # warp image + def _compute_features(self, image): i = self.interface.warp(image) - # vectorize it and mask it i_m = i.as_vector()[self.interface.i_mask] - # project it onto the appearance model - return self.project(i_m) - - def _compute_test_features(self, image): - # warp image - i = self.interface.warp(image) - # vectorize it and mask it - i_m = i.as_vector()[self.interface.i_mask] - # project it onto the appearance model + # Project image onto the appearance model return self.project(i_m) @@ -402,65 +414,27 @@ def _compute_test_features(self, image): class AppearanceWeightsNewton(AppearanceWeights): r""" """ - def _perform_regression(self, features, deltas, gamma=None, - dtype=np.float64): - return _supervised_newton(features, deltas, gamma=gamma, dtype=dtype) - + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, bias=True): + super(AppearanceWeightsNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) -# TODO: document me! -class AppearanceWeightsGaussNewton(AppearanceWeights): - r""" - """ - def _perform_regression(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - return _supervised_gauss_newton(features, deltas, gamma=gamma, - psi=psi, dtype=dtype) + self._regressor_cls = partial(IRLRegression, alpha=alpha, + bias=bias) # TODO: document me! -class _supervised_newton(object): - r""" - """ - def __init__(self, features, deltas, gamma=None, dtype=np.float64): - features = features.astype(dtype) - deltas = deltas.astype(dtype) - XX = features.T.dot(features) - XT = features.T.dot(deltas) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # descent direction - self.R = np.linalg.solve(XX, XT) - - def __call__(self, features): - return np.dot(features, self.R) - - -# TODO: document me! -class _supervised_gauss_newton(object): +class AppearanceWeightsGaussNewton(AppearanceWeights): r""" """ - def __init__(self, features, deltas, gamma=None, psi=None, - dtype=np.float64): - features = features.astype(dtype) - # ridge regression - deltas = deltas.astype(dtype) - XX = deltas.T.dot(deltas) - XT = deltas.T.dot(features) - if gamma: - np.fill_diagonal(XX, gamma + np.diag(XX)) - # average Jacobian - self.J = np.linalg.solve(XX, XT) - # average Hessian - self.H = self.J.dot(self.J.T) - if psi: - np.fill_diagonal(self.H, psi + np.diag(self.H)) - # descent direction - self.R = np.linalg.solve(self.H, self.J).T - - def __call__(self, features): - return np.dot(features, self.R) - - -# TODO: document me! -def _compute_rmse(x1, x2): - return np.sqrt(np.mean(np.sum((x1 - x2) ** 2, axis=1))) + def __init__(self, aam_interface, n_iterations=3, + compute_error=compute_normalise_point_to_point_error, + eps=10**-5, alpha=0, alpha2=0, bias=True): + super(AppearanceWeightsGaussNewton, self).__init__( + aam_interface, n_iterations=n_iterations, + compute_error=compute_error, eps=eps) + + self._regressor_cls = partial(IIRLRegression, alpha=alpha, + alpha2=alpha2, bias=bias) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 58d3404..f0c4cb5 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -125,37 +125,34 @@ def _setup_algorithms(self): for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, self.aam.shape_models, self._sampling)): - + template = am.mean() if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, - source=am.mean().landmarks['source'].lms) - # set up algorithm using standard aam interface + source=template.landmarks['source'].lms) + interface = SupervisedDescentStandardInterface( + am, md_transform, template, sampling=s) algorithm = self._sd_algorithm_cls( - SupervisedDescentStandardInterface, am, md_transform, - sampling=s, max_iters=self.n_iterations[j]) - + interface, n_iterations=self.n_iterations[j]) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): - # build linear version of orthogonal model driven transform + # Build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) - # set up algorithm using linear aam interface + interface = SupervisedDescentLinearInterface( + am, md_transform, template, sampling=s) algorithm = self._sd_algorithm_cls( - SupervisedDescentLinearInterface, am, md_transform, - sampling=s, max_iters=self.n_iterations[j]) - + interface, n_iterations=self.n_iterations[j]) elif type(self.aam) is PartsAAM: - # build orthogonal point distribution model + # Build orthogonal point distribution model pdm = OrthoPDM(sm) - # set up algorithm using parts aam interface - algorithm = self._sd_algorithm_cls( - SupervisedDescentPartsInterface, am, pdm, - sampling=s, max_iters=self.n_iterations[j], + interface = SupervisedDescentPartsInterface( + am, pdm, template, sampling=s, patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts) - + algorithm = self._sd_algorithm_cls( + interface, n_iterations=self.n_iterations[j]) else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " From 3ed665241d5cdd958cc248827e195ae5e3b4680c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 16:23:05 +0100 Subject: [PATCH 385/423] Fix incremental Gauss-Newton --- menpofit/math/regression.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/menpofit/math/regression.py b/menpofit/math/regression.py index 9b9c65d..0ec4ac6 100644 --- a/menpofit/math/regression.py +++ b/menpofit/math/regression.py @@ -64,7 +64,7 @@ def train(self, X, Y): # Note that everything is transposed from the above exchanging of roles H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J).T + self.W = np.linalg.solve(H, J) def increment(self, X, Y): # incremental least squares exchanging the roles of X and Y @@ -75,4 +75,7 @@ def increment(self, X, Y): # Note that everything is transposed from the above exchanging of roles H = J.dot(J.T) np.fill_diagonal(H, self.alpha2 + np.diag(H)) - self.W = np.linalg.solve(H, J).T + self.W = np.linalg.solve(H, J) + + def predict(self, x): + return self.W.dot(x.T).T From e1bd94b86211babba089d3e629087850f2b18c91 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 17:33:23 +0100 Subject: [PATCH 386/423] Refactoring AAM to match SD-AAM Interfaces are now concrete rather than passing a class. --- menpofit/aam/algorithm/lk.py | 57 +++++++++++++++++++----------------- menpofit/aam/fitter.py | 38 +++++++++++------------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 3750cba..5955b78 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -9,12 +9,18 @@ class LucasKanadeStandardInterface(object): r""" """ - def __init__(self, aam_algorithm, sampling=None): - self.algorithm = aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None): + self.appearance_model = appearance_model + self.transform = transform + self.template = template + self._build_sampling_mask(sampling) + + def _build_sampling_mask(self, sampling): n_true_pixels = self.template.n_true_pixels() n_channels = self.template.n_channels n_parameters = self.transform.n_parameters + sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) if sampling is None: @@ -37,18 +43,6 @@ def __init__(self, aam_algorithm, sampling=None): def shape_model(self): return self.transform.pdm.model - @property - def appearance_model(self): - return self.algorithm.appearance_model - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - @property def n(self): return self.transform.n_parameters @@ -151,16 +145,20 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, class LucasKanadePartsInterface(LucasKanadeStandardInterface): r""" """ - def __init__(self, aam_algorithm, sampling=None, patch_shape=(17, 17), - normalize_parts=no_op): - self.algorithm = aam_algorithm + def __init__(self, appearance_model, transform, template, sampling=None, + patch_shape=(17, 17), normalize_parts=no_op): self.patch_shape = patch_shape + # TODO: Refactor to patch_features self.normalize_parts = normalize_parts + super(LucasKanadePartsInterface, self).__init__( + appearance_model, transform, template, sampling=sampling) + + def _build_sampling_mask(self, sampling): if sampling is None: sampling = np.ones(self.patch_shape, dtype=np.bool) - image_shape = self.algorithm.template.pixels.shape + image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], image_shape[:3] + (1, 1)) self.i_mask = np.nonzero(image_mask.flatten())[0] @@ -210,18 +208,23 @@ def steepest_descent_images(self, nabla, dw_dp): class LucasKanade(object): r""" """ - def __init__(self, aam_interface, appearance_model, transform, - eps=10**-5, **kwargs): - # set common state for all AAM algorithms - self.appearance_model = appearance_model - self.template = appearance_model.mean() - self.transform = transform + def __init__(self, aam_interface, eps=10**-5): self.eps = eps - # set interface - self.interface = aam_interface(self, **kwargs) - # perform pre-computations + self.interface = aam_interface self._precompute() + @property + def appearance_model(self): + return self.interface.appearance_model + + @property + def transform(self): + return self.interface.transform + + @property + def template(self): + return self.interface.template + def _precompute(self): # grab number of shape and appearance parameters self.n = self.transform.n_parameters diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index f0c4cb5..2afc4b4 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -40,54 +40,50 @@ class LucasKanadeAAMFitter(AAMFitter): r""" """ def __init__(self, aam, lk_algorithm_cls=WibergInverseCompositional, - n_shape=None, n_appearance=None, sampling=None, **kwargs): + n_shape=None, n_appearance=None, sampling=None): self._model = aam self._check_n_shape(n_shape) self._check_n_appearance(n_appearance) - sampling = checks.check_sampling(sampling, self.n_scales) - self._set_up(lk_algorithm_cls, sampling, **kwargs) + self._sampling = checks.check_sampling(sampling, aam.n_scales) + self._set_up(lk_algorithm_cls) - def _set_up(self, lk_algorithm_cls, sampling, **kwargs): + def _set_up(self, lk_algorithm_cls): self.algorithms = [] for j, (am, sm, s) in enumerate(zip(self.aam.appearance_models, - self.aam.shape_models, sampling)): + self.aam.shape_models, + self._sampling)): + template = am.mean() if type(self.aam) is AAM or type(self.aam) is PatchAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, source=am.mean().landmarks['source'].lms) - # set up algorithm using standard aam interface - algorithm = lk_algorithm_cls( - LucasKanadeStandardInterface, am, md_transform, sampling=s, - **kwargs) - + interface = LucasKanadeStandardInterface(am, md_transform, + template, sampling=s) + algorithm = lk_algorithm_cls(interface) elif (type(self.aam) is LinearAAM or type(self.aam) is LinearPatchAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) - # set up algorithm using linear aam interface - algorithm = lk_algorithm_cls( - LucasKanadeLinearInterface, am, md_transform, sampling=s, - **kwargs) - + interface = LucasKanadeLinearInterface(am, md_transform, + template, sampling=s) + algorithm = lk_algorithm_cls(interface) elif type(self.aam) is PartsAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) - # set up algorithm using parts aam interface - algorithm = lk_algorithm_cls( - LucasKanadePartsInterface, am, pdm, sampling=s, + interface = LucasKanadePartsInterface( + am, pdm, template, sampling=s, patch_shape=self.aam.patch_shape[j], - normalize_parts=self.aam.normalize_parts, **kwargs) - + normalize_parts=self.aam.normalize_parts) + algorithm = lk_algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " "{}".format(AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM)) - # append algorithms to list self.algorithms.append(algorithm) From abc1dc3a5c5b7db187e34c72c79b22b31ed9a92a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 17:33:59 +0100 Subject: [PATCH 387/423] Refactor results to take a transform rather than algorithm This makes the objects much less heavyweight --- menpofit/aam/algorithm/lk.py | 4 ++-- menpofit/aam/algorithm/sd.py | 10 ++-------- menpofit/clm/algorithm/gd.py | 6 ++++-- menpofit/result.py | 15 +++++++-------- menpofit/sdm/algorithm.py | 2 +- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 5955b78..c983f55 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -120,7 +120,7 @@ def solve_all_ml(self, H, J, e): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self.transform, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -136,7 +136,7 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self.transform, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index bb3242e..a88847b 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -53,11 +53,8 @@ def warp(self, image): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): - # TODO: Faking an 'algorithm' - algorithm = lambda x: x - algorithm.transform = self.transform return AAMAlgorithmResult( - image, algorithm, shape_parameters, + image, self.transform, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -71,11 +68,8 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): - # TODO: Faking an 'algorithm' - algorithm = lambda x: x - algorithm.transform = self.transform return LinearAAMAlgorithmResult( - image, algorithm, shape_parameters, + image, self.transform, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/clm/algorithm/gd.py b/menpofit/clm/algorithm/gd.py index 6a7539b..b6fe3ca 100644 --- a/menpofit/clm/algorithm/gd.py +++ b/menpofit/clm/algorithm/gd.py @@ -125,7 +125,8 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, k += 1 # Return algorithm result - return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) + return CLMAlgorithmResult(image, self.transform, p_list, + gt_shape=gt_shape) # TODO: Document me! @@ -234,4 +235,5 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, k += 1 # Return algorithm result - return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) \ No newline at end of file + return CLMAlgorithmResult(image, self.transform, p_list, + gt_shape=gt_shape) diff --git a/menpofit/result.py b/menpofit/result.py index a8e7364..beae971 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -420,9 +420,9 @@ def as_serializableresult(self): class ParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, algorithm, shape_parameters, gt_shape=None): + def __init__(self, image, transform, shape_parameters, gt_shape=None): self.image = image - self.algorithm = algorithm + self.transform = transform self.shape_parameters = shape_parameters self._gt_shape = gt_shape @@ -436,7 +436,7 @@ def transforms(self): Generates a list containing the transforms obtained at each fitting iteration. """ - return [self.algorithm.transform.from_vector(p) + return [self.transform.from_vector(p) for p in self.shape_parameters] @property @@ -444,18 +444,18 @@ def final_transform(self): r""" Returns the final transform. """ - return self.algorithm.transform.from_vector(self.shape_parameters[-1]) + return self.transform.from_vector(self.shape_parameters[-1]) @property def initial_transform(self): r""" Returns the initial transform from which the fitting started. """ - return self.algorithm.transform.from_vector(self.shape_parameters[0]) + return self.transform.from_vector(self.shape_parameters[0]) @property def shapes(self): - return [self.algorithm.transform.from_vector(p).target + return [self.transform.from_vector(p).target for p in self.shape_parameters] @property @@ -471,9 +471,8 @@ def initial_shape(self): class NonParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, algorithm, shapes, gt_shape=None): + def __init__(self, image, shapes, gt_shape=None): self.image = image - self.algorithm = algorithm self._shapes = shapes self._gt_shape = gt_shape diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index b8affbb..96826cd 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -101,7 +101,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): shapes.append(current_shape) # return algorithm result - return NonParametricAlgorithmResult(image, self, shapes, + return NonParametricAlgorithmResult(image, shapes, gt_shape=gt_shape) def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, From bf4eff34937b45725c26e25be02530882b716d58 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 4 Aug 2015 17:34:28 +0100 Subject: [PATCH 388/423] Fix menpofit widgets No more exceptions from iterations or error types. --- menpofit/result.py | 4 ++-- menpofit/visualize/widgets/base.py | 35 ++++++++++++++++-------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/menpofit/result.py b/menpofit/result.py index beae971..ecedd93 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -171,7 +171,7 @@ def errors(self, compute_error=None): raise ValueError('Ground truth has not been set, errors cannot ' 'be computed') - def plot_errors(self, error_type='me_norm', figure_id=None, + def plot_errors(self, error_type=None, figure_id=None, new_figure=False, render_lines=True, line_colour='b', line_style='-', line_width=2, render_markers=True, marker_style='o', marker_size=4, marker_face_colour='b', @@ -245,7 +245,7 @@ def plot_errors(self, error_type='me_norm', figure_id=None, The viewer object. """ from menpo.visualize import GraphPlotter - errors_list = self.errors(error_type=error_type) + errors_list = self.errors(compute_error=error_type) return GraphPlotter(figure_id=figure_id, new_figure=new_figure, x_axis=range(len(errors_list)), y_axis=[errors_list], diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 05739ee..3871615 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -2223,7 +2223,7 @@ def plot_errors_function(name): renderer_options_wid.selected_values[0]['figure']['x_scale'] * 10, renderer_options_wid.selected_values[0]['figure']['y_scale'] * 3) renderer = fitting_results[im].plot_errors( - error_type=error_type_wid.value, + error_type=_error_type_key_to_func(error_type_wid.value), figure_id=save_figure_wid.renderer.figure_id, figure_size=new_figure_size) @@ -2517,18 +2517,8 @@ def plot_ced_fun(name): # widget closes plot_ced_but.visible = False - # Get error type error_type = error_type_wid.value - - from menpofit.result import ( - compute_root_mean_square_error, compute_point_to_point_error, - compute_normalise_point_to_point_error) - if error_type is 'me_norm': - func = compute_normalise_point_to_point_error - elif error_type is 'me': - func = compute_point_to_point_error - elif error_type is 'rmse': - func = compute_root_mean_square_error + func = _error_type_key_to_func(error_type) # Create errors list fit_errors = [f.final_error(compute_error=func) @@ -2646,8 +2636,8 @@ def update_widgets(name, value): # animation. Specifically, if the animation is activated and the user # selects the iterations tab, then the animation stops. def results_tab_fun(name, value): - if value == 1 and image_number_wid.play_toggle.value: - image_number_wid.stop_toggle.value = True + if value == 1 and image_number_wid.play_options_toggle.value: + image_number_wid.stop_options_toggle.value = True result_wid.on_trait_change(results_tab_fun, 'selected_index') # Widget titles @@ -2676,8 +2666,8 @@ def results_tab_fun(name, value): # If animation is activated and the user selects the save figure tab, # then the animation stops. def save_fig_tab_fun(name, value): - if value == 3 and image_number_wid.play_toggle.value: - image_number_wid.stop_toggle.value = True + if value == 3 and image_number_wid.play_options_toggle.value: + image_number_wid.stop_options_toggle.value = True options_box.on_trait_change(save_fig_tab_fun, 'selected_index') # Set widget's style @@ -2691,3 +2681,16 @@ def save_fig_tab_fun(name, value): # Reset value to trigger initial visualization renderer_options_wid.options_widgets[3].render_legend_checkbox.value = True + + +def _error_type_key_to_func(error_type): + from menpofit.result import ( + compute_root_mean_square_error, compute_point_to_point_error, + compute_normalise_point_to_point_error) + if error_type is 'me_norm': + func = compute_normalise_point_to_point_error + elif error_type is 'me': + func = compute_point_to_point_error + elif error_type is 'rmse': + func = compute_root_mean_square_error + return func From e3fd5c4ad3b2c3c4d9cae0be6ea825cdeafeb0e8 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 11:42:52 +0100 Subject: [PATCH 389/423] Nasty bug about truncating shape model Because we just deepcopy the shape model, we were throwing the components away too early. Now we build all the shape models, then truncate them at the end. --- menpofit/aam/base.py | 719 +++++++++++++++++++++++++++---------------- 1 file changed, 459 insertions(+), 260 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 3fe1c63..21d7efd 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -1,10 +1,21 @@ from __future__ import division +from copy import deepcopy +import warnings import numpy as np -from menpo.shape import TriMesh -from menpofit.transform import DifferentiableThinPlateSplines -from menpofit.base import name_of_callable +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpo.model import PCAModel +from menpo.transform import Scale +from menpo.shape import mean_pointcloud +from menpofit import checks +from menpofit.transform import (DifferentiableThinPlateSplines, + DifferentiablePiecewiseAffine) +from menpofit.base import name_of_callable, batch from menpofit.builder import ( - build_reference_frame, build_patch_reference_frame) + build_reference_frame, build_patch_reference_frame, + compute_features, scale_images, build_shape_model, warp_images, + align_shapes, rescale_images_to_reference_shape, densify_shapes, + extract_patches, MenpoFitBuilderWarning, compute_reference_shape) # TODO: document me! @@ -13,22 +24,8 @@ class AAM(object): Active Appearance Model class. Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - appearance_models : :map:`PCAModel` list - A list containing the appearance models of the AAM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, + ---------- + features : `callable` or ``[callable]``, optional If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. The first element of the list specifies the features to be extracted at @@ -41,35 +38,303 @@ class AAM(object): Note that from our experience, this approach of extracting features once and then creating a pyramid on top tends to lead to better performing AAMs. - + transform : :map:`PureAlignmentTransform`, optional + The :map:`PureAlignmentTransform` that will be + used to warp the images. + trilist : ``(t, 3)`` `ndarray`, optional + Triangle list that will be used to build the reference frame. If + ``None``, defaults to performing Delaunay triangulation on the points. + diagonal : `int` >= ``20``, optional + During building an AAM, all images are rescaled to ensure that the + scale of their landmarks matches the scale of the mean shape. + + If `int`, it ensures that the mean shape is scaled so that the diagonal + of the bounding box containing it matches the diagonal value. + + If ``None``, the mean shape is not rescaled. + + Note that, because the reference frame is computed from the mean + landmarks, this kwarg also specifies the diagonal length of the + reference frame (provided that features computation does not change + the image size). scales : `int` or float` or list of those, optional - - scale_shapes : `boolean` - - scale_features : `boolean` - + max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_scales``, then a number of shape components is + defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + shape components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + max_appearance_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional + If list of length ``n_scales``, then a number of appearance components + is defined per level. The first element of the list specifies the number + of components of the lowest pyramidal level and so on. + + If not a list or a list with length ``1``, then the specified number of + appearance components will be used for all levels. + + Per level: + If `int`, it specifies the exact number of components to be + retained. + + If `float`, it specifies the percentage of variance to be retained. + + If ``None``, all the available components are kept + (100% of variance). + + Returns + ------- + aam : :map:`AAMBuilder` + The AAM Builder object + + Raises + ------- + ValueError + ``diagonal`` must be >= ``20``. + ValueError + ``scales`` must be `int` or `float` or list of those. + ValueError + ``features`` must be a `function` or a list of those + containing ``1`` or ``len(scales)`` elements + ValueError + ``max_shape_components`` must be ``None`` or an `int` > 0 or + a ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements + ValueError + ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a + ``0`` <= `float` <= ``1`` or a list of those containing 1 or + ``len(scales)`` elements """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform + def __init__(self, images, group=None, verbose=False, reference_shape=None, + features=no_op, transform=DifferentiablePiecewiseAffine, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + max_appearance_components=None, batch_size=None): + + checks.check_diagonal(diagonal) + n_scales = len(scales) + scales = checks.check_scales(scales) + features = checks.check_features(features, n_scales) + max_shape_components = checks.check_max_components( + max_shape_components, n_scales, 'max_shape_components') + max_appearance_components = checks.check_max_components( + max_appearance_components, n_scales, 'max_appearance_components') + self.features = features - self.reference_shape = reference_shape + self.transform = transform + self.diagonal = diagonal self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + self.max_shape_components = max_shape_components + self.max_appearance_components = max_appearance_components + self.reference_shape = reference_shape + self.shape_models = [] + self.appearance_models = [] + + # Train AAM + self._train(images, group=group, verbose=verbose, increment=False, + batch_size=batch_size) + + def _train(self, images, group=None, verbose=False, increment=False, + shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, + batch_size=None): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest scale + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) + checks.check_trilist(image_batch[0], self.transform, + group=group) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): + if verbose: + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) + else: + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract potentially rescaled shapes + scale_shapes = [i.landmarks[group].lms for i in scaled_images] + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model( + scale_shapes, j) + self.shape_models.append(shape_model) + else: + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(scaled_images, scale_shapes, + scaled_reference_shape, + j, scale_prefix, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format( + scale_prefix)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for k, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[k] + if max_sc is not None: + sm.trim_components(max_sc) + + def increment(self, images, group=None, verbose=False, + shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, + batch_size=None): + # Literally just to fit under 80 characters, but maintain the sensible + # parameter name + aff = appearance_forgetting_factor + return self._train(images, group=group, + verbose=verbose, + shape_forgetting_factor=shape_forgetting_factor, + appearance_forgetting_factor=aff, + increment=True, batch_size=batch_size) + + def _build_shape_model(self, shapes, scale_index): + return build_shape_model(shapes) + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + # Compute aligned shapes + aligned_shapes = align_shapes(shapes) + # Increment shape model + shape_model.increment(aligned_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + prefix=prefix, verbose=verbose) @property def n_scales(self): """ - The number of scale levels of the AAM. + The number of scales of the AAM. :type: `int` """ return len(self.scales) - # TODO: Could we directly use class names instead of this? @property def _str_title(self): r""" @@ -78,7 +343,8 @@ def _str_title(self): """ return 'Active Appearance Model' - def instance(self, shape_weights=None, appearance_weights=None, level=-1): + def instance(self, shape_weights=None, appearance_weights=None, + scale_index=-1): r""" Generates a novel AAM instance given a set of shape and appearance weights. If no weights are provided, the mean AAM instance is @@ -90,22 +356,20 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): Weights of the shape model that will be used to create a novel shape instance. If ``None``, the mean shape ``(shape_weights = [0, 0, ..., 0])`` is used. - appearance_weights : ``(n_weights,)`` `ndarray` or `float` list Weights of the appearance model that will be used to create a novel appearance instance. If ``None``, the mean appearance ``(appearance_weights = [0, 0, ..., 0])`` is used. - - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[level] - am = self.appearance_models[level] + sm = self.shape_models[scale_index] + am = self.appearance_models[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel if shape_weights is None: @@ -119,24 +383,24 @@ def instance(self, shape_weights=None, appearance_weights=None, level=-1): appearance_weights *= am.eigenvalues[:n_appearance_weights] ** 0.5 appearance_instance = am.instance(appearance_weights) - return self._instance(level, shape_instance, appearance_instance) + return self._instance(scale_index, shape_instance, appearance_instance) - def random_instance(self, level=-1): + def random_instance(self, scale_index=-1): r""" Generates a novel random instance of the AAM. Parameters ----------- - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` The novel AAM instance. """ - sm = self.shape_models[level] - am = self.appearance_models[level] + sm = self.shape_models[scale_index] + am = self.appearance_models[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel shape_weights = (np.random.randn(sm.n_active_components) * @@ -146,23 +410,18 @@ def random_instance(self, level=-1): am.eigenvalues[:am.n_active_components]**0.5) appearance_instance = am.instance(appearance_weights) - return self._instance(level, shape_instance, appearance_instance) + return self._instance(scale_index, shape_instance, appearance_instance) - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean() + def _instance(self, scale_index, shape_instance, appearance_instance): + template = self.appearance_models[scale_index].mean() landmarks = template.landmarks['source'].lms - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) + reference_frame = build_reference_frame(shape_instance) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - return appearance_instance.as_unmasked().warp_to_mask( + return appearance_instance.as_unmasked(copy=False).warp_to_mask( reference_frame.mask, transform, warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, @@ -207,11 +466,11 @@ def view_appearance_models_widget(self, n_parameters=5, n_parameters : `int` or `list` of `int` or ``None``, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -239,19 +498,19 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, n_shape_parameters : `int` or `list` of `int` or None, optional The number of shape principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. n_appearance_parameters : `int` or `list` of `int` or None, optional The number of appearance principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional @@ -269,106 +528,7 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, # TODO: fix me! def __str__(self): - out = "{}\n - {} training images.\n".format(self._str_title, - self.n_training_images) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_scales): - n_channels.append( - self.appearance_models[j].template_instance.n_channels) - if j == self.n_scales - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_scales - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_scales): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_scales > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_scales, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_scales, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.appearance_models[0].n_features, - self.appearance_models[0].template_instance.n_true_pixels(), - n_channels[0], - self.appearance_models[0].template_instance._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_scales - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_scales - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, self.appearance_models[i].n_features, - self.appearance_models[i].template_instance.n_true_pixels(), - n_channels[i], - self.appearance_models[i].template_instance._str_shape, - n_channels[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n - {3} appearance components " \ - "({4:.2f}% of variance)\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100, - self.appearance_models[i].n_components, - self.appearance_models[i].variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} shape components ({10:.2f}% of " \ - "variance)\n - {11} appearance components ({12:.2f}% of " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.appearance_models[0].n_features, - self.appearance_models[0].template_instance.n_true_pixels(), - n_channels[0], - self.appearance_models[0].template_instance._str_shape, - n_channels[0], self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100, - self.appearance_models[0].n_components, - self.appearance_models[0].variance_ratio() * 100) - return out + return '' # TODO: document me! @@ -380,22 +540,18 @@ class PatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. + each scale after downscaling of the image. The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. + the lowest scale and so on. If ``callable`` the specified feature will be applied to the original image and pyramid generation will be performed on top of the feature @@ -406,30 +562,35 @@ class PatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - scale_shapes : `boolean` - - scale_features : `boolean` - """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = DifferentiableThinPlateSplines - self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, max_appearance_components=None, + batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + + super(PatchAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[scale_index]) + return warp_images(images, shapes, reference_frame, self.transform, + prefix=prefix, verbose=verbose) @property def _str_title(self): return 'Patch-Based Active Appearance Model' - def _instance(self, level, shape_instance, appearance_instance): - template = self.appearance_models[level].mean + def _instance(self, scale_index, shape_instance, appearance_instance): + template = self.appearance_models[scale_index].mean landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( @@ -469,18 +630,14 @@ class LinearAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - transform : :map:`PureAlignmentTransform` The transform used to warp the images from which the AAM was constructed. - features : `callable` or ``[callable]``, optional If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. @@ -496,27 +653,55 @@ class LinearAAM(AAM): performing AAMs. scales : `int` or float` or list of those + """ - scale_shapes : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, + max_appearance_components=None, batch_size=None): - scale_features : `boolean` + super(LinearAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=transform, diagonal=diagonal, scales=scales, + max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) - """ - def __init__(self, shape_models, appearance_models, reference_shape, - transform, features, scales, scale_shapes, scale_features, - n_landmarks): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Active Appearance Model' + + def _build_shape_model(self, shapes, scale_index): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes) + return shape_model + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, prefix=prefix, + verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -545,17 +730,13 @@ class LinearPatchAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. @@ -571,30 +752,49 @@ class LinearPatchAAM(AAM): performing AAMs. scales : `int` or float` or list of those - - scale_shapes : `boolean` - - scale_features : `boolean` - - n_landmarks: `int` - """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, scales, scale_shapes, - scale_features, n_landmarks): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.transform = DifferentiableThinPlateSplines - self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, max_appearance_components=None, + batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + + super(LinearPatchAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) + + def _build_shape_model(self, shapes, max_components, scale_index): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes) + return shape_model + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, prefix=prefix, + verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! @@ -615,6 +815,7 @@ def __str__(self): # TODO: document me! +# TODO: implement offsets support? class PartsAAM(AAM): r""" Parts based Active Appearance Model class. @@ -623,17 +824,13 @@ class PartsAAM(AAM): ----------- shape_models : :map:`PCAModel` list A list containing the shape models of the AAM. - appearance_models : :map:`PCAModel` list A list containing the appearance models of the AAM. - reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` The shape of the patches used to build the Patch Based AAM. - features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at each level after downscaling of the image. @@ -649,29 +846,31 @@ class PartsAAM(AAM): performing AAMs. normalize_parts: `callable` - scales : `int` or float` or list of those + """ - scale_shapes : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.normalize_parts = normalize_parts - scale_features : `boolean` + super(PartsAAM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + max_appearance_components=max_appearance_components, + batch_size=batch_size) - """ - def __init__(self, shape_models, appearance_models, reference_shape, - patch_shape, features, normalize_parts, scales, - scale_shapes, scale_features): - self.shape_models = shape_models - self.appearance_models = appearance_models - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return extract_patches(images, shapes, self.patch_shape[scale_index], + normalize_function=self.normalize_parts, + prefix=prefix, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance, appearance_instance): + def _instance(self, scale_index, shape_instance, appearance_instance): raise NotImplemented # TODO: implement me! From fa24f248ad8ed336e8b3aa70bc2206292f38f362 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 11:43:32 +0100 Subject: [PATCH 390/423] Fix __str__ for AAMs Also, add HolisticAAM as AAM alias --- menpofit/aam/__init__.py | 2 +- menpofit/aam/base.py | 81 +++++++++++++++++++++++++++++++--------- menpofit/sdm/fitter.py | 24 +++++++----- 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index aca4250..b91a3ae 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,4 +1,4 @@ -from .base import AAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM +from .base import HolisticAAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 21d7efd..a9ac753 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -341,7 +341,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Active Appearance Model' + return 'Holistic Active Appearance Model' def instance(self, shape_weights=None, appearance_weights=None, scale_index=-1): @@ -526,9 +526,8 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, parameters_bounds=parameters_bounds, figure_size=figure_size, mode=mode) - # TODO: fix me! def __str__(self): - return '' + return _aam_str(self) # TODO: document me! @@ -587,7 +586,7 @@ def _warp_images(self, images, shapes, reference_shape, scale_index, @property def _str_title(self): - return 'Patch-Based Active Appearance Model' + return 'Patch-based Active Appearance Model' def _instance(self, scale_index, shape_instance, appearance_instance): template = self.appearance_models[scale_index].mean @@ -611,14 +610,8 @@ def view_appearance_models_widget(self, n_parameters=5, parameters_bounds=parameters_bounds, figure_size=figure_size, mode=mode) - # TODO: fix me! def __str__(self): - out = super(PatchAAM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + return _aam_str(self) # TODO: document me! @@ -716,9 +709,8 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _aam_str(self) # TODO: document me! @@ -767,7 +759,15 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) - def _build_shape_model(self, shapes, max_components, scale_index): + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Patch-based Active Appearance Model' + + def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( @@ -809,9 +809,8 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _aam_str(self) # TODO: document me! @@ -863,6 +862,14 @@ def __init__(self, images, group=None, verbose=False, features=no_op, max_appearance_components=max_appearance_components, batch_size=batch_size) + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Parts-based Active Appearance Model' + def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): return extract_patches(images, shapes, self.patch_shape[scale_index], @@ -885,6 +892,44 @@ def view_aam_widget(self, n_shape_parameters=5, n_appearance_parameters=5, figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _aam_str(self) + + +def _aam_str(aam): + if aam.diagonal is not None: + diagonal = aam.diagonal + else: + y, x = aam.reference_shape.range() + diagonal = np.sqrt(x ** 2 + y ** 2) + + # Compute scale info strings + scales_info = [] + lvl_str_tmplt = r""" - Scale {} + - Holistic feature: {} + - {} appearance components + - {} shape components""" + for k, s in enumerate(aam.scales): + scales_info.append(lvl_str_tmplt.format( + s, name_of_callable(aam.features[k]), + aam.appearance_models[k].n_components, + aam.shape_models[k].n_components)) + # Patch based AAM + if hasattr(aam, 'patch_shape'): + for k, s in enumerate(scales_info): + s += '\n - Patch shape: {}'.format(aam.patch_shape[k]) + scales_info = '\n'.join(scales_info) + + cls_str = r"""{class_title} + - Images warped with {transform} transform + - Images scaled to diagonal: {diagonal:.2f} + - Scales: {scales} +{scales_info} +""".format(class_title=aam._str_title, + transform=name_of_callable(aam.transform), + diagonal=diagonal, + scales=aam.scales, + scales_info=scales_info) + return cls_str + +HolisticAAM = AAM diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 741e1dc..511a50d 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -251,28 +251,32 @@ def __str__(self): scales_info = [] lvl_str_tmplt = r""" - Scale {} - {} iterations - - Patch shape: {}""" + - Patch shape: {} + - Holistic feature: {} + - Patch feature: {}""" for k, s in enumerate(self.scales): - scales_info.append(lvl_str_tmplt.format(s, - self.n_iterations[k], - self._patch_shape[k])) + scales_info.append(lvl_str_tmplt.format( + s, self.n_iterations[k], self._patch_shape[k], + name_of_callable(self.features[k]), + name_of_callable(self._patch_features[k]))) scales_info = '\n'.join(scales_info) cls_str = r"""Supervised Descent Method - Regression performed using the {reg_alg} algorithm - Regression class: {reg_cls} - - Scales: {scales} -{scales_info} - Perturbations generated per shape: {n_perturbations} - Images scaled to diagonal: {diagonal:.2f} - - Custom perturbation scheme used: {is_custom_perturb_func}""".format( + - Custom perturbation scheme used: {is_custom_perturb_func} + - Scales: {scales} +{scales_info} +""".format( reg_alg=name_of_callable(self._sd_algorithm_cls), reg_cls=name_of_callable(regressor_cls), - scales=self.scales, - scales_info=scales_info, n_perturbations=self.n_perturbations, diagonal=diagonal, - is_custom_perturb_func=is_custom_perturb_func) + is_custom_perturb_func=is_custom_perturb_func, + scales=self.scales, + scales_info=scales_info) return cls_str From a7821ecc7c2ef6800c8451a49dec9e58c85b4bd0 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 13:53:32 +0100 Subject: [PATCH 391/423] Change results back to taking algorithm After speaking to @jalabort - he found this more useful. --- menpofit/aam/algorithm/lk.py | 2 +- menpofit/aam/algorithm/sd.py | 4 ++-- menpofit/aam/result.py | 1 - menpofit/atm/algorithm.py | 2 +- menpofit/clm/algorithm/gd.py | 3 +-- menpofit/result.py | 12 ++++++------ 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index c983f55..fe63075 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -120,7 +120,7 @@ def solve_all_ml(self, H, J, e): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index a88847b..7baad2f 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -54,7 +54,7 @@ def warp(self, image): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): return AAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) @@ -69,7 +69,7 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, appearance_parameters=appearance_parameters, gt_shape=gt_shape) diff --git a/menpofit/aam/result.py b/menpofit/aam/result.py index 00a500a..ae478f2 100644 --- a/menpofit/aam/result.py +++ b/menpofit/aam/result.py @@ -107,4 +107,3 @@ def costs(self): for a in self.algorithm_results: costs += a.costs return costs - diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index d92df72..387a9a5 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -101,7 +101,7 @@ def solve_shape_ml(cls, H, J, e): def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): return ATMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, gt_shape=gt_shape) diff --git a/menpofit/clm/algorithm/gd.py b/menpofit/clm/algorithm/gd.py index b6fe3ca..a140f5c 100644 --- a/menpofit/clm/algorithm/gd.py +++ b/menpofit/clm/algorithm/gd.py @@ -235,5 +235,4 @@ def run(self, image, initial_shape, max_iters=20, gt_shape=None, k += 1 # Return algorithm result - return CLMAlgorithmResult(image, self.transform, p_list, - gt_shape=gt_shape) + return CLMAlgorithmResult(image, self, p_list, gt_shape=gt_shape) diff --git a/menpofit/result.py b/menpofit/result.py index ecedd93..20cd791 100644 --- a/menpofit/result.py +++ b/menpofit/result.py @@ -420,9 +420,9 @@ def as_serializableresult(self): class ParametricAlgorithmResult(IterativeResult): r""" """ - def __init__(self, image, transform, shape_parameters, gt_shape=None): + def __init__(self, image, algorithm, shape_parameters, gt_shape=None): self.image = image - self.transform = transform + self.algorithm = algorithm self.shape_parameters = shape_parameters self._gt_shape = gt_shape @@ -436,7 +436,7 @@ def transforms(self): Generates a list containing the transforms obtained at each fitting iteration. """ - return [self.transform.from_vector(p) + return [self.algorithm.transform.from_vector(p) for p in self.shape_parameters] @property @@ -444,18 +444,18 @@ def final_transform(self): r""" Returns the final transform. """ - return self.transform.from_vector(self.shape_parameters[-1]) + return self.algorithm.transform.from_vector(self.shape_parameters[-1]) @property def initial_transform(self): r""" Returns the initial transform from which the fitting started. """ - return self.transform.from_vector(self.shape_parameters[0]) + return self.algorithm.transform.from_vector(self.shape_parameters[0]) @property def shapes(self): - return [self.transform.from_vector(p).target + return [self.algorithm.transform.from_vector(p).target for p in self.shape_parameters] @property From 7e714d56152dc34fcc278357726ac86e50068e32 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 5 Aug 2015 16:23:49 +0100 Subject: [PATCH 392/423] Partial commit - moving home due to tube strike Working on refactoring the ATMs --- menpofit/aam/algorithm/lk.py | 112 +++-- menpofit/atm/algorithm.py | 188 ++------ menpofit/atm/base.py | 833 ++++++++++++++++++----------------- menpofit/atm/fitter.py | 42 +- 4 files changed, 557 insertions(+), 618 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index fe63075..8c33efc 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -5,12 +5,30 @@ from ..result import AAMAlgorithmResult, LinearAAMAlgorithmResult +def _solve_all_map(H, J, e, Ja_prior, c, Js_prior, p, m, n): + if n is not H.shape[0] - m: + # Bidirectional Compositional case + Js_prior = np.hstack((Js_prior, Js_prior)) + p = np.hstack((p, p)) + # compute and return MAP solution + J_prior = np.hstack((Ja_prior, Js_prior)) + H += np.diag(J_prior) + Je = J_prior * np.hstack((c, p)) + J.T.dot(e) + dq = - np.linalg.solve(H, Je) + return dq[:m], dq[m:] + + +def _solve_all_ml(H, J, e, m): + # compute ML solution + dq = - np.linalg.solve(H, J.T.dot(e)) + return dq[:m], dq[m:] + + # TODO document me! -class LucasKanadeStandardInterface(object): +class LucasKanadeBaseInterface(object): r""" """ - def __init__(self, appearance_model, transform, template, sampling=None): - self.appearance_model = appearance_model + def __init__(self, transform, template, sampling=None): self.transform = transform self.template = template @@ -47,10 +65,6 @@ def shape_model(self): def n(self): return self.transform.n_parameters - @property - def m(self): - return self.appearance_model.n_active_components - @property def true_indices(self): return self.template.mask.true_indices() @@ -100,22 +114,24 @@ def solve_shape_ml(cls, H, J, e): # compute and return ML solution return -np.linalg.solve(H, J.T.dot(e)) + +class LucasKanadeStandardInterface(LucasKanadeBaseInterface): + + def __init__(self, appearance_model, transform, template, sampling=None): + super(LucasKanadeStandardInterface, self).__init__(transform, template, + sampling=sampling) + self.appearance_model = appearance_model + + @property + def m(self): + return self.appearance_model.n_active_components + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): - if self.n is not H.shape[0] - self.m: - # Bidirectional Compositional case - Js_prior = np.hstack((Js_prior, Js_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - J_prior = np.hstack((Ja_prior, Js_prior)) - H += np.diag(J_prior) - Je = J_prior * np.hstack((c, p)) + J.T.dot(e) - dq = - np.linalg.solve(H, Je) - return dq[:self.m], dq[self.m:] + return _solve_all_map(H, J, e, Ja_prior, c, Js_prior, p, + self.m, self.n) def solve_all_ml(self, H, J, e): - # compute ML solution - dq = - np.linalg.solve(H, J.T.dot(e)) - return dq[:self.m], dq[self.m:] + return _solve_all_ml(H, J, e, self.m) def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): @@ -136,23 +152,23 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, cost_functions=None, appearance_parameters=None, gt_shape=None): return LinearAAMAlgorithmResult( - image, self.transform, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, appearance_parameters=appearance_parameters, gt_shape=gt_shape) # TODO document me! -class LucasKanadePartsInterface(LucasKanadeStandardInterface): +class LucasKanadePartsBaseInterface(LucasKanadeBaseInterface): r""" """ - def __init__(self, appearance_model, transform, template, sampling=None, + def __init__(self, transform, template, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): self.patch_shape = patch_shape # TODO: Refactor to patch_features self.normalize_parts = normalize_parts - super(LucasKanadePartsInterface, self).__init__( - appearance_model, transform, template, sampling=sampling) + super(LucasKanadeBaseInterface, self).__init__( + transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): if sampling is None: @@ -175,15 +191,18 @@ def warp_jacobian(self): return np.rollaxis(self.transform.d_dp(None), -1) def warp(self, image): - return Image(image.extract_patches( - self.transform.target, patch_size=self.patch_shape, - as_single_array=True)) + parts = image.extract_patches(self.transform.target, + patch_size=self.patch_shape, + as_single_array=True) + parts = self.normalize_parts(parts) + return Image(parts, copy=False) def gradient(self, image): - nabla = fast_gradient(image.pixels.reshape((-1,) + self.patch_shape)) + pixels = image.pixels + nabla = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) # remove 1st dimension gradient which corresponds to the gradient # between parts - return nabla.reshape((2,) + image.pixels.shape) + return nabla.reshape((2,) + pixels.shape) def steepest_descent_images(self, nabla, dw_dp): # reshape nabla @@ -204,6 +223,39 @@ def steepest_descent_images(self, nabla, dw_dp): return sdi.reshape((-1, sdi.shape[-1])) +# TODO document me! +class LucasKanadePartsInterface(LucasKanadePartsBaseInterface): + r""" + """ + def __init__(self, appearance_model, transform, template, sampling=None, + patch_shape=(17, 17), normalize_parts=no_op): + self.patch_shape = patch_shape + # TODO: Refactor to patch_features + self.normalize_parts = normalize_parts + self.appearance_model = appearance_model + + super(LucasKanadePartsInterface, self).__init__( + transform, template, sampling=sampling) + + @property + def m(self): + return self.appearance_model.n_active_components + + def solve_all_map(self, H, J, e, Ja_prior, c, Js_prior, p): + return _solve_all_map(H, J, e, Ja_prior, c, Js_prior, p, + self.m, self.n) + + def solve_all_ml(self, H, J, e): + return _solve_all_ml(H, J, e, self.m) + + def algorithm_result(self, image, shape_parameters, cost_functions=None, + appearance_parameters=None, gt_shape=None): + return AAMAlgorithmResult( + image, self, shape_parameters, + cost_functions=cost_functions, + appearance_parameters=appearance_parameters, gt_shape=gt_shape) + + # TODO document me! class LucasKanade(object): r""" diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 387a9a5..90515cd 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,102 +1,17 @@ from __future__ import division import numpy as np -from menpo.image import Image -from menpo.feature import no_op -from menpo.feature import gradient as fast_gradient +from menpofit.aam.algorithm.lk import (LucasKanadeBaseInterface, + LucasKanadePartsBaseInterface) from .result import ATMAlgorithmResult, LinearATMAlgorithmResult # TODO document me! -class LucasKanadeStandardInterface(object): +class ATMLKStandardInterface(LucasKanadeBaseInterface): r""" """ - def __init__(self, lk_algorithm, sampling=None): - self.algorithm = lk_algorithm - - n_true_pixels = self.template.n_true_pixels() - n_channels = self.template.n_channels - n_parameters = self.transform.n_parameters - sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) - - if sampling is None: - sampling = 1 - sampling_pattern = xrange(0, n_true_pixels, sampling) - sampling_mask[sampling_pattern] = 1 - - self.i_mask = np.nonzero(np.tile( - sampling_mask[None, ...], (n_channels, 1)).flatten())[0] - self.dW_dp_mask = np.nonzero(np.tile( - sampling_mask[None, ..., None], (2, 1, n_parameters))) - self.nabla_mask = np.nonzero(np.tile( - sampling_mask[None, None, ...], (2, n_channels, 1))) - self.nabla2_mask = np.nonzero(np.tile( - sampling_mask[None, None, None, ...], (2, 2, n_channels, 1))) - - @property - def template(self): - return self.algorithm.template - - @property - def transform(self): - return self.algorithm.transform - - @property - def n(self): - return self.transform.n_parameters - - @property - def true_indices(self): - return self.template.mask.true_indices() - - @property - def shape_model(self): - return self.transform.pdm.model - - def warp_jacobian(self): - dW_dp = np.rollaxis(self.transform.d_dp(self.true_indices), -1) - return dW_dp[self.dW_dp_mask].reshape((dW_dp.shape[0], -1, - dW_dp.shape[2])) - - def warp(self, image): - return image.warp_to_mask(self.template.mask, - self.transform) - - def gradient(self, img): - nabla = fast_gradient(img) - nabla.set_boundary_pixels() - return nabla.as_vector().reshape((2, img.n_channels, -1)) - - def steepest_descent_images(self, nabla, dW_dp): - # reshape gradient - # nabla: n_dims x n_channels x n_pixels - nabla = nabla[self.nabla_mask].reshape(nabla.shape[:2] + (-1,)) - # compute steepest descent images - # nabla: n_dims x n_channels x n_pixels - # warp_jacobian: n_dims x x n_pixels x n_params - # sdi: n_channels x n_pixels x n_params - sdi = 0 - a = nabla[..., None] * dW_dp[:, None, ...] - for d in a: - sdi += d - # reshape steepest descent images - # sdi: (n_channels x n_pixels) x n_params - return sdi.reshape((-1, sdi.shape[2])) - - @classmethod - def solve_shape_map(cls, H, J, e, J_prior, p): - if p.shape[0] is not H.shape[0]: - # Bidirectional Compositional case - J_prior = np.hstack((J_prior, J_prior)) - p = np.hstack((p, p)) - # compute and return MAP solution - H += np.diag(J_prior) - Je = J_prior * p + J.T.dot(e) - return - np.linalg.solve(H, Je) - - @classmethod - def solve_shape_ml(cls, H, J, e): - # compute and return ML solution - return -np.linalg.solve(H, J.T.dot(e)) + def __init__(self, transform, template, sampling=None): + super(ATMLKStandardInterface, self).__init__(transform, template, + sampling=sampling) def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): @@ -106,7 +21,7 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, # TODO document me! -class LucasKanadeLinearInterface(LucasKanadeStandardInterface): +class ATMLKLinearInterface(ATMLKStandardInterface): r""" """ @property @@ -116,87 +31,42 @@ def shape_model(self): def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): return LinearATMAlgorithmResult( - image, self.algorithm, shape_parameters, + image, self, shape_parameters, cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! -class LucasKanadePartsInterface(LucasKanadeStandardInterface): +class ATMLKPartsInterface(LucasKanadePartsBaseInterface): r""" """ - def __init__(self, lk_algorithm, patch_shape=(17, 17), - normalize_parts=no_op, sampling=None): - self.algorithm = lk_algorithm - self.patch_shape = patch_shape - self.normalize_parts = normalize_parts - - if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) - - image_shape = self.algorithm.template.pixels.shape - image_mask = np.tile(sampling[None, None, None, ...], - image_shape[:3] + (1, 1)) - self.i_mask = np.nonzero(image_mask.flatten())[0] - self.nabla_mask = np.nonzero(np.tile( - image_mask[None, ...], (2, 1, 1, 1, 1, 1))) - self.nabla2_mask = np.nonzero(np.tile( - image_mask[None, None, ...], (2, 2, 1, 1, 1, 1, 1))) - - @property - def shape_model(self): - return self.transform.model - - def warp_jacobian(self): - return np.rollaxis(self.transform.d_dp(None), -1) - - def warp(self, image): - parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, - as_single_array=True) - parts = self.normalize_parts(parts) - return Image(parts) - - def gradient(self, image): - pixels = image.pixels - g = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) - # remove 1st dimension gradient which corresponds to the gradient - # between parts - return g.reshape((2,) + pixels.shape) - - def steepest_descent_images(self, nabla, dw_dp): - # reshape nabla - # nabla: dims x parts x off x ch x (h x w) - nabla = nabla[self.nabla_mask].reshape( - nabla.shape[:-2] + (-1,)) - # compute steepest descent images - # nabla: dims x parts x off x ch x (h x w) - # ds_dp: dims x parts x x params - # sdi: parts x off x ch x (h x w) x params - sdi = 0 - a = nabla[..., None] * dw_dp[..., None, None, None, :] - for d in a: - sdi += d - - # reshape steepest descent images - # sdi: (parts x offsets x ch x w x h) x params - return sdi.reshape((-1, sdi.shape[-1])) + def algorithm_result(self, image, shape_parameters, cost_functions=None, + gt_shape=None): + return LinearATMAlgorithmResult( + image, self, shape_parameters, + cost_functions=cost_functions, gt_shape=gt_shape) # TODO document me! class LucasKanade(object): - def __init__(self, lk_atm_interface_cls, template, transform, - eps=10**-5, **kwargs): - # set common state for all ATM algorithms - self.template = template - self.transform = transform + def __init__(self, atm_interface, eps=10**-5): self.eps = eps - # set interface - self.interface = lk_atm_interface_cls(self, **kwargs) - # perform pre-computations + self.interface = atm_interface() self._precompute() - def _precompute(self, **kwargs): + @property + def appearance_model(self): + return self.interface.appearance_model + + @property + def transform(self): + return self.interface.transform + + @property + def template(self): + return self.interface.template + + def _precompute(self): # grab number of shape and appearance parameters self.n = self.transform.n_parameters diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index c11455f..95d675d 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -1,104 +1,257 @@ from __future__ import division +from copy import deepcopy +import warnings import numpy as np -from menpo.shape import TriMesh -from menpofit.transform import DifferentiableThinPlateSplines -from menpofit.base import name_of_callable -from menpofit.aam.builder import ( - build_patch_reference_frame, build_reference_frame) +from menpo.feature import no_op +from menpo.visualize import print_dynamic +from menpo.model import PCAModel +from menpo.transform import Scale +from menpo.shape import mean_pointcloud +from menpofit import checks +from menpofit.transform import (DifferentiableThinPlateSplines, + DifferentiablePiecewiseAffine) +from menpofit.base import name_of_callable, batch +from menpofit.builder import ( + build_reference_frame, build_patch_reference_frame, + compute_features, scale_images, build_shape_model, warp_images, + align_shapes, rescale_images_to_reference_shape, densify_shapes, + extract_patches, MenpoFitBuilderWarning, compute_reference_shape) +# TODO: document me! class ATM(object): r""" Active Template Model class. + """ + def __init__(self, images, group=None, verbose=False, reference_shape=None, + features=no_op, transform=DifferentiablePiecewiseAffine, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + batch_size=None): + + checks.check_diagonal(diagonal) + n_scales = len(scales) + scales = checks.check_scales(scales) + features = checks.check_features(features, n_scales) + max_shape_components = checks.check_max_components( + max_shape_components, n_scales, 'max_shape_components') - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the ATM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. + self.features = features + self.transform = transform + self.diagonal = diagonal + self.scales = scales + self.max_shape_components = max_shape_components + self.reference_shape = reference_shape + self.shape_models = [] + self.warped_templates = [] - scales : `int` or float` or list of those, optional + # Train ATM + self._train(images, group=group, verbose=verbose, increment=False, + batch_size=batch_size) - scale_shapes : `boolean` + def _train(self, images, group=None, verbose=False, increment=False, + shape_forgetting_factor=1.0, batch_size=None): + r""" + Builds an Active Template Model from a list of landmarked images. - scale_features : `boolean` + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. - """ - def __init__(self, shape_models, warped_templates, reference_shape, - transform, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest scale + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The mean ' + 'of the first batch will be the reference ' + 'shape. If the batch mean is not ' + 'representative of the true mean, this may ' + 'cause issues.', MenpoFitBuilderWarning) + checks.check_trilist(image_batch[0], self.transform, + group=group) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): + if verbose: + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) + else: + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract potentially rescaled shapes + scale_shapes = [i.landmarks[group].lms for i in scaled_images] + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model( + scale_shapes, j) + self.shape_models.append(shape_model) + else: + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(scaled_images, scale_shapes, + scaled_reference_shape, + j, scale_prefix, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format( + scale_prefix)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for k, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[k] + if max_sc is not None: + sm.trim_components(max_sc) + + def increment(self, images, group=None, verbose=False, + shape_forgetting_factor=1.0, batch_size=None): + return self._train(images, group=group, + verbose=verbose, + shape_forgetting_factor=shape_forgetting_factor, + increment=True, batch_size=batch_size) + + def _build_shape_model(self, shapes, scale_index): + return build_shape_model(shapes) + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + # Compute aligned shapes + aligned_shapes = align_shapes(shapes) + # Increment shape model + shape_model.increment(aligned_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + reference_frame = build_reference_frame(reference_shape) + return warp_images(images, shapes, reference_frame, self.transform, + prefix=prefix, verbose=verbose) @property def n_scales(self): """ - The number of scale level of the ATM. + The number of scales of the AAM. :type: `int` """ return len(self.scales) - # TODO: Could we directly use class names instead of this? @property def _str_title(self): r""" Returns a string containing name of the model. - :type: `string` """ - return 'Active Template Model' + return 'Holistic Active Template Model' - def instance(self, shape_weights=None, level=-1): + def instance(self, shape_weights=None, scale_index=-1): r""" - Generates a novel ATM instance given a set of shape weights. If no - weights are provided, the mean shape instance is returned. - - Parameters - ----------- - shape_weights : ``(n_weights,)`` `ndarray` or `float` list - Weights of the shape model that will be used to create - a novel shape instance. If ``None``, the mean shape - ``(shape_weights = [0, 0, ..., 0])`` is used. - - level : `int`, optional - The pyramidal level to be used. - Returns ------- image : :map:`Image` - The novel ATM instance. + The novel AAM instance. """ - sm = self.shape_models[level] + sm = self.shape_models[scale_index] + template = self.warped_templates[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel if shape_weights is None: @@ -107,47 +260,42 @@ def instance(self, shape_weights=None, level=-1): shape_weights *= sm.eigenvalues[:n_shape_weights] ** 0.5 shape_instance = sm.instance(shape_weights) - return self._instance(level, shape_instance) + return self._instance(shape_instance, template) - def random_instance(self, level=-1): + def random_instance(self, scale_index=-1): r""" Generates a novel random instance of the ATM. Parameters ----------- - level : `int`, optional - The pyramidal level to be used. + scale_index : `int`, optional + The scale to be used. Returns ------- image : :map:`Image` - The novel ATM instance. + The novel AAM instance. """ - sm = self.shape_models[level] + sm = self.shape_models[scale_index] + template = self.warped_templates[scale_index] # TODO: this bit of logic should to be transferred down to PCAModel shape_weights = (np.random.randn(sm.n_active_components) * sm.eigenvalues[:sm.n_active_components]**0.5) shape_instance = sm.instance(shape_weights) - return self._instance(level, shape_instance) + return self._instance(scale_index, shape_instance, template) - def _instance(self, level, shape_instance): - template = self.warped_templates[level] + def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms - if type(landmarks) == TriMesh: - trilist = landmarks.trilist - else: - trilist = None - reference_frame = build_reference_frame(shape_instance, - trilist=trilist) + reference_frame = build_reference_frame(shape_instance) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - return template.warp_to_mask(reference_frame.mask, transform, - warp_landmarks=True) + return template.as_unmasked(copy=False).warp_to_mask( + reference_frame.mask, transform, warp_landmarks=True) def view_shape_models_widget(self, n_parameters=5, parameters_bounds=(-3.0, 3.0), @@ -185,25 +333,32 @@ def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): r""" - Visualizes the ATM object using the - menpo.visualize.widgets.visualize_atm widget. - + Visualizes both the shape and appearance models of the AAM object using + the `menpo.visualize.widgets.visualize_aam` widget. Parameters ----------- n_shape_parameters : `int` or `list` of `int` or None, optional The number of shape principal components to be used for the parameters sliders. - If `int`, then the number of sliders per level is the minimum + If `int`, then the number of sliders per scale is the minimum between `n_parameters` and the number of active components per - level. - If `list` of `int`, then a number of sliders is defined per level. - If ``None``, all the active components per level will have a slider. + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. + n_appearance_parameters : `int` or `list` of `int` or None, optional + The number of appearance principal components to be used for the + parameters sliders. + If `int`, then the number of sliders per scale is the minimum + between `n_parameters` and the number of active components per + scale. + If `list` of `int`, then a number of sliders is defined per scale. + If ``None``, all the active components per scale will have a slider. parameters_bounds : (`float`, `float`), optional The minimum and maximum bounds, in std units, for the sliders. mode : {``single``, ``multiple``}, optional If ``'single'``, only a single slider is constructed along with a drop down menu. - If ``'multiple'``, a slider is constructed for each pp window. + If ``'multiple'``, a slider is constructed for each parameter. figure_size : (`int`, `int`), optional The size of the plotted figures. """ @@ -212,165 +367,39 @@ def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=parameters_bounds, figure_size=figure_size, mode=mode) - # TODO: fix me! def __str__(self): - out = "{}\n - {} training shapes.\n".format(self._str_title, - self.n_training_shapes) - # small strings about number of channels, channels string and downscale - n_channels = [] - down_str = [] - for j in range(self.n_scales): - n_channels.append( - self.warped_templates[j].n_channels) - if j == self.n_scales - 1: - down_str.append('(no downscale)') - else: - down_str.append('(downscale by {})'.format( - self.downscale**(self.n_scales - j - 1))) - # string about features and channels - if self.pyramid_on_features: - feat_str = "- Feature is {} with ".format( - name_of_callable(self.features)) - if n_channels[0] == 1: - ch_str = ["channel"] - else: - ch_str = ["channels"] - else: - feat_str = [] - ch_str = [] - for j in range(self.n_scales): - feat_str.append("- Feature is {} with ".format( - name_of_callable(self.features[j]))) - if n_channels[j] == 1: - ch_str.append("channel") - else: - ch_str.append("channels") - out = "{} - {} Warp.\n".format(out, name_of_callable(self.transform)) - if self.n_scales > 1: - if self.scaled_shape_models: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}.\n - Each level has a scaled shape " \ - "model (reference frame).\n".format(out, self.n_scales, - self.downscale) - - else: - out = "{} - Gaussian pyramid with {} levels and downscale " \ - "factor of {}:\n - Shape models (reference frames) " \ - "are not scaled.\n".format(out, self.n_scales, - self.downscale) - if self.pyramid_on_features: - out = "{} - Pyramid was applied on feature space.\n " \ - "{}{} {} per image.\n".format(out, feat_str, - n_channels[0], ch_str[0]) - if not self.scaled_shape_models: - out = "{} - Reference frames of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.warped_templates[0].n_true_pixels() * - n_channels[0], - self.warped_templates[0].n_true_pixels(), - n_channels[0], - self.warped_templates[0]._str_shape, - n_channels[0]) - else: - out = "{} - Features were extracted at each pyramid " \ - "level.\n".format(out) - for i in range(self.n_scales - 1, -1, -1): - out = "{} - Level {} {}: \n".format(out, self.n_scales - i, - down_str[i]) - if not self.pyramid_on_features: - out = "{} {}{} {} per image.\n".format( - out, feat_str[i], n_channels[i], ch_str[i]) - if (self.scaled_shape_models or - (not self.pyramid_on_features)): - out = "{} - Reference frame of length {} " \ - "({} x {}C, {} x {}C)\n".format( - out, - self.warped_templates[i].n_true_pixels() * - n_channels[i], - self.warped_templates[i].n_true_pixels(), - n_channels[i], - self.warped_templates[i]._str_shape, - n_channels[i]) - out = "{0} - {1} shape components ({2:.2f}% of " \ - "variance)\n".format( - out, self.shape_models[i].n_components, - self.shape_models[i].variance_ratio() * 100) - else: - if self.pyramid_on_features: - feat_str = [feat_str] - out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \ - " - Reference frame of length {4} ({5} x {6}C, " \ - "{7} x {8}C)\n - {9} shape components ({10:.2f}% of " \ - "variance)\n".format( - out, feat_str[0], n_channels[0], ch_str[0], - self.warped_templates[0].n_true_pixels() * n_channels[0], - self.warped_templates[0].n_true_pixels(), - n_channels[0], - self.warped_templates[0]._str_shape, - n_channels[0], self.shape_models[0].n_components, - self.shape_models[0].variance_ratio() * 100) - return out + return _atm_str(self) +# TODO: document me! class PatchATM(ATM): r""" - Patch Based Active Template Model class. - - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the ATM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based AAM. - - features : `callable` or ``[callable]`` - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those + Patch based Based Active Appearance Model class. + """ - scale_shapes : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - scale_features : `boolean` + super(PatchATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + batch_size=batch_size) - """ - def __init__(self, shape_models, warped_templates, reference_shape, - patch_shape, features, scales, scale_shapes, scale_features): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = DifferentiableThinPlateSplines - self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + reference_frame = build_patch_reference_frame( + reference_shape, patch_shape=self.patch_shape[scale_index]) + return warp_images(images, shapes, reference_frame, self.transform, + prefix=prefix, verbose=verbose) @property def _str_title(self): - return 'Patch-Based Active Template Model' + return 'Patch-based Active Template Model' - def _instance(self, level, shape_instance): - template = self.warped_templates[level] + def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( @@ -379,227 +408,223 @@ def _instance(self, level, shape_instance): transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) - return template.warp_to_mask(reference_frame.mask, transform, - warp_landmarks=True) + return template.as_unmasked().warp_to_mask( + reference_frame.mask, transform, warp_landmarks=True) - # TODO: fix me! def __str__(self): - out = super(PatchBasedATM, self).__str__() - out_splitted = out.splitlines() - out_splitted[0] = self._str_title - out_splitted.insert(5, " - Patch size is {}W x {}H.".format( - self.patch_shape[1], self.patch_shape[0])) - return '\n'.join(out_splitted) + return _atm_str(self) # TODO: document me! class LinearATM(ATM): r""" Linear Active Template Model class. + """ - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - transform : :map:`PureAlignmentTransform` - The transform used to warp the images from which the AAM was - constructed. - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those - - scale_shapes : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, + batch_size=None): - scale_features : `boolean` + super(LinearATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=transform, diagonal=diagonal, scales=scales, + max_shape_components=max_shape_components, batch_size=batch_size) - """ - def __init__(self, shape_models, warped_templates, reference_shape, - transform, features, scales, scale_shapes, scale_features, - n_landmarks): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = transform - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Active Template Model' + + def _build_shape_model(self, shapes, scale_index): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_reference_frame(mean_aligned_shape) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes) + return shape_model + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, prefix=prefix, + verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance): + def _instance(self, shape_instance, template): raise NotImplemented # TODO: implement me! - def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _atm_str(self) # TODO: document me! class LinearPatchATM(ATM): r""" Linear Patch based Active Template Model class. + """ - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based AAM. - - features : `callable` or ``[callable]`` - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - scales : `int` or float` or list of those - - scale_shapes : `boolean` - - scale_features : `boolean` + def __init__(self, images, group=None, verbose=False, features=no_op, + diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - n_landmarks: `int` + super(LinearPatchATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + batch_size=batch_size) - """ - def __init__(self, shape_models, warped_templates, reference_shape, - patch_shape, features, scales, scale_shapes, - scale_features, n_landmarks): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.transform = DifferentiableThinPlateSplines - self.patch_shape = patch_shape - self.features = features - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.n_landmarks = n_landmarks + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Linear Patch-based Active Template Model' + + def _build_shape_model(self, shapes, scale_index): + mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) + self.n_landmarks = mean_aligned_shape.n_points + self.reference_frame = build_patch_reference_frame( + mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + dense_shapes = densify_shapes(shapes, self.reference_frame, + self.transform) + # build dense shape model + shape_model = build_shape_model(dense_shapes) + return shape_model + + def _increment_shape_model(self, shapes, shape_model, + forgetting_factor=1.0): + aligned_shapes = align_shapes(shapes) + dense_shapes = densify_shapes(aligned_shapes, self.reference_frame, + self.transform) + # Increment shape model + shape_model.increment(dense_shapes, + forgetting_factor=forgetting_factor) + + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return warp_images(images, shapes, self.reference_frame, + self.transform, prefix=prefix, + verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance): + def _instance(self, shape_instance, template): raise NotImplemented # TODO: implement me! - def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _atm_str(self) # TODO: document me! +# TODO: implement offsets support? class PartsATM(ATM): r""" Parts based Active Template Model class. + """ - Parameters - ----------- - shape_models : :map:`PCAModel` list - A list containing the shape models of the AAM. - - warped_templates : :map:`MaskedImage` list - A list containing the warped templates models of the ATM. - - reference_shape : :map:`PointCloud` - The reference shape that was used to resize all training images to a - consistent object size. - - patch_shape : tuple of `int` - The shape of the patches used to build the Patch Based AAM. - - features : `callable` or ``[callable]`` - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - normalize_parts: `callable` - - scales : `int` or float` or list of those + def __init__(self, images, group=None, verbose=False, features=no_op, + normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + batch_size=None): + self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.normalize_parts = normalize_parts - scale_shapes : `boolean` + super(PartsATM, self).__init__( + images, group=group, verbose=verbose, features=features, + transform=DifferentiableThinPlateSplines, diagonal=diagonal, + scales=scales, max_shape_components=max_shape_components, + batch_size=batch_size) - scale_features : `boolean` + @property + def _str_title(self): + r""" + Returns a string containing name of the model. + :type: `string` + """ + return 'Parts-based Active Template Model' - """ - def __init__(self, shape_models, warped_templates, reference_shape, - patch_shape, features, normalize_parts, scales, - scale_shapes, scale_features): - self.shape_models = shape_models - self.warped_templates = warped_templates - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.reference_shape = reference_shape - self.scales = scales - self.scale_shapes = scale_shapes - self.scale_features = scale_features + def _warp_images(self, images, shapes, reference_shape, scale_index, + prefix, verbose): + return extract_patches(images, shapes, self.patch_shape[scale_index], + normalize_function=self.normalize_parts, + prefix=prefix, verbose=verbose) # TODO: implement me! - def _instance(self, level, shape_instance): + def _instance(self, shape_instance, template): raise NotImplemented # TODO: implement me! - def view_atm_widget(self, n_shape_parameters=5, n_appearance_parameters=5, + def view_atm_widget(self, n_shape_parameters=5, parameters_bounds=(-3.0, 3.0), mode='multiple', figure_size=(10, 8)): raise NotImplemented - # TODO: implement me! def __str__(self): - raise NotImplemented + return _atm_str(self) + + +def _atm_str(atm): + if atm.diagonal is not None: + diagonal = atm.diagonal + else: + y, x = atm.reference_shape.range() + diagonal = np.sqrt(x ** 2 + y ** 2) + + # Compute scale info strings + scales_info = [] + lvl_str_tmplt = r""" - Scale {} + - Holistic feature: {} + - {} appearance components + - {} shape components""" + for k, s in enumerate(atm.scales): + scales_info.append(lvl_str_tmplt.format( + s, name_of_callable(atm.features[k]), + atm.appearance_models[k].n_components, + atm.shape_models[k].n_components)) + # Patch based ATM + if hasattr(atm, 'patch_shape'): + for k, s in enumerate(scales_info): + s += '\n - Patch shape: {}'.format(atm.patch_shape[k]) + scales_info = '\n'.join(scales_info) + + cls_str = r"""{class_title} + - Images warped with {transform} transform + - Images scaled to diagonal: {diagonal:.2f} + - Scales: {scales} +{scales_info} +""".format(class_title=atm._str_title, + transform=name_of_callable(atm.transform), + diagonal=diagonal, + scales=atm.scales, + scales_info=scales_info) + return cls_str + +HolisticATM = ATM diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 4a5ce71..0861e11 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -1,11 +1,12 @@ from __future__ import division +from menpofit import checks from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM from .algorithm import ( - LucasKanadeStandardInterface, LucasKanadeLinearInterface, - LucasKanadePartsInterface, InverseCompositional) + ATMLKStandardInterface, ATMLKPartsInterface, ATMLKLinearInterface, + InverseCompositional) from .result import ATMFitterResult @@ -14,56 +15,47 @@ class LucasKanadeATMFitter(ModelFitter): r""" """ def __init__(self, atm, algorithm_cls=InverseCompositional, - n_shape=None, sampling=None, **kwargs): + n_shape=None, sampling=None): self._model = atm - self._check_n_shape(n_shape) - self._set_up(algorithm_cls, sampling, **kwargs) + checks.set_models_components(atm.shape_models, n_shape) + self._sampling = checks.check_sampling(sampling, atm.n_scales) + self._set_up(algorithm_cls) @property def atm(self): return self._model - def _set_up(self, algorithm_cls, sampling, **kwargs): + def _set_up(self, algorithm_cls): self.algorithms = [] - for j, (wt, sm) in enumerate(zip(self.atm.warped_templates, - self.atm.shape_models)): + for j, (wt, sm, s) in enumerate(zip(self.atm.warped_templates, + self.atm.shape_models, + self._sampling)): if type(self.atm) is ATM or type(self.atm) is PatchATM: - # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.atm.transform, source=wt.landmarks['source'].lms) - # set up algorithm using standard aam interface - algorithm = algorithm_cls(LucasKanadeStandardInterface, wt, - md_transform, sampling=sampling, - **kwargs) - + interface = ATMLKStandardInterface(md_transform, wt, sampling=s) + algorithm = algorithm_cls(interface) elif (type(self.atm) is LinearATM or type(self.atm) is LinearPatchATM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.atm.reference_shape) - # set up algorithm using linear aam interface - algorithm = algorithm_cls(LucasKanadeLinearInterface, wt, - md_transform, sampling=sampling, - **kwargs) - + interface = ATMLKLinearInterface(md_transform, wt, sampling=s) + algorithm = algorithm_cls(interface) elif type(self.atm) is PartsATM: - # build orthogonal point distribution model pdm = OrthoPDM(sm) - # set up algorithm using parts aam interface + interface = ATMLKPartsInterface(pdm, wt, sampling=s) algorithm = algorithm_cls( - LucasKanadePartsInterface, wt, pdm, sampling=sampling, + interface, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) - else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " "{}".format(ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM)) - - # append algorithms to list self.algorithms.append(algorithm) def _fitter_result(self, image, algorithm_results, affine_correction, From 7a1cbf394f8d9c557d51df78c52c9598bcca5858 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:35:03 +0100 Subject: [PATCH 393/423] Remove xrange - for Python 3 compat --- menpofit/aam/algorithm/lk.py | 4 +- menpofit/atm/builder.py | 770 ----------------------------------- 2 files changed, 2 insertions(+), 772 deletions(-) delete mode 100644 menpofit/atm/builder.py diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 8c33efc..720be4c 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -42,9 +42,9 @@ def _build_sampling_mask(self, sampling): sampling_mask = np.zeros(n_true_pixels, dtype=np.bool) if sampling is None: - sampling = xrange(0, n_true_pixels, 1) + sampling = range(0, n_true_pixels, 1) elif isinstance(sampling, np.int): - sampling = xrange(0, n_true_pixels, sampling) + sampling = range(0, n_true_pixels, sampling) sampling_mask[sampling] = 1 diff --git a/menpofit/atm/builder.py b/menpofit/atm/builder.py deleted file mode 100644 index da049f0..0000000 --- a/menpofit/atm/builder.py +++ /dev/null @@ -1,770 +0,0 @@ -from __future__ import division -from copy import deepcopy -from menpo.transform import Scale -from menpofit.transform import ( - DifferentiablePiecewiseAffine, DifferentiableThinPlateSplines) -from menpo.shape import mean_pointcloud -from menpo.image import Image -from menpo.feature import no_op -from menpo.visualize import print_dynamic -from menpofit import checks -from menpofit.aam.builder import ( - align_shapes, densify_shapes, - build_reference_frame, build_patch_reference_frame) -from menpofit.builder import build_shape_model, compute_reference_shape - - -# TODO: document me! -class ATMBuilder(object): - r""" - Class that builds Active Template Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, 0.5), - scale_shapes=False, scale_features=True, - max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def build(self, shapes, template, group=None, verbose=False): - r""" - Builds a Multilevel Active Template Model given a list of shapes and a - template image. - - Parameters - ---------- - shapes : list of :map:`PointCloud` - The set of shapes from which to build the shape model of the ATM. - template : :map:`Image` or subclass - The image to be used as template. - group : `str`, optional - The key of the landmark set of the template that should be used. If - ``None``, and if there is only one set of landmarks, this set will - be used. - verbose : `bool`, optional - Flag that controls information and progress printing. - - Returns - ------- - atm : :map:`ATM` - The ATM object. Shape and appearance models are stored from lowest - to highest level. - """ - # compute reference_shape - reference_shape = compute_reference_shape(shapes, self.diagonal, - verbose=verbose) - - # normalize the template size using the reference_shape scaling - template = template.rescale_to_pointcloud( - reference_shape, group=group) - - # build models at each scale - if verbose: - print_dynamic('- Building models\n') - shape_models = [] - warped_templates = [] - # for each pyramid level (high --> low) - for j, s in enumerate(self.scales): - if verbose: - if len(self.scales) > 1: - level_str = ' - Level {}: '.format(j) - else: - level_str = ' - ' - - # obtain shape representation - if j == 0 or self.scale_shapes: - if j == 0: - level_shapes = shapes - level_reference_shape = reference_shape - else: - scale_transform = Scale(scale_factor=s, n_dims=2) - level_shapes = [scale_transform.apply(s) for s in shapes] - level_reference_shape = scale_transform.apply( - reference_shape) - # obtain shape model - if verbose: - print_dynamic('{}Building shape model'.format(level_str)) - shape_model = self._build_shape_model( - level_shapes, self.max_shape_components[j], j) - # add shape model to the list - shape_models.append(shape_model) - else: - # copy precious shape model and add it to the list - shape_models.append(deepcopy(shape_model)) - - if verbose: - print_dynamic('{}Building template model'.format(level_str)) - # obtain template representation - if j == 0: - # compute features at highest level - feature_template = self.features[j](template) - level_template = feature_template - elif self.scale_features: - # scale features at other levels - level_template = feature_template.rescale(s) - else: - # scale template and compute features at other levels - scaled_template = template.rescale(s) - level_template = self.features[j](scaled_template) - - # extract potentially rescaled template shape - level_template_shape = level_template.landmarks[group].lms - - # obtain warped template - warped_template = self._warp_template(level_template, - level_template_shape, - level_reference_shape, j) - # add warped template to the list - warped_templates.append(warped_template) - - if verbose: - print_dynamic('{}Done\n'.format(level_str)) - - # reverse the list of shape and warped templates so that they are - # ordered from lower to higher resolution - shape_models.reverse() - warped_templates.reverse() - self.scales.reverse() - - return self._build_atm(shape_models, warped_templates, reference_shape) - - @classmethod - def _build_shape_model(cls, shapes, max_components, level): - return build_shape_model(shapes, max_components=max_components) - - def _warp_template(self, template, template_shape, reference_shape, level): - # build reference frame - reference_frame = build_reference_frame(reference_shape) - # compute transforms - t = self.transform(reference_frame.landmarks['source'].lms, - template_shape) - # warp template - warped_template = template.warp_to_mask(reference_frame.mask, t) - # attach landmarks - warped_template.landmarks['source'] = reference_frame.landmarks[ - 'source'] - return warped_template - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return ATM(shape_models, warped_templates, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features) - - -class PatchATMBuilder(ATMBuilder): - r""" - Class that builds Multilevel Patch-Based Active Template Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=True, - scale_features=True, max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _warp_template(self, template, template_shape, reference_shape, level): - # build reference frame - reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[level]) - # compute transforms - t = self.transform(reference_frame.landmarks['source'].lms, - template_shape) - # warp template - warped_template = template.warp_to_mask(reference_frame.mask, t) - # attach landmarks - warped_template.landmarks['source'] = reference_frame.landmarks[ - 'source'] - return warped_template - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return PatchATM(shape_models, warped_templates, reference_shape, - self.patch_shape, self.features, self.scales, - self.scale_shapes, self.scale_features) - - -# TODO: document me! -class LinearATMBuilder(ATMBuilder): - r""" - Class that builds Linear Active Template Models. - - Parameters - ---------- - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - transform : :map:`PureAlignmentTransform`, optional - The :map:`PureAlignmentTransform` that will be - used to warp the images. - - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. If - ``None``, defaults to performing Delaunay triangulation on the points. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, features=no_op, transform=DifferentiablePiecewiseAffine, - trilist=None, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.features = features - self.transform = transform - self.trilist = trilist - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_reference_frame(mean_aligned_shape) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model( - dense_shapes, max_components=max_components) - return shape_model - - def _warp_template(self, template, template_shape, reference_shape, level): - # compute transforms - t = self.transform(self.reference_frame.landmarks['source'].lms, - template_shape) - # warp template - warped_template = template.warp_to_mask(self.reference_frame.mask, t) - # attach landmarks - warped_template.landmarks['source'] = self.reference_frame.landmarks[ - 'source'] - return warped_template - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return LinearATM(shape_models, warped_templates, reference_shape, - self.transform, self.features, self.scales, - self.scale_shapes, self.scale_features, - self.n_landmarks) - - -# TODO: document me! -class LinearPatchATMBuilder(LinearATMBuilder): - r""" - Class that builds Linear Patch based Active Template Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - diagonal=None, scales=(1, .5), scale_shapes=False, - scale_features=True, max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.transform = DifferentiableThinPlateSplines - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _build_shape_model(self, shapes, max_components, level): - mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) - self.n_landmarks = mean_aligned_shape.n_points - self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[level]) - dense_shapes = densify_shapes(shapes, self.reference_frame, - self.transform) - # build dense shape model - shape_model = build_shape_model(dense_shapes, - max_components=max_components) - return shape_model - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return LinearPatchATM(shape_models, warped_templates, - reference_shape, self.patch_shape, - self.features, self.scales, self.scale_shapes, - self.scale_features, self.n_landmarks) - - -# TODO: document me! -# TODO: implement offsets support? -class PartsATMBuilder(ATMBuilder): - r""" - Class that builds Parts based Active Template Models. - - Parameters - ---------- - patch_shape: (`int`, `int`) or list or list of (`int`, `int`) - - features : `callable` or ``[callable]``, optional - If list of length ``n_scales``, feature extraction is performed at - each level after downscaling of the image. - The first element of the list specifies the features to be extracted at - the lowest pyramidal level and so on. - - If ``callable`` the specified feature will be applied to the original - image and pyramid generation will be performed on top of the feature - image. Also see the `pyramid_on_features` property. - - Note that from our experience, this approach of extracting features - once and then creating a pyramid on top tends to lead to better - performing AAMs. - - normalize_parts : `callable`, optional - - diagonal : `int` >= ``20``, optional - During building an AAM, all images are rescaled to ensure that the - scale of their landmarks matches the scale of the mean shape. - - If `int`, it ensures that the mean shape is scaled so that the diagonal - of the bounding box containing it matches the diagonal value. - - If ``None``, the mean shape is not rescaled. - - Note that, because the reference frame is computed from the mean - landmarks, this kwarg also specifies the diagonal length of the - reference frame (provided that features computation does not change - the image size). - - scales : `int` or float` or list of those, optional - - scale_shapes : `boolean`, optional - - scale_features : `boolean`, optional - - max_shape_components : ``None`` or `int` > 0 or ``0`` <= `float` <= ``1`` or list of those, optional - If list of length ``n_scales``, then a number of shape components is - defined per level. The first element of the list specifies the number - of components of the lowest pyramidal level and so on. - - If not a list or a list with length ``1``, then the specified number of - shape components will be used for all levels. - - Per level: - If `int`, it specifies the exact number of components to be - retained. - - If `float`, it specifies the percentage of variance to be retained. - - If ``None``, all the available components are kept - (100% of variance). - - Returns - ------- - atm : :map:`ATMBuilder` - The ATM Builder object - - Raises - ------- - ValueError - ``diagonal`` must be >= ``20``. - ValueError - ``scales`` must be `int` or `float` or list of those. - ValueError - ``patch_shape`` must be (`int`, `int`) or list of (`int`, `int`) - containing 1 or `len(scales)` elements. - ValueError - ``features`` must be a `function` or a list of those - containing ``1`` or ``len(scales)`` elements - ValueError - ``max_shape_components`` must be ``None`` or an `int` > 0 or - a ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - ValueError - ``max_appearance_components`` must be ``None`` or an `int` > ``0`` or a - ``0`` <= `float` <= ``1`` or a list of those containing 1 or - ``len(scales)`` elements - """ - def __init__(self, patch_shape=(17, 17), features=no_op, - normalize_parts=no_op, diagonal=None, scales=(1, .5), - scale_shapes=False, scale_features=True, - max_shape_components=None): - # check parameters - checks.check_diagonal(diagonal) - scales = checks.check_scales(scales) - patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - max_shape_components = checks.check_max_components( - max_shape_components, len(scales), 'max_shape_components') - # set parameters - self.patch_shape = patch_shape - self.features = features - self.normalize_parts = normalize_parts - self.diagonal = diagonal - self.scales = list(scales) - self.scale_shapes = scale_shapes - self.scale_features = scale_features - self.max_shape_components = max_shape_components - - def _warp_template(self, template, template_shape, reference_shape, level): - parts = template.extract_patches(template_shape, - patch_size=self.patch_shape[level], - as_single_array=True) - if self.normalize_parts: - parts = self.normalize_parts(parts) - - return Image(parts) - - def _build_atm(self, shape_models, warped_templates, reference_shape): - return PartsATM(shape_models, warped_templates, reference_shape, - self.patch_shape, self.features, - self.normalize_parts, self.scales, - self.scale_shapes, self.scale_features) - - -from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM From 31c5ed2d17ecdb1e41045bb704bce78641302675 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:35:32 +0100 Subject: [PATCH 394/423] Incorrect call to super - fixed --- menpofit/aam/algorithm/lk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 720be4c..c21f934 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -167,7 +167,7 @@ def __init__(self, transform, template, sampling=None, # TODO: Refactor to patch_features self.normalize_parts = normalize_parts - super(LucasKanadeBaseInterface, self).__init__( + super(LucasKanadePartsBaseInterface, self).__init__( transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): From 2ac167bdbad583adbd0aecd460690596678e724f Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:35:45 +0100 Subject: [PATCH 395/423] Fix noise variance bug This fix is like not modelling the noise at all - its a reasonable fix but we should think carefully about it. --- menpofit/aam/algorithm/lk.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index c21f934..bacf76d 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -298,8 +298,9 @@ def _precompute(self): self.dW_dp = self.interface.warp_jacobian() # compute shape model prior - s2 = (self.appearance_model.noise_variance() / - self.interface.shape_model.noise_variance()) + # TODO: Is this correct? It's like modelling no noise at all + sm_noise_variance = self.interface.shape_model.noise_variance() or 1 + s2 = self.appearance_model.noise_variance() / sm_noise_variance L = self.interface.shape_model.eigenvalues self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) # compute appearance model prior From 79943a4730f6bf8164b3f1c80026ab5995e729d5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:37:03 +0100 Subject: [PATCH 396/423] Refactor trilist check method Handle images and shapes --- menpofit/aam/base.py | 6 +++--- menpofit/checks.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index a9ac753..409de66 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -116,7 +116,7 @@ class AAM(object): """ def __init__(self, images, group=None, verbose=False, reference_shape=None, features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): checks.check_diagonal(diagonal) @@ -190,8 +190,8 @@ def _train(self, images, group=None, verbose=False, increment=False, 'shape. If the batch mean is not ' 'representative of the true mean, this may ' 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(image_batch[0], self.transform, - group=group) + checks.check_landmark_trilist(image_batch[0], self.transform, + group=group) self.reference_shape = compute_reference_shape( [i.landmarks[group].lms for i in image_batch], self.diagonal, verbose=verbose) diff --git a/menpofit/checks.py b/menpofit/checks.py index 5c954c7..1bfab89 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -16,11 +16,14 @@ def check_diagonal(diagonal): return diagonal -def check_trilist(image, transform, group=None): - trilist = image.landmarks[group].lms +def check_landmark_trilist(image, transform, group=None): + shape = image.landmarks[group].lms + check_trilist(shape, transform) - if not isinstance(trilist, TriMesh) and isinstance(transform, - PiecewiseAffine): + +def check_trilist(shape, transform): + if not isinstance(shape, TriMesh) and isinstance(transform, + PiecewiseAffine): warnings.warn('The given images do not have an explicit triangulation ' 'applied. A Delaunay Triangulation will be computed ' 'and used for warping. This may be suboptimal and cause ' From 8d6e355fce5bbefd5949650d5dd2a2fe86597da0 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:37:35 +0100 Subject: [PATCH 397/423] Remove mention of downscale etc from visualize Helps visualize just work at the moment --- menpofit/visualize/widgets/base.py | 60 ++++-------------------------- 1 file changed, 8 insertions(+), 52 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 3871615..2fbd2b9 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -961,39 +961,17 @@ def update_info(aam, instance, level, group): aam_mean = lvl_app_mod.mean() n_channels = aam_mean.n_channels tmplt_inst = lvl_app_mod.template_instance - feat = (aam.features if aam.pyramid_on_features - else aam.features[level]) + feat = aam.features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( name_of_callable(feat), n_channels, 's' * (n_channels > 1)) - # create info str - if n_levels == 1: - tmp_shape_models = '' - tmp_pyramid = '' - else: # n_scales > 1 - # shape models info - if aam.scaled_shape_models: - tmp_shape_models = "Each level has a scaled shape model " \ - "(reference frame)" - else: - tmp_shape_models = "Shape models (reference frames) are " \ - "not scaled" - # pyramid info - if aam.pyramid_on_features: - tmp_pyramid = "Pyramid was applied on feature space" - else: - tmp_pyramid = "Features were extracted at each pyramid level" - # update info widgets text_per_line = [ - "> {} training images".format(aam.n_training_images), - "> {}".format(tmp_shape_models), "> Warp using {} transform".format(aam.transform.__name__), - "> {}".format(tmp_pyramid), - "> Level {}/{} (downscale={:.1f})".format( - level + 1, aam.n_scales, aam.downscale), + "> Level {}/{}".format( + level + 1, aam.n_scales), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -1377,39 +1355,17 @@ def update_info(atm, instance, level, group): lvl_shape_mod = atm.shape_models[level] tmplt_inst = atm.warped_templates[level] n_channels = tmplt_inst.n_channels - feat = (atm.features if atm.pyramid_on_features - else atm.features[level]) + feat = atm.features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( name_of_callable(feat), n_channels, 's' * (n_channels > 1)) - # create info str - if n_levels == 1: - tmp_shape_models = '' - tmp_pyramid = '' - else: # n_scales > 1 - # shape models info - if atm.scaled_shape_models: - tmp_shape_models = "Each level has a scaled shape model " \ - "(reference frame)" - else: - tmp_shape_models = "Shape models (reference frames) are " \ - "not scaled" - # pyramid info - if atm.pyramid_on_features: - tmp_pyramid = "Pyramid was applied on feature space" - else: - tmp_pyramid = "Features were extracted at each pyramid level" - # update info widgets text_per_line = [ - "> {} training shapes".format(atm.n_training_shapes), - "> {}".format(tmp_shape_models), "> Warp using {} transform".format(atm.transform.__name__), - "> {}".format(tmp_pyramid), - "> Level {}/{} (downscale={:.1f})".format( - level + 1, atm.n_scales, atm.downscale), + "> Level {}/{}".format( + level + 1, atm.n_scales), "> {} landmark points".format( instance.landmarks[group].lms.n_points), "> {} shape components ({:.2f}% of variance)".format( @@ -2467,8 +2423,8 @@ def update_info(name, value): text_per_line = [ "> {} iterations".format(fitting_results[im].n_iters)] if hasattr(fitting_results[im], 'n_scales'): # Multilevel result - text_per_line.append("> {} levels with downscale of {:.1f}".format( - fitting_results[im].n_scales, fitting_results[im].downscale)) + text_per_line.append("> {} scales".format( + fitting_results[im].n_scales)) info_wid.set_widget_state(n_lines=len(text_per_line), text_per_line=text_per_line) From bda03788f62150f06f452cc410aa03db252038fe Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:38:02 +0100 Subject: [PATCH 398/423] Remove unused methods --- menpofit/base.py | 87 +----------------------------------------------- 1 file changed, 1 insertion(+), 86 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index cdedcec..638aca1 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -2,7 +2,7 @@ from functools import partial import itertools import numpy as np -from menpo.visualize import progress_bar_str, print_dynamic, print_progress +from menpo.visualize import progress_bar_str, print_dynamic def name_of_callable(c): @@ -26,91 +26,6 @@ def batch(iterable, n): yield chunk -def is_pyramid_on_features(features): - r""" - True if feature extraction happens once and then a gaussian pyramid - is taken. False if a gaussian pyramid is taken and then features are - extracted at each level. - """ - return callable(features) - - -def create_pyramid(images, n_levels, downscale, features, verbose=False): - r""" - Function that creates a generator function for Gaussian pyramid. The - pyramid can be created either on the feature space or the original - (intensities) space. - - Parameters - ---------- - images: list of :map:`Image` - The set of landmarked images from which to build the AAM. - - n_scales: int - The number of multi-resolution pyramidal levels to be used. - - downscale: float - The downscale factor that will be used to create the different - pyramidal levels. - - features: ``callable`` ``[callable]`` - If a single callable, then the feature calculation will happen once - followed by a gaussian pyramid. If a list of callables then a - gaussian pyramid is generated with features extracted at each level - (after downsizing and blurring). - - Returns - ------- - list of generators : - The generator function of the Gaussian pyramid. - - """ - will_take_a_while = is_pyramid_on_features(features) - if will_take_a_while and verbose: - images = print_progress(images, show_bar=False, show_count=False, - prefix='- Computing top-level feature space') - pyramids = [] - for img in images: - pyramids.append(pyramid_of_feature_images(n_levels, downscale, - features, img)) - return pyramids - - -def pyramid_of_feature_images(n_levels, downscale, features, image): - r""" - Generates a gaussian pyramid of feature images for a single image. - """ - if is_pyramid_on_features(features): - # compute feature image at the top - feature_image = features(image) - # create pyramid on the feature image - return feature_image.gaussian_pyramid(n_levels=n_levels, - downscale=downscale) - else: - # create pyramid on intensities image - # feature will be computed per level - pyramid = image.gaussian_pyramid(n_levels=n_levels, - downscale=downscale) - # add the feature generation here - return feature_images(pyramid, features) - - -# adds feature extraction to a generator of images -def feature_images(images, features): - for feature, level in zip(reversed(features), images): - yield feature(level) - - -class DeformableModel(object): - - def __init__(self, features): - self.features = features - - @property - def pyramid_on_features(self): - return is_pyramid_on_features(self.features) - - def build_grid(shape): r""" """ From 9fb6561aba79f7955893ed81de1acef1c09ad067 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:38:16 +0100 Subject: [PATCH 399/423] Fix bugs using wrong index in AAM --- menpofit/aam/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 409de66..70e875d 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -292,8 +292,8 @@ def _train(self, images, group=None, verbose=False, increment=False, # Because we just copy the shape model, we need to wait to trim # it after building each model. This ensures we can have a different # number of components per level - for k, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[k] + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] if max_sc is not None: sm.trim_components(max_sc) @@ -916,8 +916,9 @@ def _aam_str(aam): aam.shape_models[k].n_components)) # Patch based AAM if hasattr(aam, 'patch_shape'): - for k, s in enumerate(scales_info): - s += '\n - Patch shape: {}'.format(aam.patch_shape[k]) + for k in range(len(scales_info)): + scales_info[k] += '\n - Patch shape: {}'.format( + aam.patch_shape[k]) scales_info = '\n'.join(scales_info) cls_str = r"""{class_title} From ae3884ce57d8f46f2f6e2040d31ebf281d13344e Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 12:38:30 +0100 Subject: [PATCH 400/423] Update ATM to work with AAM layout Now ATMs act like AAMs - including the new interface usage so that the algorithms just take a single interface. A lot of the code is similar to the AAM - but not identical! --- menpofit/atm/__init__.py | 4 +- menpofit/atm/algorithm.py | 12 ++- menpofit/atm/base.py | 173 ++++++++++++++++++-------------------- menpofit/atm/fitter.py | 18 ++-- 4 files changed, 94 insertions(+), 113 deletions(-) diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index cea05ea..2ed3a89 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,5 +1,3 @@ -from .builder import ( - ATMBuilder, PatchATMBuilder, LinearATMBuilder, - LinearPatchATMBuilder, PartsATMBuilder) +from .base import HolisticATM, PartsATM, PatchATM, LinearATM, LinearPatchATM from .fitter import LucasKanadeATMFitter from .algorithm import ForwardCompositional, InverseCompositional diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 90515cd..3a020c6 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -41,7 +41,7 @@ class ATMLKPartsInterface(LucasKanadePartsBaseInterface): """ def algorithm_result(self, image, shape_parameters, cost_functions=None, gt_shape=None): - return LinearATMAlgorithmResult( + return ATMAlgorithmResult( image, self, shape_parameters, cost_functions=cost_functions, gt_shape=gt_shape) @@ -51,13 +51,9 @@ class LucasKanade(object): def __init__(self, atm_interface, eps=10**-5): self.eps = eps - self.interface = atm_interface() + self.interface = atm_interface self._precompute() - @property - def appearance_model(self): - return self.interface.appearance_model - @property def transform(self): return self.interface.transform @@ -77,7 +73,9 @@ def _precompute(self): self.dW_dp = self.interface.warp_jacobian() # compute shape model prior - s2 = 1 / self.interface.shape_model.noise_variance() + # TODO: Is this correct? It's like modelling no noise at all + noise_variance = self.interface.shape_model.noise_variance() or 1 + s2 = 1.0 / noise_variance L = self.interface.shape_model.eigenvalues self.s2_inv_L = np.hstack((np.ones((4,)), s2 / L)) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 95d675d..b8f13d4 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -4,7 +4,6 @@ import numpy as np from menpo.feature import no_op from menpo.visualize import print_dynamic -from menpo.model import PCAModel from menpo.transform import Scale from menpo.shape import mean_pointcloud from menpofit import checks @@ -14,7 +13,7 @@ from menpofit.builder import ( build_reference_frame, build_patch_reference_frame, compute_features, scale_images, build_shape_model, warp_images, - align_shapes, rescale_images_to_reference_shape, densify_shapes, + align_shapes, densify_shapes, extract_patches, MenpoFitBuilderWarning, compute_reference_shape) @@ -23,9 +22,10 @@ class ATM(object): r""" Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, reference_shape=None, - features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + def __init__(self, template, shapes, group=None, verbose=False, + reference_shape=None, features=no_op, + transform=DifferentiablePiecewiseAffine, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, batch_size=None): checks.check_diagonal(diagonal) @@ -45,11 +45,11 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, self.warped_templates = [] # Train ATM - self._train(images, group=group, verbose=verbose, increment=False, - batch_size=batch_size) + self._train(template, shapes, group=group, verbose=verbose, + increment=False, batch_size=batch_size) - def _train(self, images, group=None, verbose=False, increment=False, - shape_forgetting_factor=1.0, batch_size=None): + def _train(self, template, shapes, group=None, verbose=False, + increment=False, shape_forgetting_factor=1.0, batch_size=None): r""" Builds an Active Template Model from a list of landmarked images. @@ -74,11 +74,11 @@ def _train(self, images, group=None, verbose=False, increment=False, if batch_size is not None: # Create a generator of fixed sized batches. Will still work even # on an infinite list. - image_batches = batch(images, batch_size) + shape_batches = batch(shapes, batch_size) else: - image_batches = [list(images)] + shape_batches = [list(shapes)] - for k, image_batch in enumerate(image_batches): + for k, shape_batch in enumerate(shape_batches): # After the first batch, we are incrementing the model if k > 0: increment = True @@ -95,16 +95,14 @@ def _train(self, images, group=None, verbose=False, increment=False, 'shape. If the batch mean is not ' 'representative of the true mean, this may ' 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(image_batch[0], self.transform, - group=group) + checks.check_trilist(shape_batch[0], self.transform) self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) + shape_batch, self.diagonal, verbose=verbose) - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, - verbose=verbose) + if k == 0: + # Rescale the template the reference shape + template = template.rescale_to_pointcloud( + self.reference_shape, group=group) # build models at each scale if verbose: @@ -126,7 +124,7 @@ def _train(self, images, group=None, verbose=False, increment=False, # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale - feature_images = compute_features(image_batch, + feature_images = compute_features([template], self.features[j], prefix=scale_prefix, verbose=verbose) @@ -136,11 +134,14 @@ def _train(self, images, group=None, verbose=False, increment=False, scaled_images = scale_images(feature_images, self.scales[j], prefix=scale_prefix, verbose=verbose) + # Extract potentially rescaled shapes + scale_transform = Scale(scale_factor=self.scales[j], + n_dims=2) + scale_shapes = [scale_transform.apply(s) + for s in shape_batch] else: scaled_images = feature_images - - # Extract potentially rescaled shapes - scale_shapes = [i.landmarks[group].lms for i in scaled_images] + scale_shapes = shape_batch # Build the shape model if verbose: @@ -148,8 +149,7 @@ def _train(self, images, group=None, verbose=False, increment=False, if not increment: if j == 0: - shape_model = self._build_shape_model( - scale_shapes, j) + shape_model = self._build_shape_model(scale_shapes, j) self.shape_models.append(shape_model) else: self.shape_models.append(deepcopy(shape_model)) @@ -164,32 +164,10 @@ def _train(self, images, group=None, verbose=False, increment=False, # reference frame. scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( self.reference_shape) - warped_images = self._warp_images(scaled_images, scale_shapes, - scaled_reference_shape, - j, scale_prefix, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format( - scale_prefix)) - - if not increment: - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - self.appearance_models.append(appearance_model) - else: - # increment appearance model - self.appearance_models[j].increment( - warped_images, - forgetting_factor=appearance_forgetting_factor) - # trim appearance model if required - if self.max_appearance_components is not None: - self.appearance_models[j].trim_components( - self.max_appearance_components[j]) + warped_template = self._warp_template(scaled_images[0], group, + scaled_reference_shape, + j, scale_prefix, verbose) + self.warped_templates.append(warped_template[0]) if verbose: print_dynamic('{}Done\n'.format(scale_prefix)) @@ -197,14 +175,14 @@ def _train(self, images, group=None, verbose=False, increment=False, # Because we just copy the shape model, we need to wait to trim # it after building each model. This ensures we can have a different # number of components per level - for k, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[k] + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] if max_sc is not None: sm.trim_components(max_sc) - def increment(self, images, group=None, verbose=False, + def increment(self, template, shapes, group=None, verbose=False, shape_forgetting_factor=1.0, batch_size=None): - return self._train(images, group=group, + return self._train(template, shapes, group=group, verbose=verbose, shape_forgetting_factor=shape_forgetting_factor, increment=True, batch_size=batch_size) @@ -220,10 +198,11 @@ def _increment_shape_model(self, shapes, shape_model, shape_model.increment(aligned_shapes, forgetting_factor=forgetting_factor) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): reference_frame = build_reference_frame(reference_shape) - return warp_images(images, shapes, reference_frame, self.transform, + shape = template.landmarks[group].lms + return warp_images([template], [shape], reference_frame, self.transform, prefix=prefix, verbose=verbose) @property @@ -284,7 +263,7 @@ def random_instance(self, scale_index=-1): sm.eigenvalues[:sm.n_active_components]**0.5) shape_instance = sm.instance(shape_weights) - return self._instance(scale_index, shape_instance, template) + return self._instance(shape_instance, template) def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms @@ -377,22 +356,24 @@ class PatchATM(ATM): Patch based Based Active Appearance Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - max_shape_components=None, batch_size=None): + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(PatchATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): reference_frame = build_patch_reference_frame( reference_shape, patch_shape=self.patch_shape[scale_index]) - return warp_images(images, shapes, reference_frame, self.transform, + shape = template.landmarks[group].lms + return warp_images([template], [shape], reference_frame, self.transform, prefix=prefix, verbose=verbose) @property @@ -421,13 +402,13 @@ class LinearATM(ATM): Linear Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - transform=DifferentiableThinPlateSplines, diagonal=None, - scales=(0.5, 1.0), max_shape_components=None, + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, transform=DifferentiableThinPlateSplines, + diagonal=None, scales=(0.5, 1.0), max_shape_components=None, batch_size=None): super(LinearATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=transform, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -458,9 +439,10 @@ def _increment_shape_model(self, shapes, shape_model, shape_model.increment(dense_shapes, forgetting_factor=forgetting_factor) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): - return warp_images(images, shapes, self.reference_frame, + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): + shape = template.landmarks[group].lms + return warp_images([template], [shape], self.reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -484,13 +466,14 @@ class LinearPatchATM(ATM): Linear Patch based Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), - max_shape_components=None, batch_size=None): + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_shape=(17, 17), max_shape_components=None, + batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) super(LinearPatchATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -523,9 +506,10 @@ def _increment_shape_model(self, shapes, shape_model, shape_model.increment(dense_shapes, forgetting_factor=forgetting_factor) - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): - return warp_images(images, shapes, self.reference_frame, + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): + shape = template.landmarks[group].lms + return warp_images([template], [shape], self.reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -550,15 +534,15 @@ class PartsATM(ATM): Parts based Active Template Model class. """ - def __init__(self, images, group=None, verbose=False, features=no_op, - normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, - batch_size=None): + def __init__(self, template, shapes, group=None, verbose=False, + features=no_op, normalize_parts=no_op, diagonal=None, + scales=(0.5, 1.0), patch_shape=(17, 17), + max_shape_components=None, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts super(PartsATM, self).__init__( - images, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -571,9 +555,11 @@ def _str_title(self): """ return 'Parts-based Active Template Model' - def _warp_images(self, images, shapes, reference_shape, scale_index, - prefix, verbose): - return extract_patches(images, shapes, self.patch_shape[scale_index], + def _warp_template(self, template, group, reference_shape, scale_index, + prefix, verbose): + shape = template.landmarks[group].lms + return extract_patches([template], [shape], + self.patch_shape[scale_index], normalize_function=self.normalize_parts, prefix=prefix, verbose=verbose) @@ -602,17 +588,18 @@ def _atm_str(atm): scales_info = [] lvl_str_tmplt = r""" - Scale {} - Holistic feature: {} - - {} appearance components + - Template shape: {} - {} shape components""" for k, s in enumerate(atm.scales): scales_info.append(lvl_str_tmplt.format( s, name_of_callable(atm.features[k]), - atm.appearance_models[k].n_components, + atm.warped_templates[k].shape, atm.shape_models[k].n_components)) # Patch based ATM if hasattr(atm, 'patch_shape'): - for k, s in enumerate(scales_info): - s += '\n - Patch shape: {}'.format(atm.patch_shape[k]) + for k in range(len(scales_info)): + scales_info[k] += '\n - Patch shape: {}'.format( + atm.patch_shape[k]) scales_info = '\n'.join(scales_info) cls_str = r"""{class_title} diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 0861e11..a7b334e 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -32,25 +32,23 @@ def _set_up(self, algorithm_cls): self._sampling)): if type(self.atm) is ATM or type(self.atm) is PatchATM: - md_transform = OrthoMDTransform( - sm, self.atm.transform, - source=wt.landmarks['source'].lms) + source_lmarks = wt.landmarks['source'].lms + md_transform = OrthoMDTransform(sm, self.atm.transform, + source=source_lmarks) interface = ATMLKStandardInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) elif (type(self.atm) is LinearATM or type(self.atm) is LinearPatchATM): - # build linear version of orthogonal model driven transform - md_transform = LinearOrthoMDTransform( - sm, self.atm.reference_shape) + md_transform = LinearOrthoMDTransform(sm, + self.atm.reference_shape) interface = ATMLKLinearInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) elif type(self.atm) is PartsATM: pdm = OrthoPDM(sm) - interface = ATMLKPartsInterface(pdm, wt, sampling=s) - algorithm = algorithm_cls( - interface, - patch_shape=self.atm.patch_shape[j], + interface = ATMLKPartsInterface( + pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) + algorithm = algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " From 18135e0d4ea81507195b0096ec4e4c7c3cf67649 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 13:23:34 +0100 Subject: [PATCH 401/423] Refactor LK to match new scheme Just use the correct ordering for scales etc. Also, use _prepare_image rather than _prepare_template to make sure everything is consistent --- menpofit/lk/fitter.py | 87 ++++++++++++++++------------------------- menpofit/lk/residual.py | 1 + 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 11130c2..2319248 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -1,10 +1,11 @@ from __future__ import division from menpo.feature import no_op from menpofit.transform import DifferentiableAlignmentAffine -from menpofit.fitter import MultiFitter, noisy_target_alignment_transform +from menpofit.fitter import (MultiFitter, noisy_shape_from_shape, + noisy_shape_from_bounding_box) from menpofit import checks from .algorithm import InverseCompositional -from .residual import SSD, FourierSSD +from .residual import SSD from .result import LucasKanadeFitterResult @@ -14,76 +15,54 @@ class LucasKanadeFitter(MultiFitter): """ def __init__(self, template, group=None, features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, - scales=(1, .5), scale_features=True, - algorithm_cls=InverseCompositional, residual_cls=SSD, - **kwargs): - # check parameters + scales=(0.5, 1.0), algorithm_cls=InverseCompositional, + residual_cls=SSD): + checks.check_diagonal(diagonal) scales = checks.check_scales(scales) features = checks.check_features(features, len(scales)) - scale_features = checks.check_scale_features(scale_features, features) - # set parameters + self.features = features self.transform_cls = transform_cls self.diagonal = diagonal self.scales = list(scales) - self.scales.reverse() - self.scale_features = scale_features + # Make template masked for warping + template = template.as_masked(copy=False) - self.templates, self.sources = self._prepare_template( - template, group=group) + if self.diagonal: + template = template.rescale_landmarks_to_diagonal_range( + self.diagonal, group=group) + self.reference_shape = template.landmarks[group].lms - self.reference_shape = self.sources[0] + self.templates, self.sources = self._prepare_template(template, + group=group) + self._set_up(algorithm_cls, residual_cls) + def _set_up(self, algorithm_cls, residual_cls): self.algorithms = [] for j, (t, s) in enumerate(zip(self.templates, self.sources)): transform = self.transform_cls(s, s) - if ('kernel_func' in kwargs and - (residual_cls is SSD or - residual_cls is FourierSSD)): - kernel_func = kwargs.pop('kernel_func') - kernel = kernel_func(t.shape) - residual = residual_cls(kernel=kernel) - else: - residual = residual_cls() - algorithm = algorithm_cls(t, transform, residual, **kwargs) + residual = residual_cls() + algorithm = algorithm_cls(t, transform, residual) self.algorithms.append(algorithm) def _prepare_template(self, template, group=None): - template = template.crop_to_landmarks(group=group) - template = template.as_masked() - - # rescale template to diagonal range - if self.diagonal: - template = template.rescale_landmarks_to_diagonal_range( - self.diagonal, group=group) - - # obtain image representation - templates = [] - for j, s in enumerate(self.scales[::-1]): - if j == 0: - # compute features at highest level - feature_template = self.features[j](template) - elif self.scale_features: - # scale features at other levels - feature_template = templates[0].rescale(s) - else: - # scale image and compute features at other levels - scaled_template = template.rescale(s) - feature_template = self.features[j](scaled_template) - templates.append(feature_template) - templates.reverse() - - # get sources per level - sources = [i.landmarks[group].lms for i in templates] - + gt_shape = template.landmarks[group].lms + templates, _, sources = self._prepare_image(template, gt_shape, + gt_shape=gt_shape) return templates, sources - def noisy_shape_from_shape(self, gt_shape, noise_std=0.04): - transform = noisy_target_alignment_transform( - self.reference_shape, gt_shape, - alignment_transform_cls=self.transform_cls, noise_std=noise_std) - return transform.apply(self.reference_shape) + def noisy_shape_from_bounding_box(self, bounding_box, noise_type='uniform', + noise_percentage=0.1, rotation=False): + return noisy_shape_from_bounding_box( + self.reference_shape, bounding_box, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) + + def noisy_shape_from_shape(self, shape, noise_type='uniform', + noise_percentage=0.1, rotation=False): + return noisy_shape_from_shape( + self.reference_shape, shape, noise_type=noise_type, + noise_percentage=noise_percentage, rotation=rotation) def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): diff --git a/menpofit/lk/residual.py b/menpofit/lk/residual.py index d406c0f..9a44f20 100755 --- a/menpofit/lk/residual.py +++ b/menpofit/lk/residual.py @@ -5,6 +5,7 @@ import scipy.linalg from menpo.feature import gradient + # TODO: Do we want residuals to support masked templates? class Residual(object): """ From ab67285019e9c86f44c3954c212c1dd0d2542788 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 6 Aug 2015 13:27:30 +0100 Subject: [PATCH 402/423] Rename Patch{ATM,AAM} to Masked and Parts to Patch Since we use Patch all the time in SDM etc, we should probably use it for the 'parts' AAM which is more similar to SDM. Since 'Patch' was being used merely for an AAM where the appearance model has a disjoint mask applied - we change the name to 'Masked' --- menpofit/aam/__init__.py | 2 +- menpofit/aam/base.py | 24 ++++++++++++------------ menpofit/aam/fitter.py | 22 +++++++++++----------- menpofit/atm/__init__.py | 2 +- menpofit/atm/base.py | 24 ++++++++++++------------ menpofit/atm/fitter.py | 12 ++++++------ 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/menpofit/aam/__init__.py b/menpofit/aam/__init__.py index b91a3ae..cbc979c 100644 --- a/menpofit/aam/__init__.py +++ b/menpofit/aam/__init__.py @@ -1,4 +1,4 @@ -from .base import HolisticAAM, LinearAAM, LinearPatchAAM, PartsAAM, PatchAAM +from .base import HolisticAAM, LinearAAM, LinearMaskedAAM, PatchAAM, MaskedAAM from .fitter import ( LucasKanadeAAMFitter, SupervisedDescentAAMFitter, holistic_sampling_from_scale, holistic_sampling_from_step) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 70e875d..8ec78c9 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -531,9 +531,9 @@ def __str__(self): # TODO: document me! -class PatchAAM(AAM): +class MaskedAAM(AAM): r""" - Patch based Based Active Appearance Model class. + Masked Active Appearance Model class. Parameters ----------- @@ -570,7 +570,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(PatchAAM, self).__init__( + super(MaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -586,7 +586,7 @@ def _warp_images(self, images, shapes, reference_shape, scale_index, @property def _str_title(self): - return 'Patch-based Active Appearance Model' + return 'Masked Active Appearance Model' def _instance(self, scale_index, shape_instance, appearance_instance): template = self.appearance_models[scale_index].mean @@ -714,9 +714,9 @@ def __str__(self): # TODO: document me! -class LinearPatchAAM(AAM): +class LinearMaskedAAM(AAM): r""" - Linear Patch based Active Appearance Model class. + Linear Masked Active Appearance Model class. Parameters ----------- @@ -752,7 +752,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(LinearPatchAAM, self).__init__( + super(LinearMaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -765,7 +765,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Linear Patch-based Active Appearance Model' + return 'Linear Masked Active Appearance Model' def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) @@ -815,9 +815,9 @@ def __str__(self): # TODO: document me! # TODO: implement offsets support? -class PartsAAM(AAM): +class PatchAAM(AAM): r""" - Parts based Active Appearance Model class. + Patch-based Active Appearance Model class. Parameters ----------- @@ -855,7 +855,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts - super(PartsAAM, self).__init__( + super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -868,7 +868,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Parts-based Active Appearance Model' + return 'Patch-based Active Appearance Model' def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 2afc4b4..ff42ea6 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -8,7 +8,7 @@ from menpofit.sdm import SupervisedDescentFitter from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform import menpofit.checks as checks -from .base import AAM, PatchAAM, LinearAAM, LinearPatchAAM, PartsAAM +from .base import AAM, MaskedAAM, LinearAAM, LinearMaskedAAM, PatchAAM from .algorithm.lk import ( LucasKanadeStandardInterface, LucasKanadeLinearInterface, LucasKanadePartsInterface, WibergInverseCompositional) @@ -54,7 +54,7 @@ def _set_up(self, lk_algorithm_cls): self._sampling)): template = am.mean() - if type(self.aam) is AAM or type(self.aam) is PatchAAM: + if type(self.aam) is AAM or type(self.aam) is MaskedAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, @@ -63,14 +63,14 @@ def _set_up(self, lk_algorithm_cls): template, sampling=s) algorithm = lk_algorithm_cls(interface) elif (type(self.aam) is LinearAAM or - type(self.aam) is LinearPatchAAM): + type(self.aam) is LinearMaskedAAM): # build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) interface = LucasKanadeLinearInterface(am, md_transform, template, sampling=s) algorithm = lk_algorithm_cls(interface) - elif type(self.aam) is PartsAAM: + elif type(self.aam) is PatchAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) interface = LucasKanadePartsInterface( @@ -81,8 +81,8 @@ def _set_up(self, lk_algorithm_cls): else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " - "{}".format(AAM, PatchAAM, LinearAAM, - LinearPatchAAM, PartsAAM)) + "{}".format(AAM, MaskedAAM, LinearAAM, + LinearMaskedAAM, PatchAAM)) self.algorithms.append(algorithm) @@ -122,7 +122,7 @@ def _setup_algorithms(self): self.aam.shape_models, self._sampling)): template = am.mean() - if type(self.aam) is AAM or type(self.aam) is PatchAAM: + if type(self.aam) is AAM or type(self.aam) is MaskedAAM: # build orthonormal model driven transform md_transform = OrthoMDTransform( sm, self.aam.transform, @@ -132,7 +132,7 @@ def _setup_algorithms(self): algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) elif (type(self.aam) is LinearAAM or - type(self.aam) is LinearPatchAAM): + type(self.aam) is LinearMaskedAAM): # Build linear version of orthogonal model driven transform md_transform = LinearOrthoMDTransform( sm, self.aam.reference_shape) @@ -140,7 +140,7 @@ def _setup_algorithms(self): am, md_transform, template, sampling=s) algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) - elif type(self.aam) is PartsAAM: + elif type(self.aam) is PatchAAM: # Build orthogonal point distribution model pdm = OrthoPDM(sm) interface = SupervisedDescentPartsInterface( @@ -152,8 +152,8 @@ def _setup_algorithms(self): else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " - "{}".format(AAM, PatchAAM, LinearAAM, - LinearPatchAAM, PartsAAM)) + "{}".format(AAM, MaskedAAM, LinearAAM, + LinearMaskedAAM, PatchAAM)) # append algorithms to list self.algorithms.append(algorithm) diff --git a/menpofit/atm/__init__.py b/menpofit/atm/__init__.py index 2ed3a89..6705775 100644 --- a/menpofit/atm/__init__.py +++ b/menpofit/atm/__init__.py @@ -1,3 +1,3 @@ -from .base import HolisticATM, PartsATM, PatchATM, LinearATM, LinearPatchATM +from .base import HolisticATM, PatchATM, MaskedATM, LinearATM, LinearMaskedATM from .fitter import LucasKanadeATMFitter from .algorithm import ForwardCompositional, InverseCompositional diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index b8f13d4..57d961f 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -351,9 +351,9 @@ def __str__(self): # TODO: document me! -class PatchATM(ATM): +class MaskedATM(ATM): r""" - Patch based Based Active Appearance Model class. + Masked Based Active Appearance Model class. """ def __init__(self, template, shapes, group=None, verbose=False, @@ -362,7 +362,7 @@ def __init__(self, template, shapes, group=None, verbose=False, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(PatchATM, self).__init__( + super(MaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -378,7 +378,7 @@ def _warp_template(self, template, group, reference_shape, scale_index, @property def _str_title(self): - return 'Patch-based Active Template Model' + return 'Masked Active Template Model' def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms @@ -461,9 +461,9 @@ def __str__(self): # TODO: document me! -class LinearPatchATM(ATM): +class LinearMaskedATM(ATM): r""" - Linear Patch based Active Template Model class. + Linear Masked Active Template Model class. """ def __init__(self, template, shapes, group=None, verbose=False, @@ -472,7 +472,7 @@ def __init__(self, template, shapes, group=None, verbose=False, batch_size=None): self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) - super(LinearPatchATM, self).__init__( + super(LinearMaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -484,7 +484,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Linear Patch-based Active Template Model' + return 'Linear Masked Active Template Model' def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) @@ -529,9 +529,9 @@ def __str__(self): # TODO: document me! # TODO: implement offsets support? -class PartsATM(ATM): +class PatchATM(ATM): r""" - Parts based Active Template Model class. + Patch-based Active Template Model class. """ def __init__(self, template, shapes, group=None, verbose=False, @@ -541,7 +541,7 @@ def __init__(self, template, shapes, group=None, verbose=False, self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) self.normalize_parts = normalize_parts - super(PartsATM, self).__init__( + super(PatchATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, @@ -553,7 +553,7 @@ def _str_title(self): Returns a string containing name of the model. :type: `string` """ - return 'Parts-based Active Template Model' + return 'Patch-based Active Template Model' def _warp_template(self, template, group, reference_shape, scale_index, prefix, verbose): diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index a7b334e..dc126e4 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -3,7 +3,7 @@ from menpofit.fitter import ModelFitter from menpofit.modelinstance import OrthoPDM from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform -from .base import ATM, PatchATM, LinearATM, LinearPatchATM, PartsATM +from .base import ATM, MaskedATM, LinearATM, LinearMaskedATM, PatchATM from .algorithm import ( ATMLKStandardInterface, ATMLKPartsInterface, ATMLKLinearInterface, InverseCompositional) @@ -31,19 +31,19 @@ def _set_up(self, algorithm_cls): self.atm.shape_models, self._sampling)): - if type(self.atm) is ATM or type(self.atm) is PatchATM: + if type(self.atm) is ATM or type(self.atm) is MaskedATM: source_lmarks = wt.landmarks['source'].lms md_transform = OrthoMDTransform(sm, self.atm.transform, source=source_lmarks) interface = ATMLKStandardInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) elif (type(self.atm) is LinearATM or - type(self.atm) is LinearPatchATM): + type(self.atm) is LinearMaskedATM): md_transform = LinearOrthoMDTransform(sm, self.atm.reference_shape) interface = ATMLKLinearInterface(md_transform, wt, sampling=s) algorithm = algorithm_cls(interface) - elif type(self.atm) is PartsATM: + elif type(self.atm) is PatchATM: pdm = OrthoPDM(sm) interface = ATMLKPartsInterface( pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], @@ -52,8 +52,8 @@ def _set_up(self, algorithm_cls): else: raise ValueError("AAM object must be of one of the " "following classes: {}, {}, {}, {}, " - "{}".format(ATM, PatchATM, LinearATM, - LinearPatchATM, PartsATM)) + "{}".format(ATM, MaskedATM, LinearATM, + LinearMaskedATM, PatchATM)) self.algorithms.append(algorithm) def _fitter_result(self, image, algorithm_results, affine_correction, From facea21e74625ed01acde7c476b7efddd2136b6a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 7 Aug 2015 10:52:20 +0100 Subject: [PATCH 403/423] Update travis and appveyor --- .travis.yml | 12 +++++++++--- appveyor.yml | 13 +++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b463bc3..cae360d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,11 @@ sudo: false +os: +- linux + +git: + depth: 200 + env: global: - BINSTAR_USER: menpo @@ -8,11 +14,11 @@ env: - PYTHON_VERSION: 3.4 install: - - wget https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py -O condaci.py - - python condaci.py setup +- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py -O condaci.py +- python condaci.py setup script: - - ~/miniconda/bin/python condaci.py build ./conda +- ~/miniconda/bin/python condaci.py build ./conda notifications: slack: diff --git a/appveyor.yml b/appveyor.yml index 2f668bb..0e70524 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,18 +10,18 @@ environment: - PYTHON_VERSION: 3.4 matrix: - fast_finish: true + fast_finish: true platform: - - x86 - - x64 +- x86 +- x64 init: - - ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py' C:\\condaci.py; echo "Done" - - cmd: python C:\\condaci.py setup +- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py' C:\\condaci.py; echo "Done" +- cmd: python C:\\condaci.py setup install: - - cmd: C:\\Miniconda\\python C:\\condaci.py build ./conda +- cmd: C:\\Miniconda\\python C:\\condaci.py build ./conda notifications: - provider: Slack @@ -31,3 +31,4 @@ notifications: on_build_status_changed: true on_build_success: false on_build_failure: false + From 0353e72f0a9ac6670633ef887688498de058675c Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 7 Aug 2015 11:04:53 +0100 Subject: [PATCH 404/423] Fix a bunch of relative import errors --- menpofit/clm/algorithm/__init__.py | 2 +- menpofit/clm/base.py | 2 +- menpofit/clm/expert/__init__.py | 4 ++-- menpofit/feature/__init__.py | 2 +- menpofit/math/__init__.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/menpofit/clm/algorithm/__init__.py b/menpofit/clm/algorithm/__init__.py index 1df3fab..a4ba38c 100644 --- a/menpofit/clm/algorithm/__init__.py +++ b/menpofit/clm/algorithm/__init__.py @@ -1,3 +1,3 @@ -from gd import ( +from .gd import ( GradientDescentCLMAlgorithm, ActiveShapeModel, RegularisedLandmarkMeanShift) diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 7feabf3..1bc41ae 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -6,7 +6,7 @@ from menpofit.builder import ( normalization_wrt_reference_shape, compute_features, scale_images, build_shape_model, increment_shape_model) -from expert import ExpertEnsemble, CorrelationFilterExpertEnsemble +from .expert import ExpertEnsemble, CorrelationFilterExpertEnsemble # TODO: Document me! diff --git a/menpofit/clm/expert/__init__.py b/menpofit/clm/expert/__init__.py index 673e02b..c0e7ae7 100644 --- a/menpofit/clm/expert/__init__.py +++ b/menpofit/clm/expert/__init__.py @@ -1,2 +1,2 @@ -from ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble -from base import IncrementalCorrelationFilterThinWrapper +from .ensemble import ExpertEnsemble, CorrelationFilterExpertEnsemble +from .base import IncrementalCorrelationFilterThinWrapper diff --git a/menpofit/feature/__init__.py b/menpofit/feature/__init__.py index 03a2607..2d00a6a 100644 --- a/menpofit/feature/__init__.py +++ b/menpofit/feature/__init__.py @@ -1,2 +1,2 @@ -from features import ( +from .features import ( centralize, normalize_norm, normalize_std, normalize_var, probability_map) diff --git a/menpofit/math/__init__.py b/menpofit/math/__init__.py index aa9091f..fa310c7 100644 --- a/menpofit/math/__init__.py +++ b/menpofit/math/__init__.py @@ -1 +1 @@ -from .regression import IRLRegression, IIRLRegression \ No newline at end of file +from .regression import IRLRegression, IIRLRegression From d53a9c18240d1ab51061ba682934d93e1705ebc6 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 10 Aug 2015 13:05:37 +0100 Subject: [PATCH 405/423] Refactor training to be more consistent Now everyone method has essentially the same _train method. Then, this method calls a _train_batch method that is different for each method and actually performs training. --- menpofit/aam/base.py | 280 +++++++++++++++++++----------------- menpofit/atm/base.py | 216 ++++++++++++++-------------- menpofit/clm/base.py | 56 +++++--- menpofit/sdm/fitter.py | 312 +++++++++++++++++++++-------------------- 4 files changed, 451 insertions(+), 413 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 8ec78c9..e5257c5 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -139,33 +139,18 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, self.appearance_models = [] # Train AAM - self._train(images, group=group, verbose=verbose, increment=False, + self._train(images, increment=False, group=group, verbose=verbose, batch_size=batch_size) - def _train(self, images, group=None, verbose=False, increment=False, + def _train(self, images, increment=False, group=None, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, - batch_size=None): + verbose=False, batch_size=None): r""" - Builds an Active Appearance Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest scale """ # If batch_size is not None, then we may have a generator, else we # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. if batch_size is not None: # Create a generator of fixed sized batches. Will still work even # on an infinite list. @@ -174,6 +159,23 @@ def _train(self, images, group=None, verbose=False, increment=False, image_batches = [list(images)] for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + checks.check_landmark_trilist(image_batch[0], + self.transform, group=group) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + # After the first batch, we are incrementing the model if k > 0: increment = True @@ -181,121 +183,135 @@ def _train(self, images, group=None, verbose=False, increment=False, if verbose: print('Computing batch {}'.format(k)) - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - checks.check_landmark_trilist(image_batch[0], self.transform, - group=group) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, + # Train each batch + self._train_batch( + image_batch, increment=increment, group=group, + shape_forgetting_factor=shape_forgetting_factor, + appearance_forgetting_factor=appearance_forgetting_factor, verbose=verbose) - # build models at each scale + def _train_batch(self, image_batch, increment=False, group=None, + verbose=False, shape_forgetting_factor=1.0, + appearance_forgetting_factor=1.0): + r""" + Builds an Active Appearance Model from a list of landmarked images. + + Parameters + ---------- + images : list of :map:`MaskedImage` + The set of landmarked images from which to build the AAM. + group : `string`, optional + The key of the landmark set that should be used. If ``None``, + and if there is only one set of landmarks, this set will be used. + verbose : `boolean`, optional + Flag that controls information and progress printing. + + Returns + ------- + aam : :map:`AAM` + The AAM object. Shape and appearance models are stored from + lowest to highest scale + """ + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # build models at each scale + if verbose: + print_dynamic('- Building models\n') + + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): if verbose: - print_dynamic('- Building models\n') - - feature_images = [] - # for each scale (low --> high) - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features(image_batch, - self.features[j], - prefix=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - prefix=scale_prefix, - verbose=verbose) - else: - scaled_images = feature_images - - # Extract potentially rescaled shapes - scale_shapes = [i.landmarks[group].lms for i in scaled_images] - - # Build the shape model - if verbose: - print_dynamic('{}Building shape model'.format(scale_prefix)) - - if not increment: - if j == 0: - shape_model = self._build_shape_model( - scale_shapes, j) - self.shape_models.append(shape_model) - else: - self.shape_models.append(deepcopy(shape_model)) - else: - self._increment_shape_model( - scale_shapes, self.shape_models[j], - forgetting_factor=shape_forgetting_factor) - - # Obtain warped images - we use a scaled version of the - # reference shape, computed here. This is because the mean - # moves when we are incrementing, and we need a consistent - # reference frame. - scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( - self.reference_shape) - warped_images = self._warp_images(scaled_images, scale_shapes, - scaled_reference_shape, - j, scale_prefix, verbose) - - # obtain appearance model - if verbose: - print_dynamic('{}Building appearance model'.format( - scale_prefix)) - - if not increment: - appearance_model = PCAModel(warped_images) - # trim appearance model if required - if self.max_appearance_components is not None: - appearance_model.trim_components( - self.max_appearance_components[j]) - # add appearance model to the list - self.appearance_models.append(appearance_model) + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract potentially rescaled shapes + scale_shapes = [i.landmarks[group].lms for i in scaled_images] + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model( + scale_shapes, j) + self.shape_models.append(shape_model) else: - # increment appearance model - self.appearance_models[j].increment( - warped_images, - forgetting_factor=appearance_forgetting_factor) - # trim appearance model if required - if self.max_appearance_components is not None: - self.appearance_models[j].trim_components( - self.max_appearance_components[j]) - - if verbose: - print_dynamic('{}Done\n'.format(scale_prefix)) - - # Because we just copy the shape model, we need to wait to trim - # it after building each model. This ensures we can have a different - # number of components per level - for j, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[j] - if max_sc is not None: - sm.trim_components(max_sc) + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_images = self._warp_images(scaled_images, scale_shapes, + scaled_reference_shape, + j, scale_prefix, verbose) + + # obtain appearance model + if verbose: + print_dynamic('{}Building appearance model'.format( + scale_prefix)) + + if not increment: + appearance_model = PCAModel(warped_images) + # trim appearance model if required + if self.max_appearance_components is not None: + appearance_model.trim_components( + self.max_appearance_components[j]) + # add appearance model to the list + self.appearance_models.append(appearance_model) + else: + # increment appearance model + self.appearance_models[j].increment( + warped_images, + forgetting_factor=appearance_forgetting_factor) + # trim appearance model if required + if self.max_appearance_components is not None: + self.appearance_models[j].trim_components( + self.max_appearance_components[j]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] + if max_sc is not None: + sm.trim_components(max_sc) def increment(self, images, group=None, verbose=False, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, @@ -303,11 +319,11 @@ def increment(self, images, group=None, verbose=False, # Literally just to fit under 80 characters, but maintain the sensible # parameter name aff = appearance_forgetting_factor - return self._train(images, group=group, + return self._train(images, increment=True, group=group, verbose=verbose, shape_forgetting_factor=shape_forgetting_factor, appearance_forgetting_factor=aff, - increment=True, batch_size=batch_size) + batch_size=batch_size) def _build_shape_model(self, shapes, scale_index): return build_shape_model(shapes) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 57d961f..f6fa2ff 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -45,32 +45,17 @@ def __init__(self, template, shapes, group=None, verbose=False, self.warped_templates = [] # Train ATM - self._train(template, shapes, group=group, verbose=verbose, - increment=False, batch_size=batch_size) + self._train(template, shapes, increment=False, group=group, + verbose=verbose, batch_size=batch_size) - def _train(self, template, shapes, group=None, verbose=False, - increment=False, shape_forgetting_factor=1.0, batch_size=None): + def _train(self, template, shapes, increment=False, group=None, + shape_forgetting_factor=1.0, verbose=False, batch_size=None): r""" - Builds an Active Template Model from a list of landmarked images. - - Parameters - ---------- - images : list of :map:`MaskedImage` - The set of landmarked images from which to build the AAM. - group : `string`, optional - The key of the landmark set that should be used. If ``None``, - and if there is only one set of landmarks, this set will be used. - verbose : `boolean`, optional - Flag that controls information and progress printing. - - Returns - ------- - aam : :map:`AAM` - The AAM object. Shape and appearance models are stored from - lowest to highest scale """ # If batch_size is not None, then we may have a generator, else we # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. if batch_size is not None: # Create a generator of fixed sized batches. Will still work even # on an infinite list. @@ -79,6 +64,26 @@ def _train(self, template, shapes, group=None, verbose=False, shape_batches = [list(shapes)] for k, shape_batch in enumerate(shape_batches): + if k == 0: + # Rescale the template the reference shape + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + checks.check_trilist(shape_batch[0], self.transform) + self.reference_shape = compute_reference_shape( + shape_batch, self.diagonal, verbose=verbose) + + # Rescale the template the reference shape + template = template.rescale_to_pointcloud( + self.reference_shape, group=group) + # After the first batch, we are incrementing the model if k > 0: increment = True @@ -86,99 +91,92 @@ def _train(self, template, shapes, group=None, verbose=False, if verbose: print('Computing batch {}'.format(k)) - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - checks.check_trilist(shape_batch[0], self.transform) - self.reference_shape = compute_reference_shape( - shape_batch, self.diagonal, verbose=verbose) + # Train each batch + self._train_batch(template, shape_batch, increment=increment, + group=group, + shape_forgetting_factor=shape_forgetting_factor, + verbose=verbose) - if k == 0: - # Rescale the template the reference shape - template = template.rescale_to_pointcloud( - self.reference_shape, group=group) + def _train_batch(self, template, shape_batch, increment=False, group=None, + shape_forgetting_factor=1.0, verbose=False): + r""" + Builds an Active Template Model from a list of landmarked images. + """ + # build models at each scale + if verbose: + print_dynamic('- Building models\n') - # build models at each scale + feature_images = [] + # for each scale (low --> high) + for j in range(self.n_scales): if verbose: - print_dynamic('- Building models\n') - - feature_images = [] - # for each scale (low --> high) - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' - else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features([template], - self.features[j], - prefix=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - prefix=scale_prefix, - verbose=verbose) - # Extract potentially rescaled shapes - scale_transform = Scale(scale_factor=self.scales[j], - n_dims=2) - scale_shapes = [scale_transform.apply(s) - for s in shape_batch] + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) else: - scaled_images = feature_images - scale_shapes = shape_batch - - # Build the shape model - if verbose: - print_dynamic('{}Building shape model'.format(scale_prefix)) - - if not increment: - if j == 0: - shape_model = self._build_shape_model(scale_shapes, j) - self.shape_models.append(shape_model) - else: - self.shape_models.append(deepcopy(shape_model)) + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features([template], + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + # Extract potentially rescaled shapes + scale_transform = Scale(scale_factor=self.scales[j], + n_dims=2) + scale_shapes = [scale_transform.apply(s) + for s in shape_batch] + else: + scaled_images = feature_images + scale_shapes = shape_batch + + # Build the shape model + if verbose: + print_dynamic('{}Building shape model'.format(scale_prefix)) + + if not increment: + if j == 0: + shape_model = self._build_shape_model(scale_shapes, j) + self.shape_models.append(shape_model) else: - self._increment_shape_model( - scale_shapes, self.shape_models[j], - forgetting_factor=shape_forgetting_factor) - - # Obtain warped images - we use a scaled version of the - # reference shape, computed here. This is because the mean - # moves when we are incrementing, and we need a consistent - # reference frame. - scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( - self.reference_shape) - warped_template = self._warp_template(scaled_images[0], group, - scaled_reference_shape, - j, scale_prefix, verbose) - self.warped_templates.append(warped_template[0]) - - if verbose: - print_dynamic('{}Done\n'.format(scale_prefix)) - - # Because we just copy the shape model, we need to wait to trim - # it after building each model. This ensures we can have a different - # number of components per level - for j, sm in enumerate(self.shape_models): - max_sc = self.max_shape_components[j] - if max_sc is not None: - sm.trim_components(max_sc) + self.shape_models.append(deepcopy(shape_model)) + else: + self._increment_shape_model( + scale_shapes, self.shape_models[j], + forgetting_factor=shape_forgetting_factor) + + # Obtain warped images - we use a scaled version of the + # reference shape, computed here. This is because the mean + # moves when we are incrementing, and we need a consistent + # reference frame. + scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply( + self.reference_shape) + warped_template = self._warp_template(scaled_images[0], group, + scaled_reference_shape, + j, scale_prefix, verbose) + self.warped_templates.append(warped_template[0]) + + if verbose: + print_dynamic('{}Done\n'.format(scale_prefix)) + + # Because we just copy the shape model, we need to wait to trim + # it after building each model. This ensures we can have a different + # number of components per level + for j, sm in enumerate(self.shape_models): + max_sc = self.max_shape_components[j] + if max_sc is not None: + sm.trim_components(max_sc) def increment(self, template, shapes, group=None, verbose=False, shape_forgetting_factor=1.0, batch_size=None): diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 1bc41ae..fb64599 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -1,11 +1,13 @@ from __future__ import division +import warnings from menpo.feature import no_op from menpo.visualize import print_dynamic from menpofit import checks from menpofit.base import batch from menpofit.builder import ( normalization_wrt_reference_shape, compute_features, scale_images, - build_shape_model, increment_shape_model) + build_shape_model, increment_shape_model, MenpoFitBuilderWarning, + compute_reference_shape, rescale_images_to_reference_shape) from .expert import ExpertEnsemble, CorrelationFilterExpertEnsemble @@ -28,7 +30,7 @@ def __init__(self, images, group=None, verbose=False, batch_size=None, diagonal=None, scales=(0.5, 1), features=no_op, # shape_model_cls=build_normalised_pca_shape_model, expert_ensemble_cls=CorrelationFilterExpertEnsemble, - max_shape_components=None, + max_shape_components=None, reference_shape=None, shape_forgetting_factor=1.0): self.diagonal = checks.check_diagonal(diagonal) self.scales = checks.check_scales(scales) @@ -41,9 +43,13 @@ def __init__(self, images, group=None, verbose=False, batch_size=None, self.max_shape_components = checks.check_max_components( max_shape_components, self.n_scales, 'max_shape_components') self.shape_forgetting_factor = shape_forgetting_factor + self.reference_shape = reference_shape + self.shape_models = [] + self.expert_ensembles = [] # Train CLM - self.train(images, group=group, verbose=verbose, batch_size=batch_size) + self._train(images, increment=False, group=group, verbose=verbose, + batch_size=batch_size) @property def n_scales(self): @@ -54,18 +60,13 @@ def n_scales(self): """ return len(self.scales) - def _train_batch(self, image_batch, increment, group=None, verbose=False): + def _train_batch(self, image_batch, increment=False, group=None, + verbose=False): r""" """ - # If increment is False, we need to initialise/reset both shape models - # and ensembles of experts - if not increment: - self.shape_models = [] - self.expert_ensembles = [] - - # normalize images and compute reference shape - self.reference_shape, image_batch = normalization_wrt_reference_shape( - image_batch, group, self.diagonal, verbose=verbose) + # normalize images + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, verbose=verbose) # build models at each scale if verbose: @@ -139,7 +140,7 @@ def _train_batch(self, image_batch, increment, group=None, verbose=False): if verbose: print_dynamic('{}Done\n'.format(prefix)) - def _train(self, images, increment, group=None, verbose=False, + def _train(self, images, increment=False, group=None, verbose=False, batch_size=None): r""" """ @@ -155,27 +156,36 @@ def _train(self, images, increment, group=None, verbose=False, image_batches = [list(images)] for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + # After the first batch, we are incrementing the model if k > 0: increment = True if verbose: - print('Batch {}'.format(k)) + print('Computing batch {}'.format(k)) # Train each batch - self._train_batch(image_batch, increment, group=group, + self._train_batch(image_batch, increment=increment, group=group, verbose=verbose) - def train(self, images, group=None, verbose=False, batch_size=None): - r""" - """ - return self._train(images, False, group=group, verbose=verbose, - batch_size=batch_size) - def increment(self, images, group=None, verbose=False, batch_size=None): r""" """ - return self._train(images, True, group=group, verbose=verbose, + return self._train(images, increment=True, group=group, verbose=verbose, batch_size=batch_size) def view_shape_models_widget(self, n_parameters=5, diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 511a50d..93d4e70 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -29,7 +29,8 @@ def __init__(self, images, group=None, bounding_box_group=None, batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) - scales, n_scales = checks.check_scales(scales) + n_scales = len(scales) + scales = checks.check_scales(scales) patch_features = checks.check_features(patch_features, n_scales) holistic_features = checks.check_features(holistic_feature, n_scales) patch_shape = checks.check_patch_shape(patch_shape, n_scales) @@ -49,8 +50,9 @@ def __init__(self, images, group=None, bounding_box_group=None, self._setup_algorithms() # Now, train the model! - self._train(images, group=group, bounding_box_group=bounding_box_group, - verbose=verbose, increment=False, batch_size=batch_size) + self._train(images,increment=False, group=group, + bounding_box_group=bounding_box_group, verbose=verbose, + batch_size=batch_size) def _setup_algorithms(self): for j in range(self.n_scales): @@ -59,13 +61,12 @@ def _setup_algorithms(self): patch_shape=self._patch_shape[j], n_iterations=self.n_iterations[j])) - def perturb_from_bounding_box(self, bounding_box): - return self._perturb_from_bounding_box(self.reference_shape, - bounding_box) - - def _train(self, images, group=None, bounding_box_group=None, - verbose=False, increment=False, batch_size=None): - + def _train(self, images, increment=False, group=None, + bounding_box_group=None, verbose=False, batch_size=None): + r""" + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. # If batch_size is not None, then we may have a generator, else we # assume we have a list. if batch_size is not None: @@ -76,154 +77,163 @@ def _train(self, images, group=None, bounding_box_group=None, image_batches = [list(images)] for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + # We set landmarks on the images to archive the perturbations, so + # when the default 'None' is used, we need to grab the actual + # label to sort out the ambiguity + if group is None: + group = image_batch[0].landmarks.group_labels[0] + # After the first batch, we are incrementing the model if k > 0: increment = True if verbose: - print('Computing batch {} - ({})'.format(k, len(image_batch))) - - # In the case where group is None, we need to get the only key so - # that we can attach landmarks below and not get a complaint about - # using None - if group is None: - group = image_batch[0].landmarks.group_labels[0] + print('Computing batch {}'.format(k)) - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The mean ' - 'of the first batch will be the reference ' - 'shape. If the batch mean is not ' - 'representative of the true mean, this may ' - 'cause issues.', MenpoFitBuilderWarning) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # Rescale to existing reference shape - image_batch = rescale_images_to_reference_shape( - image_batch, group, self.reference_shape, + # Train each batch + self._train_batch( + image_batch, increment=increment, group=group, + bounding_box_group=bounding_box_group, verbose=verbose) - # No bounding box is given, so we will use the ground truth box - if bounding_box_group is None: - # It's important to use bb_group for batching, so that we - # generate ground truth bounding boxes for each batch, every - # time - bb_group = '__gt_bb_' - for i in image_batch: - gt_s = i.landmarks[group].lms - perturb_bbox_group = bb_group + '0' - i.landmarks[perturb_bbox_group] = gt_s.bounding_box() - else: - bb_group = bounding_box_group - - # Find all bounding boxes on the images with the given bounding - # box key - all_bb_keys = list(image_batch[0].landmarks.keys_matching( - '*{}*'.format(bb_group))) - n_perturbations = len(all_bb_keys) - - # If there is only one example bounding box, then we will generate - # more perturbations based on the bounding box. - if n_perturbations == 1: - msg = '- Generating {} new initial bounding boxes ' \ - 'per image'.format(self.n_perturbations) - wrap = partial(print_progress, prefix=msg, verbose=verbose) - - for i in wrap(image_batch): - # We assume that the first bounding box is a valid - # perturbation thus create n_perturbations - 1 new bounding - # boxes - for j in range(1, self.n_perturbations): - gt_s = i.landmarks[group].lms.bounding_box() - bb = i.landmarks[all_bb_keys[0]].lms - - # This is customizable by passing in the correct method - p_s = self._perturb_from_bounding_box(gt_s, bb) - perturb_bbox_group = '{}_{}'.format(bb_group, j) - i.landmarks[perturb_bbox_group] = p_s - elif n_perturbations != self.n_perturbations: - warnings.warn('The original value of n_perturbation {} ' - 'will be reset to {} in order to agree with ' - 'the provided bounding_box_group.'. - format(self.n_perturbations, n_perturbations), - MenpoFitBuilderWarning) - self.n_perturbations = n_perturbations - - # Re-grab all the bounding box keys for iterating over when - # calculating perturbations - all_bb_keys = list(image_batch[0].landmarks.keys_matching( - '*{}*'.format(bb_group))) - - # for each scale (low --> high) - current_shapes = [] - for j in range(self.n_scales): - if verbose: - if len(self.scales) > 1: - scale_prefix = ' - Scale {}: '.format(j) - else: - scale_prefix = ' - ' - else: - scale_prefix = None - - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: - # Compute features only if this is the first pass through - # the loop or the features at this scale are different from - # the features at the previous scale - feature_images = compute_features(image_batch, - self.features[j], - level_str=scale_prefix, - verbose=verbose) - # handle scales - if self.scales[j] != 1: - # Scale feature images only if scale is different than 1 - scaled_images = scale_images(feature_images, self.scales[j], - level_str=scale_prefix, - verbose=verbose) - else: - scaled_images = feature_images - - # Extract scaled ground truth shapes for current scale - scaled_shapes = [i.landmarks[group].lms for i in scaled_images] - - if j == 0: - msg = '{}Generating {} perturbations per image'.format( - scale_prefix, self.n_perturbations) - wrap = partial(print_progress, prefix=msg, - end_with_newline=False, verbose=verbose) - - # Extract perturbations at the very bottom level - for i in wrap(scaled_images): - c_shapes = [] - for perturb_bbox_group in all_bb_keys: - bbox = i.landmarks[perturb_bbox_group].lms - c_s = align_shape_with_bounding_box( - self.reference_shape, bbox) - c_shapes.append(c_s) - current_shapes.append(c_shapes) - - # train supervised descent algorithm - if not increment: - current_shapes = self.algorithms[j].train( - scaled_images, scaled_shapes, current_shapes, - level_str=scale_prefix, verbose=verbose) + def _train_batch(self, image_batch, increment=False, group=None, + bounding_box_group=None, verbose=False): + # Rescale to existing reference shape + image_batch = rescale_images_to_reference_shape( + image_batch, group, self.reference_shape, + verbose=verbose) + + # No bounding box is given, so we will use the ground truth box + if bounding_box_group is None: + # It's important to use bb_group for batching, so that we + # generate ground truth bounding boxes for each batch, every + # time + bb_group = '__gt_bb_' + for i in image_batch: + gt_s = i.landmarks[group].lms + perturb_bbox_group = bb_group + '0' + i.landmarks[perturb_bbox_group] = gt_s.bounding_box() + else: + bb_group = bounding_box_group + + # Find all bounding boxes on the images with the given bounding + # box key + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) + n_perturbations = len(all_bb_keys) + + # If there is only one example bounding box, then we will generate + # more perturbations based on the bounding box. + if n_perturbations == 1: + msg = '- Generating {} new initial bounding boxes ' \ + 'per image'.format(self.n_perturbations) + wrap = partial(print_progress, prefix=msg, verbose=verbose) + + for i in wrap(image_batch): + # We assume that the first bounding box is a valid + # perturbation thus create n_perturbations - 1 new bounding + # boxes + for j in range(1, self.n_perturbations): + gt_s = i.landmarks[group].lms.bounding_box() + bb = i.landmarks[all_bb_keys[0]].lms + + # This is customizable by passing in the correct method + p_s = self._perturb_from_bounding_box(gt_s, bb) + perturb_bbox_group = '{}_{}'.format(bb_group, j) + i.landmarks[perturb_bbox_group] = p_s + elif n_perturbations != self.n_perturbations: + warnings.warn('The original value of n_perturbation {} ' + 'will be reset to {} in order to agree with ' + 'the provided bounding_box_group.'. + format(self.n_perturbations, n_perturbations), + MenpoFitBuilderWarning) + self.n_perturbations = n_perturbations + + # Re-grab all the bounding box keys for iterating over when + # calculating perturbations + all_bb_keys = list(image_batch[0].landmarks.keys_matching( + '*{}*'.format(bb_group))) + + # for each scale (low --> high) + current_shapes = [] + for j in range(self.n_scales): + if verbose: + if len(self.scales) > 1: + scale_prefix = ' - Scale {}: '.format(j) else: - current_shapes = self.algorithms[j].increment( - scaled_images, scaled_shapes, current_shapes, - level_str=scale_prefix, verbose=verbose) - - # Scale current shapes to next resolution, don't bother - # scaling final level - if j != (self.n_scales - 1): - transform = Scale(self.scales[j + 1] / self.scales[j], - n_dims=2) - for image_shapes in current_shapes: - for shape in image_shapes: - transform.apply_inplace(shape) + scale_prefix = ' - ' + else: + scale_prefix = None + + # Handle features + if j == 0 or self.features[j] is not self.features[j - 1]: + # Compute features only if this is the first pass through + # the loop or the features at this scale are different from + # the features at the previous scale + feature_images = compute_features(image_batch, + self.features[j], + prefix=scale_prefix, + verbose=verbose) + # handle scales + if self.scales[j] != 1: + # Scale feature images only if scale is different than 1 + scaled_images = scale_images(feature_images, self.scales[j], + prefix=scale_prefix, + verbose=verbose) + else: + scaled_images = feature_images + + # Extract scaled ground truth shapes for current scale + scaled_shapes = [i.landmarks[group].lms for i in scaled_images] + + if j == 0: + msg = '{}Generating {} perturbations per image'.format( + scale_prefix, self.n_perturbations) + wrap = partial(print_progress, prefix=msg, + end_with_newline=False, verbose=verbose) + + # Extract perturbations at the very bottom level + for i in wrap(scaled_images): + c_shapes = [] + for perturb_bbox_group in all_bb_keys: + bbox = i.landmarks[perturb_bbox_group].lms + c_s = align_shape_with_bounding_box( + self.reference_shape, bbox) + c_shapes.append(c_s) + current_shapes.append(c_shapes) + + # train supervised descent algorithm + if not increment: + current_shapes = self.algorithms[j].train( + scaled_images, scaled_shapes, current_shapes, + prefix=scale_prefix, verbose=verbose) + else: + current_shapes = self.algorithms[j].increment( + scaled_images, scaled_shapes, current_shapes, + prefix=scale_prefix, verbose=verbose) + + # Scale current shapes to next resolution, don't bother + # scaling final level + if j != (self.n_scales - 1): + transform = Scale(self.scales[j + 1] / self.scales[j], + n_dims=2) + for image_shapes in current_shapes: + for shape in image_shapes: + transform.apply_inplace(shape) def increment(self, images, group=None, bounding_box_group=None, verbose=False, batch_size=None): @@ -232,6 +242,10 @@ def increment(self, images, group=None, bounding_box_group=None, verbose=verbose, increment=True, batch_size=batch_size) + def perturb_from_bounding_box(self, bounding_box): + return self._perturb_from_bounding_box(self.reference_shape, + bounding_box) + def _fitter_result(self, image, algorithm_results, affine_correction, gt_shape=None): return MultiFitterResult(image, self, algorithm_results, From 15b34c23ab30da771ed5deb3a33ed371fcf6e710 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 10 Aug 2015 13:07:18 +0100 Subject: [PATCH 406/423] Move _train to top in CLM - consistent with AAM etc --- menpofit/clm/base.py | 90 ++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index fb64599..4d8159d 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -5,9 +5,9 @@ from menpofit import checks from menpofit.base import batch from menpofit.builder import ( - normalization_wrt_reference_shape, compute_features, scale_images, - build_shape_model, increment_shape_model, MenpoFitBuilderWarning, - compute_reference_shape, rescale_images_to_reference_shape) + compute_features, scale_images, build_shape_model, increment_shape_model, + MenpoFitBuilderWarning, compute_reference_shape, + rescale_images_to_reference_shape) from .expert import ExpertEnsemble, CorrelationFilterExpertEnsemble @@ -60,6 +60,48 @@ def n_scales(self): """ return len(self.scales) + def _train(self, images, increment=False, group=None, verbose=False, + batch_size=None): + r""" + """ + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + # If batch_size is not None, then we may have a generator, else we + # assume we have a list. + if batch_size is not None: + # Create a generator of fixed sized batches. Will still work even + # on an infinite list. + image_batches = batch(images, batch_size) + else: + image_batches = [list(images)] + + for k, image_batch in enumerate(image_batches): + if k == 0: + if self.reference_shape is None: + # If no reference shape was given, use the mean of the first + # batch + if batch_size is not None: + warnings.warn('No reference shape was provided. The ' + 'mean of the first batch will be the ' + 'reference shape. If the batch mean is ' + 'not representative of the true mean, ' + 'this may cause issues.', + MenpoFitBuilderWarning) + self.reference_shape = compute_reference_shape( + [i.landmarks[group].lms for i in image_batch], + self.diagonal, verbose=verbose) + + # After the first batch, we are incrementing the model + if k > 0: + increment = True + + if verbose: + print('Computing batch {}'.format(k)) + + # Train each batch + self._train_batch(image_batch, increment=increment, group=group, + verbose=verbose) + def _train_batch(self, image_batch, increment=False, group=None, verbose=False): r""" @@ -140,48 +182,6 @@ def _train_batch(self, image_batch, increment=False, group=None, if verbose: print_dynamic('{}Done\n'.format(prefix)) - def _train(self, images, increment=False, group=None, verbose=False, - batch_size=None): - r""" - """ - # If batch_size is not None, then we may have a generator, else we - # assume we have a list. - # If batch_size is not None, then we may have a generator, else we - # assume we have a list. - if batch_size is not None: - # Create a generator of fixed sized batches. Will still work even - # on an infinite list. - image_batches = batch(images, batch_size) - else: - image_batches = [list(images)] - - for k, image_batch in enumerate(image_batches): - if k == 0: - if self.reference_shape is None: - # If no reference shape was given, use the mean of the first - # batch - if batch_size is not None: - warnings.warn('No reference shape was provided. The ' - 'mean of the first batch will be the ' - 'reference shape. If the batch mean is ' - 'not representative of the true mean, ' - 'this may cause issues.', - MenpoFitBuilderWarning) - self.reference_shape = compute_reference_shape( - [i.landmarks[group].lms for i in image_batch], - self.diagonal, verbose=verbose) - - # After the first batch, we are incrementing the model - if k > 0: - increment = True - - if verbose: - print('Computing batch {}'.format(k)) - - # Train each batch - self._train_batch(image_batch, increment=increment, group=group, - verbose=verbose) - def increment(self, images, group=None, verbose=False, batch_size=None): r""" """ From b0a02a277420fbf360f864e13fbdfaaeb317fe75 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:24:42 +0100 Subject: [PATCH 407/423] Fix silly bug in PatchAAM Wasn't passing parameters up to super class properly. --- menpofit/aam/algorithm/lk.py | 5 ++--- menpofit/aam/base.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index bacf76d..181c155 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -229,13 +229,12 @@ class LucasKanadePartsInterface(LucasKanadePartsBaseInterface): """ def __init__(self, appearance_model, transform, template, sampling=None, patch_shape=(17, 17), normalize_parts=no_op): - self.patch_shape = patch_shape # TODO: Refactor to patch_features - self.normalize_parts = normalize_parts self.appearance_model = appearance_model super(LucasKanadePartsInterface, self).__init__( - transform, template, sampling=sampling) + transform, template, patch_shape=patch_shape, + normalize_parts=normalize_parts, sampling=sampling) @property def m(self): diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index e5257c5..a448cb3 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -873,7 +873,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, super(PatchAAM, self).__init__( images, group=group, verbose=verbose, features=features, - transform=DifferentiableThinPlateSplines, diagonal=diagonal, + transform=None, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -937,13 +937,18 @@ def _aam_str(aam): aam.patch_shape[k]) scales_info = '\n'.join(scales_info) + if aam.transform is not None: + transform_str = 'Images warped with {transform} transform' + else: + transform_str = 'No image warping performed' + cls_str = r"""{class_title} - - Images warped with {transform} transform - Images scaled to diagonal: {diagonal:.2f} + - {transform} - Scales: {scales} {scales_info} """.format(class_title=aam._str_title, - transform=name_of_callable(aam.transform), + transform=transform_str, diagonal=diagonal, scales=aam.scales, scales_info=scales_info) From ff07836d8b878647816a1a3668181b14618b3fc9 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:25:18 +0100 Subject: [PATCH 408/423] Small inhancement for no_op When holistic feature is no_op, don't copy images or even bother running through the loop, just skip it. --- menpofit/aam/base.py | 7 +++++-- menpofit/clm/base.py | 7 +++++-- menpofit/sdm/fitter.py | 7 +++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index a448cb3..5e6d52b 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -232,8 +232,11 @@ def _train_batch(self, image_batch, increment=False, group=None, else: scale_prefix = None - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: + # Handle holistic features + if j == 0 and self.features[j] == no_op: + # Saves a lot of memory + feature_images = image_batch + elif j == 0 or self.features[j] is not self.features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 4d8159d..42f3614 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -122,8 +122,11 @@ def _train_batch(self, image_batch, increment=False, group=None, else: prefix = ' - ' - # handle features - if i == 0 or self.features[i] is not self.features[i-1]: + # Handle holistic features + if i == 0 and self.features[i] == no_op: + # Saves a lot of memory + feature_images = image_batch + elif i == 0 or self.features[i] is not self.features[i - 1]: # compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 93d4e70..010dcb8 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -179,8 +179,11 @@ def _train_batch(self, image_batch, increment=False, group=None, else: scale_prefix = None - # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: + # Handle holistic features + if j == 0 and self.features[j] == no_op: + # Saves a lot of memory + feature_images = image_batch + elif j == 0 or self.features[j] is not self.features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale From d592ddbbeb0c38a3ace799360fda37320c64034a Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:25:54 +0100 Subject: [PATCH 409/423] Fix printing a bit Includes non-verbose bug for CLMs --- menpofit/builder.py | 2 +- menpofit/clm/base.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index b148916..9cce3b1 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -162,7 +162,7 @@ def warp_images(images, shapes, reference_frame, transform, prefix='', def extract_patches(images, shapes, patch_shape, normalize_function=no_op, prefix='', verbose=False): wrap = partial(print_progress, - prefix='{}Warping images'.format(prefix), + prefix='{}Extracting patches'.format(prefix), end_with_newline=not prefix, verbose=verbose) parts_images = [] diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index 42f3614..e335957 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -118,9 +118,11 @@ def _train_batch(self, image_batch, increment=False, group=None, for i in range(self.n_scales): if verbose: if self.n_scales > 1: - prefix = ' - Scale {}: '.format(i) + prefix = ' - Scale {}: '.format(j) else: - prefix = ' - ' + prefix = ' - ' + else: + prefix = None # Handle holistic features if i == 0 and self.features[i] == no_op: From 4ea9c52cd856548507315788fc9dbca308dbf8ac Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:26:31 +0100 Subject: [PATCH 410/423] Change fitting methods to fit_from_shape/bb This renames the fitting methods and adds a new method that can fit from bounding boxes. --- menpofit/builder.py | 30 +++++++++++++----------------- menpofit/fitter.py | 12 ++++++++++-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/menpofit/builder.py b/menpofit/builder.py index 9cce3b1..a174780 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -173,9 +173,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images.append(Image(parts)) return parts_images - -def build_reference_frame(landmarks, boundary=3, group='source', - trilist=None): +def build_reference_frame(landmarks, boundary=3, group='source'): r""" Builds a reference frame from a particular set of landmarks. @@ -183,22 +181,14 @@ def build_reference_frame(landmarks, boundary=3, group='source', ---------- landmarks : :map:`PointCloud` The landmarks that will be used to build the reference frame. - boundary : `int`, optional The number of pixels to be left as a safe margin on the boundaries of the reference frame (has potential effects on the gradient computation). - group : `string`, optional Group that will be assigned to the provided set of landmarks on the reference frame. - trilist : ``(t, 3)`` `ndarray`, optional - Triangle list that will be used to build the reference frame. - - If ``None``, defaults to performing Delaunay triangulation on the - points. - Returns ------- reference_frame : :map:`Image` @@ -206,14 +196,13 @@ def build_reference_frame(landmarks, boundary=3, group='source', """ reference_frame = _build_reference_frame(landmarks, boundary=boundary, group=group) - if trilist is not None: - reference_frame.landmarks[group] = TriMesh( - reference_frame.landmarks['source'].lms.points, trilist=trilist) + source_landmarks = reference_frame.landmarks['source'].lms + if isinstance(source_landmarks, TriMesh): + trilist = source_landmarks.trilist + else: + trilist = None - # TODO: revise kwarg trilist in method constrain_mask_to_landmarks, - # perhaps the trilist should be directly obtained from the group landmarks reference_frame.constrain_mask_to_landmarks(group=group, trilist=trilist) - return reference_frame @@ -337,3 +326,10 @@ def increment_shape_model(shape_model, shapes, forgetting_factor=None, if max_components is not None: shape_model.trim_components(max_components) return shape_model + + +class MenpoFitBuilderWarning(Warning): + r""" + A warning that some part of building the model may cause issues. + """ + pass diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 7f1992d..9194747 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -21,8 +21,8 @@ def n_scales(self): """ return len(self.scales) - def fit(self, image, initial_shape, max_iters=20, gt_shape=None, - crop_image=0.5, **kwargs): + def fit_from_shape(self, image, initial_shape, max_iters=20, gt_shape=None, + crop_image=None, **kwargs): r""" Fits the multilevel fitter to an image. @@ -78,6 +78,14 @@ def fit(self, image, initial_shape, max_iters=20, gt_shape=None, return fitter_result + def fit_from_bb(self, image, bounding_box, max_iters=20, gt_shape=None, + crop_image=None, **kwargs): + initial_shape = align_shape_with_bounding_box(self.reference_shape, + bounding_box) + return self.fit_from_shape(image, initial_shape, max_iters=max_iters, + gt_shape=gt_shape, crop_image=crop_image, + **kwargs) + def _prepare_image(self, image, initial_shape, gt_shape=None, crop_image=0.5): r""" From 00b28cce4eb875d2826795e3339f1a092940d598 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 14:36:03 +0100 Subject: [PATCH 411/423] Rename Parts interfaces to patch This is consistent with the previous renaming of parts aam to patch aam --- menpofit/aam/algorithm/lk.py | 8 ++++---- menpofit/aam/fitter.py | 4 ++-- menpofit/atm/algorithm.py | 4 ++-- menpofit/atm/fitter.py | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 181c155..5cf0806 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -158,7 +158,7 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, # TODO document me! -class LucasKanadePartsBaseInterface(LucasKanadeBaseInterface): +class LucasKanadePatchBaseInterface(LucasKanadeBaseInterface): r""" """ def __init__(self, transform, template, sampling=None, @@ -167,7 +167,7 @@ def __init__(self, transform, template, sampling=None, # TODO: Refactor to patch_features self.normalize_parts = normalize_parts - super(LucasKanadePartsBaseInterface, self).__init__( + super(LucasKanadePatchBaseInterface, self).__init__( transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): @@ -224,7 +224,7 @@ def steepest_descent_images(self, nabla, dw_dp): # TODO document me! -class LucasKanadePartsInterface(LucasKanadePartsBaseInterface): +class LucasKanadePatchInterface(LucasKanadePatchBaseInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, @@ -232,7 +232,7 @@ def __init__(self, appearance_model, transform, template, sampling=None, # TODO: Refactor to patch_features self.appearance_model = appearance_model - super(LucasKanadePartsInterface, self).__init__( + super(LucasKanadePatchInterface, self).__init__( transform, template, patch_shape=patch_shape, normalize_parts=normalize_parts, sampling=sampling) diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index ff42ea6..57da7f2 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -11,7 +11,7 @@ from .base import AAM, MaskedAAM, LinearAAM, LinearMaskedAAM, PatchAAM from .algorithm.lk import ( LucasKanadeStandardInterface, LucasKanadeLinearInterface, - LucasKanadePartsInterface, WibergInverseCompositional) + LucasKanadePatchInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, SupervisedDescentPartsInterface, ProjectOutNewton) @@ -73,7 +73,7 @@ def _set_up(self, lk_algorithm_cls): elif type(self.aam) is PatchAAM: # build orthogonal point distribution model pdm = OrthoPDM(sm) - interface = LucasKanadePartsInterface( + interface = LucasKanadePatchInterface( am, pdm, template, sampling=s, patch_shape=self.aam.patch_shape[j], normalize_parts=self.aam.normalize_parts) diff --git a/menpofit/atm/algorithm.py b/menpofit/atm/algorithm.py index 3a020c6..0e5d5db 100644 --- a/menpofit/atm/algorithm.py +++ b/menpofit/atm/algorithm.py @@ -1,7 +1,7 @@ from __future__ import division import numpy as np from menpofit.aam.algorithm.lk import (LucasKanadeBaseInterface, - LucasKanadePartsBaseInterface) + LucasKanadePatchBaseInterface) from .result import ATMAlgorithmResult, LinearATMAlgorithmResult @@ -36,7 +36,7 @@ def algorithm_result(self, image, shape_parameters, cost_functions=None, # TODO document me! -class ATMLKPartsInterface(LucasKanadePartsBaseInterface): +class ATMLKPatchInterface(LucasKanadePatchBaseInterface): r""" """ def algorithm_result(self, image, shape_parameters, cost_functions=None, diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index dc126e4..5370996 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -5,7 +5,7 @@ from menpofit.transform import OrthoMDTransform, LinearOrthoMDTransform from .base import ATM, MaskedATM, LinearATM, LinearMaskedATM, PatchATM from .algorithm import ( - ATMLKStandardInterface, ATMLKPartsInterface, ATMLKLinearInterface, + ATMLKStandardInterface, ATMLKPatchInterface, ATMLKLinearInterface, InverseCompositional) from .result import ATMFitterResult @@ -45,7 +45,7 @@ def _set_up(self, algorithm_cls): algorithm = algorithm_cls(interface) elif type(self.atm) is PatchATM: pdm = OrthoPDM(sm) - interface = ATMLKPartsInterface( + interface = ATMLKPatchInterface( pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], normalize_parts=self.atm.normalize_parts) algorithm = algorithm_cls(interface) From fe6cdec701f97514eaf284b54a63681c157afed5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 11 Aug 2015 16:23:02 +0100 Subject: [PATCH 412/423] Universally rename patch_shape to patch_size This is consistent with Menpo's extract_patches method and with CLM --- menpofit/aam/algorithm/lk.py | 14 +++---- menpofit/aam/algorithm/sd.py | 8 ++-- menpofit/aam/base.py | 32 ++++++++-------- menpofit/aam/fitter.py | 6 +-- menpofit/atm/base.py | 26 ++++++------- menpofit/atm/fitter.py | 2 +- menpofit/builder.py | 12 +++--- menpofit/checks.py | 18 ++++----- menpofit/clm/expert/ensemble.py | 10 ++--- menpofit/sdm/algorithm.py | 66 ++++++++++++++++----------------- menpofit/sdm/fitter.py | 16 ++++---- 11 files changed, 103 insertions(+), 107 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 5cf0806..58ce0bf 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -162,8 +162,8 @@ class LucasKanadePatchBaseInterface(LucasKanadeBaseInterface): r""" """ def __init__(self, transform, template, sampling=None, - patch_shape=(17, 17), normalize_parts=no_op): - self.patch_shape = patch_shape + patch_size=(17, 17), normalize_parts=no_op): + self.patch_size = patch_size # TODO: Refactor to patch_features self.normalize_parts = normalize_parts @@ -172,7 +172,7 @@ def __init__(self, transform, template, sampling=None, def _build_sampling_mask(self, sampling): if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) + sampling = np.ones(self.patch_size, dtype=np.bool) image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], @@ -192,14 +192,14 @@ def warp_jacobian(self): def warp(self, image): parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, + patch_size=self.patch_size, as_single_array=True) parts = self.normalize_parts(parts) return Image(parts, copy=False) def gradient(self, image): pixels = image.pixels - nabla = fast_gradient(pixels.reshape((-1,) + self.patch_shape)) + nabla = fast_gradient(pixels.reshape((-1,) + self.patch_size)) # remove 1st dimension gradient which corresponds to the gradient # between parts return nabla.reshape((2,) + pixels.shape) @@ -228,12 +228,12 @@ class LucasKanadePatchInterface(LucasKanadePatchBaseInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_shape=(17, 17), normalize_parts=no_op): + patch_size=(17, 17), normalize_parts=no_op): # TODO: Refactor to patch_features self.appearance_model = appearance_model super(LucasKanadePatchInterface, self).__init__( - transform, template, patch_shape=patch_shape, + transform, template, patch_size=patch_size, normalize_parts=normalize_parts, sampling=sampling) @property diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index 7baad2f..810f3a1 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -78,8 +78,8 @@ class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_shape=(17, 17), normalize_parts=no_op): - self.patch_shape = patch_shape + patch_size=(17, 17), normalize_parts=no_op): + self.patch_size = patch_size # TODO: Refactor to patch_features self.normalize_parts = normalize_parts @@ -88,7 +88,7 @@ def __init__(self, appearance_model, transform, template, sampling=None, def _build_sampling_mask(self, sampling): if sampling is None: - sampling = np.ones(self.patch_shape, dtype=np.bool) + sampling = np.ones(self.patch_size, dtype=np.bool) image_shape = self.template.pixels.shape image_mask = np.tile(sampling[None, None, None, ...], @@ -101,7 +101,7 @@ def shape_model(self): def warp(self, image): parts = image.extract_patches(self.transform.target, - patch_size=self.patch_shape, + patch_size=self.patch_size, as_single_array=True) parts = self.normalize_parts(parts) return Image(parts, copy=False) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 5e6d52b..cc608cb 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -563,7 +563,7 @@ class MaskedAAM(AAM): reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` + patch_size : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at @@ -584,10 +584,10 @@ class MaskedAAM(AAM): """ def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -599,7 +599,7 @@ def __init__(self, images, group=None, verbose=False, features=no_op, def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[scale_index]) + reference_shape, patch_size=self.patch_size[scale_index]) return warp_images(images, shapes, reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -612,7 +612,7 @@ def _instance(self, scale_index, shape_instance, appearance_instance): landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( - shape_instance, patch_shape=self.patch_shape) + shape_instance, patch_size=self.patch_size) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) @@ -746,7 +746,7 @@ class LinearMaskedAAM(AAM): reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` + patch_size : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at @@ -766,10 +766,10 @@ class LinearMaskedAAM(AAM): """ def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_shape=(17, 17), + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedAAM, self).__init__( images, group=group, verbose=verbose, features=features, @@ -790,7 +790,7 @@ def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + mean_aligned_shape, patch_size=self.patch_size[scale_index]) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model @@ -847,7 +847,7 @@ class PatchAAM(AAM): reference_shape : :map:`PointCloud` The reference shape that was used to resize all training images to a consistent object size. - patch_shape : tuple of `int` + patch_size : tuple of `int` The shape of the patches used to build the Patch Based AAM. features : `callable` or ``[callable]`` If list of length ``n_scales``, feature extraction is performed at @@ -869,9 +869,9 @@ class PatchAAM(AAM): def __init__(self, images, group=None, verbose=False, features=no_op, normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, + patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) self.normalize_parts = normalize_parts super(PatchAAM, self).__init__( @@ -891,7 +891,7 @@ def _str_title(self): def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): - return extract_patches(images, shapes, self.patch_shape[scale_index], + return extract_patches(images, shapes, self.patch_size[scale_index], normalize_function=self.normalize_parts, prefix=prefix, verbose=verbose) @@ -934,10 +934,10 @@ def _aam_str(aam): aam.appearance_models[k].n_components, aam.shape_models[k].n_components)) # Patch based AAM - if hasattr(aam, 'patch_shape'): + if hasattr(aam, 'patch_size'): for k in range(len(scales_info)): - scales_info[k] += '\n - Patch shape: {}'.format( - aam.patch_shape[k]) + scales_info[k] += '\n - Patch size: {}'.format( + aam.patch_size[k]) scales_info = '\n'.join(scales_info) if aam.transform is not None: diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 57da7f2..5ab2e82 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -75,7 +75,7 @@ def _set_up(self, lk_algorithm_cls): pdm = OrthoPDM(sm) interface = LucasKanadePatchInterface( am, pdm, template, sampling=s, - patch_shape=self.aam.patch_shape[j], + patch_size=self.aam.patch_size[j], normalize_parts=self.aam.normalize_parts) algorithm = lk_algorithm_cls(interface) else: @@ -102,7 +102,7 @@ def __init__(self, images, aam, group=None, bounding_box_group=None, checks.set_models_components(aam.shape_models, n_shape) self._sampling = checks.check_sampling(sampling, aam.n_scales) - # patch_feature and patch_shape are not actually + # patch_feature and patch_size are not actually # used because they are fully defined by the AAM already. Therefore, # we just leave them as their 'defaults' because they won't be used. super(SupervisedDescentAAMFitter, self).__init__( @@ -145,7 +145,7 @@ def _setup_algorithms(self): pdm = OrthoPDM(sm) interface = SupervisedDescentPartsInterface( am, pdm, template, sampling=s, - patch_shape=self.aam.patch_shape[j], + patch_size=self.aam.patch_size[j], normalize_parts=self.aam.normalize_parts) algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index f6fa2ff..7e8c83f 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -356,9 +356,9 @@ class MaskedATM(ATM): def __init__(self, template, shapes, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, + patch_size=(17, 17), max_shape_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, @@ -369,7 +369,7 @@ def __init__(self, template, shapes, group=None, verbose=False, def _warp_template(self, template, group, reference_shape, scale_index, prefix, verbose): reference_frame = build_patch_reference_frame( - reference_shape, patch_shape=self.patch_shape[scale_index]) + reference_shape, patch_size=self.patch_size[scale_index]) shape = template.landmarks[group].lms return warp_images([template], [shape], reference_frame, self.transform, prefix=prefix, verbose=verbose) @@ -382,7 +382,7 @@ def _instance(self, shape_instance, template): landmarks = template.landmarks['source'].lms reference_frame = build_patch_reference_frame( - shape_instance, patch_shape=self.patch_shape) + shape_instance, patch_size=self.patch_size) transform = self.transform( reference_frame.landmarks['source'].lms, landmarks) @@ -466,9 +466,9 @@ class LinearMaskedATM(ATM): def __init__(self, template, shapes, group=None, verbose=False, features=no_op, diagonal=None, scales=(0.5, 1.0), - patch_shape=(17, 17), max_shape_components=None, + patch_size=(17, 17), max_shape_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedATM, self).__init__( template, shapes, group=group, verbose=verbose, features=features, @@ -488,7 +488,7 @@ def _build_shape_model(self, shapes, scale_index): mean_aligned_shape = mean_pointcloud(align_shapes(shapes)) self.n_landmarks = mean_aligned_shape.n_points self.reference_frame = build_patch_reference_frame( - mean_aligned_shape, patch_shape=self.patch_shape[scale_index]) + mean_aligned_shape, patch_size=self.patch_size[scale_index]) dense_shapes = densify_shapes(shapes, self.reference_frame, self.transform) # build dense shape model @@ -534,9 +534,9 @@ class PatchATM(ATM): def __init__(self, template, shapes, group=None, verbose=False, features=no_op, normalize_parts=no_op, diagonal=None, - scales=(0.5, 1.0), patch_shape=(17, 17), + scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): - self.patch_shape = checks.check_patch_shape(patch_shape, len(scales)) + self.patch_size = checks.check_patch_size(patch_size, len(scales)) self.normalize_parts = normalize_parts super(PatchATM, self).__init__( @@ -557,7 +557,7 @@ def _warp_template(self, template, group, reference_shape, scale_index, prefix, verbose): shape = template.landmarks[group].lms return extract_patches([template], [shape], - self.patch_shape[scale_index], + self.patch_size[scale_index], normalize_function=self.normalize_parts, prefix=prefix, verbose=verbose) @@ -594,10 +594,10 @@ def _atm_str(atm): atm.warped_templates[k].shape, atm.shape_models[k].n_components)) # Patch based ATM - if hasattr(atm, 'patch_shape'): + if hasattr(atm, 'patch_size'): for k in range(len(scales_info)): - scales_info[k] += '\n - Patch shape: {}'.format( - atm.patch_shape[k]) + scales_info[k] += '\n - Patch size: {}'.format( + atm.patch_size[k]) scales_info = '\n'.join(scales_info) cls_str = r"""{class_title} diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index 5370996..b4147c3 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -46,7 +46,7 @@ def _set_up(self, algorithm_cls): elif type(self.atm) is PatchATM: pdm = OrthoPDM(sm) interface = ATMLKPatchInterface( - pdm, wt, sampling=s, patch_shape=self.atm.patch_shape[j], + pdm, wt, sampling=s, patch_size=self.atm.patch_size[j], normalize_parts=self.atm.normalize_parts) algorithm = algorithm_cls(interface) else: diff --git a/menpofit/builder.py b/menpofit/builder.py index a174780..7f952d4 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -159,7 +159,7 @@ def warp_images(images, shapes, reference_frame, transform, prefix='', # TODO: document me! -def extract_patches(images, shapes, patch_shape, normalize_function=no_op, +def extract_patches(images, shapes, patch_size, normalize_function=no_op, prefix='', verbose=False): wrap = partial(print_progress, prefix='{}Extracting patches'.format(prefix), @@ -167,7 +167,7 @@ def extract_patches(images, shapes, patch_shape, normalize_function=no_op, parts_images = [] for i, s in wrap(zip(images, shapes)): - parts = i.extract_patches(s, patch_size=patch_shape, + parts = i.extract_patches(s, patch_size=patch_size, as_single_array=True) parts = normalize_function(parts) parts_images.append(Image(parts)) @@ -207,7 +207,7 @@ def build_reference_frame(landmarks, boundary=3, group='source'): def build_patch_reference_frame(landmarks, boundary=3, group='source', - patch_shape=(17, 17)): + patch_size=(17, 17)): r""" Builds a reference frame from a particular set of landmarks. @@ -225,7 +225,7 @@ def build_patch_reference_frame(landmarks, boundary=3, group='source', Group that will be assigned to the provided set of landmarks on the reference frame. - patch_shape : tuple of ints, optional + patch_size : tuple of ints, optional Tuple specifying the shape of the patches. Returns @@ -233,12 +233,12 @@ def build_patch_reference_frame(landmarks, boundary=3, group='source', patch_based_reference_frame : :map:`Image` The patch based reference frame. """ - boundary = np.max(patch_shape) + boundary + boundary = np.max(patch_size) + boundary reference_frame = _build_reference_frame(landmarks, boundary=boundary, group=group) # mask reference frame - reference_frame.build_mask_around_landmarks(patch_shape, group=group) + reference_frame.build_mask_around_landmarks(patch_size, group=group) return reference_frame diff --git a/menpofit/checks.py b/menpofit/checks.py index 1bfab89..cd9934d 100644 --- a/menpofit/checks.py +++ b/menpofit/checks.py @@ -86,17 +86,17 @@ def check_scale_features(scale_features, features): # TODO: document me! -def check_patch_shape(patch_shape, n_scales): - if len(patch_shape) == 2 and isinstance(patch_shape[0], int): - return [patch_shape] * n_scales - elif len(patch_shape) == 1: - return check_patch_shape(patch_shape[0], 1) - elif len(patch_shape) == n_scales: - l1 = check_patch_shape(patch_shape[0], 1) - l2 = check_patch_shape(patch_shape[1:], n_scales-1) +def check_patch_size(patch_size, n_scales): + if len(patch_size) == 2 and isinstance(patch_size[0], int): + return [patch_size] * n_scales + elif len(patch_size) == 1: + return check_patch_size(patch_size[0], 1) + elif len(patch_size) == n_scales: + l1 = check_patch_size(patch_size[0], 1) + l2 = check_patch_size(patch_size[1:], n_scales-1) return l1 + l2 else: - raise ValueError("patch_shape must be a list/tuple of int or a " + raise ValueError("patch_size must be a list/tuple of int or a " "list/tuple of lit/tuple of int/float with the " "same length as the number of scales") diff --git a/menpofit/clm/expert/ensemble.py b/menpofit/clm/expert/ensemble.py index a120345..fbd663f 100644 --- a/menpofit/clm/expert/ensemble.py +++ b/menpofit/clm/expert/ensemble.py @@ -218,10 +218,10 @@ def _train(self, images, shapes, prefix='', verbose=False, # Increment correlation filter correlation_filter, auto_correlation, cross_correlation = ( self._icf.increment(self.auto_correlations[i], - self.cross_correlations[i], - self.n_images, - patches, - self.response)) + self.cross_correlations[i], + self.n_images, + patches, + self.response)) else: # Train correlation filter correlation_filter, auto_correlation, cross_correlation = ( @@ -258,5 +258,3 @@ def generate_cosine_mask(patch_size): cy = np.hanning(patch_size[0]) cx = np.hanning(patch_size[1]) return cy[..., None].dot(cx[None, ...]) - - diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 96826cd..939843a 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -17,18 +17,18 @@ class SupervisedDescentAlgorithm(object): def __init__(self): self.regressors = [] - def train(self, images, gt_shapes, current_shapes, level_str='', + def train(self, images, gt_shapes, current_shapes, prefix='', verbose=False): return self._train(images, gt_shapes, current_shapes, increment=False, - level_str=level_str, verbose=verbose) + prefix=prefix, verbose=verbose) - def increment(self, images, gt_shapes, current_shapes, level_str='', + def increment(self, images, gt_shapes, current_shapes, prefix='', verbose=False): return self._train(images, gt_shapes, current_shapes, increment=True, - level_str=level_str, verbose=verbose) + prefix=prefix, verbose=verbose) def _train(self, images, gt_shapes, current_shapes, increment=False, - level_str='', verbose=False): + prefix='', verbose=False): if not increment: # Reset the regressors @@ -44,13 +44,13 @@ def _train(self, images, gt_shapes, current_shapes, increment=False, for k in range(self.n_iterations): # generate regression data features = features_per_image( - images, current_shapes, self.patch_shape, self.features, - level_str='{}(Iteration {}) - '.format(level_str, k), + images, current_shapes, self.patch_size, self.features, + prefix='{}(Iteration {}) - '.format(prefix, k), verbose=verbose) if verbose: print_dynamic('{}(Iteration {}) - Performing regression'.format( - level_str, k)) + prefix, k)) if not increment: r = self._regressor_cls() @@ -65,7 +65,7 @@ def _train(self, images, gt_shapes, current_shapes, increment=False, self._print_regression_info(template_shape, gt_shapes, n_perturbations, delta_x, estimated_delta_x, k, - level_str=level_str) + prefix=prefix) j = 0 for shapes in current_shapes: @@ -90,7 +90,7 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): for r in self.regressors: # compute regression features features = features_per_patch(image, current_shape, - self.patch_shape, self.features) + self.patch_size, self.features) # solve for increments on the shape vector dx = r.predict(features) @@ -106,9 +106,9 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, delta_x, estimated_delta_x, level_index, - level_str=''): + prefix=''): print_dynamic('{}(Iteration {}) - Calculating errors'.format( - level_str, level_index)) + prefix, level_index)) errors = [] for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)): s1 = template_shape.from_vector(dx) @@ -120,22 +120,21 @@ def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, median = np.median(errors) print_dynamic('{}(Iteration {}) - Training error -> ' 'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'. - format(level_str, level_index, mean, std, median)) + format(prefix, level_index, mean, std, median)) # TODO: document me! class Newton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, + def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True): super(Newton, self).__init__() self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) - self.patch_shape = patch_shape + self.patch_size = patch_size self.features = features - self.patch_shape = patch_shape self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps @@ -145,26 +144,25 @@ def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, class GaussNewton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_shape=(17, 17), n_iterations=3, + def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True, alpha2=0): super(GaussNewton, self).__init__() self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, alpha2=alpha2) - self.patch_shape = patch_shape + self.patch_size = patch_size self.features = features - self.patch_shape = patch_shape self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps # TODO: document me! -def features_per_patch(image, shape, patch_shape, features_callable): +def features_per_patch(image, shape, patch_size, features_callable): """r """ - patches = image.extract_patches(shape, patch_size=patch_shape, + patches = image.extract_patches(shape, patch_size=patch_size, as_single_array=True) patch_features = [features_callable(p[0]).ravel() for p in patches] @@ -172,10 +170,10 @@ def features_per_patch(image, shape, patch_shape, features_callable): # TODO: document me! -def features_per_shape(image, shapes, patch_shape, features_callable): +def features_per_shape(image, shapes, patch_size, features_callable): """r """ - patch_features = [features_per_patch(image, s, patch_shape, + patch_features = [features_per_patch(image, s, patch_size, features_callable) for s in shapes] @@ -183,15 +181,15 @@ def features_per_shape(image, shapes, patch_shape, features_callable): # TODO: document me! -def features_per_image(images, shapes, patch_shape, features_callable, - level_str='', verbose=False): +def features_per_image(images, shapes, patch_size, features_callable, + prefix='', verbose=False): """r """ wrap = partial(print_progress, - prefix='{}Extracting patches'.format(level_str), - end_with_newline=not level_str, verbose=verbose) + prefix='{}Extracting patches'.format(prefix), + end_with_newline=not prefix, verbose=verbose) - patch_features = [features_per_shape(i, shapes[j], patch_shape, + patch_features = [features_per_shape(i, shapes[j], patch_size, features_callable) for j, i in enumerate(wrap(images))] patch_features = np.asarray(patch_features) @@ -237,16 +235,16 @@ def obtain_delta_x(gt_shapes, current_shapes): def compute_features_info(image, shape, features_callable, - patch_shape=(17, 17)): + patch_size=(17, 17)): # TODO: include offsets support? - patches = image.extract_patches(shape, patch_size=patch_shape, + patches = image.extract_patches(shape, patch_size=patch_size, as_single_array=True) # TODO: include offsets support? - features_patch_shape = features_callable(patches[0, 0]).shape - features_patch_length = np.prod(features_patch_shape) - features_shape = patches.shape[:1] + features_patch_shape + features_patch_size = features_callable(patches[0, 0]).shape + features_patch_length = np.prod(features_patch_size) + features_shape = patches.shape[:1] + features_patch_size features_length = np.prod(features_shape) - return (features_patch_shape, features_patch_length, + return (features_patch_size, features_patch_length, features_shape, features_length) diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 010dcb8..de7addf 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -23,7 +23,7 @@ class SupervisedDescentFitter(MultiFitter): def __init__(self, images, group=None, bounding_box_group=None, reference_shape=None, sd_algorithm_cls=Newton, holistic_feature=no_op, patch_features=no_op, - patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): @@ -33,14 +33,14 @@ def __init__(self, images, group=None, bounding_box_group=None, scales = checks.check_scales(scales) patch_features = checks.check_features(patch_features, n_scales) holistic_features = checks.check_features(holistic_feature, n_scales) - patch_shape = checks.check_patch_shape(patch_shape, n_scales) + patch_size = checks.check_patch_size(patch_size, n_scales) # set parameters self.algorithms = [] self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls self.features = holistic_features self._patch_features = patch_features - self._patch_shape = patch_shape + self._patch_size = patch_size self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations @@ -58,7 +58,7 @@ def _setup_algorithms(self): for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( features=self._patch_features[j], - patch_shape=self._patch_shape[j], + patch_size=self._patch_size[j], n_iterations=self.n_iterations[j])) def _train(self, images, increment=False, group=None, @@ -268,12 +268,12 @@ def __str__(self): scales_info = [] lvl_str_tmplt = r""" - Scale {} - {} iterations - - Patch shape: {} + - Patch size: {} - Holistic feature: {} - Patch feature: {}""" for k, s in enumerate(self.scales): scales_info.append(lvl_str_tmplt.format( - s, self.n_iterations[k], self._patch_shape[k], + s, self.n_iterations[k], self._patch_size[k], name_of_callable(self.features[k]), name_of_callable(self._patch_features[k]))) scales_info = '\n'.join(scales_info) @@ -305,7 +305,7 @@ class RegularizedSDM(SupervisedDescentFitter): def __init__(self, images, group=None, bounding_box_group=None, alpha=1.0, reference_shape=None, holistic_feature=no_op, patch_features=no_op, - patch_shape=(17, 17), diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, batch_size=None, verbose=False): @@ -314,7 +314,7 @@ def __init__(self, images, group=None, bounding_box_group=None, reference_shape=reference_shape, sd_algorithm_cls=partial(Newton, alpha=alpha), holistic_feature=holistic_feature, patch_features=patch_features, - patch_shape=patch_shape, diagonal=diagonal, scales=scales, + patch_size=patch_size, diagonal=diagonal, scales=scales, n_iterations=n_iterations, n_perturbations=n_perturbations, perturb_from_bounding_box=perturb_from_bounding_box, batch_size=batch_size, verbose=verbose) From 471e06339ccf85cde457eb212b120c13b0cc2574 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 13 Aug 2015 11:52:19 +0100 Subject: [PATCH 413/423] Renaming features parameters features -> holistic_features normalize_parts -> patch_normalisation normalise_callable -> patch_normalisation patch_features -> patch_features At the moment, only SDMs have patch_features due to the lack of distinction on menpo between image and vector features. --- menpofit/aam/algorithm/lk.py | 12 +++--- menpofit/aam/algorithm/sd.py | 7 ++-- menpofit/aam/base.py | 67 +++++++++++++++++------------- menpofit/aam/fitter.py | 6 +-- menpofit/atm/base.py | 44 +++++++++++--------- menpofit/atm/fitter.py | 2 +- menpofit/builder.py | 6 +-- menpofit/clm/base.py | 11 ++--- menpofit/clm/expert/ensemble.py | 10 ++--- menpofit/fitter.py | 8 ++-- menpofit/lk/fitter.py | 7 ++-- menpofit/sdm/algorithm.py | 15 +++---- menpofit/sdm/fitter.py | 30 ++++++------- menpofit/visualize/widgets/base.py | 4 +- 14 files changed, 120 insertions(+), 109 deletions(-) diff --git a/menpofit/aam/algorithm/lk.py b/menpofit/aam/algorithm/lk.py index 58ce0bf..3dd6960 100644 --- a/menpofit/aam/algorithm/lk.py +++ b/menpofit/aam/algorithm/lk.py @@ -162,10 +162,9 @@ class LucasKanadePatchBaseInterface(LucasKanadeBaseInterface): r""" """ def __init__(self, transform, template, sampling=None, - patch_size=(17, 17), normalize_parts=no_op): + patch_size=(17, 17), patch_normalisation=no_op): self.patch_size = patch_size - # TODO: Refactor to patch_features - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(LucasKanadePatchBaseInterface, self).__init__( transform, template, sampling=sampling) @@ -194,7 +193,7 @@ def warp(self, image): parts = image.extract_patches(self.transform.target, patch_size=self.patch_size, as_single_array=True) - parts = self.normalize_parts(parts) + parts = self.patch_normalisation(parts) return Image(parts, copy=False) def gradient(self, image): @@ -228,13 +227,12 @@ class LucasKanadePatchInterface(LucasKanadePatchBaseInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_size=(17, 17), normalize_parts=no_op): - # TODO: Refactor to patch_features + patch_size=(17, 17), patch_normalisation=no_op): self.appearance_model = appearance_model super(LucasKanadePatchInterface, self).__init__( transform, template, patch_size=patch_size, - normalize_parts=normalize_parts, sampling=sampling) + patch_normalisation=patch_normalisation, sampling=sampling) @property def m(self): diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index 810f3a1..ea073a8 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -78,10 +78,9 @@ class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, - patch_size=(17, 17), normalize_parts=no_op): + patch_size=(17, 17), patch_normalisation=no_op): self.patch_size = patch_size - # TODO: Refactor to patch_features - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(SupervisedDescentPartsInterface, self).__init__( appearance_model, transform, template, sampling=sampling) @@ -103,7 +102,7 @@ def warp(self, image): parts = image.extract_patches(self.transform.target, patch_size=self.patch_size, as_single_array=True) - parts = self.normalize_parts(parts) + parts = self.patch_normalisation(parts) return Image(parts, copy=False) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index cc608cb..83e634f 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -115,20 +115,21 @@ class AAM(object): ``len(scales)`` elements """ def __init__(self, images, group=None, verbose=False, reference_shape=None, - features=no_op, transform=DifferentiablePiecewiseAffine, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, + holistic_features=no_op, + transform=DifferentiablePiecewiseAffine, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): checks.check_diagonal(diagonal) n_scales = len(scales) scales = checks.check_scales(scales) - features = checks.check_features(features, n_scales) + holistic_features = checks.check_features(holistic_features, n_scales) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') max_appearance_components = checks.check_max_components( max_appearance_components, n_scales, 'max_appearance_components') - self.features = features + self.holistic_features = holistic_features self.transform = transform self.diagonal = diagonal self.scales = scales @@ -233,15 +234,15 @@ def _train_batch(self, image_batch, increment=False, group=None, scale_prefix = None # Handle holistic features - if j == 0 and self.features[j] == no_op: + if j == 0 and self.holistic_features[j] == no_op: # Saves a lot of memory feature_images = image_batch - elif j == 0 or self.features[j] is not self.features[j - 1]: + elif j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features(image_batch, - self.features[j], + self.holistic_features[j], prefix=scale_prefix, verbose=verbose) # handle scales @@ -583,14 +584,15 @@ class MaskedAAM(AAM): scale_shapes : `boolean` """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), - max_shape_components=None, max_appearance_components=None, - batch_size=None): + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedAAM, self).__init__( - images, group=group, verbose=verbose, features=features, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, @@ -667,14 +669,16 @@ class LinearAAM(AAM): scales : `int` or float` or list of those """ - def __init__(self, images, group=None, verbose=False, features=no_op, + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, transform=DifferentiableThinPlateSplines, diagonal=None, scales=(0.5, 1.0), max_shape_components=None, max_appearance_components=None, batch_size=None): super(LinearAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, scales=scales, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=transform, + diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -765,14 +769,15 @@ class LinearMaskedAAM(AAM): scales : `int` or float` or list of those """ - def __init__(self, images, group=None, verbose=False, features=no_op, - diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), - max_shape_components=None, max_appearance_components=None, - batch_size=None): + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), + patch_size=(17, 17), max_shape_components=None, + max_appearance_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedAAM, self).__init__( - images, group=group, verbose=verbose, features=features, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, @@ -867,17 +872,19 @@ class PatchAAM(AAM): scales : `int` or float` or list of those """ - def __init__(self, images, group=None, verbose=False, features=no_op, - normalize_parts=no_op, diagonal=None, scales=(0.5, 1.0), - patch_size=(17, 17), max_shape_components=None, - max_appearance_components=None, batch_size=None): + def __init__(self, images, group=None, verbose=False, + holistic_features=no_op, patch_normalisation=no_op, + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), + max_shape_components=None, max_appearance_components=None, + batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(PatchAAM, self).__init__( - images, group=group, verbose=verbose, features=features, - transform=None, diagonal=diagonal, - scales=scales, max_shape_components=max_shape_components, + images, group=group, verbose=verbose, + holistic_features=holistic_features, transform=None, + diagonal=diagonal, scales=scales, + max_shape_components=max_shape_components, max_appearance_components=max_appearance_components, batch_size=batch_size) @@ -892,7 +899,7 @@ def _str_title(self): def _warp_images(self, images, shapes, reference_shape, scale_index, prefix, verbose): return extract_patches(images, shapes, self.patch_size[scale_index], - normalize_function=self.normalize_parts, + normalise_function=self.patch_normalisation, prefix=prefix, verbose=verbose) # TODO: implement me! @@ -930,7 +937,7 @@ def _aam_str(aam): - {} shape components""" for k, s in enumerate(aam.scales): scales_info.append(lvl_str_tmplt.format( - s, name_of_callable(aam.features[k]), + s, name_of_callable(aam.holistic_features[k]), aam.appearance_models[k].n_components, aam.shape_models[k].n_components)) # Patch based AAM diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 5ab2e82..3e74017 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -76,7 +76,7 @@ def _set_up(self, lk_algorithm_cls): interface = LucasKanadePatchInterface( am, pdm, template, sampling=s, patch_size=self.aam.patch_size[j], - normalize_parts=self.aam.normalize_parts) + patch_normalisation=self.aam.patch_normalisation) algorithm = lk_algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " @@ -109,7 +109,7 @@ def __init__(self, images, aam, group=None, bounding_box_group=None, images, group=group, bounding_box_group=bounding_box_group, reference_shape=self.aam.reference_shape, sd_algorithm_cls=sd_algorithm_cls, - holistic_feature=self.aam.features, + holistic_feature=self.aam.holistic_features, diagonal=self.aam.diagonal, scales=self.aam.scales, n_iterations=n_iterations, n_perturbations=n_perturbations, @@ -146,7 +146,7 @@ def _setup_algorithms(self): interface = SupervisedDescentPartsInterface( am, pdm, template, sampling=s, patch_size=self.aam.patch_size[j], - normalize_parts=self.aam.normalize_parts) + patch_normalisation=self.aam.patch_normalisation) algorithm = self._sd_algorithm_cls( interface, n_iterations=self.n_iterations[j]) else: diff --git a/menpofit/atm/base.py b/menpofit/atm/base.py index 7e8c83f..3b53565 100644 --- a/menpofit/atm/base.py +++ b/menpofit/atm/base.py @@ -23,7 +23,7 @@ class ATM(object): Active Template Model class. """ def __init__(self, template, shapes, group=None, verbose=False, - reference_shape=None, features=no_op, + reference_shape=None, holistic_features=no_op, transform=DifferentiablePiecewiseAffine, diagonal=None, scales=(0.5, 1.0), max_shape_components=None, batch_size=None): @@ -31,11 +31,11 @@ def __init__(self, template, shapes, group=None, verbose=False, checks.check_diagonal(diagonal) n_scales = len(scales) scales = checks.check_scales(scales) - features = checks.check_features(features, n_scales) + holistic_features = checks.check_features(holistic_features, n_scales) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') - self.features = features + self.holistic_features = holistic_features self.transform = transform self.diagonal = diagonal self.scales = scales @@ -118,12 +118,12 @@ def _train_batch(self, template, shape_batch, increment=False, group=None, scale_prefix = None # Handle features - if j == 0 or self.features[j] is not self.features[j - 1]: + if j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features([template], - self.features[j], + self.holistic_features[j], prefix=scale_prefix, verbose=verbose) # handle scales @@ -355,13 +355,14 @@ class MaskedATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, diagonal=None, scales=(0.5, 1.0), + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(MaskedATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -401,13 +402,14 @@ class LinearATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, transform=DifferentiableThinPlateSplines, - diagonal=None, scales=(0.5, 1.0), max_shape_components=None, - batch_size=None): + holistic_features=no_op, + transform=DifferentiableThinPlateSplines, diagonal=None, + scales=(0.5, 1.0), max_shape_components=None, batch_size=None): super(LinearATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, - transform=transform, diagonal=diagonal, scales=scales, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=transform, + diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @property @@ -465,13 +467,14 @@ class LinearMaskedATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, diagonal=None, scales=(0.5, 1.0), + holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) super(LinearMaskedATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -533,14 +536,15 @@ class PatchATM(ATM): """ def __init__(self, template, shapes, group=None, verbose=False, - features=no_op, normalize_parts=no_op, diagonal=None, - scales=(0.5, 1.0), patch_size=(17, 17), + holistic_features=no_op, patch_normalisation=no_op, + diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, batch_size=None): self.patch_size = checks.check_patch_size(patch_size, len(scales)) - self.normalize_parts = normalize_parts + self.patch_normalisation = patch_normalisation super(PatchATM, self).__init__( - template, shapes, group=group, verbose=verbose, features=features, + template, shapes, group=group, verbose=verbose, + holistic_features=holistic_features, transform=DifferentiableThinPlateSplines, diagonal=diagonal, scales=scales, max_shape_components=max_shape_components, batch_size=batch_size) @@ -558,7 +562,7 @@ def _warp_template(self, template, group, reference_shape, scale_index, shape = template.landmarks[group].lms return extract_patches([template], [shape], self.patch_size[scale_index], - normalize_function=self.normalize_parts, + normalise_function=self.patch_normalisation, prefix=prefix, verbose=verbose) # TODO: implement me! @@ -590,7 +594,7 @@ def _atm_str(atm): - {} shape components""" for k, s in enumerate(atm.scales): scales_info.append(lvl_str_tmplt.format( - s, name_of_callable(atm.features[k]), + s, name_of_callable(atm.holistic_features[k]), atm.warped_templates[k].shape, atm.shape_models[k].n_components)) # Patch based ATM diff --git a/menpofit/atm/fitter.py b/menpofit/atm/fitter.py index b4147c3..0450571 100644 --- a/menpofit/atm/fitter.py +++ b/menpofit/atm/fitter.py @@ -47,7 +47,7 @@ def _set_up(self, algorithm_cls): pdm = OrthoPDM(sm) interface = ATMLKPatchInterface( pdm, wt, sampling=s, patch_size=self.atm.patch_size[j], - normalize_parts=self.atm.normalize_parts) + patch_normalisation=self.atm.patch_normalisation) algorithm = algorithm_cls(interface) else: raise ValueError("AAM object must be of one of the " diff --git a/menpofit/builder.py b/menpofit/builder.py index 7f952d4..6fdeec1 100644 --- a/menpofit/builder.py +++ b/menpofit/builder.py @@ -159,7 +159,7 @@ def warp_images(images, shapes, reference_frame, transform, prefix='', # TODO: document me! -def extract_patches(images, shapes, patch_size, normalize_function=no_op, +def extract_patches(images, shapes, patch_size, normalise_function=no_op, prefix='', verbose=False): wrap = partial(print_progress, prefix='{}Extracting patches'.format(prefix), @@ -169,8 +169,8 @@ def extract_patches(images, shapes, patch_size, normalize_function=no_op, for i, s in wrap(zip(images, shapes)): parts = i.extract_patches(s, patch_size=patch_size, as_single_array=True) - parts = normalize_function(parts) - parts_images.append(Image(parts)) + parts = normalise_function(parts) + parts_images.append(Image(parts, copy=False)) return parts_images def build_reference_frame(landmarks, boundary=3, group='source'): diff --git a/menpofit/clm/base.py b/menpofit/clm/base.py index e335957..780f874 100644 --- a/menpofit/clm/base.py +++ b/menpofit/clm/base.py @@ -27,14 +27,15 @@ class CLM(object): The CLM object """ def __init__(self, images, group=None, verbose=False, batch_size=None, - diagonal=None, scales=(0.5, 1), features=no_op, + diagonal=None, scales=(0.5, 1), holistic_features=no_op, # shape_model_cls=build_normalised_pca_shape_model, expert_ensemble_cls=CorrelationFilterExpertEnsemble, max_shape_components=None, reference_shape=None, shape_forgetting_factor=1.0): self.diagonal = checks.check_diagonal(diagonal) self.scales = checks.check_scales(scales) - self.features = checks.check_features(features, self.n_scales) + self.holistic_features = checks.check_features(holistic_features, + self.n_scales) # self.shape_model_cls = checks.check_algorithm_cls( # shape_model_cls, self.n_scales, ShapeModel) self.expert_ensemble_cls = checks.check_algorithm_cls( @@ -125,15 +126,15 @@ def _train_batch(self, image_batch, increment=False, group=None, prefix = None # Handle holistic features - if i == 0 and self.features[i] == no_op: + if i == 0 and self.holistic_features[i] == no_op: # Saves a lot of memory feature_images = image_batch - elif i == 0 or self.features[i] is not self.features[i - 1]: + elif i == 0 or self.holistic_features[i] is not self.holistic_features[i - 1]: # compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features(image_batch, - self.features[i], + self.holistic_features[i], prefix=prefix, verbose=verbose) # handle scales diff --git a/menpofit/clm/expert/ensemble.py b/menpofit/clm/expert/ensemble.py index fbd663f..1843ee7 100644 --- a/menpofit/clm/expert/ensemble.py +++ b/menpofit/clm/expert/ensemble.py @@ -93,7 +93,7 @@ def _extract_patch(self, image, landmark): # patch: (offsets x ch) x h x w patch = patch.reshape((-1,) + patch.shape[-2:]) # Normalise patch - return self.normalise_callable(patch) + return self.patch_normalisation(patch) def _extract_patches(self, image, shape): r""" @@ -107,7 +107,7 @@ def _extract_patches(self, image, shape): # patches: n_patches x (n_offsets x n_channels) x height x width patches = patches.reshape((patches.shape[0], -1) + patches.shape[-2:]) # Normalise patches - return self.normalise_callable(patches) + return self.patch_normalisation(patches) def predict_response(self, image, shape): r""" @@ -134,7 +134,7 @@ class CorrelationFilterExpertEnsemble(ConvolutionBasedExpertEnsemble): def __init__(self, images, shapes, verbose=False, prefix='', icf_cls=IncrementalCorrelationFilterThinWrapper, patch_size=(17, 17), context_size=(34, 34), - response_covariance=3, normalise_callable=normalize_norm, + response_covariance=3, patch_normalisation=normalize_norm, cosine_mask=True, sample_offsets=None): # TODO: check parameters? # Set parameters @@ -142,7 +142,7 @@ def __init__(self, images, shapes, verbose=False, prefix='', self.patch_size = patch_size self.context_size = context_size self.response_covariance = response_covariance - self.normalise_callable = normalise_callable + self.patch_normalisation = patch_normalisation self.cosine_mask = cosine_mask self.sample_offsets = sample_offsets @@ -168,7 +168,7 @@ def _extract_patch(self, image, landmark): # patch: (offsets x ch) x h x w patch = patch.reshape((-1,) + patch.shape[-2:]) # Normalise patch - patch = self.normalise_callable(patch) + patch = self.patch_normalisation(patch) if self.cosine_mask: # Apply cosine mask if require patch = self._cosine_mask * patch diff --git a/menpofit/fitter.py b/menpofit/fitter.py index 9194747..6d01b7e 100644 --- a/menpofit/fitter.py +++ b/menpofit/fitter.py @@ -141,11 +141,11 @@ def _prepare_image(self, image, initial_shape, gt_shape=None, images = [] for i in range(self.n_scales): # Handle features - if i == 0 or self.features[i] is not self.features[i - 1]: + if i == 0 or self.holistic_features[i] is not self.holistic_features[i - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale - feature_image = self.features[i](image) + feature_image = self.holistic_features[i](image) # Handle scales if self.scales[i] != 1: @@ -246,14 +246,14 @@ def reference_shape(self): return self._model.reference_shape @property - def features(self): + def holistic_features(self): r""" The feature extracted at each pyramidal level during AAM building. Stored in ascending pyramidal order. :type: `list` """ - return self._model.features + return self._model.holistic_features @property def scales(self): diff --git a/menpofit/lk/fitter.py b/menpofit/lk/fitter.py index 2319248..b1a639a 100644 --- a/menpofit/lk/fitter.py +++ b/menpofit/lk/fitter.py @@ -13,16 +13,17 @@ class LucasKanadeFitter(MultiFitter): r""" """ - def __init__(self, template, group=None, features=no_op, + def __init__(self, template, group=None, holistic_features=no_op, transform_cls=DifferentiableAlignmentAffine, diagonal=None, scales=(0.5, 1.0), algorithm_cls=InverseCompositional, residual_cls=SSD): checks.check_diagonal(diagonal) scales = checks.check_scales(scales) - features = checks.check_features(features, len(scales)) + holistic_features = checks.check_features(holistic_features, + len(scales)) - self.features = features + self.holistic_features = holistic_features self.transform_cls = transform_cls self.diagonal = diagonal self.scales = list(scales) diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index 939843a..b2326be 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -44,7 +44,7 @@ def _train(self, images, gt_shapes, current_shapes, increment=False, for k in range(self.n_iterations): # generate regression data features = features_per_image( - images, current_shapes, self.patch_size, self.features, + images, current_shapes, self.patch_size, self.patch_features, prefix='{}(Iteration {}) - '.format(prefix, k), verbose=verbose) @@ -89,8 +89,8 @@ def run(self, image, initial_shape, gt_shape=None, **kwargs): # Cascaded Regression loop for r in self.regressors: # compute regression features - features = features_per_patch(image, current_shape, - self.patch_size, self.features) + features = features_per_patch(image, current_shape, self.patch_size, + self.patch_features) # solve for increments on the shape vector dx = r.predict(features) @@ -127,14 +127,15 @@ def _print_regression_info(self, template_shape, gt_shapes, n_perturbations, class Newton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, + def __init__(self, patch_features=no_op, patch_size=(17, 17), + n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True): super(Newton, self).__init__() self._regressor_cls = partial(IRLRegression, alpha=alpha, bias=bias) self.patch_size = patch_size - self.features = features + self.patch_features = patch_features self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps @@ -144,7 +145,7 @@ def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, class GaussNewton(SupervisedDescentAlgorithm): r""" """ - def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, + def __init__(self, patch_features=no_op, patch_size=(17, 17), n_iterations=3, compute_error=compute_normalise_point_to_point_error, eps=10**-5, alpha=0, bias=True, alpha2=0): super(GaussNewton, self).__init__() @@ -152,7 +153,7 @@ def __init__(self, features=no_op, patch_size=(17, 17), n_iterations=3, self._regressor_cls = partial(IIRLRegression, alpha=alpha, bias=bias, alpha2=alpha2) self.patch_size = patch_size - self.features = features + self.patch_features = patch_features self.n_iterations = n_iterations self._compute_error = compute_error self.eps = eps diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index de7addf..555335d 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -22,7 +22,7 @@ class SupervisedDescentFitter(MultiFitter): """ def __init__(self, images, group=None, bounding_box_group=None, reference_shape=None, sd_algorithm_cls=Newton, - holistic_feature=no_op, patch_features=no_op, + holistic_features=no_op, patch_features=no_op, patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, @@ -32,15 +32,15 @@ def __init__(self, images, group=None, bounding_box_group=None, n_scales = len(scales) scales = checks.check_scales(scales) patch_features = checks.check_features(patch_features, n_scales) - holistic_features = checks.check_features(holistic_feature, n_scales) + holistic_features = checks.check_features(holistic_features, n_scales) patch_size = checks.check_patch_size(patch_size, n_scales) # set parameters self.algorithms = [] self.reference_shape = reference_shape self._sd_algorithm_cls = sd_algorithm_cls - self.features = holistic_features - self._patch_features = patch_features - self._patch_size = patch_size + self.holistic_features = holistic_features + self.patch_features = patch_features + self.patch_size = patch_size self.diagonal = diagonal self.scales = scales self.n_perturbations = n_perturbations @@ -57,8 +57,8 @@ def __init__(self, images, group=None, bounding_box_group=None, def _setup_algorithms(self): for j in range(self.n_scales): self.algorithms.append(self._sd_algorithm_cls( - features=self._patch_features[j], - patch_size=self._patch_size[j], + patch_features=self.patch_features[j], + patch_size=self.patch_size[j], n_iterations=self.n_iterations[j])) def _train(self, images, increment=False, group=None, @@ -180,15 +180,15 @@ def _train_batch(self, image_batch, increment=False, group=None, scale_prefix = None # Handle holistic features - if j == 0 and self.features[j] == no_op: + if j == 0 and self.holistic_features[j] == no_op: # Saves a lot of memory feature_images = image_batch - elif j == 0 or self.features[j] is not self.features[j - 1]: + elif j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]: # Compute features only if this is the first pass through # the loop or the features at this scale are different from # the features at the previous scale feature_images = compute_features(image_batch, - self.features[j], + self.holistic_features[j], prefix=scale_prefix, verbose=verbose) # handle scales @@ -273,9 +273,9 @@ def __str__(self): - Patch feature: {}""" for k, s in enumerate(self.scales): scales_info.append(lvl_str_tmplt.format( - s, self.n_iterations[k], self._patch_size[k], - name_of_callable(self.features[k]), - name_of_callable(self._patch_features[k]))) + s, self.n_iterations[k], self.patch_size[k], + name_of_callable(self.holistic_features[k]), + name_of_callable(self.patch_features[k]))) scales_info = '\n'.join(scales_info) cls_str = r"""Supervised Descent Method @@ -304,7 +304,7 @@ class RegularizedSDM(SupervisedDescentFitter): def __init__(self, images, group=None, bounding_box_group=None, alpha=1.0, reference_shape=None, - holistic_feature=no_op, patch_features=no_op, + holistic_features=no_op, patch_features=no_op, patch_size=(17, 17), diagonal=None, scales=(0.5, 1.0), n_iterations=6, n_perturbations=30, perturb_from_bounding_box=noisy_shape_from_bounding_box, @@ -313,7 +313,7 @@ def __init__(self, images, group=None, bounding_box_group=None, images, group=group, bounding_box_group=bounding_box_group, reference_shape=reference_shape, sd_algorithm_cls=partial(Newton, alpha=alpha), - holistic_feature=holistic_feature, patch_features=patch_features, + holistic_features=holistic_features, patch_features=patch_features, patch_size=patch_size, diagonal=diagonal, scales=scales, n_iterations=n_iterations, n_perturbations=n_perturbations, perturb_from_bounding_box=perturb_from_bounding_box, diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 2fbd2b9..ac881c6 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -961,7 +961,7 @@ def update_info(aam, instance, level, group): aam_mean = lvl_app_mod.mean() n_channels = aam_mean.n_channels tmplt_inst = lvl_app_mod.template_instance - feat = aam.features[level] + feat = aam.holistic_features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( @@ -1355,7 +1355,7 @@ def update_info(atm, instance, level, group): lvl_shape_mod = atm.shape_models[level] tmplt_inst = atm.warped_templates[level] n_channels = tmplt_inst.n_channels - feat = atm.features[level] + feat = atm.holistic_features[level] # Feature string tmp_feat = 'Feature is {} with {} channel{}'.format( From bc57cbd9db6787e5d3ada6e42fed9e8866a98cc5 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Thu, 13 Aug 2015 11:54:24 +0100 Subject: [PATCH 414/423] Rename partsinterface to patchinterface Consistent with other renaming --- menpofit/aam/algorithm/sd.py | 4 ++-- menpofit/aam/fitter.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/menpofit/aam/algorithm/sd.py b/menpofit/aam/algorithm/sd.py index ea073a8..18270d9 100644 --- a/menpofit/aam/algorithm/sd.py +++ b/menpofit/aam/algorithm/sd.py @@ -74,7 +74,7 @@ def algorithm_result(self, image, shape_parameters, # TODO document me! -class SupervisedDescentPartsInterface(SupervisedDescentStandardInterface): +class SupervisedDescentPatchInterface(SupervisedDescentStandardInterface): r""" """ def __init__(self, appearance_model, transform, template, sampling=None, @@ -82,7 +82,7 @@ def __init__(self, appearance_model, transform, template, sampling=None, self.patch_size = patch_size self.patch_normalisation = patch_normalisation - super(SupervisedDescentPartsInterface, self).__init__( + super(SupervisedDescentPatchInterface, self).__init__( appearance_model, transform, template, sampling=sampling) def _build_sampling_mask(self, sampling): diff --git a/menpofit/aam/fitter.py b/menpofit/aam/fitter.py index 3e74017..e7b7d98 100644 --- a/menpofit/aam/fitter.py +++ b/menpofit/aam/fitter.py @@ -14,7 +14,7 @@ LucasKanadePatchInterface, WibergInverseCompositional) from .algorithm.sd import ( SupervisedDescentStandardInterface, SupervisedDescentLinearInterface, - SupervisedDescentPartsInterface, ProjectOutNewton) + SupervisedDescentPatchInterface, ProjectOutNewton) from .result import AAMFitterResult @@ -143,7 +143,7 @@ def _setup_algorithms(self): elif type(self.aam) is PatchAAM: # Build orthogonal point distribution model pdm = OrthoPDM(sm) - interface = SupervisedDescentPartsInterface( + interface = SupervisedDescentPatchInterface( am, pdm, template, sampling=s, patch_size=self.aam.patch_size[j], patch_normalisation=self.aam.patch_normalisation) From 5330aaa663241d1907af2ac2496a2b32df65ffa3 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 18 Aug 2015 13:15:24 +0100 Subject: [PATCH 415/423] Fixes for Jupyter 4.0 Empty strings are no longer allowed as colours - use None instead. --- menpofit/visualize/widgets/options.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/menpofit/visualize/widgets/options.py b/menpofit/visualize/widgets/options.py index ccfda5e..9ace94a 100644 --- a/menpofit/visualize/widgets/options.py +++ b/menpofit/visualize/widgets/options.py @@ -250,8 +250,8 @@ def _check_parameters(self, parameters, bounds): def style(self, box_style=None, border_visible=False, border_color='black', border_style='solid', border_width=1, border_radius=0, padding=0, margin=0, font_family='', font_size=None, font_style='', - font_weight='', slider_width='', slider_handle_colour='', - slider_background_colour='', buttons_style=''): + font_weight='', slider_width='', slider_handle_colour=None, + slider_background_colour=None, buttons_style=''): r""" Function that defines the styling of the widget. @@ -376,8 +376,8 @@ def predefined_style(self, style): border_width=1, border_radius=0, padding='0.2cm', margin='0.3cm', font_family='', font_size=None, font_style='', font_weight='', slider_width='', - slider_handle_colour='', slider_background_colour='', - buttons_style='') + slider_handle_colour=None, + slider_background_colour=None, buttons_style='') elif (style == 'info' or style == 'success' or style == 'danger' or style == 'warning'): self.style(box_style=style, border_visible=True, @@ -387,7 +387,8 @@ def predefined_style(self, style): font_size=None, font_style='', font_weight='', slider_width='', slider_handle_colour=_map_styles_to_hex_colours(style), - slider_background_colour='', buttons_style='primary') + slider_background_colour=None, + buttons_style='primary') else: raise ValueError('style must be minimal or info or success or ' 'danger or warning') @@ -1381,10 +1382,10 @@ def style(self, box_style=None, border_visible=False, border_color='black', self.index_animation.play_options_toggle.button_style = '' _format_box(self.index_animation.loop_interval_box, '', False, 'black', 'solid', 1, 10, '0.1cm', '0.1cm') - self.index_animation.index_wid.slider.slider_color = '' - self.index_animation.index_wid.slider.background_color = '' - self.index_slider.slider_color = '' - self.index_slider.background_color = '' + self.index_animation.index_wid.slider.slider_color = None + self.index_animation.index_wid.slider.background_color = None + self.index_slider.slider_color = None + self.index_slider.background_color = None self.common_figure.button_style = '' else: self.index_animation.play_stop_toggle.button_style = 'success' From 2a573676c921742579aa935a8bb82abcbcc4dae3 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Tue, 18 Aug 2015 13:16:23 +0100 Subject: [PATCH 416/423] Update to v0.4.4 condaci Important to stop appveyor from hangin --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cae360d..3402231 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - PYTHON_VERSION: 3.4 install: -- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py -O condaci.py +- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.4/condaci.py -O condaci.py - python condaci.py setup script: diff --git a/appveyor.yml b/appveyor.yml index 0e70524..46a731e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,7 @@ platform: - x64 init: -- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.2/condaci.py' C:\\condaci.py; echo "Done" +- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.4/condaci.py' C:\\condaci.py; echo "Done" - cmd: python C:\\condaci.py setup install: From 7d0806ae65834e5306c9b707d684ec550d9d6933 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 19 Aug 2015 09:29:59 +0100 Subject: [PATCH 417/423] Update condaci Fixes WIndows builds. --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3402231..bf90d1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - PYTHON_VERSION: 3.4 install: -- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.4/condaci.py -O condaci.py +- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.5/condaci.py -O condaci.py - python condaci.py setup script: diff --git a/appveyor.yml b/appveyor.yml index 46a731e..c08ac8c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,7 +17,7 @@ platform: - x64 init: -- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.4/condaci.py' C:\\condaci.py; echo "Done" +- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.5/condaci.py' C:\\condaci.py; echo "Done" - cmd: python C:\\condaci.py setup install: From cc3509d66f77b983934917b71173f38f897ca7db Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 19 Aug 2015 14:50:45 +0100 Subject: [PATCH 418/423] Upgrade to latest condaci --- .travis.yml | 2 +- appveyor.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf90d1b..15bca0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - PYTHON_VERSION: 3.4 install: -- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.5/condaci.py -O condaci.py +- wget https://raw.githubusercontent.com/menpo/condaci/v0.4.6/condaci.py -O condaci.py - python condaci.py setup script: diff --git a/appveyor.yml b/appveyor.yml index c08ac8c..14fa231 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -17,11 +17,14 @@ platform: - x64 init: -- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.5/condaci.py' C:\\condaci.py; echo "Done" +- ps: Start-FileDownload 'https://raw.githubusercontent.com/menpo/condaci/v0.4.6/condaci.py' C:\\condaci.py; echo "Done" - cmd: python C:\\condaci.py setup install: -- cmd: C:\\Miniconda\\python C:\\condaci.py build ./conda +- cmd: set tmp_cmd=python C:\\condaci.py miniconda_dir +- cmd: "FOR /F %%i IN (' %tmp_cmd% ') DO SET APPV_MINICONDA_DIR=%%i" +- cmd: set APPV_MINICONDA_EXE="%APPV_MINICONDA_DIR%\python.exe" +- cmd: "%APPV_MINICONDA_EXE% C:\\condaci.py build conda" notifications: - provider: Slack From f86830c6698090b68e3d4bd9df76f7fc2e0c7ef9 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 21 Aug 2015 11:18:14 +0100 Subject: [PATCH 419/423] More fixes for Jupyter 4.0 Removal of *Widget classes --- menpofit/visualize/widgets/base.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index ac881c6..1f5cb0d 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -326,11 +326,11 @@ def plot_variance(name): mode_dict = OrderedDict() mode_dict['Deformation'] = 1 mode_dict['Vectors'] = 2 - mode_wid = ipywidgets.RadioButtonsWidget(options=mode_dict, - description='Mode:', value=1) + mode_wid = ipywidgets.RadioButtons(options=mode_dict, + description='Mode:', value=1) mode_wid.on_trait_change(render_function, 'value') - mean_wid = ipywidgets.CheckboxWidget(value=False, - description='Render mean shape') + mean_wid = ipywidgets.Checkbox(value=False, + description='Render mean shape') mean_wid.on_trait_change(render_function, 'value') # Function that controls mean shape checkbox visibility @@ -346,7 +346,7 @@ def mean_visible(name, value): mode=mode, params_bounds=parameters_bounds, params_step=0.1, plot_variance_visible=True, plot_variance_function=plot_variance, style=model_parameters_style) - axes_mode_wid = ipywidgets.RadioButtonsWidget( + axes_mode_wid = ipywidgets.RadioButtons( options={'Image': 1, 'Point cloud': 2}, description='Axes mode:', value=2) axes_mode_wid.on_trait_change(render_function, 'value') @@ -381,7 +381,7 @@ def update_widgets(name, value): radio_str["Level {} (high)".format(l)] = l else: radio_str["Level {}".format(l)] = l - level_wid = ipywidgets.RadioButtonsWidget( + level_wid = ipywidgets.RadioButtons( options=radio_str, description='Pyramid:', value=0) level_wid.on_trait_change(update_widgets, 'value') level_wid.on_trait_change(render_function, 'value') @@ -718,7 +718,7 @@ def update_widgets(name, value): radio_str["Level {} (high)".format(l)] = l else: radio_str["Level {}".format(l)] = l - level_wid = ipywidgets.RadioButtonsWidget( + level_wid = ipywidgets.RadioButtons( options=radio_str, description='Pyramid:', value=0) level_wid.on_trait_change(update_widgets, 'value') level_wid.on_trait_change(render_function, 'value') @@ -1128,7 +1128,7 @@ def update_widgets(name, value): radio_str["Level {} (high)".format(l)] = l else: radio_str["Level {}".format(l)] = l - level_wid = ipywidgets.RadioButtonsWidget( + level_wid = ipywidgets.RadioButtons( options=radio_str, description='Pyramid:', value=0) level_wid.on_trait_change(update_widgets, 'value') level_wid.on_trait_change(render_function, 'value') @@ -1476,7 +1476,7 @@ def update_widgets(name, value): radio_str["Level {} (high)".format(l)] = l else: radio_str["Level {}".format(l)] = l - level_wid = ipywidgets.RadioButtonsWidget( + level_wid = ipywidgets.RadioButtons( options=radio_str, description='Pyramid:', value=0) level_wid.on_trait_change(update_widgets, 'value') level_wid.on_trait_change(render_function, 'value') From 1f406b58ba52e426d2d8e021d0381af996e44dc2 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Fri, 21 Aug 2015 13:04:09 +0100 Subject: [PATCH 420/423] Move len(scales) to AFTER it has been checked --- menpofit/aam/base.py | 2 +- menpofit/sdm/fitter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index 83e634f..ef75f7d 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -121,8 +121,8 @@ def __init__(self, images, group=None, verbose=False, reference_shape=None, max_appearance_components=None, batch_size=None): checks.check_diagonal(diagonal) - n_scales = len(scales) scales = checks.check_scales(scales) + n_scales = len(scales) holistic_features = checks.check_features(holistic_features, n_scales) max_shape_components = checks.check_max_components( max_shape_components, n_scales, 'max_shape_components') diff --git a/menpofit/sdm/fitter.py b/menpofit/sdm/fitter.py index 555335d..cd81ca7 100644 --- a/menpofit/sdm/fitter.py +++ b/menpofit/sdm/fitter.py @@ -29,8 +29,8 @@ def __init__(self, images, group=None, bounding_box_group=None, batch_size=None, verbose=False): # check parameters checks.check_diagonal(diagonal) - n_scales = len(scales) scales = checks.check_scales(scales) + n_scales = len(scales) patch_features = checks.check_features(patch_features, n_scales) holistic_features = checks.check_features(holistic_features, n_scales) patch_size = checks.check_patch_size(patch_size, n_scales) From 4b127f7718fd3c02f55abcd8de55a84987862287 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 24 Aug 2015 11:58:12 +0100 Subject: [PATCH 421/423] ipywidgets rather than IPython (jupyter 4) --- menpofit/visualize/widgets/base.py | 2 +- menpofit/visualize/widgets/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/menpofit/visualize/widgets/base.py b/menpofit/visualize/widgets/base.py index 1f5cb0d..9bd0170 100644 --- a/menpofit/visualize/widgets/base.py +++ b/menpofit/visualize/widgets/base.py @@ -3,7 +3,7 @@ import matplotlib.pyplot as plt from matplotlib import collections as mc -import IPython.html.widgets as ipywidgets +import ipywidgets import IPython.display as ipydisplay from menpo.visualize.widgets import (RendererOptionsWidget, diff --git a/menpofit/visualize/widgets/options.py b/menpofit/visualize/widgets/options.py index 9ace94a..1e71edb 100644 --- a/menpofit/visualize/widgets/options.py +++ b/menpofit/visualize/widgets/options.py @@ -1,6 +1,6 @@ from collections import OrderedDict -import IPython.html.widgets as ipywidgets +import ipywidgets from menpo.visualize.widgets.tools import (_format_box, _format_font, _map_styles_to_hex_colours) From c6fea14fc110eddf27749cd1be980219d92abe87 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Mon, 24 Aug 2015 13:12:06 +0100 Subject: [PATCH 422/423] Fix passing single scale --- menpofit/aam/base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/menpofit/aam/base.py b/menpofit/aam/base.py index ef75f7d..47cee1e 100644 --- a/menpofit/aam/base.py +++ b/menpofit/aam/base.py @@ -588,7 +588,8 @@ def __init__(self, images, group=None, verbose=False, holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_size = checks.check_patch_size(patch_size, len(scales)) + n_scales = len(checks.check_scales(scales)) + self.patch_size = checks.check_patch_size(patch_size, n_scales) super(MaskedAAM, self).__init__( images, group=group, verbose=verbose, @@ -773,7 +774,8 @@ def __init__(self, images, group=None, verbose=False, holistic_features=no_op, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_size = checks.check_patch_size(patch_size, len(scales)) + n_scales = len(checks.check_scales(scales)) + self.patch_size = checks.check_patch_size(patch_size, n_scales) super(LinearMaskedAAM, self).__init__( images, group=group, verbose=verbose, @@ -877,7 +879,8 @@ def __init__(self, images, group=None, verbose=False, diagonal=None, scales=(0.5, 1.0), patch_size=(17, 17), max_shape_components=None, max_appearance_components=None, batch_size=None): - self.patch_size = checks.check_patch_size(patch_size, len(scales)) + n_scales = len(checks.check_scales(scales)) + self.patch_size = checks.check_patch_size(patch_size, n_scales) self.patch_normalisation = patch_normalisation super(PatchAAM, self).__init__( From 1a7a2f4ca999010a43dca2de76f7b70528388e16 Mon Sep 17 00:00:00 2001 From: Patrick Snape Date: Wed, 26 Aug 2015 15:04:14 +0100 Subject: [PATCH 423/423] Change asarray to stacking This helps when you have different number of features per patch, otherwise, you get an object array back. (SDM) --- menpofit/base.py | 1 - menpofit/sdm/algorithm.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/menpofit/base.py b/menpofit/base.py index 638aca1..aa32cb3 100644 --- a/menpofit/base.py +++ b/menpofit/base.py @@ -2,7 +2,6 @@ from functools import partial import itertools import numpy as np -from menpo.visualize import progress_bar_str, print_dynamic def name_of_callable(c): diff --git a/menpofit/sdm/algorithm.py b/menpofit/sdm/algorithm.py index b2326be..37cab04 100644 --- a/menpofit/sdm/algorithm.py +++ b/menpofit/sdm/algorithm.py @@ -167,7 +167,7 @@ def features_per_patch(image, shape, patch_size, features_callable): as_single_array=True) patch_features = [features_callable(p[0]).ravel() for p in patches] - return np.asarray(patch_features).ravel() + return np.hstack(patch_features) # TODO: document me! @@ -178,7 +178,7 @@ def features_per_shape(image, shapes, patch_size, features_callable): features_callable) for s in shapes] - return np.asarray(patch_features) + return np.vstack(patch_features) # TODO: document me! @@ -193,8 +193,7 @@ def features_per_image(images, shapes, patch_size, features_callable, patch_features = [features_per_shape(i, shapes[j], patch_size, features_callable) for j, i in enumerate(wrap(images))] - patch_features = np.asarray(patch_features) - return patch_features.reshape((-1, patch_features.shape[-1])) + return np.vstack(patch_features) def compute_delta_x(gt_shape, current_shapes):