Skip to content

Commit

Permalink
added a threshold argument for early stopping based on convergence
Browse files Browse the repository at this point in the history
  • Loading branch information
@mpwillia
mpwillia committed Sep 17, 2017
1 parent 085fcf3 commit 41ec679
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 51 deletions.
30 changes: 17 additions & 13 deletions demo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from PIL import Image
import time
import argparse
import os
import pyflow

parser = argparse.ArgumentParser(
Expand All @@ -17,31 +18,34 @@
help='Visualize (i.e. save) output of flow.')
args = parser.parse_args()

im1 = np.array(Image.open('examples/car1.jpg'))
im2 = np.array(Image.open('examples/car2.jpg'))
examples_dir = "./examples"
im1 = np.array(Image.open(os.path.join(examples_dir, 'img1.jpg')))
im2 = np.array(Image.open(os.path.join(examples_dir, 'img2.jpg')))
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.

# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
alpha = 0.012 # default 0.012
ratio = 0.75 # default 0.75
minWidth = 20 # default 20
nOuterFPIterations = 7 # default 7
nInnerFPIterations = 1 # default 1
nSORIterations = 30 # default 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
threshold = 0.000005

s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType, verbose = True)
nSORIterations, colType, verbose = True, threshold = threshold)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' % (
e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
np.save('examples/outFlow.npy', flow)
np.save(os.path.join(examples_dir, 'outFlow.npy'), flow)

if args.viz:
always_viz = True
if args.viz or always_viz:
import cv2
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
Expand All @@ -50,5 +54,5 @@
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imwrite('examples/outFlow_new.png', rgb)
cv2.imwrite('examples/car2Warped_new.jpg', im2W[:, :, ::-1] * 255)
cv2.imwrite(os.path.join(examples_dir, 'outFlow_new.png'), rgb)
cv2.imwrite(os.path.join(examples_dir, 'img2Warped_new.jpg'), im2W[:, :, ::-1] * 255)
8 changes: 5 additions & 3 deletions pyflow.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,15 @@ cdef extern from "src/Coarse2FineFlowWrapper.h":
double alpha, double ratio, int minWidth,
int nOuterFPIterations, int nInnerFPIterations,
int nSORIterations, int colType,
int h, int w, int c, bool verbose);
int h, int w, int c, bool verbose, double threshold);

def coarse2fine_flow(np.ndarray[double, ndim=3, mode="c"] Im1 not None,
np.ndarray[double, ndim=3, mode="c"] Im2 not None,
double alpha=1, double ratio=0.5, int minWidth=40,
int nOuterFPIterations=3, int nInnerFPIterations=1,
int nSORIterations=20, int colType=0,
bool verbose = False):
bool verbose = False,
double threshold = 0.0):
"""
Input Format:
double * vx, double * vy, double * warpI2,
Expand Down Expand Up @@ -50,5 +51,6 @@ def coarse2fine_flow(np.ndarray[double, ndim=3, mode="c"] Im1 not None,
alpha, ratio, minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations, colType,
h, w, c,
verbose)
verbose,
threshold)
return vx, vy, warpI2
4 changes: 2 additions & 2 deletions src/Coarse2FineFlowWrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ void Coarse2FineFlowWrapper(double * vx, double * vy, double * warpI2,
int nOuterFPIterations, int nInnerFPIterations,
int nSORIterations, int colType,
int h, int w, int c,
bool verbose) {
bool verbose, double threshold) {
DImage ImFormatted1, ImFormatted2;
DImage vxFormatted, vyFormatted, warpI2Formatted;

Expand All @@ -33,7 +33,7 @@ void Coarse2FineFlowWrapper(double * vx, double * vy, double * warpI2,
alpha, ratio, minWidth,
nOuterFPIterations, nInnerFPIterations,
nSORIterations,
verbose);
verbose, threshold);

// copy formatted output to a contiguous memory to be returned
memcpy(vx, vxFormatted.pData, h * w * sizeof(double));
Expand Down
2 changes: 1 addition & 1 deletion src/Coarse2FineFlowWrapper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ extern void Coarse2FineFlowWrapper(double * vx, double * vy, double * warpI2,
int nOuterFPIterations, int nInnerFPIterations,
int nSORIterations, int colType,
int h, int w, int c,
bool verbose);
bool verbose, double threshold);
87 changes: 57 additions & 30 deletions src/OpticalFlow.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "ImageProcessing.h"
#include "GaussianPyramid.h"
#include <cstdlib>
#include <cmath>
#include <iostream>

using namespace std;
Expand Down Expand Up @@ -195,7 +196,7 @@ void OpticalFlow::genInImageMask(DImage &mask, const DImage &flow,int interval)
//--------------------------------------------------------------------------------------------------------
void OpticalFlow::SmoothFlowSOR(const DImage &Im1, const DImage &Im2, DImage &warpIm2, DImage &u, DImage &v,
double alpha, int nOuterFPIterations, int nInnerFPIterations, int nSORIterations,
bool verbose)
bool verbose, double threshold)
{
DImage mask,imdx,imdy,imdt;
int imWidth,imHeight,nChannels,nPixels;
Expand Down Expand Up @@ -380,6 +381,8 @@ void OpticalFlow::SmoothFlowSOR(const DImage &Im1, const DImage &Im2, DImage &wa

du.reset();
dv.reset();

bool use_converge_est = (threshold > 0.0);

for(int k = 0; k<nSORIterations; k++) {
double total_du_change = 0;
Expand Down Expand Up @@ -423,36 +426,60 @@ void OpticalFlow::SmoothFlowSOR(const DImage &Im1, const DImage &Im2, DImage &wa
sigma1 *= -alpha;
sigma2 *= -alpha;
coeff *= alpha;
// compute du
sigma1 += imdxy.data()[offset]*dv.data()[offset];

double new_du = (1-omega)*du.data()[offset] + omega/(imdx2.data()[offset] + alpha*0.05 + coeff)*(imdtdx.data()[offset] - sigma1);
double du_change = new_du - du.data()[offset];
if(du_change < 0)
du_change *= -1;
total_du_change += du_change;
du.data()[offset] = new_du;
// compute dv
sigma2 += imdxy.data()[offset]*du.data()[offset];

double new_dv = (1-omega)*dv.data()[offset] + omega/(imdy2.data()[offset] + alpha*0.05 + coeff)*(imdtdy.data()[offset] - sigma2);
double dv_change = new_dv - dv.data()[offset];
if(dv_change < 0)
dv_change *= -1;
total_dv_change += dv_change;
dv.data()[offset] = new_dv;


if(use_converge_est) {

// compute du
sigma1 += imdxy.data()[offset]*dv.data()[offset];

double new_du = (1-omega)*du.data()[offset] + omega/(imdx2.data()[offset] + alpha*0.05 + coeff)*(imdtdx.data()[offset] - sigma1);
total_du_change += abs(new_du - du.data()[offset]);

/*
double du_change = new_du - du.data()[offset];
if(du_change < 0)
du_change *= -1;
total_du_change += du_change;
*/

du.data()[offset] = new_du;
// compute dv
sigma2 += imdxy.data()[offset]*du.data()[offset];

double new_dv = (1-omega)*dv.data()[offset] + omega/(imdy2.data()[offset] + alpha*0.05 + coeff)*(imdtdy.data()[offset] - sigma2);
total_dv_change += abs(new_dv - dv.data()[offset]);
/*
double dv_change = new_dv - dv.data()[offset];
if(dv_change < 0)
dv_change *= -1;
total_dv_change += dv_change;
*/

dv.data()[offset] = new_dv;
} else {

// compute du
sigma1 += imdxy.data()[offset]*dv.data()[offset];
du.data()[offset] = (1-omega)*du.data()[offset] + omega/(imdx2.data()[offset] + alpha*0.05 + coeff)*(imdtdx.data()[offset] - sigma1);

// compute dv
sigma2 += imdxy.data()[offset]*du.data()[offset];
dv.data()[offset] = (1-omega)*dv.data()[offset] + omega/(imdy2.data()[offset] + alpha*0.05 + coeff)*(imdtdy.data()[offset] - sigma2);

}
} //End of image width loop
} // End of image height loop

// Compute the average change in the derivitives for each pixel
total_du_change /= imHeight * imWidth;
total_dv_change /= imHeight * imWidth;

// Apply basic convergence thresholding
double threshold = 0.005;
if(total_du_change < threshold && total_dv_change < threshold) {
break;
if(use_converge_est) {
// Compute the average change in the derivitives for each pixel
total_du_change /= imHeight * imWidth;
total_dv_change /= imHeight * imWidth;

// Apply basic convergence thresholding
if(total_du_change < threshold && total_dv_change < threshold) {
//cout << "Breaking early" << endl;
break;
}
}
} // End of SOR Iterations Loops
} // End of Inner FP Iterations Loop
Expand Down Expand Up @@ -973,7 +1000,7 @@ void OpticalFlow::testLaplacian(int dim)
// function to perfomr coarse to fine optical flow estimation
//--------------------------------------------------------------------------------------
void OpticalFlow::Coarse2FineFlow(DImage &vx, DImage &vy, DImage &warpI2,const DImage &Im1, const DImage &Im2, double alpha, double ratio, int minWidth,
int nOuterFPIterations, int nInnerFPIterations, int nCGIterations, bool verbose)
int nOuterFPIterations, int nInnerFPIterations, int nCGIterations, bool verbose, double threshold)
{
// first build the pyramid of the two images
GaussianPyramid GPyramid1;
Expand Down Expand Up @@ -1034,7 +1061,7 @@ void OpticalFlow::Coarse2FineFlow(DImage &vx, DImage &vy, DImage &warpI2,const D
//SmoothFlowPDE(Image1,Image2,WarpImage2,vx,vy,alpha*pow((1/ratio),k),nOuterFPIterations,nInnerFPIterations,nCGIterations,GMPara);

//SmoothFlowPDE(Image1,Image2,WarpImage2,vx,vy,alpha,nOuterFPIterations,nInnerFPIterations,nCGIterations);
SmoothFlowSOR(Image1,Image2,WarpImage2,vx,vy,alpha,nOuterFPIterations+k,nInnerFPIterations,nCGIterations+k*3, verbose);
SmoothFlowSOR(Image1,Image2,WarpImage2,vx,vy,alpha,nOuterFPIterations+k,nInnerFPIterations,nCGIterations+k*3, verbose, threshold);

//GMPara.display();
if(verbose)
Expand Down
4 changes: 2 additions & 2 deletions src/OpticalFlow.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class OpticalFlow
double alpha,int nOuterFPIterations,int nInnerFPIterations,int nCGIterations, bool verbose = false);

static void SmoothFlowSOR(const DImage& Im1,const DImage& Im2, DImage& warpIm2, DImage& vx, DImage& vy,
double alpha,int nOuterFPIterations,int nInnerFPIterations,int nSORIterations, bool verbose = false);
double alpha,int nOuterFPIterations,int nInnerFPIterations,int nSORIterations, bool verbose = false, double threshold = 0.0);

static void estGaussianMixture(const DImage& Im1,const DImage& Im2,GaussianMixture& para,double prior = 0.9);
static void estLaplacianNoise(const DImage& Im1,const DImage& Im2,Vector<double>& para, bool verbose = false);
Expand All @@ -46,7 +46,7 @@ class OpticalFlow

// function of coarse to fine optical flow
static void Coarse2FineFlow(DImage& vx,DImage& vy,DImage &warpI2,const DImage& Im1,const DImage& Im2,double alpha,double ratio,int minWidth,
int nOuterFPIterations,int nInnerFPIterations,int nCGIterations, bool verbose);
int nOuterFPIterations,int nInnerFPIterations,int nCGIterations, bool verbose, double threshold = 0.0);

static void Coarse2FineFlowLevel(DImage& vx,DImage& vy,DImage &warpI2,const DImage& Im1,const DImage& Im2,double alpha,double ratio,int nLevels,
int nOuterFPIterations,int nInnerFPIterations,int nCGIterations, bool verbose);
Expand Down

0 comments on commit 41ec679

Please sign in to comment.