DenseNet feature pyramid computation

DENSENET_MERGE_TODO

@@ -0,0 +1,15 @@
+
+list of the issues blocking the merge of the DenseNet feature branch:
+
+- critical
+-- replacement of GPL'd code (including removal from history)
+-- update build process / Makefile to match current practice 
+-- tests
+-- trivial: remove DenseNet README.md header
+-- trivial: remove this todo file
+
+- unclear neccessity, semantics, and/or priority
+-- general cleanup of commit sequence (probably mostly squashing)
+-- input interface changes (i.e. jpeg filname as input -> ?)
+-- output interface changes (?, but probably something: image support size, multiple layer output, alignment, etc.)
+-- if still reading image files after any iface changes and removal of GPL code, use XX instead of YY

Makefile

@@ -44,6 +44,10 @@ NONGEN_CXX_SRCS := $(shell find \
 	-name "*.cpp" -or -name "*.hpp" -or -name "*.cu" -or -name "*.cuh")
 LINT_REPORT := $(BUILD_DIR)/cpp_lint.log
 FAILED_LINT_REPORT := $(BUILD_DIR)/cpp_lint.error_log
+# STITCHPYRAMID is for stitching multiresolution feature pyramids. (exclude test files)
+STITCHPYRAMID_SRC := $(shell find src/stitch_pyramid ! -name "test_*.cpp" -name "*.cpp")
+STITCHPYRAMID_HDRS := $(shell find src/stitch_pyramid -name "*.h")
+STITCHPYRAMID_SO := src/stitch_pyramid/libPyramidStitcher.so
 # PY$(PROJECT)_SRC is the python wrapper for $(PROJECT)
 PY$(PROJECT)_SRC := python/$(PROJECT)/py$(PROJECT).cpp
 PY$(PROJECT)_SO := python/$(PROJECT)/py$(PROJECT).so
@@ -96,13 +100,18 @@ LIBRARIES := cudart cublas curand \
 PYTHON_LIBRARIES := boost_python python2.7
 WARNINGS := -Wall
 
-COMMON_FLAGS := -DNDEBUG -O2 $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
+COMMON_FLAGS := -O2 $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
+#COMMON_FLAGS := -O3 -DENABLE_ALLOC_TRACE $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
+#COMMON_FLAGS := -DNDEBUG -O2 $(foreach includedir,$(INCLUDE_DIRS),-I$(includedir))
 CXXFLAGS += -pthread -fPIC $(COMMON_FLAGS)
 NVCCFLAGS := -ccbin=$(CXX) -Xcompiler -fPIC $(COMMON_FLAGS)
 LDFLAGS += $(foreach librarydir,$(LIBRARY_DIRS),-L$(librarydir)) \
 		$(foreach library,$(LIBRARIES),-l$(library))
 PYTHON_LDFLAGS := $(LDFLAGS) $(foreach library,$(PYTHON_LIBRARIES),-l$(library))
 
+#SHARED_LDFLAGS := -shared -Wl,--no-undefined # for gcc, this is probably a more sane default: any undefined syms will give a link error
+SHARED_LDFLAGS := -shared
+
 
 ##############################
 # Define build targets
@@ -139,22 +148,42 @@ tools: init $(TOOL_BINS)
 
 examples: init $(EXAMPLE_BINS)
 
+
+stitch: $(STITCHPYRAMID_SO)
+
+.PHONY : stitch
+
+$(STITCHPYRAMID_SO): $(STITCHPYRAMID_HDRS) $(STITCHPYRAMID_SRC) 
+	$(CXX) $(SHARED_LDFLAGS) -o $(STITCHPYRAMID_SO) $(STITCHPYRAMID_SRC) $(CXXFLAGS) -ljpeg
+
 py$(PROJECT): py
 
-py: init $(STATIC_NAME) $(PY$(PROJECT)_SRC) $(PROTO_GEN_PY)
-	$(CXX) -shared -o $(PY$(PROJECT)_SO) $(PY$(PROJECT)_SRC) \
+py: init $(STATIC_NAME) $(PY$(PROJECT)_SRC) $(PROTO_GEN_PY) $(STITCHPYRAMID_SO) 
+	$(CXX) $(SHARED_LDFLAGS) -o $(PY$(PROJECT)_SO) $(PY$(PROJECT)_SRC) -L./src/stitch_pyramid -lPyramidStitcher -I./src/stitch_pyramid \
 		$(STATIC_NAME) $(CXXFLAGS) $(PYTHON_LDFLAGS)
 	@echo
 
 mat$(PROJECT): mat
 
-mat: init $(STATIC_NAME) $(MAT$(PROJECT)_SRC)
-	$(MATLAB_DIR)/bin/mex $(MAT$(PROJECT)_SRC) $(STATIC_NAME) \
-		CXXFLAGS="\$$CXXFLAGS $(CXXFLAGS) $(WARNINGS)" \
-		CXXLIBS="\$$CXXLIBS $(LDFLAGS)" \
+mat: init $(STATIC_NAME) $(MAT$(PROJECT)_SRC) $(STITCHPYRAMID_SO)
+	$(MATLAB_DIR)/bin/mex -g $(MAT$(PROJECT)_SRC) $(STATIC_NAME) \
+		CXXFLAGS="\$$CXXFLAGS $(CXXFLAGS) $(WARNINGS)" -I./python/caffe \
+		CXXLIBS="\$$CXXLIBS $(LDFLAGS)" -L./src/stitch_pyramid -lPyramidStitcher \
 		-o $(MAT$(PROJECT)_SO)
 	@echo
 
+mkoctfile_mat$(PROJECT): mkoctfile_mat
+
+#	CXXFLAGS="-Wall -fpic -O2" mkoctfile --mex matlab/caffe/matcaffe.cpp libcaffe.a -pthread -I/usr/local/include -I/usr/include/python2.7 -I/usr/local/lib/python2.7/dist-packages/numpy/core/include -I./src -I./include -I/usr/local/cuda/include -I/opt/intel/mkl/include -Wall -L/usr/lib -L/usr/local/lib -L/usr/local/cuda/lib64 -L/usr/local/cuda/lib -L/opt/intel/mkl/lib -L/opt/intel/mkl/lib/intel64 -lcudart -lcublas -lcurand -lprotobuf -lopencv_core -lopencv_highgui -lglog -lmkl_rt -lmkl_intel_thread -lleveldb -lsnappy -lpthread -lboost_system -lopencv_imgproc -L/home/moskewcz/git_work/caffe/python/caffe/stitch_pyramid -lPyramidStitcher -I./python/caffe -o matlab/caffe/caffe
+
+
+mkoctfile_mat: init $(STATIC_NAME) $(MAT$(PROJECT)_SRC) $(STITCHPYRAMID_SO)
+	CXXFLAGS="$$CXXFLAGS $(CXXFLAGS)" \
+	DL_LDFLAGS="$$DL_LDFLAGS $(SHARED_LDFLAGS)" \
+	LFLAGS="$$LFLAGS $(LDFLAGS) -L./src/stitch_pyramid -lPyramidStitcher" \
+	mkoctfile --mex -g -v $(MAT$(PROJECT)_SRC) $(STATIC_NAME) -o $(MAT$(PROJECT)_SO)
+	@echo
+
 $(NAME): init $(PROTO_OBJS) $(OBJS)
 	$(CXX) -shared -o $(NAME) $(OBJS) $(CXXFLAGS) $(LDFLAGS) $(WARNINGS)
 	@echo

README.md

@@ -1,3 +1,75 @@
+# -- WARNING: THIS IS AN FORK/FEATURE BRANCH OF [CAFFE](http://github.com/BVLC/caffe) (PR PENDING). --
+## DenseNet
+
+[DenseNet: Implementing Efficient ConvNet Descriptor Pyramids](http://arxiv.org/abs/1404.1869)<br>
+Forrest Iandola, Matt Moskewicz, Sergey Karayev, Ross Girshick, Trevor Darrell, and Kurt Keutzer.<br>
+Arxiv technical report, April 2014.
+
+<b>Licensing</b><br>
+Except where noted in individual files, all new code files / changes in this branch are:
+Copyright (c) 2013 Matthew Moskewicz and Forrest Iandola
+and are BSD 2-Clause licensed with the same as the original source (see [LICENSE](LICENSE)).
+
+The two example images are taken from the PASCAL vision benchmark set.
+
+<b>DenseNet APIs in Matlab and Python</b><br>
+The DenseNet API is fairly similar to the popular `featpyramid.m` HOG extraction API from the [voc-release5 Deformable Parts Model code](https://github.com/rbgirshick/voc-dpm/blob/master/features/featpyramid.m). Our primary API function is called `convnet_featpyramid()`.
+
+<b>Running DenseNet in Matlab</b><br>
+`caffe/matlab/caffe/featpyramid_matcaffe_demo.m` is a good example to start with. Or, we can walk through it together here:
+
+```matlab
+    %Caffe setup:
+    model_def_file = 'CAFFE_ROOT/python/caffe/imagenet/imagenet_rcnn_batch_1_input_1100x1100_output_conv5.prototxt'
+    % NOTE: you'll have to get the pre-trained ILSVRC network
+    model_file = 'path/to/alexnet_train_iter_470000';
+    caffe('init', model_def_file, model_file);
+    caffe('set_mode_gpu') %CPU mode works too
+    caffe('set_phase_test')
+
+    %using DenseNet: 
+     image = 'myImage.jpg' %must be JPEG
+     pyra = convnet_featpyramid(image)
+```
+
+<b>Running DenseNet in Matlab (advanced users)</b><br>
+```matlab
+    % (you need to do Caffe setup first, as shown in above example)
+    image = 'myImage.jpg'
+
+    %optional parameters (code will still run with incomplete or nonexistant pyra_params):
+    pyra_params.interval = 5; %octaves per pyramid scale
+    pyra_params.img_padding = 16 %padding around image (in pixels)
+    pyra_params.feat_minWidth = 6; %select smallest scale in pyramid (in output feature dimensions)
+    pyra_params.feat_minHeight = 6; %in output feature dimensions
+    pyra = convnet_featpyramid(image, pyra_params)
+
+    %taking a look at the output pyramid:
+          scales: [40x1 double] %resolution of each pyramid scale
+            feat: {40x1 cell} %descriptors (one cell array per scale)
+         imwidth: 353 %input image size in pixels
+        imheight: 500
+       feat_padx: 1 %border padding around descriptors (img_padding/sbin)
+       feat_pady: 1
+            sbin: 16 %approx. downsampling factor from pixels to descriptors
+            padx: 1 %extra copy of feat_pad{x,y}. silly...should remove?
+            pady: 1
+      num_levels: 40 %num scales in pyramid
+    valid_levels: [40x1 logical]
+```
+
+<b>Running DenseNet in Python</b><br>
+The Python API is similar to the Matlab API described above. 
+`caffe/python/caffe/featpyramid_demo.py` is a good starting point for using DenseNet in Python.
+
+
+Other notes:
+- As with many other operations in Caffe, you'll need to download a pretrained Alexnet CNN prior to running our DenseNet demo.
+- For most of our default examples, we use the 'Alexnet' network and output descriptors from the conv5 layer. You can adjust these decisions by editing the 'prototxt' files used at setup time.
+
+
+## Original Caffe README.md follows
+
 [Caffe: Convolutional Architecture for Fast Feature Extraction](http://caffe.berkeleyvision.org)
 
 Created by [Yangqing Jia](http://daggerfs.com), UC Berkeley EECS department.

include/caffe/featpyra_common.hpp

@@ -0,0 +1,79 @@
+#include "caffe/imagenet_mean.hpp"
+#include "boost/shared_ptr.hpp"
+#include "caffe/caffe.hpp"
+#include <vector>
+
+namespace caffe {
+
+  using namespace std;
+  using boost::shared_ptr;
+
+//switch RGB to BGR indexing (for Caffe convention)
+  inline int get_BGR(int channel_RGB) { assert( channel_RGB < 3 ); return 2 - channel_RGB; }
+
+  struct densenet_params_t {
+    uint32_t interval; // # scales per octave
+    uint32_t img_padding; // in image pixels (at scale=1.0). could later allow per-dim img_padx/img_pady as alternative.
+    uint32_t feat_minWidth; //smallest desired output scale, in terms of descriptor dimensions
+    uint32_t feat_minHeight;
+    densenet_params_t( void ) { // default values
+      interval = 10;
+      img_padding = 16;
+      feat_minWidth = 1;
+      feat_minHeight = 1;
+    }
+  };
+
+  typedef vector< float > vect_float;
+  typedef shared_ptr< vect_float > p_vect_float;
+  typedef vector< p_vect_float > vect_p_vect_float;
+
+  typedef vector< uint32_t > vect_uint32_t;
+  typedef shared_ptr< vect_uint32_t > p_vect_uint32_t;
+
+  // a sketch of a possible shared output type for python/matlab interfaces. missing dims for feats.
+  struct densenet_output_t {
+    p_vect_float imwidth; // not including padding
+    p_vect_float imheight;
+    p_vect_uint32_t feat_padx;
+    p_vect_uint32_t feat_pady;
+    p_vect_float scales;
+    vect_p_vect_float feats;
+    uint32_t nb_planes;
+  };
+
+  static void raw_do_forward( shared_ptr<Net<float> > net_, vect_p_vect_float const & bottom ) {
+    vector<Blob<float>*>& input_blobs = net_->input_blobs();
+    CHECK_EQ(bottom.size(), input_blobs.size());
+    for (unsigned int i = 0; i < input_blobs.size(); ++i) {
+      assert( bottom[i]->size() == uint32_t(input_blobs[i]->count()) );
+      const float* const data_ptr = &bottom[i]->front();
+      switch (Caffe::mode()) {
+      case Caffe::CPU:
+	memcpy(input_blobs[i]->mutable_cpu_data(), data_ptr,
+	       sizeof(float) * input_blobs[i]->count());
+	break;
+      case Caffe::GPU:
+	cudaMemcpy(input_blobs[i]->mutable_gpu_data(), data_ptr,
+		   sizeof(float) * input_blobs[i]->count(), cudaMemcpyHostToDevice);
+	break;
+      default:
+	LOG(FATAL) << "Unknown Caffe mode.";
+      }  // switch (Caffe::mode())
+    }
+    //const vector<Blob<float>*>& output_blobs = net_->ForwardPrefilled();
+    net_->ForwardPrefilled();
+  }
+
+  // get sbin as product of strides. note that a stride of 0 in the
+  // caffe strides vector indictes a layer has no stride. such layers
+  // are ignored in this calculation
+  static uint32_t get_sbin( shared_ptr<Net<float> > net_ ) {
+    vect_uint32_t const & strides = net_->layer_strides();
+    uint32_t ret = 1;
+    for( vect_uint32_t::const_iterator i = strides.begin(); i != strides.end(); ++i ) { if( *i ) { ret *= (*i); } }
+    return ret;
+  }
+
+  template< typename T > inline std::string str(T const & i) { std::stringstream s; s << i; return s.str(); } // convert T i to string
+}

include/caffe/imagenet_mean.hpp

@@ -0,0 +1,18 @@
+#ifndef IMAGENET_MEAN_H
+#define IMAGENET_MEAN_H
+
+//mean of all imagenet classification images
+// calculation:
+// 1. mean of all imagenet images, per pixel location
+// 2. take the mean image, and get the mean pixel for R,G,B
+// (did it this way because we already had the 'mean of all images, per pixel location')
+
+
+#define IMAGENET_MEAN_R 122.67f
+#define IMAGENET_MEAN_G 116.66f
+#define IMAGENET_MEAN_B 104.00f
+
+static float const IMAGENET_MEAN_RGB[3] = {IMAGENET_MEAN_R, IMAGENET_MEAN_G, IMAGENET_MEAN_B};
+static float const IMAGENET_MEAN_BGR[3] = {IMAGENET_MEAN_B, IMAGENET_MEAN_G, IMAGENET_MEAN_R};
+
+#endif

include/caffe/net.hpp

@@ -62,6 +62,8 @@ class Net {
   inline const string& name() { return name_; }
   // returns the layer names
   inline const vector<string>& layer_names() { return layer_names_; }
+  // returns the layer strides
+  inline const vector<uint32_t>& layer_strides() { return layer_strides_; }
   // returns the blob names
   inline const vector<string>& blob_names() { return blob_names_; }
   // returns the blobs
@@ -91,6 +93,7 @@ class Net {
   // Individual layers in the net
   vector<shared_ptr<Layer<Dtype> > > layers_;
   vector<string> layer_names_;
+  vector<uint32_t> layer_strides_;
   vector<bool> layer_need_backward_;
   // blobs stores the blobs that store intermediate results between the
   // layers.

matlab/caffe/convnet_featpyramid.m

@@ -0,0 +1,43 @@
+
+% wrapper for DenseNet convnet_pyramid.
+% provides some extra output params that you wouldn't get with caffe('convnet_featpyramid', ...)
+
+% YOU MUST CALL caffe('init', ...) BEFORE RUNNING THIS.
+function pyra = convnet_featpyramid(imgFname, pyra_params)
+
+    %set defaults for params not passed by user
+    if( ~exist('pyra_params') || ~isfield(pyra_params, 'interval') )
+        pyra_params.interval = 5;
+    end
+    if( ~exist('pyra_params') || ~isfield(pyra_params, 'img_padding') )
+        pyra_params.img_padding = 16;
+    end
+    if( ~exist('pyra_params') || ~isfield(pyra_params, 'feat_minWidth') )
+        pyra_params.feat_minWidth = 1;
+    end
+    if( ~exist('pyra_params') || ~isfield(pyra_params, 'feat_minHeight') )
+        pyra_params.feat_minHeight = 1;
+    end
+
+    % compute the pyramid: 
+    pyra = caffe('convnet_featpyramid', imgFname, pyra_params);
+
+    % add DPM-style fields:
+    pyra.padx = pyra.feat_padx; % for DPM conventions
+    pyra.pady = pyra.feat_pady;
+    pyra.num_levels = length(pyra.scales);
+    pyra.valid_levels = true(pyra.num_levels, 1);
+
+    pyra.imsize = [pyra.imheight pyra.imwidth];
+    pyra.feat = permute_feat(pyra.feat); % [d h w] -> [h w d]     
+    pyra.scales = double(pyra.scales); %get_detection_trees prefers double
+end
+
+% input:  pyra.feat{:}, with dims [d h w]
+% output: pyra.feat{:}, with dims [h w d]
+function feat = permute_feat(feat)
+    for featIdx = 1:length(feat)
+        feat{featIdx} = permute( feat{featIdx}, [2 3 1] );
+    end
+end
+