[WIP] - Feature train orb

CMakeLists.txt

@@ -8,9 +8,9 @@ set(DEBUG_DEFINES " -DDEBUG_EM ")
 set(DEBUG_FLAGS " -g -sASSERTIONS=1 --profiling -s DEMANGLE_SUPPORT=1 ")
 set(JPEG_FLAG " -s USE_LIBJPEG=1 ")
 set(MEMORY_OPTION " -s TOTAL_MEMORY=268435456 -s ALLOW_MEMORY_GROWTH=1 ")
-set(ES6_BUILD " -s EXPORT_ES6=1 -s USE_ES6_IMPORT_META=0 -sEXPORT_NAME=jsfeatCpp -s MODULARIZE=1 ")
+set(ES6_BUILD " -s EXPORT_ES6=1 -s USE_ES6_IMPORT_META=0 -sENVIRONMENT=web -sEXPORT_NAME=jsfeatCpp -s MODULARIZE=1 ")
 set(SINGLE_FILE " -s SINGLE_FILE=1")
-set(POST_JS " --post-js ../js/jsfeatCpp_post.api.js ")
+set(POST_JS " --post-js ../js/jsfeatCpp_post.api.js ") 
 set(LIBAR " libar.bc ")
 set(DEBUG_LIBAR " libar_debug.bc ")
 set(EXPORTED_RUNTIME_METHODS " -sEXPORTED_RUNTIME_METHODS=FS ")

emscripten/bindings.cpp

@@ -134,6 +134,7 @@ EMSCRIPTEN_BINDINGS(webarkit) {
     // Extern jsfeat functions
 
     function("load_jpeg_data", &load_jpeg_data);
+    function("train_orb_pattern", &train_orb_pattern);
     function("yape06_detect", &yape06_detect);
 
 };

emscripten/webarkitJsfeat.cpp

@@ -7,6 +7,8 @@
 #include <jsfeat.h>
 #include <stdio.h>
 
+#include <cmath>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -87,6 +89,168 @@ emscripten::val load_jpeg_data(std::string filename) {
   return out;
 };
 
+void train_orb_pattern_internal(const char* filename) {
+  char* ext;
+  char buf1[512], buf2[512];
+
+  AR2JpegImageT* jpegImage;
+
+  auto lev = 0, i = 0;
+  auto sc = 1.0;
+  auto max_pattern_size = 512;
+  auto max_per_level = 300;
+  auto sc_inc = std::sqrt(2.0);  // magic number ;)
+  auto new_width = 0, new_height = 0;
+  // var lev_corners, lev_descr;
+  auto corners_num = 0;
+
+  // if (!filename) return emscripten::val::null();
+  ext = arUtilGetFileExtensionFromPath(filename, 1);
+  if (!ext) {
+    webarkitLOGe(
+        "Error: unable to determine extension of file '%s'. Exiting.\n",
+        filename);
+  }
+  if (strcmp(ext, "jpeg") == 0 || strcmp(ext, "jpg") == 0 ||
+      strcmp(ext, "jpe") == 0) {
+    webarkitLOGi("Waiting for the jpeg...");
+    webarkitLOGi("Reading JPEG file...");
+    ar2UtilDivideExt(filename, buf1, buf2);
+    jpegImage = ar2ReadJpegImage(buf1, buf2);
+    if (jpegImage == NULL) {
+      webarkitLOGe(
+          "Error: unable to read JPEG image from file '%s'. Exiting.\n",
+          filename);
+    }
+    webarkitLOGi("   Done.");
+
+    if (jpegImage->nc != 1 && jpegImage->nc != 3) {
+      ARLOGe(
+          "Error: Input JPEG image is in neither RGB nor grayscale format. "
+          "%d bytes/pixel %sformat is unsupported. Exiting.\n",
+          jpegImage->nc, (jpegImage->nc == 4 ? "(possibly CMYK) " : ""));
+    }
+    webarkitLOGi("JPEG image number of channels: '%d'", jpegImage->nc);
+    webarkitLOGi("JPEG image width is: '%d'", jpegImage->xsize);
+    webarkitLOGi("JPEG image height is: '%d'", jpegImage->ysize);
+    webarkitLOGi("JPEG image, dpi is: '%d'", jpegImage->dpi);
+
+    if (jpegImage->dpi == 0.0f) {
+      webarkitLOGw(
+          "JPEG image '%s' does not contain embedded resolution data, and no "
+          "resolution specified on command-line.",
+          filename);
+    }
+
+  } else if (strcmp(ext, "png") == 0) {
+    webarkitLOGe(
+        "Error: file has extension '%s', which is not supported for "
+        "reading. Exiting.\n",
+        ext);
+    free(ext);
+  }
+  webarkitLOGi("Image done!");
+
+  JSLOGi("Starting detection routine...");
+
+  Orb orb;
+  Imgproc imgproc;
+  detectors::Detectors detectors;
+  auto width = jpegImage->xsize;
+  auto height = jpegImage->ysize;
+  std::unique_ptr<Matrix_t> lev0_img = std::make_unique<Matrix_t>(width, height, ComboTypes::U8C1_t);
+  std::unique_ptr<Matrix_t> lev_img = std::make_unique<Matrix_t>(width, height, ComboTypes::U8C1_t);
+  Array<std::unique_ptr<Matrix_t>> pattern_corners;
+
+  auto sc0 = std::min(max_pattern_size / height, max_pattern_size / width);
+  // new_width = (jpegImage->ysize * sc0) | 0;
+  // new_height = (jpegImage->xsize * sc0) | 0;
+  auto num_train_levels = 4;
+
+  JSLOGi("Converting the RGB image to GRAY...");
+
+  imgproc.grayscale_internal<u_char, Matrix_t>(jpegImage->image, width, height, lev0_img.get(), Colors::COLOR_RGB2GRAY);
+
+  JSLOGi("Image converted to GRAY.");
+
+  Array<KeyPoints> lev_corners(num_train_levels);
+  // Array<std::unique_ptr<KeyPoints>> lev_corners;
+  Array<std::unique_ptr<Matrix_t>> pattern_descriptors;
+
+  for (lev = 0; lev < num_train_levels; ++lev) {
+    // what we should do with this code?
+    // pattern_corners[lev] = [];
+    // lev_corners = pattern_corners[lev];
+
+    // preallocate corners array
+    // i = (new_width * new_height) >> lev;
+    i = (width * height) >> lev;
+    JSLOGi("Level %i with %i keypoints.", lev, i);
+    lev_corners[lev].set_size(i);
+    lev_corners[lev].allocate();
+    while (--i >= 0) {
+      // lev_corners[lev].set_size(i);
+      // lev_corners[lev].allocate();
+      // lev_corners[lev] = std::make_unique<KeyPoints>(i);
+      // lev_corners.push_back(std::unique_ptr<KeyPoints>(new KeyPoints(i)));
+    }
+
+    pattern_descriptors.push_back(std::unique_ptr<Matrix_t>(new Matrix_t(32, max_per_level, ComboTypes::U8C1_t)));
+  }
+
+  std::cout << "Size of first lev_corners: " << lev_corners[0].kpoints.size() << std::endl;
+
+  imgproc.gaussian_blur_internal(lev0_img.get(), lev_img.get(), 5, 0.0);  // this is more robust
+
+  JSLOGi("After Gaussian blur");
+
+  corners_num = detectors.detect_keypoints(lev_img.get(), &lev_corners[0], max_per_level);
+
+  orb.describe_internal(lev_img.get(), lev_corners[0].kpoints, corners_num, pattern_descriptors[0].get());
+
+  JSLOGi("train %i x %i points: %i", lev_img.get()->get_cols(), lev_img.get()->get_rows(), corners_num);
+
+  sc /= sc_inc;
+
+  for (lev = 1; lev < num_train_levels; ++lev) {
+    //lev_corners = pattern_corners[lev];
+    //lev_descr = pattern_descriptors[lev];
+
+    //new_width = (lev0_img.cols * sc) | 0;
+    new_width = (lev0_img.get()->get_cols() * sc) ;
+    //new_height = (lev0_img.rows * sc) | 0;
+    new_height = (lev0_img.get()->get_rows() * sc);
+
+    imgproc.resample(lev0_img.get(), lev_img.get(), new_width, new_height);
+    imgproc.gaussian_blur_internal(lev_img.get(), lev_img.get(), 5, 0.0);
+    corners_num = detectors.detect_keypoints(lev_img.get(), &lev_corners[lev], max_per_level);
+    orb.describe_internal(lev_img.get(), lev_corners[lev].kpoints, corners_num, pattern_descriptors[lev].get());
+
+    // fix the coordinates due to scale level
+    // will fix this later...
+    /*for (i = 0; i < corners_num; ++i) {
+      lev_corners[i].x *= 1. / sc;
+      lev_corners[i].y *= 1. / sc;
+    }*/
+
+    for (i = 0; i < corners_num; ++i) {
+      lev_corners[lev].kpoints[i].x *= 1. / sc;
+      lev_corners[lev].kpoints[i].y *= 1. / sc;
+    }
+
+    JSLOGi("train %i x %i points: %i", lev_img.get()->get_cols(), lev_img.get()->get_rows(), corners_num);
+
+    sc /= sc_inc;
+  }
+
+  free(ext);
+  free(jpegImage);
+};
+
+void train_orb_pattern(std::string filename) {
+  train_orb_pattern_internal(filename.c_str());
+}
+
 emscripten::val yape06_detect(emscripten::val inputSrc, int w, int h) {
   auto src = emscripten::convertJSArrayToNumberVector<u_char>(inputSrc);
   Imgproc imgproc;
@@ -115,7 +279,6 @@ emscripten::val yape06_detect(emscripten::val inputSrc, int w, int h) {
 
   return outObj;
 };
-
 }
 
 #include "bindings.cpp"

examples/js/loader.js

@@ -1,15 +1,14 @@
-import jsfeatCpp from "./../../build/jsfeatES6cpp_debug.js"
+//import jsfeatCpp from "./../../build/jsfeatES6cpp_debug.js"
+import jsfeatCpp from "./../../build/jsfeatES6cpp.js"
 var jsfeat = await jsfeatCpp();
 
 var jpeg_count = 0;
 export function loadJpeg(url, callback, errorCallback) {
     var filename = '/load_jpeg_' + jpeg_count++ + '.jpg';
     var writeCallback = function (errorCode) {
-        //if (!Module._loadCamera) {
         if (!jsfeat.load_jpeg_data) {
             if (callback) callback(id); setTimeout(writeCallback, 10);
         } else {
-            //var id = Module._loadCamera(filename);
             var id = jsfeat.load_jpeg_data(filename)
             if (callback) callback(id);
         }
@@ -37,6 +36,39 @@ export function loadJpeg(url, callback, errorCallback) {
     }
 }
 
+export function trainOrbPattern(url, callback, errorCallback) {
+    var filename = '/load_jpeg_' + jpeg_count++ + '.jpg';
+    var writeCallback = function (errorCode) {
+        if (!jsfeat.train_orb_pattern) {
+            if (callback) callback(id); setTimeout(writeCallback, 10);
+        } else {
+            var id = jsfeat.train_orb_pattern(filename)
+            if (callback) callback(id);
+        }
+    };
+    if (typeof url === 'object') { // Maybe it's a byte array
+        writeByteArrayToFS(filename, url, writeCallback);
+    } else if (url.indexOf("\n") > -1) { // Or a string with the jpeg path
+        writeStringToFS(filename, url, writeCallback);
+    } else {
+        fetch(url)
+            .then(response => {
+                if (!response.ok) {
+                    throw new Error('Network response was not OK');
+                }
+                return response.arrayBuffer();
+            })
+            .then(buff => {
+                let buffer = new Uint8Array(buff)
+                writeByteArrayToFS(filename, buffer, writeCallback);
+            })
+
+            .catch(error => {
+                errorCallback(error)
+            });
+    }
+}
+
 // transfer image
 
 function writeStringToFS(target, string, callback) {

examples/train_orb_example.html

@@ -0,0 +1,18 @@
+<html>
+
+<body>
+
+    <script type="module">
+        //import jsfeatCpp from "./../build/jsfeatES6cpp_debug.js"
+        import jsfeatCpp from "./../build/jsfeatES6cpp.js"
+        import {  trainOrbPattern } from "./js/loader.js"
+
+        const jsfeat = await jsfeatCpp();
+
+        trainOrbPattern("pinball.jpg", () => {}, ()=>{});
+
+    </script>
+
+</body>
+
+</html>

run_docker.txt

@@ -0,0 +1,7 @@
+// before all:
+git submodule update --init
+//Assumend that you have emscripten engine installed under docker, you may run:
+// for the first time
+docker exec emscripten ./build.sh emscripten-all
+// and then when WebarkitLib is compiled:
+docker exec emscripten ./build.sh emscripten

src/feature_detection/detectors.h

@@ -0,0 +1,81 @@
+#ifndef DETECTORS_H
+#define DETECTORS_H
+
+#include <keypoint_t/keypoint_t.h>
+#include <keypoints/keypoints.h>
+#include <keypoints_filter/keypoints_filter.h>
+#include <math/math.h>
+#include <matrix_t/matrix_t.h>
+#include <types/types.h>
+#include <yape06/yape06.h>
+
+namespace jsfeat {
+
+namespace detectors {
+
+class Detectors : public Yape06, public Math, public KeyPointsFilter {
+ public:
+  Detectors() {}
+  ~Detectors() {}
+
+  int detect_keypoints(Matrix_t* img, KeyPoints* corners, int max_allowed) {
+    // detect features
+    auto kpc = detect_internal(img, corners, 17);
+    auto count = kpc.count;
+    std::cout << "Count inside detect_keypoints: " << count << std::endl;
+    // sort by score and reduce the count if needed
+    if (count > max_allowed) {
+      // qsort_internal<KeyPoint_t, bool>(corners.kpoints, 0, count - 1, [](KeyPoint_t i, KeyPoint_t j){return (i.score < j.score);});
+      retainBest(corners->kpoints, count);
+      count = max_allowed;
+    }
+
+    // calculate dominant orientation for each keypoint
+    for (auto i = 0; i < count; ++i) {
+      corners->kpoints[i].angle = ic_angle(img, corners->kpoints[i].x, corners->kpoints[i].y);
+    }
+
+    return count;
+  }
+
+ private:
+  // function(a, b) { return (b.score < a.score); }
+  // bool myfunction(KeyPoint_t i, KeyPoint_t j) { return (i.score < j.score); }
+  // central difference using image moments to find dominant orientation
+  // var u_max = new Int32Array([15, 15, 15, 15, 14, 14, 14, 13, 13, 12, 11, 10, 9, 8, 6, 3, 0]);
+  float ic_angle(Matrix_t* img, int px, int py) {
+    Array<u_int> u_max{15, 15, 15, 15, 14, 14, 14, 13, 13, 12, 11, 10, 9, 8, 6, 3, 0};
+    auto half_k = 15;  // half patch size
+    auto m_01 = 0, m_10 = 0;
+    auto src = img->u8;
+    auto step = img->get_cols();
+    auto u = 0, v = 0, center_off = (py * step + px) | 0;
+    auto v_sum = 0, d = 0, val_plus = 0, val_minus = 0;
+
+    // Treat the center line differently, v=0
+    for (u = -half_k; u <= half_k; ++u)
+      m_10 += u * src[center_off + u];
+
+    // Go line by line in the circular patch
+    for (v = 1; v <= half_k; ++v) {
+      // Proceed over the two lines
+      v_sum = 0;
+      d = u_max[v];
+      for (u = -d; u <= d; ++u) {
+        val_plus = src[center_off + u + v * step];
+        val_minus = src[center_off + u - v * step];
+        v_sum += (val_plus - val_minus);
+        m_10 += u * (val_plus + val_minus);
+      }
+      m_01 += v * v_sum;
+    }
+
+    return std::atan2(m_01, m_10);
+  }
+};
+
+}  // namespace detectors
+
+}  // namespace jsfeat
+
+#endif