Merge pull request apache#15 from dato-code/build_flexible_type

Pull in SFrame subtree and build flexible_type

.gitignore

@@ -96,3 +96,4 @@ tags
 
 
 miniconda.sh
+deps_version

Makefile

@@ -31,10 +31,8 @@ endif
 ifneq ($(WIN32), 1)
 	CFLAGS += -fPIC
 	SHARED_LIB_EXT = so
-	STATIC_LIB_EXT = a
 else
 	SHARED_LIB_EXT = dll
-	STATIC_LIB_EXT = lib
 endif
 CFLAGS += -I./mshadow/ -I./dmlc-core/include -Iinclude $(MSHADOW_CFLAGS)
 LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS)
@@ -43,7 +41,10 @@ ifeq ($(DEBUG), 1)
 else
 	NVCCFLAGS = --use_fast_math -g -O3 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS)
 endif
-ROOTDIR = $(CURDIR)
+
+ifndef ROOTDIR
+	ROOTDIR = $(CURDIR)
+endif
 
 ifndef LINT_LANG
 	LINT_LANG="all"
@@ -81,9 +82,6 @@ endif
 
 ifneq ($(USE_CUDA_PATH), NONE)
 	NVCC = $(USE_CUDA_PATH)/bin/nvcc
-	CUDA_DEP = $(wildcard $(USE_CUDA_PATH)/lib64/libcudart.so*)
-	CUDA_DEP += $(wildcard $(USE_CUDA_PATH)/lib64/libcurand.so*)
-	CUDA_DEP += $(wildcard $(USE_CUDA_PATH)/lib64/libcublas.so*)
 endif
 
 # ps-lite
@@ -96,11 +94,16 @@ ifeq ($(USE_DIST_KVSTORE), 1)
 	LDFLAGS += $(PS_LDFLAGS_A)
 endif
 
+# SFrame flexible_type
+FLEXIBLE_TYPE = $(ROOTDIR)/flexible_type
+LIB_DEP += $(FLEXIBLE_TYPE)/build/libflexible_type.a
+
+# plugins
 include $(MXNET_PLUGINS)
 
 .PHONY: clean all test lint doc clean_all rcpplint rcppexport roxygen
 
-all: lib/libmxnet.$(STATIC_LIB_EXT) lib/libmxnet.$(SHARED_LIB_EXT) $(BIN)
+all: lib/libmxnet.$(SHARED_LIB_EXT) $(BIN)
 
 SRC = $(wildcard src/*.cc src/*/*.cc)
 OBJ = $(patsubst %.cc, build/%.o, $(SRC))
@@ -165,6 +168,11 @@ build/plugin/%.o: plugin/%.cc
 	$(CXX) -std=c++0x $(CFLAGS) -MM -MT build/plugin/$*.o $< >build/plugin/$*.d
 	$(CXX) -std=c++0x -c $(CFLAGS) -c $< -o $@
 
+build/plugin/%.o: plugin/%.cpp
+	@mkdir -p $(@D)
+	$(CXX) -std=c++0x $(CFLAGS) -MM -MT build/plugin/$*.o $< >build/plugin/$*.d
+	$(CXX) -std=c++0x -c $(CFLAGS) -c $< -o $@
+
 # A nvcc bug cause this to generate "generic/xxx.h" dependencies from torch headers.
 # $(NVCC) $(NVCCFLAGS) -Xcompiler "$(CFLAGS)" -M -MT build/plugin/$*_gpu.o $< >build/plugin/$*_gpu.d
 build/plugin/%_gpu.o: plugin/%.cu
@@ -181,16 +189,13 @@ $(EXTRA_OPERATORS)/build/%_gpu.o: $(EXTRA_OPERATORS)/%.cu
 	$(NVCC) $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $(EXTRA_OPERATORS)/build/$*_gpu.o $< >$(EXTRA_OPERATORS)/build/$*_gpu.d
 	$(NVCC) -c -o $@ $(NVCCFLAGS) -Xcompiler "$(CFLAGS) -Isrc/operator" $<
 
-# NOTE: to statically link libmxnet.a we need the option
-# --Wl,--whole-archive -lmxnet --Wl,--no-whole-archive
-lib/libmxnet.$(STATIC_LIB_EXT): $(ALL_DEP)
-	@mkdir -p $(@D)
-	ar crv $@ $(filter %.o, $?)
-
 lib/libmxnet.$(SHARED_LIB_EXT): $(ALL_DEP)
 	@mkdir -p $(@D)
 	$(CXX) $(CFLAGS) -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS)
 
+$(FLEXIBLE_TYPE)/build/libflexible_type.a:
+	+ cd $(FLEXIBLE_TYPE); make CXX=$(CXX); cd $(ROOTDIR)
+
 $(PS_PATH)/build/libps.a:
 	$(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps
 	ln -fs $(PS_PATH)/tracker .
@@ -240,12 +245,14 @@ clean:
 	$(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~
 	cd $(DMLC_CORE); make clean; cd -
 	cd $(PS_PATH); make clean; cd -
+	cd $(FLEXIBLE_TYPE); make clean; cd -
 	$(RM) -r $(EXTRA_OPERATORS)/build
 else
 clean:
 	$(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~
 	cd $(DMLC_CORE); make clean; cd -
 	cd $(PS_PATH); make clean; cd -
+	cd $(FLEXIBLE_TYPE); make clean; cd -
 endif
 
 clean_all: clean

configure

@@ -6,11 +6,11 @@ PROJECT_HOME=$PWD
 DEPS_PREFIX=$PWD/deps/local
 
 if [[ $OSTYPE == linux* ]]; then
-        DEFAULT_DATO_DEPS_VERSION=4
+        DEFAULT_DATO_DEPS_VERSION=8
 elif [[ $OSTYPE == darwin* ]]; then
-        DEFAULT_DATO_DEPS_VERSION=4
+        DEFAULT_DATO_DEPS_VERSION=8
 elif [[ $OSTYPE == msys ]]; then
-        DEFAULT_DATO_DEPS_VERSION=4
+        DEFAULT_DATO_DEPS_VERSION=8
 fi
 
 ## Support code

flexible_type/Makefile

@@ -0,0 +1,30 @@
+ifndef CXX
+CXX = g++
+endif
+
+SFRAME_SRC = $(PWD)/../plugin/SFrameSubtree/oss_src
+DMLC_SRC = $(PWD)/../dmlc-core/include
+DEPS = $(PWD)/../deps/local/include
+
+CFLAGS = -std=c++11 -msse2 -fPIC -O3 -Wall -finline-functions -I$(DEPS) -I$(SFRAME_SRC) -I$(DMLC_SRC) -include $(PWD)/flexible_type.h
+
+all: flexible_type 
+
+clean:
+	rm -rf build
+
+flexible_type: build/libflexible_type.a
+
+test: build/flexible_type_test
+OBJS = $(addprefix build/, flexible_type/flexible_type.o image/image_type.o)
+
+build/libflexible_type.a: $(OBJS)
+	ar crv $@ $(filter %.o, $?)
+
+build/%.o: $(SFRAME_SRC)/%.cpp
+	@mkdir -p $(@D)
+	$(CXX) $(CFLAGS) -std=c++0x -MM -MT build/$*.o $< >build/$*.d
+	$(CXX) $(CFLAGS) -c $< -o $@
+
+build/flexible_type_test: test/flexible_type_test.cpp
+	$(CXX) $(CFLAGS) $< -o $@ -L build -lflexible_type

flexible_type/flexible_type.h

@@ -0,0 +1,34 @@
+/*!
+ * Copyright (c) 2015 by Contributors
+ * \file flexible_type.h
+ * \brief
+ * \author Bing Xu
+*/
+#ifndef MXNET_FLEXIBLE_TYPE_H_
+#define MXNET_FLEXIBLE_TYPE_H_
+
+#define _ASSERTIONS_H_
+#define GRAPHLAB_LOG_LOG_HPP
+
+#include <iostream>
+#include <dmlc/logging.h>
+
+// -----------------------
+#define ASSERT_NE CHECK_NE
+#define ASSERT_GE CHECK_GE
+#define ASSERT_GT CHECK_GT
+#define ASSERT_LE CHECK_LE
+#define ASSERT_LT CHECK_LT
+#define ASSERT_EQ CHECK_EQ
+#define ASSERT_MSG(FALSE, MSG, ...) \
+  LOG(FATAL) << MSG;
+#define ASSERT_FALSE(VAR) \
+  CHECK_EQ(VAR, false);
+#define DASSERT_FALSE(cond) ASSERT_FALSE(cond)
+#define log_and_throw(msg) \
+  LOG(FATAL) << msg;
+#define logstream(LOGLEVEL) std::cerr
+
+#include<flexible_type/flexible_type.hpp>
+
+#endif  // MXNET_FLEXIBLE_TYPE_H_

flexible_type/test/flexible_type_test.cpp

@@ -0,0 +1,46 @@
+#include "../flexible_type.h"
+#include <image/image_type.hpp>
+#include <dmlc/logging.h>
+#include <fstream>
+
+using namespace graphlab;
+
+void test_int() { 
+  int y = 1;
+  flexible_type x = y;
+  CHECK_EQ((int)(x), y);
+  CHECK_EQ(x.get_type(), flex_type_enum::INTEGER);
+}
+
+void test_float() { 
+  float y = 3.0;
+  flexible_type x = y;
+  CHECK_EQ((float)(x), y);
+  CHECK_EQ(x.get_type(), flex_type_enum::FLOAT);
+}
+
+void test_array() { 
+  std::vector<double> y{1.0, 2.0, 3.0};
+  flexible_type x = y;
+  CHECK_EQ(x.get_type(), flex_type_enum::VECTOR);
+  CHECK_EQ(x.size(), y.size());
+  for (size_t i = 0; i < x.size(); ++i) {
+    CHECK_EQ(x[i], y[i]);
+  }
+}
+
+void test_image() {
+  flex_image x;
+  CHECK_EQ(x.is_decoded(), false);
+  CHECK_EQ(x.m_height, 0);
+  CHECK_EQ(x.m_width, 0);
+  CHECK_EQ(x.m_channels, 0);
+}
+
+int main() {
+  test_int();
+  test_float();
+  test_array();
+  test_image();
+  return 0;
+}

include/mxnet/c_api.h

@@ -227,6 +227,24 @@ MXNET_DLL int MXNDArrayLoad(const char* fname,
 MXNET_DLL int MXNDArraySyncCopyFromCPU(NDArrayHandle handle,
                                        const void *data,
                                        size_t size);
+/*!
+ * \brief Perform a synchonize copy by using as SFrame callback
+ * This function will call WaitToWrite before the copy is performed.
+ * This is useful to copy data from existing memory region that are
+ * not wrapped by NDArray(thus dependency not being tracked).
+ *
+ * \param callback_handle pointer to struct of callback function state 
+ * \param data the data source to copy from
+ * \param size the element size to copy
+ */
+struct SFrameCallbackHandle {
+  NDArrayHandle handle;
+  size_t idx;
+  size_t batch_size;
+};
+MXNET_DLL int MXNDArraySyncCopyFromSFrame(const void *data,
+                                          size_t size,
+                                          void* callback_handle);
 /*!
  * \brief Perform a synchronize copyto a continugous CPU memory region.
  *

include/mxnet/flexible_type.h

@@ -0,0 +1,29 @@
+/*!
+ * Copyright (c) 2015 by Contributors
+ * \file flexible_type.h
+ * \brief
+ * \author Bing Xu
+*/
+#ifndef MXNET_FLEXIBLE_TYPE_H_
+#define MXNET_FLEXIBLE_TYPE_H_
+
+#define _ASSERTIONS_H_
+#define GRAPHLAB_LOG_LOG_HPP
+// -----------------------
+#define ASSERT_NE CHECK_NE
+#define ASSERT_GE CHECK_GE
+#define ASSERT_GT CHECK_GT
+#define ASSERT_LE CHECK_LE
+#define ASSERT_LT CHECK_LT
+#define ASSERT_EQ CHECK_EQ
+#define ASSERT_MSG(FALSE, MSG, ...) \
+  LOG(FATAL) << MSG;
+#define ASSERT_FALSE(VAR) \
+  CHECK_EQ(VAR, false);
+#define DASSERT_FALSE(cond) ASSERT_FALSE(cond)
+#define log_and_throw(msg) \
+  LOG(FATAL) << msg;
+#include<flexible_type/flexible_type.hpp>
+
+
+#endif  // MXNET_FLEXIBLE_TYPE_H_

include/mxnet/ndarray.h

@@ -7,7 +7,6 @@
 #define MXNET_NDARRAY_H_
 
 #include <dmlc/base.h>
-#include <dmlc/logging.h>
 #include <dmlc/io.h>
 #include <dmlc/type_traits.h>
 #include <dmlc/registry.h>
@@ -18,6 +17,7 @@
 #include "./base.h"
 #include "./storage.h"
 #include "./engine.h"
+#include "./flexible_type.h"
 
 // check c++11
 #if DMLC_USE_CXX11 == 0
@@ -207,6 +207,21 @@ class NDArray {
    * \param size the size of the source array, in sizeof(DType) not raw btyes.
    */
   void SyncCopyFromCPU(const void *data, size_t size) const;
+  /*!
+   * \brief Do a synchnoize copy from SFrame
+   * This function will call WaitToWrite before the copy is performed.
+   * This is useful to copy data from existing memory region that are
+   * not wrapped by NDArray(thus dependency not being tracked).
+   *
+   * \param data sframe flexible type source
+   * \param size field lenth of data
+   * \param idx index in batch
+   * \param batch_size total batch size
+   */
+  void SyncCopyFromSFrame(const graphlab::flexible_type *data,
+                          size_t size,
+                          size_t idx,
+                          size_t batch_size) const;
   /*!
    * \brief Do a synchronize copy to a continugous CPU memory region.
    *

make/dato-config.mk

@@ -16,6 +16,8 @@
 #  $ make -j8
 #-------------------------------------------------------------------------------
 
+ROOTDIR = ${PROJECT_HOME}
+
 #---------------------
 # choice of compiler
 #--------------------
@@ -34,6 +36,7 @@ ADD_LDFLAGS += -L${DEPS}/lib -L${DEPS}/lib64 ${SHARED_LINKER_FLAGS}
 
 # the additional compile flags you want to add
 ADD_CFLAGS = ${ADD_CFLAGS}
+ADD_CFLAGS += -Iplugin/SFrameSubtree/oss_src
 ADD_CFLAGS += -I${DEPS}/include
 
 #---------------------------------------------
@@ -100,3 +103,7 @@ USE_S3 = 0
 
 # path to folders containing projects specific operators that you don't want to put in src/operators
 EXTRA_OPERATORS =
+
+#----------------------------
+# plugins
+#----------------------------