feature/pip build

.travis.yml

@@ -3,6 +3,7 @@ cache:
   - pip
   - ccache
   - yarn
+  - npm
 sudo: required
 dist: trusty
 os:

README.md

@@ -1,34 +1,14 @@
 # VisualDL
 
 
-### How to use
-#### Step 1: build frontend
-```shell
-cd frontend
-npm install
-npm run build
+### How to install
 ```
-
-this step will generate a dist directory under frontend
-
-### Step 2: copy frontend/dist to server/visualdl/frontend/dist
-```shell
-mkdir -p server/visualdl/frontend/dist
-cp -r frontend/dist server/visualdl/frontend/dist
-```
-
-#### Step 3: build and install Python package
-```shell
-cd server/
-sh build.sh
-cd dist
-sudo pip install --upgrade visualdl-0.0.1-py2-none-any.whl
+python setup.py bdist_wheel
+pip install --upgrade dist/visualdl-0.0.1-py2-none-any.whl
 ```
 
 
 ### Step 3: run
 ```
-# cd to visualdl install dir
-cd /usr/local/lib/python2.7/site-packages/visualdl/
-python visual_dl.py --port=8888
+visualDL --logdir=<some log> --port=8888
 ```

VERSION_NUMBER

@@ -0,0 +1 @@
+0.0.1

build.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -ex
+
+TOP_DIR=$(pwd)
+FRONTEND_DIR=$TOP_DIR/frontend
+BACKEND_DIR=$TOP_DIR/visualdl
+BUILD_DIR=$TOP_DIR/build
+
+mkdir -p $BUILD_DIR
+
+build_frontend() {
+    cd $FRONTEND_DIR
+    if [ ! -d "dist" ]; then
+      npm install
+      npm run build
+    fi
+}
+
+build_frontend_fake() {
+    cd $FRONTEND_DIR
+    mkdir -p dist
+}
+
+build_backend() {
+    cd $BUILD_DIR
+    cmake ..
+    make -j2
+}
+
+build_onnx_graph() {
+    # TODO(ChunweiYan) check protoc version here
+    cd $TOP_DIR/visualdl/server/onnx
+    protoc onnx.proto --python_out .
+}
+
+package() {
+    cp -rf $FRONTEND_DIR/dist $TOP_DIR/visualdl/server/
+    cp $BUILD_DIR/visualdl/logic/core.so $TOP_DIR/visualdl
+    cp $BUILD_DIR/visualdl/logic/core.so $TOP_DIR/visualdl/python/
+}
+
+ARG=$1
+echo "ARG: " $ARG
+
+
+if [ $ARG = "travis-CI" ]; then
+    build_frontend_fake
+else
+    build_frontend
+fi
+
+build_backend
+build_onnx_graph
+package

setup.py

@@ -0,0 +1,92 @@
+from __future__ import absolute_import
+
+import os
+import sys
+from distutils.spawn import find_executable
+from distutils import sysconfig, dep_util, log
+import setuptools.command.build_py
+import setuptools
+from setuptools import setup, find_packages
+import subprocess
+
+TOP_DIR = os.path.realpath(os.path.dirname(__file__))
+PYTHON_SDK_DIR = os.path.join(TOP_DIR, 'visualdl/python')
+BUILD_DIR = os.path.join(TOP_DIR, 'build')
+MODE = os.environ.get('VS_BUILD_MODE', 'RELEASE')
+
+
+def read(name):
+    return open(os.path.join(TOP_DIR, name)).read()
+
+
+def readlines(name):
+    return read(name).split('\n')
+
+
+VERSION_NUMBER = read('VERSION_NUMBER')
+LICENSE = readlines('LICENSE')[0].strip()
+
+install_requires = ['Flask', 'numpy', 'Pillow', 'protobuf']
+execute_requires = ['npm', 'node', 'protoc', 'bash']
+
+
+def die(msg):
+    log.error(msg)
+    sys.exit(1)
+
+
+def CHECK(cond, msg):
+    if not cond:
+        die(msg)
+
+
+for exe in execute_requires:
+    CHECK(find_executable(exe), "{} should be installed.".format(exe))
+
+
+class BaseCommand(setuptools.Command):
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+
+class build_py(setuptools.command.build_py.build_py):
+    def run(self):
+        cmd = ['bash', 'build.sh']
+        if MODE == "travis-CI":
+            cmd.append('travis-CI')
+        subprocess.check_call(cmd)
+        return setuptools.command.build_py.build_py.run(self)
+
+
+cmdclass = {
+    'build_py': build_py,
+}
+
+packages = [
+    'visualdl',
+    'visualdl.python',
+    'visualdl.server',
+    'visualdl.server.mock',
+    'visualdl.server.onnx',
+]
+
+setup(
+    name="visualdl",
+    version=VERSION_NUMBER,
+    author="PaddlePaddle and Echarts team.",
+    description="Visualize Deep Learning.",
+    license=LICENSE,
+    keywords="visualization deeplearning",
+    long_description=read('README.md'),
+    install_requires=install_requires,
+    package_data={'visualdl.server': ['dist/*', 'dist/fonts/*'],
+                  'visualdl':['core.so'],
+                  'visualdl.python':['core.so']},
+    packages=packages,
+    scripts=['visualdl/server/visualDL'],
+    cmdclass=cmdclass)

tests.sh

@@ -2,15 +2,39 @@
 set -ex
 
 mode=$1
-readonly cur=$(pwd)
-readonly core_path=$cur/build/visualdl/logic
-readonly python_path=$cur/visualdl/python
+readonly TOP_DIR=$(pwd)
+readonly core_path=$TOP_DIR/build/visualdl/logic
+readonly python_path=$TOP_DIR/visualdl/python
 readonly max_file_size=1000000 # 1MB
 
 export PYTHONPATH="${core_path}:${python_path}"
 
+# install the visualdl wheel first
+package() {
+    # some bug with frontend build
+    # a environment variable to skip frontend build
+    export VS_BUILD_MODE="travis-CI"
+
+    cd $TOP_DIR/visualdl/server
+    # manully install protobuf3
+    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
+    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
+    export PATH="$PATH:$(pwd)/protoc3/bin"
+    chmod +x protoc3/bin/*
+
+    sudo pip install numpy
+    sudo pip install Flask
+    sudo pip install Pillow
+    sudo pip install protobuf
+
+    #sudo apt-get install protobuf-compiler
+    cd $TOP_DIR
+    python setup.py bdist_wheel
+    sudo pip install dist/visualdl-0.0.1-py2-none-any.whl
+}
+
 backend_test() {
-    cd $cur
+    cd $TOP_DIR
     sudo pip install numpy
     sudo pip install Pillow
     mkdir -p build
@@ -21,7 +45,7 @@ backend_test() {
 }
 
 frontend_test() {
-    cd $cur
+    cd $TOP_DIR
     cd frontend
     npm install
     npm run build
@@ -31,29 +55,20 @@ server_test() {
     sudo pip install google
     sudo pip install protobuf==3.1.0
 
-    cd $cur/server
-    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
-    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
-    export PATH=$PATH:protoc3/bin
-    sudo chmod +x protoc3/bin/protoc
-    sudo chown `whoami` protoc3/bin/protoc
-
-    bash build.sh
-
-    cd visualdl
+    cd $TOP_DIR/visualdl/server
     bash graph_test.sh
 
-    cd $cur/server/visualdl
+    cd $TOP_DIR/visualdl/server
     python lib_test.py
 }
 
 # check the size of files in the repo.
 # reject PR that has some big data included.
 bigfile_reject() {
-    cd $cur
+    cd $TOP_DIR
     # it failed to exclude .git, remove it first.
     rm -rf .git
-    local largest_file=$(find . -path .git -prune -o -printf '%s %p\n' | sort -nr | head -n1)
+    local largest_file="$(find . -path .git -prune -o -printf '%s %p\n' | sort -nr | head -n1)"
     local size=$(echo $largest_file | awk '{print $1}')
     if [ "$size" -ge "$max_file_size" ]; then
         echo $largest_file
@@ -68,10 +83,16 @@ echo "mode" $mode
 if [ $mode = "backend" ]; then
     backend_test
 elif [ $mode = "all" ]; then
+    # bigfile_reject should be tested first, or some files downloaded may fail this test.
     bigfile_reject
+    package
     frontend_test
     backend_test
     server_test
+elif [ $mode = "local" ]; then
+    #frontend_test
+    backend_test
+    server_test
 else
     frontend_test
 fi

visualdl/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+
+from .python.storage import *

visualdl/python/__init__.py

@@ -1 +0,0 @@
-from storage import *

visualdl/python/storage.py

@@ -1,8 +1,6 @@
-__all__ = [
-    'StorageReader',
-    'StorageWriter',
-]
-import core
+from __future__ import absolute_import
+
+from visualdl import core
 
 dtypes = ("float", "double", "int32", "int64")
 

visualdl/python/test_storage.py

@@ -5,13 +5,16 @@
 import numpy as np
 from PIL import Image
 
-import storage
+import sys, pprint
+pprint.pprint(sys.path)
+
+from visualdl import LogWriter, LogReader
 
 
 class StorageTest(unittest.TestCase):
     def setUp(self):
         self.dir = "./tmp/storage_test"
-        self.writer = storage.LogWriter(
+        self.writer = LogWriter(
             self.dir, sync_cycle=1).as_mode("train")
 
     def test_scalar(self):
@@ -22,7 +25,7 @@ def test_scalar(self):
             scalar.add_record(i, float(i))
 
         print 'test read'
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             scalar = reader.scalar("model/scalar/min")
             self.assertEqual(scalar.caption(), "train")
@@ -50,7 +53,7 @@ def test_image(self):
                     image_writer.set_sample(index, shape, list(data))
             image_writer.finish_sampling()
 
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             image_reader = reader.image(tag)
             self.assertEqual(image_reader.caption(), tag)
@@ -77,7 +80,7 @@ def test_check_image(self):
         shape = [image.size[1], image.size[0], 3]
         origin_data = np.array(image.getdata()).flatten()
 
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
 
             image_writer.start_sampling()
@@ -110,14 +113,14 @@ def test_with_syntax(self):
             for i in range(10):
                 scalar.add_record(i, float(i))
 
-        self.reader = storage.LogReader(self.dir)
+        self.reader = LogReader(self.dir)
         with self.reader.mode("train") as reader:
             scalar = reader.scalar("model/scalar/average")
             self.assertEqual(scalar.caption(), "train")
 
     def test_modes(self):
         dir = "./tmp/storagetest0"
-        store = storage.LogWriter(
+        store = LogWriter(
             self.dir, sync_cycle=1)
 
         scalars = []