Merge pull request #13 from ChunweiYan/feature/add_image_component

CMakeLists.txt

@@ -23,27 +23,30 @@ include(external/zlib)      # download, build, install zlib
 include(external/gflags)    # download, build, install gflags
 include(external/glog)      # download, build, install glog
 include(external/gtest)     # download, build, install gtest
+include(external/eigen)     # download eigen
 include(external/pybind11)  # download pybind11
 include(external/protobuf)  # download, build, install protobuf
 include(external/python)    # find python and set path
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
-include_directories(${PROJECT_SOURCE_DIR}/thirdparty/local/include)
+# TODO(ChunweiYan) debug, remote latter
+#include_directories(/home/superjom/project/VisualDL/build/third_party/eigen3/src/extern_eigen3)
 
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/storage)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/logic)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/visualdl/python)
 
 add_executable(vl_test
         ${PROJECT_SOURCE_DIR}/visualdl/test.cc
-        ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/logic/sdk_test.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/storage/storage_test.cc
         ${PROJECT_SOURCE_DIR}/visualdl/utils/test_concurrency.cc
+        ${PROJECT_SOURCE_DIR}/visualdl/utils/test_image.cc
         ${PROJECT_SOURCE_DIR}/visualdl/utils/concurrency.h
         ${PROJECT_SOURCE_DIR}/visualdl/utils/filesystem.h
         )
-target_link_libraries(vl_test sdk storage entry im gtest glog protobuf gflags pthread)
+target_link_libraries(vl_test sdk storage entry tablet im gtest glog protobuf gflags pthread)
 
 enable_testing ()
 

cmake/external/eigen.cmake

@@ -0,0 +1,30 @@
+INCLUDE(ExternalProject)
+
+SET(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3)
+
+INCLUDE_DIRECTORIES(${EIGEN_SOURCE_DIR}/src/extern_eigen3)
+
+ExternalProject_Add(
+    extern_eigen3
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    GIT_REPOSITORY  "https://github.com/RLovelett/eigen.git"
+    GIT_TAG         "master"
+    PREFIX          ${EIGEN_SOURCE_DIR}
+    UPDATE_COMMAND  ""
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND     ""
+    INSTALL_COMMAND   ""
+    TEST_COMMAND      ""
+)
+
+if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
+    set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/eigen3_dummy.c)
+    file(WRITE ${dummyfile} "const char * dummy_eigen3 = \"${dummyfile}\";")
+    add_library(eigen3 STATIC ${dummyfile})
+else()
+    add_library(eigen3 INTERFACE)
+endif()
+
+add_dependencies(eigen3 extern_eigen3)
+
+LIST(APPEND external_project_dependencies eigen3)

server/visualdl/lib.py

@@ -0,0 +1,124 @@
+import pprint
+import re
+import urllib
+from tempfile import NamedTemporaryFile
+
+import numpy as np
+from PIL import Image
+
+import storage
+
+
+def get_modes(storage):
+    return storage.modes()
+
+
+def get_scalar_tags(storage, mode):
+    result = {}
+    for mode in storage.modes():
+        reader = storage.as_mode(mode)
+        tags = reader.tags('scalar')
+        if tags:
+            result[mode] = {}
+            for tag in tags:
+                result[mode][tag] = {
+                    'displayName': reader.scalar(tag).caption(),
+                    'description': "",
+                }
+    return result
+
+
+def get_scalar(storage, mode, tag):
+    reader = storage.as_mode(mode)
+    scalar = reader.scalar(tag)
+
+    records = scalar.records()
+    ids = scalar.ids()
+    timestamps = scalar.timestamps()
+
+    result = zip(timestamps, ids, records)
+    return result
+
+
+def get_image_tags(storage):
+    result = {}
+
+    for mode in storage.modes():
+        reader = storage.as_mode(mode)
+        tags = reader.tags('image')
+        if tags:
+            result[mode] = {}
+            for tag in tags:
+                image = reader.image(tag)
+                for i in xrange(max(1, image.num_samples())):
+                    caption = tag if image.num_samples() <= 1 else '%s/%d'%(tag, i)
+                    result[mode][caption] = {
+                        'displayName': caption,
+                        'description': "",
+                        'samples': 1,
+                    }
+    return result
+
+
+def get_image_tag_steps(storage, mode, tag):
+    print 'image_tag_steps,mode,tag:', mode, tag
+    # remove suffix '/x'
+    res = re.search(r".*/([0-9]+$)", tag)
+    sample_index = 0
+    origin_tag = tag
+    if res:
+        tag = tag[:tag.rfind('/')]
+        sample_index = int(res.groups()[0])
+
+    reader = storage.as_mode(mode)
+    image = reader.image(tag)
+    res = []
+
+    for step_index in range(image.num_records()):
+        record = image.record(step_index, sample_index)
+        shape = record.shape()
+        assert shape, "%s,%s" % (mode, tag)
+        query = urllib.urlencode({
+            'sample': 0,
+            'index': step_index,
+            'tag': origin_tag,
+            'run': mode,
+        })
+        res.append({
+            'height': shape[0],
+            'width': shape[1],
+            'step': record.step_id(),
+            'wall_time': image.timestamp(step_index),
+            'query': query,
+        })
+    return res
+
+
+def get_invididual_image(storage, mode, tag, step_index):
+    reader = storage.as_mode(mode)
+    res = re.search(r".*/([0-9]+$)", tag)
+    # remove suffix '/x'
+    if res:
+        offset = int(res.groups()[0])
+        tag = tag[:tag.rfind('/')]
+
+    image = reader.image(tag)
+    record = image.record(step_index, offset)
+
+    data = np.array(record.data(), dtype='uint8').reshape(record.shape())
+    tempfile = NamedTemporaryFile(mode='w+b', suffix='.png')
+    with Image.fromarray(data) as im:
+        im.save(tempfile)
+    tempfile.seek(0, 0)
+    return tempfile
+
+
+if __name__ == '__main__':
+    reader = storage.StorageReader('./tmp/mock')
+    tags = get_image_tags(reader)
+
+    tags = get_image_tag_steps(reader, 'train', 'layer1/layer2/image0/0')
+    pprint.pprint(tags)
+
+    image = get_invididual_image(reader, "train", 'layer1/layer2/image0/0', 2)
+    print image

server/visualdl/lib_test.py

@@ -0,0 +1,54 @@
+import lib
+import unittest
+import storage
+import pprint
+from storage_mock import add_scalar, add_image
+
+
+class LibTest(unittest.TestCase):
+    def setUp(self):
+        dir = "./tmp/mock"
+        writer = storage.StorageWriter(dir, sync_cycle=20)
+
+        add_scalar(writer, "train", "layer/scalar0/min", 1000, 1)
+        add_scalar(writer, "test", "layer/scalar0/min", 1000, 10)
+        add_scalar(writer, "valid", "layer/scalar0/min", 1000, 10)
+
+        add_scalar(writer, "train", "layer/scalar0/max", 1000, 1)
+        add_scalar(writer, "test", "layer/scalar0/max", 1000, 10)
+        add_scalar(writer, "valid", "layer/scalar0/max", 1000, 10)
+
+        add_image(writer, "train", "layer/image0", 7, 10, 1)
+        add_image(writer, "test", "layer/image0", 7, 10, 3)
+
+        add_image(writer, "train", "layer/image1", 7, 10, 1, shape=[30,30,2])
+        add_image(writer, "test", "layer/image1", 7, 10, 1, shape=[30,30,2])
+
+        self.reader = storage.StorageReader(dir)
+
+    def test_modes(self):
+        modes = lib.get_modes(self.reader)
+        self.assertEqual(sorted(modes), sorted(["train", "test", "valid"]))
+
+    def test_scalar(self):
+
+        for mode in "train test valid".split():
+            tags = lib.get_scalar_tags(self.reader, mode)
+            print 'scalar tags:'
+            pprint.pprint(tags)
+            self.assertEqual(len(tags), 3)
+            self.assertEqual(sorted(tags.keys()), sorted("train test valid".split()))
+
+    def test_image(self):
+        tags = lib.get_image_tags(self.reader)
+        self.assertEqual(len(tags), 2)
+
+        tags = lib.get_image_tag_steps(self.reader, 'train', 'layer/image0/0')
+        pprint.pprint(tags)
+
+        image = lib.get_invididual_image(self.reader, "train", 'layer/image0/0', 2)
+        print image
+
+
+if __name__ == '__main__':
+    unittest.main()

server/visualdl/mock.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -ex
+
+export PYTHONPATH="/home/superjom/project/VisualDL/build/visualdl/logic:/home/superjom/project/VisualDL/visualdl/python"
+
+python lib_test.py

server/visualdl/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+
+export PYTHONPATH="$(pwd)/..:/home/superjom/project/VisualDL/build/visualdl/logic:/home/superjom/project/VisualDL/visualdl/python"
+export FLASK_APP=visual_dl.py
+export FLASK_DEBUG=1
+
+python visual_dl.py --logdir ./tmp/mock --host 172.23.233.68 --port 8043

server/visualdl/storage_mock.py

@@ -0,0 +1,49 @@
+import random
+import time
+import unittest
+
+import numpy as np
+
+
+def add_scalar(writer, mode, tag, num_steps, skip):
+    my_writer = writer.as_mode(mode)
+    scalar = my_writer.scalar(tag)
+    for i in range(num_steps):
+        if i % skip == 0:
+            scalar.add_record(i, random.random())
+
+
+def add_image(writer,
+              mode,
+              tag,
+              num_samples,
+              num_passes,
+              step_cycle,
+              shape=[50, 50, 3]):
+    writer_ = writer.as_mode(mode)
+    image_writer = writer_.image(tag, num_samples, step_cycle)
+
+    for pass_ in xrange(num_passes):
+        image_writer.start_sampling()
+        for ins in xrange(2 * num_samples):
+            index = image_writer.is_sample_taken()
+            if index != -1:
+                data = np.random.random(shape) * 256
+                data = np.ndarray.flatten(data)
+                assert shape
+                assert len(data) > 0
+                image_writer.set_sample(index, shape, list(data))
+        image_writer.finish_sampling()
+
+
+if __name__ == '__main__':
+    add_scalar("train", "layer/scalar0/min", 1000, 1)
+    add_scalar("test", "layer/scalar0/min", 1000, 10)
+    add_scalar("valid", "layer/scalar0/min", 1000, 10)
+
+    add_scalar("train", "layer/scalar0/max", 1000, 1)
+    add_scalar("test", "layer/scalar0/max", 1000, 10)
+    add_scalar("valid", "layer/scalar0/max", 1000, 10)
+
+    add_image("train", "layer/image0", 7, 10, 1)
+    add_image("test", "layer/image0", 7, 10, 3)