Merge master into wasm (#1)

* fix(user_db): unwanted implicit instantiation of UserDbFormat template

Fixes rime#188: UserDbFormat<BaseDb>::extension not working

* fix(config_compiler): "/" mistaken as path separator in merged map key (rime#192)

Fixes rime#190

* fix(ConfigFileUpdate): no need to create user build if shared build is up-to-date

* fix(SchemaUpdate): read compiled schema from shared build if there is no user build

* chore(release tag): deprecating tag name prefix 'rime-' in favor of conventional 'v'

BREAKING CHANGE: After 1.3.0 release, we'll no longer be creating tags in the format 'rime-X.Y.Z'. Downstream packagers please change automated scripts accordingly.

* chore(CMakeLists.txt): bump version to 1.3.0

* chore(release): 1.3.0 🎉

* chore(tags): adopt semver "X.Y.Z" without prefix  [ci skip]

* fix(config_file_update): clean up deprecated user copy (rime#193)

* fix(config_file_update): trash deprecated user copy created by older rime version

* fix(config_file_update): create trash directory when needed to trash config files

* fix(config_file_update): prefer rime-installed user copy to shared minimal version if numbers match'

* fix(thirdparty/src/leveldb): do not link to snappy library

* chore(bump-version.sh): npm version script

* chore(release): 1.3.1 🎉

* docs(README): replace rime/brise with rime/plum

* Add ENABLE_ASAN option

This enable the Address Sanitizer memory detect when turns on, make us
easy to find out the memory problems.

* Fix a heap-use-after-free error found by asan

ctx->composition().Forward() might invalidated the iterator when did a
push_back, we can't use `seg` anymore.

* chore(engine.cc): Google code style; more informative name

* chore(REAME.md): require boost>=1.48, for boost::locale  [ci skip]

* fix(config_compiler): support creating list in-place by __patch and __merge

* fix(CMakeLists.txt): do not link binaries when building static library

* refactor(editor): extract helper class key_binding_processor

* refactor(navigator): key bindings

* fix(config_compiler): ambiguous operator overload with cmake option ENABLE_LOGGING=OFF

Fixes rime#211

* feat(language): shared user dictionary per language (Closes rime#184) (rime#214)

* fix(table_translator): enable encoding uniquified commit history

* feat: always_show_comments option (rime#220)

* feat: add always_show_comments option

* fix: address feedback
Line wraps

.npmrc

@@ -1,2 +1,2 @@
-tag-version-prefix="rime-"
+tag-version-prefix=""
 message="chore(release): %s :tada:"

CHANGELOG.md

@@ -1,3 +1,39 @@
+<a name="1.3.1"></a>
+## [1.3.1](https://github.com/rime/librime/compare/1.3.0...1.3.1) (2018-04-01)
+
+
+### Bug Fixes
+
+* **config_file_update:** clean up deprecated user copy ([#193](https://github.com/rime/librime/issues/193)) ([8d8d2e6](https://github.com/rime/librime/commit/8d8d2e6))
+* **thirdparty/src/leveldb:** do not link to snappy library ([6f6056a](https://github.com/rime/librime/commit/6f6056a))
+
+
+
+<a name="1.3.0"></a>
+# 1.3.0 (2018-03-09)
+
+
+### Bug Fixes
+
+* **CMakeLists.txt, build.bat:** install header files (public API) ([06c9e86](https://github.com/rime/librime/commit/06c9e86))
+* **config_compiler:** "/" mistaken as path separator in merged map key ([#192](https://github.com/rime/librime/issues/192)) ([831ffba](https://github.com/rime/librime/commit/831ffba)), closes [#190](https://github.com/rime/librime/issues/190)
+* **ConfigFileUpdate:** no need to create user build if shared build is up-to-date ([cafd5c4](https://github.com/rime/librime/commit/cafd5c4))
+* **SchemaUpdate:** read compiled schema from shared build if there is no user build ([45a04dd](https://github.com/rime/librime/commit/45a04dd))
+* **simplifier:** fix typo ([9e1114e](https://github.com/rime/librime/commit/9e1114e)), closes [#183](https://github.com/rime/librime/issues/183)
+* **user_db:** unwanted implicit instantiation of UserDbFormat template ([3cbc9cb](https://github.com/rime/librime/commit/3cbc9cb)), closes [#188](https://github.com/rime/librime/issues/188)
+
+
+### Chores
+
+* **release tag:** deprecating tag name prefix 'rime-' in favor of semver 'X.Y.Z'
+
+
+### BREAKING CHANGES
+
+* **release tag:** After 1.3.0 release, we'll no longer be creating tags in the format 'rime-X.Y.Z'. Downstream packagers please change automated scripts accordingly.
+
+
+
 <a name="1.2.10"></a>
 ## 1.2.10 (2018-02-21)
 

CMakeLists.txt

@@ -4,7 +4,7 @@ set(CMAKE_USER_MAKE_RULES_OVERRIDE_CXX ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cxx_fla
 project(rime)
 cmake_minimum_required(VERSION 2.8.11)
 
-set(rime_version 1.2.10)
+set(rime_version 1.3.1)
 set(rime_soversion 1)
 
 add_definitions(-DRIME_VERSION="${rime_version}")
@@ -17,8 +17,9 @@ option(BUILD_SEPARATE_LIBS "Build a separate rime-gears library" OFF)
 option(ENABLE_LOGGING "Enable logging with google-glog library" ON)
 option(BOOST_USE_CXX11 "Boost has been built with C++11 support" OFF)
 option(BOOST_USE_SIGNALS2 "Boost use signals2 instead of signals" ON)
+option(ENABLE_ASAN "Enable Address Sanitizer (Unix Only)" OFF)
 
-SET(rime_data_dir "/share/rime-data" CACHE STRING "Target directory for Rime data")
+set(rime_data_dir "/share/rime-data" CACHE STRING "Target directory for Rime data")
 
 if(WIN32)
   set(ext ".exe")
@@ -27,6 +28,15 @@ endif(WIN32)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${PROJECT_SOURCE_DIR}/cmake")
 set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} "${PROJECT_SOURCE_DIR}/thirdparty")
 
+if (ENABLE_ASAN)
+  set(asan_cflags "-fsanitize=address -fno-omit-frame-pointer")
+  set(asan_lflags "-fsanitize=address -lasan")
+  set(CMAKE_C_FLAGS "${asan_cflags} ${CMAKE_C_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${asan_cflags} ${CMAKE_CXX_FLAGS}")
+  set(CMAKE_EXE_LINKER_FLAGS "${asan_lflags} ${CMAKE_EXE_LINKER_FLAGS}")
+  set(CMAKE_SHARED_LINKER_FLAGS "${asan_lflags} ${CMAKE_SHARED_LINKER_FLAGS}")
+endif()
+
 set(Boost_USE_STATIC_LIBS ${BUILD_STATIC})
 set(Gflags_STATIC ${BUILD_STATIC})
 set(Glog_STATIC ${BUILD_STATIC})
@@ -186,8 +196,11 @@ else()
 endif()
 
 add_subdirectory(src)
-add_subdirectory(tools)
 
-if(GTEST_FOUND)
-  add_subdirectory(test)
+if(BUILD_SHARED_LIBS)
+  add_subdirectory(tools)
+
+  if(GTEST_FOUND)
+    add_subdirectory(test)
+  endif()
 endif()

README.md

@@ -34,7 +34,7 @@ Build dependencies
 ---
   - compiler with C++11 support
   - cmake>=2.8
-  - libboost>=1.46
+  - libboost>=1.48
   - libglog (optional)
   - libleveldb
   - libmarisa
@@ -79,10 +79,9 @@ Plugins
 
 Related works
 ===
-  - [brise](https://github.com/rime/brise): Rime schema repository
-  - Combo Pinyin: an innovative chord-typing practice to input Pinyin
+  - [plum](https://github.com/rime/plum): Rime configuration manager and input schema repository
+  - [Combo Pinyin](https://github.com/rime/home/wiki/ComboPinyin): an innovative chord-typing practice to input Pinyin
   - essay: the vocabulary and language model for Rime
-  - [rimekit](https://github.com/lotem/rimekit): configuration tools for Rime (under construction)
   - [SCU](https://github.com/neolee/SCU/): Squirrel Configuration Utilities
 
 Credits

bump-version.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+version=$(node -p 'require("./package.json").version')
+
+sed -i'~' 's/set(\(rime_version\) .*)/set(\1 '$version')/' CMakeLists.txt
+rm 'CMakeLists.txt~'
+git add CMakeLists.txt
+
+conventional-changelog -p angular -i CHANGELOG.md -s
+git add CHANGELOG.md

data/test/config_merge_test.yaml

@@ -46,3 +46,11 @@ merge_tree:
   zerg:
     # overwrite existing list
     ground_units: []
+
+create_list_with_inplace_patch:
+  # map node without data key-value (exclude compiler directives) can be converted to list
+  all_ground_units:
+    __patch:
+      - __append: [scv, marine, firebat, vulture, tank]
+      - __append: {__include: starcraft/protoss/ground_units}
+      - __append: {__include: starcraft/zerg/ground_units}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "librime",
-  "version": "1.2.10",
+  "version": "1.3.1",
   "description": "Rime Input Method Engine",
   "main": "index.js",
   "directories": {
@@ -9,7 +9,7 @@
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "version": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md"
+    "version": "./bump-version.sh"
   },
   "repository": {
     "type": "git",

src/rime/common.h

@@ -37,6 +37,8 @@
 // call a pointer to member function on this
 #define RIME_THIS_CALL(f) (this->*(f))
 
+#define RIME_THIS_CALL_AS(T, f) ((T*)this->*(f))
+
 namespace rime {
 
 using std::function;

src/rime/config/config_compiler.cc

@@ -9,6 +9,14 @@
 
 namespace rime {
 
+std::ostream& operator<< (std::ostream& stream, const Reference& reference) {
+  return stream << reference.repr();
+}
+
+std::ostream& operator<< (std::ostream& stream, const Dependency& dependency) {
+  return stream << dependency.repr();
+}
+
 struct ConfigDependencyGraph {
   map<string, of<ConfigResource>> resources;
   vector<of<ConfigItemRef>> node_stack;
@@ -97,8 +105,13 @@ static bool AppendToList(an<ConfigItemRef> target, an<ConfigList> list) {
     return false;
   auto existing_list = As<ConfigList>(**target);
   if (!existing_list) {
-    LOG(ERROR) << "trying to append list to other value";
-    return false;
+    if (!(**target)->empty()) {
+      LOG(ERROR) << "trying to append list to incompatible node type";
+      return false;
+    }
+    // convert empty node (usually map with only compiler directives) to list;
+    // refer to test case RimeConfigMergeTest.CreateListWithInplacePatch
+    existing_list = target->AsList();
   }
   if (list->empty())
     return true;
@@ -155,33 +168,39 @@ inline static string StripOperator(const string& key, bool adding) {
 }
 
 // defined in config_data.cc
-bool TraverseCopyOnWrite(an<ConfigItemRef> root, const string& path,
-                         function<bool (an<ConfigItemRef> target)> writer);
+an<ConfigItemRef> TypeCheckedCopyOnWrite(an<ConfigItemRef> parent,
+                                         const string& key);
+an<ConfigItemRef> TraverseCopyOnWrite(an<ConfigItemRef> head,
+                                      const string& path);
 
-static bool EditNode(an<ConfigItemRef> target,
+static bool EditNode(an<ConfigItemRef> head,
                      const string& key,
                      const an<ConfigItem>& value,
                      bool merge_tree) {
-  DLOG(INFO) << "EditNode(" << key << "," << merge_tree << ")";
+  DLOG(INFO) << "edit node: " << key << ", merge_tree: " << merge_tree;
   bool appending = IsAppending(key);
   bool merging = IsMerging(key, value, merge_tree);
-  auto writer = [=](an<ConfigItemRef> target) {
-    if ((appending || merging) && **target) {
-      DLOG(INFO) << "writer: editing node";
-      return !value ||
-      (appending && (AppendToString(target, As<ConfigValue>(value)) ||
-                     AppendToList(target, As<ConfigList>(value)))) ||
-      (merging && MergeTree(target, As<ConfigMap>(value)));
-    } else {
-      DLOG(INFO) << "writer: overwriting node";
-      *target = value;
-      return true;
-    }
-  };
   string path = StripOperator(key, appending || merging);
   DLOG(INFO) << "appending: " << appending << ", merging: " << merging
              << ", path: " << path;
-  return TraverseCopyOnWrite(target, path, writer);
+  auto find_target_node =
+      merge_tree ? &TypeCheckedCopyOnWrite : &TraverseCopyOnWrite;
+  auto target = find_target_node(head, path);
+  if (!target) {
+    // error finding target node; cannot write
+    return false;
+  }
+  if ((appending || merging) && **target) {
+    DLOG(INFO) << "writer: editing node";
+    return !value ||  // no-op
+        (appending && (AppendToString(target, As<ConfigValue>(value)) ||
+                       AppendToList(target, As<ConfigList>(value)))) ||
+        (merging && MergeTree(target, As<ConfigMap>(value)));
+  } else {
+    DLOG(INFO) << "writer: overwriting node";
+    *target = value;
+    return true;
+  }
 }
 
 bool PatchLiteral::Resolve(ConfigCompiler* compiler) {

src/rime/config/config_compiler.h

@@ -5,6 +5,7 @@
 #ifndef RIME_CONFIG_COMPILER_H_
 #define RIME_CONFIG_COMPILER_H_
 
+#include <ostream>
 #include <rime/common.h>
 #include <rime/config/config_data.h>
 #include <rime/config/config_types.h>
@@ -35,10 +36,7 @@ struct Reference {
   string repr() const;
 };
 
-template <class StreamT>
-StreamT& operator<< (StreamT& stream, const Reference& reference) {
-  return stream << reference.repr();
-}
+std::ostream& operator<< (std::ostream& stream, const Reference& reference);
 
 class ConfigCompilerPlugin;
 class ResourceResolver;

src/rime/config/config_compiler_impl.h

@@ -5,6 +5,7 @@
 #ifndef RIME_CONFIG_COMPILER_IMPL_H_
 #define RIME_CONFIG_COMPILER_IMPL_H_
 
+#include <ostream>
 #include <rime/common.h>
 #include <rime/config/config_compiler.h>
 #include <rime/config/config_types.h>
@@ -32,10 +33,7 @@ struct Dependency {
   virtual bool Resolve(ConfigCompiler* compiler) = 0;
 };
 
-template <class StreamT>
-StreamT& operator<< (StreamT& stream, const Dependency& dependency) {
-  return stream << dependency.repr();
-}
+std::ostream& operator<< (std::ostream& stream, const Dependency& dependency);
 
 struct PendingChild : Dependency {
   string child_path;

src/rime/config/config_data.cc

@@ -159,37 +159,52 @@ class ConfigDataRootRef : public ConfigItemRef {
   ConfigData* data_;
 };
 
-bool TraverseCopyOnWrite(an<ConfigItemRef> root, const string& path,
-                         function<bool (an<ConfigItemRef> target)> writer) {
+an<ConfigItemRef> TypeCheckedCopyOnWrite(an<ConfigItemRef> parent,
+                                         const string& key) {
+  // special case to allow editing current node by __append: __merge: /+: /=:
+  if (key.empty()) {
+    return parent;
+  }
+  bool is_list = ConfigData::IsListItemReference(key);
+  auto expected_node_type = is_list ? ConfigItem::kList : ConfigItem::kMap;
+  an<ConfigItem> existing_node = *parent;
+  if (existing_node && existing_node->type() != expected_node_type) {
+    LOG(ERROR) << "copy on write failed; incompatible node type: " << key;
+    return nullptr;
+  }
+  return Cow(parent, key);
+}
+
+an<ConfigItemRef> TraverseCopyOnWrite(an<ConfigItemRef> head,
+                                      const string& path) {
   DLOG(INFO) << "TraverseCopyOnWrite(" << path << ")";
   if (path.empty() || path == "/") {
-    return writer(root);
+    return head;
   }
-  an<ConfigItemRef> head = root;
   vector<string> keys = ConfigData::SplitPath(path);
   size_t n = keys.size();
   for (size_t i = 0; i < n; ++i) {
     const auto& key = keys[i];
-    bool is_list = ConfigData::IsListItemReference(key);
-    auto expected_node_type = is_list ? ConfigItem::kList : ConfigItem::kMap;
-    an<ConfigItem> existing_node = *head;
-    if (existing_node && existing_node->type() != expected_node_type) {
-      LOG(ERROR) << "copy on write failed; incompatible node type: " << key;
-      return false;
+    if (auto child = TypeCheckedCopyOnWrite(head, key)) {
+      head = child;
+    } else {
+      LOG(ERROR) << "while writing to " << path;
+      return nullptr;
     }
-    head = Cow(head, key);
   }
-  return writer(head);
+  return head;
 }
 
 bool ConfigData::TraverseWrite(const string& path, an<ConfigItem> item) {
   LOG(INFO) << "write: " << path;
   auto root = New<ConfigDataRootRef>(this);
-  return TraverseCopyOnWrite(root, path, [=](an<ConfigItemRef> target) {
+  if (auto target = TraverseCopyOnWrite(root, path)) {
     *target = item;
     set_modified();
     return true;
-  });
+  } else {
+    return false;
+  }
 }
 
 vector<string> ConfigData::SplitPath(const string& path) {

src/rime/dict/level_db.cc

@@ -345,10 +345,14 @@ bool LevelDb::CommitTransaction() {
 }
 
 template <>
-const string UserDbFormat<LevelDb>::extension(".userdb");
+string UserDbComponent<LevelDb>::extension() const {
+  return ".userdb";
+}
 
 template <>
-const string UserDbFormat<LevelDb>::snapshot_extension(".userdb.txt");
+string UserDbComponent<LevelDb>::snapshot_extension() const {
+  return ".userdb.txt";
+}
 
 template <>
 UserDbWrapper<LevelDb>::UserDbWrapper(const string& db_name)