From 85b945fcc2321890b82b3ad6aebb4dc17a288cb6 Mon Sep 17 00:00:00 2001 From: Chen Gong Date: Mon, 4 Sep 2017 21:54:36 +0800 Subject: [PATCH 1/6] feat(dict): no conditional compilation on arm --- src/rime/dict/mapped_file.h | 4 ---- src/rime/dict/table.h | 16 ++++++---------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/rime/dict/mapped_file.h b/src/rime/dict/mapped_file.h index 71a9fdcc4..a79a41d34 100644 --- a/src/rime/dict/mapped_file.h +++ b/src/rime/dict/mapped_file.h @@ -131,11 +131,7 @@ class MappedFile : boost::noncopyable { // member function definitions -# if defined(__arm__) # define RIME_ALIGNED(size, T) ((size + alignof(T) - 1) & ~(alignof(T) - 1)) -# else -# define RIME_ALIGNED(size, T) (size) -# endif template T* MappedFile::Allocate(size_t count) { diff --git a/src/rime/dict/table.h b/src/rime/dict/table.h index c4d4e3a2f..013ee61bf 100644 --- a/src/rime/dict/table.h +++ b/src/rime/dict/table.h @@ -15,13 +15,13 @@ #include #include -#define RIME_TABLE_UNION(U, V, Ta, a, Tb, b) \ +#define RIME_TABLE_UNION(U, V, A, a, B, b) \ struct U { \ V value; \ - const Ta& a() const { return *reinterpret_cast(this); } \ - const Tb& b() const { return *reinterpret_cast(this); } \ - Ta& a() { return *reinterpret_cast(this); } \ - Tb& b() { return *reinterpret_cast(this); } \ + const A& a() const { return *reinterpret_cast(this); } \ + const B& b() const { return *reinterpret_cast(this); } \ + A& a() { return *reinterpret_cast(this); } \ + B& b() { return *reinterpret_cast(this); } \ } namespace rime { @@ -40,11 +40,7 @@ using Syllabary = Array; using Code = List; -#if defined(__arm__) using Weight = double; -#else -using Weight = float; -#endif struct Entry { StringType text; @@ -75,7 +71,7 @@ using TrunkIndex = Array; using TailIndex = Array; -//union Phraseindex { +//union PhraseIndex { // TrunkIndex trunk; // TailIndex tail; //}; From fdf817282a46dd1635cea3b1e9182b2b5e004a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E6=88=8E=E6=B0=8F?= Date: Wed, 4 Oct 2017 15:41:35 +0800 Subject: [PATCH 2/6] chore(dict): bump format version; drop compatability with older format --- src/rime/dict/prism.cc | 2 +- src/rime/dict/reverse_lookup_dictionary.cc | 2 +- src/rime/dict/table.cc | 79 ++++++++-------------- src/rime/dict/table.h | 30 ++------ 4 files changed, 36 insertions(+), 77 deletions(-) diff --git a/src/rime/dict/prism.cc b/src/rime/dict/prism.cc index d0be61cb3..dbd4b10f3 100644 --- a/src/rime/dict/prism.cc +++ b/src/rime/dict/prism.cc @@ -22,7 +22,7 @@ struct node_t { } // namespace -const char kPrismFormat[] = "Rime::Prism/1.0"; +const char kPrismFormat[] = "Rime::Prism/2.0"; const char kPrismFormatPrefix[] = "Rime::Prism/"; const size_t kPrismFormatPrefixLen = sizeof(kPrismFormatPrefix) - 1; diff --git a/src/rime/dict/reverse_lookup_dictionary.cc b/src/rime/dict/reverse_lookup_dictionary.cc index 1565f2134..64f5926b1 100644 --- a/src/rime/dict/reverse_lookup_dictionary.cc +++ b/src/rime/dict/reverse_lookup_dictionary.cc @@ -17,7 +17,7 @@ namespace rime { -const char kReverseFormat[] = "Rime::Reverse/1.0"; +const char kReverseFormat[] = "Rime::Reverse/2.0"; const char kReverseFormatPrefix[] = "Rime::Reverse/"; const size_t kReverseFormatPrefixLen = sizeof(kReverseFormatPrefix) - 1; diff --git a/src/rime/dict/table.cc b/src/rime/dict/table.cc index 03fa5d84e..21c080457 100644 --- a/src/rime/dict/table.cc +++ b/src/rime/dict/table.cc @@ -15,8 +15,8 @@ namespace rime { -const char kTableFormat_v1[] = "Rime::Table/1.0"; -const char kTableFormat_v2[] = "Rime::Table/2.0"; +const char kTableFormatLatest[] = "Rime::Table/3.0"; +int kTableFormatLowestCompatible = 3.0; const char kTableFormatPrefix[] = "Rime::Table/"; const size_t kTableFormatPrefixLen = sizeof(kTableFormatPrefix) - 1; @@ -244,31 +244,31 @@ TableAccessor TableQuery::Access(SyllableId syllable_id, return TableAccessor(); } -string Table::GetString_v1(const table::StringType& x) { - return x.str().c_str(); -} +// string Table::GetString_v1(const table::StringType& x) { +// return x.str().c_str(); +// } -bool Table::AddString_v1(const string& src, table::StringType* dest, - double /*weight*/) { - return CopyString(src, &dest->str()); -} +// bool Table::AddString_v1(const string& src, table::StringType* dest, +// double /*weight*/) { +// return CopyString(src, &dest->str()); +// } -string Table::GetString_v2(const table::StringType& x) { +string Table::GetString(const table::StringType& x) { return string_table_->GetString(x.str_id()); } -bool Table::AddString_v2(const string& src, table::StringType* dest, - double weight) { +bool Table::AddString(const string& src, table::StringType* dest, + double weight) { string_table_builder_->Add(src, weight, &dest->str_id()); return true; } -bool Table::OnBuildStart_v2() { +bool Table::OnBuildStart() { string_table_builder_.reset(new StringTableBuilder); return true; } -bool Table::OnBuildFinish_v2() { +bool Table::OnBuildFinish() { string_table_builder_->Build(); // saving string table image size_t image_size = string_table_builder_->BinarySize(); @@ -283,31 +283,12 @@ bool Table::OnBuildFinish_v2() { return true; } -bool Table::OnLoad_v2() { +bool Table::OnLoad() { string_table_.reset(new StringTable(metadata_->string_table.get(), metadata_->string_table_size)); return true; } -void Table::SelectTableFormat(double format_version) { - if (format_version > 2.0 - DBL_EPSILON) { - format_.format_name = kTableFormat_v2; - format_.GetString = &Table::GetString_v2; - format_.AddString = &Table::AddString_v2; - format_.OnBuildStart = &Table::OnBuildStart_v2; - format_.OnBuildFinish = &Table::OnBuildFinish_v2; - format_.OnLoad = &Table::OnLoad_v2; - } - else { - format_.format_name = kTableFormat_v1; - format_.GetString = &Table::GetString_v1; - format_.AddString = &Table::AddString_v1; - format_.OnBuildStart = nullptr; - format_.OnBuildFinish = nullptr; - format_.OnLoad = nullptr; - } -} - Table::Table(const string& file_name) : MappedFile(file_name) { } @@ -337,8 +318,12 @@ bool Table::Load() { return false; } double format_version = atof(&metadata_->format[kTableFormatPrefixLen]); - SelectTableFormat(format_version); - format_.format_name = metadata_->format; + if (format_version < kTableFormatLowestCompatible - DBL_EPSILON) { + LOG(ERROR) << "table format version " << format_version + << " is no longer supported. please upgrade to version " + << kTableFormatLatest; + return false; + } syllabary_ = metadata_->syllabary.get(); if (!syllabary_) { @@ -353,10 +338,7 @@ bool Table::Load() { return false; } - if (format_.OnLoad && !RIME_THIS_CALL(format_.OnLoad)()) { - return false; - } - return true; + return OnLoad(); } bool Table::Save() { @@ -376,8 +358,6 @@ uint32_t Table::dict_file_checksum() const { bool Table::Build(const Syllabary& syllabary, const Vocabulary& vocabulary, size_t num_entries, uint32_t dict_file_checksum) { - SelectTableFormat(2.0); - const size_t kReservedSize = 4096; size_t num_syllables = syllabary.size(); size_t estimated_file_size = kReservedSize + 32 * num_syllables + 64 * num_entries; @@ -400,7 +380,7 @@ bool Table::Build(const Syllabary& syllabary, const Vocabulary& vocabulary, metadata_->num_syllables = num_syllables; metadata_->num_entries = num_entries; - if (format_.OnBuildStart && !RIME_THIS_CALL(format_.OnBuildStart)()) { + if (!OnBuildStart()) { return false; } @@ -413,7 +393,7 @@ bool Table::Build(const Syllabary& syllabary, const Vocabulary& vocabulary, else { size_t i = 0; for (const string& syllable : syllabary) { - RIME_THIS_CALL(format_.AddString)(syllable, &syllabary_->at[i++], 0.0); + AddString(syllable, &syllabary_->at[i++], 0.0); } } metadata_->syllabary = syllabary_; @@ -426,12 +406,12 @@ bool Table::Build(const Syllabary& syllabary, const Vocabulary& vocabulary, } metadata_->index = index_; - if (format_.OnBuildFinish && !RIME_THIS_CALL(format_.OnBuildFinish)()) { + if (!OnBuildFinish()) { return false; } // at last, complete the metadata - std::strncpy(metadata_->format, format_.format_name, + std::strncpy(metadata_->format, kTableFormatLatest, table::Metadata::kFormatMaxLength); return true; } @@ -572,8 +552,7 @@ bool Table::BuildEntryList(const DictEntryList& src, bool Table::BuildEntry(const DictEntry& dict_entry, table::Entry* entry) { if (!entry) return false; - if (!RIME_THIS_CALL(format_.AddString)(dict_entry.text, &entry->text, - dict_entry.weight)) { + if (!AddString(dict_entry.text, &entry->text, dict_entry.weight)) { LOG(ERROR) << "Error creating table entry '" << dict_entry.text << "'; file size: " << file_size(); return false; @@ -595,7 +574,7 @@ string Table::GetSyllableById(SyllableId syllable_id) { syllable_id < 0 || syllable_id >= static_cast(syllabary_->size)) return string(); - return RIME_THIS_CALL(format_.GetString)(syllabary_->at[syllable_id]); + return GetString(syllabary_->at[syllable_id]); } TableAccessor Table::QueryWords(SyllableId syllable_id) { @@ -661,7 +640,7 @@ bool Table::Query(const SyllableGraph& syll_graph, size_t start_pos, } string Table::GetEntryText(const table::Entry& entry) { - return RIME_THIS_CALL(format_.GetString)(entry.text); + return GetString(entry.text); } } // namespace rime diff --git a/src/rime/dict/table.h b/src/rime/dict/table.h index 013ee61bf..ef177e689 100644 --- a/src/rime/dict/table.h +++ b/src/rime/dict/table.h @@ -170,38 +170,18 @@ class Table : public MappedFile { bool BuildEntryList(const DictEntryList& src, List* dest); bool BuildEntry(const DictEntry& dict_entry, table::Entry* entry); - string GetString_v1(const table::StringType& x); - bool AddString_v1(const string& src, table::StringType* dest, + string GetString(const table::StringType& x); + bool AddString(const string& src, table::StringType* dest, double weight); - - // v2 - string GetString_v2(const table::StringType& x); - bool AddString_v2(const string& src, table::StringType* dest, - double weight); - bool OnBuildStart_v2(); - bool OnBuildFinish_v2(); - bool OnLoad_v2(); - - void SelectTableFormat(double format_version); + bool OnBuildStart(); + bool OnBuildFinish(); + bool OnLoad(); protected: table::Metadata* metadata_ = nullptr; table::Syllabary* syllabary_ = nullptr; table::Index* index_ = nullptr; - struct TableFormat { - const char* format_name; - - string (Table::*GetString)(const table::StringType& x); - bool (Table::*AddString)(const string& src, table::StringType* dest, - double weight); - - bool (Table::*OnBuildStart)(); - bool (Table::*OnBuildFinish)(); - bool (Table::*OnLoad)(); - } format_; - - // v2 the string_table_; the string_table_builder_; }; From 8ea08b3e7d991ed1d90dec7d6f07ea8f6f90c8b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E6=88=8E=E6=B0=8F?= Date: Wed, 4 Oct 2017 15:42:07 +0800 Subject: [PATCH 3/6] feat(dict): use resource resolver to find dictionary files --- src/rime/dict/db.cc | 17 ++++++++------ src/rime/dict/dictionary.cc | 26 +++++++++++++++++----- src/rime/dict/dictionary.h | 5 +++++ src/rime/dict/preset_vocabulary.cc | 20 +++++++++-------- src/rime/dict/reverse_lookup_dictionary.cc | 25 ++++++++++----------- src/rime/dict/reverse_lookup_dictionary.h | 6 +++-- 6 files changed, 63 insertions(+), 36 deletions(-) diff --git a/src/rime/dict/db.cc b/src/rime/dict/db.cc index f7a4eaf52..314a03fad 100644 --- a/src/rime/dict/db.cc +++ b/src/rime/dict/db.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -20,15 +21,17 @@ bool DbAccessor::MatchesPrefix(const string& key) { // Db members +static const ResourceType kDbResourceType = { + "db", "", "" +}; + Db::Db(const string& name) : name_(name) { - boost::filesystem::path path(name); - if (path.has_parent_path()) { - file_name_ = name; - } - else { - boost::filesystem::path dir(Service::instance().deployer().user_data_dir); - file_name_ = (dir / path).string(); + static ResourceResolver db_resource_resolver(kDbResourceType); + if (db_resource_resolver.root_path().empty()) { + db_resource_resolver.set_root_path( + Service::instance().deployer().user_data_dir); } + file_name_ = db_resource_resolver.ResolvePath(name).string(); } bool Db::Exists() const { diff --git a/src/rime/dict/dictionary.cc b/src/rime/dict/dictionary.cc index e6f177044..ee29a057c 100644 --- a/src/rime/dict/dictionary.cc +++ b/src/rime/dict/dictionary.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -279,7 +280,22 @@ bool Dictionary::loaded() const { // DictionaryComponent members -DictionaryComponent::DictionaryComponent() { +static const ResourceType kPrismResourceType = { + "prism", "", ".prism.bin" +}; + +static const ResourceType kTableResourceType = { + "table", "", ".table.bin" +}; + +DictionaryComponent::DictionaryComponent() + : prism_resource_resolver_( + Service::instance().CreateResourceResolver(kPrismResourceType)), + table_resource_resolver_( + Service::instance().CreateResourceResolver(kTableResourceType)) { +} + +DictionaryComponent::~DictionaryComponent() { } Dictionary* DictionaryComponent::Create(const Ticket& ticket) { @@ -308,13 +324,13 @@ DictionaryComponent::CreateDictionaryWithName(const string& dict_name, boost::filesystem::path path(Service::instance().deployer().user_data_dir); auto table = table_map_[dict_name].lock(); if (!table) { - table = New((path / dict_name).string() + ".table.bin"); - table_map_[dict_name] = table; + auto file_path = table_resource_resolver_->ResolvePath(dict_name).string(); + table_map_[dict_name] = table = New
(file_path); } auto prism = prism_map_[prism_name].lock(); if (!prism) { - prism = New((path / prism_name).string() + ".prism.bin"); - prism_map_[prism_name] = prism; + auto file_path = prism_resource_resolver_->ResolvePath(prism_name).string(); + prism_map_[prism_name] = prism = New(file_path); } return new Dictionary(dict_name, table, prism); } diff --git a/src/rime/dict/dictionary.h b/src/rime/dict/dictionary.h index 3c52f7958..2851cf7b4 100644 --- a/src/rime/dict/dictionary.h +++ b/src/rime/dict/dictionary.h @@ -108,9 +108,12 @@ class Dictionary : public Class { an prism_; }; +class ResourceResolver; + class DictionaryComponent : public Dictionary::Component { public: DictionaryComponent(); + ~DictionaryComponent(); Dictionary* Create(const Ticket& ticket); Dictionary* CreateDictionaryWithName(const string& dict_name, const string& prism_name); @@ -118,6 +121,8 @@ class DictionaryComponent : public Dictionary::Component { private: map> prism_map_; map> table_map_; + the prism_resource_resolver_; + the table_resource_resolver_; }; } // namespace rime diff --git a/src/rime/dict/preset_vocabulary.cc b/src/rime/dict/preset_vocabulary.cc index 3d2d960b8..f1d6c759b 100644 --- a/src/rime/dict/preset_vocabulary.cc +++ b/src/rime/dict/preset_vocabulary.cc @@ -7,12 +7,19 @@ #include #include #include +#include #include #include #include namespace rime { +static const ResourceType kVocabularyResourceType = { + "vocabulary", "", ".txt" +}; + +static const string kDefaultVocabulary = "essay"; + struct VocabularyDb : public TextDb { explicit VocabularyDb(const string& path); an cursor; @@ -20,7 +27,7 @@ struct VocabularyDb : public TextDb { }; VocabularyDb::VocabularyDb(const string& path) - : TextDb(path, "vocabulary", VocabularyDb::format) { + : TextDb(path, kVocabularyResourceType.name, VocabularyDb::format) { } static bool rime_vocabulary_entry_parser(const Tsv& row, @@ -50,14 +57,9 @@ const TextFormat VocabularyDb::format = { }; string PresetVocabulary::DictFilePath() { - auto& deployer(Service::instance().deployer()); - boost::filesystem::path path(deployer.user_data_dir); - path /= "essay.txt"; - if (!boost::filesystem::exists(path)) { - path = deployer.shared_data_dir; - path /= "essay.txt"; - } - return path.string(); + the resource_resolver( + Service::instance().CreateResourceResolver(kVocabularyResourceType)); + return resource_resolver->ResolvePath(kDefaultVocabulary).string(); } PresetVocabulary::PresetVocabulary() { diff --git a/src/rime/dict/reverse_lookup_dictionary.cc b/src/rime/dict/reverse_lookup_dictionary.cc index 64f5926b1..c6404e090 100644 --- a/src/rime/dict/reverse_lookup_dictionary.cc +++ b/src/rime/dict/reverse_lookup_dictionary.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -24,13 +25,8 @@ const size_t kReverseFormatPrefixLen = sizeof(kReverseFormatPrefix) - 1; static const char* kStemKeySuffix = "\x1fstem"; -static string reverse_db_file_name(const string& dict_name) { - boost::filesystem::path dir(Service::instance().deployer().user_data_dir); - return (dir / dict_name).string() + ".reverse.bin"; -} - -ReverseDb::ReverseDb(const string& dict_name) - : MappedFile(reverse_db_file_name(dict_name)) { +ReverseDb::ReverseDb(const string& file_name) + : MappedFile(file_name) { } bool ReverseDb::Load() { @@ -200,10 +196,6 @@ ReverseLookupDictionary::ReverseLookupDictionary(an db) : db_(db) { } -ReverseLookupDictionary::ReverseLookupDictionary(const string& dict_name) - : db_(new ReverseDb(dict_name)) { -} - bool ReverseLookupDictionary::Load() { return db_ && (db_->IsOpen() || db_->Load()); } @@ -233,7 +225,13 @@ an ReverseLookupDictionary::GetDictSettings() { return settings; } -ReverseLookupDictionaryComponent::ReverseLookupDictionaryComponent() { +static const ResourceType kReverseDbResourceType = { + "reverse_db", "", ".reverse.bin" +}; + +ReverseLookupDictionaryComponent::ReverseLookupDictionaryComponent() + : resource_resolver_( + Service::instance().CreateResourceResolver(kReverseDbResourceType)) { } ReverseLookupDictionary* @@ -248,7 +246,8 @@ ReverseLookupDictionaryComponent::Create(const Ticket& ticket) { } auto db = db_pool_[dict_name].lock(); if (!db) { - db = New(dict_name); + auto file_path = resource_resolver_->ResolvePath(dict_name).string(); + db = New(file_path); db_pool_[dict_name] = db; } return new ReverseLookupDictionary(db); diff --git a/src/rime/dict/reverse_lookup_dictionary.h b/src/rime/dict/reverse_lookup_dictionary.h index 9efed2a01..8c328a972 100644 --- a/src/rime/dict/reverse_lookup_dictionary.h +++ b/src/rime/dict/reverse_lookup_dictionary.h @@ -38,7 +38,7 @@ class DictSettings; class ReverseDb : public MappedFile { public: - explicit ReverseDb(const string& dict_name); + explicit ReverseDb(const string& file_name); bool Load(); bool Lookup(const string& text, string* result); @@ -62,7 +62,6 @@ class ReverseLookupDictionary : public Class { public: explicit ReverseLookupDictionary(an db); - explicit ReverseLookupDictionary(const string& dict_name); bool Load(); bool ReverseLookup(const string& text, string* result); bool LookupStems(const string& text, string* result); @@ -72,6 +71,8 @@ class ReverseLookupDictionary an db_; }; +class ResourceResolver; + class ReverseLookupDictionaryComponent : public ReverseLookupDictionary::Component { public: @@ -79,6 +80,7 @@ class ReverseLookupDictionaryComponent ReverseLookupDictionary* Create(const Ticket& ticket); private: map> db_pool_; + the resource_resolver_; }; } // namespace rime From 8264748cf99a9cf22408bdcf024f8475613499a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E6=88=8E=E6=B0=8F?= Date: Wed, 4 Oct 2017 15:42:35 +0800 Subject: [PATCH 4/6] refactor(dict_compiler): use resource resolver --- src/rime/dict/dict_compiler.cc | 46 ++++++++++++++++++++---------- src/rime/dict/dict_compiler.h | 9 +----- src/rime/lever/deployment_tasks.cc | 18 +----------- 3 files changed, 33 insertions(+), 40 deletions(-) diff --git a/src/rime/dict/dict_compiler.cc b/src/rime/dict/dict_compiler.cc index ce6ce09a1..eefc843ef 100644 --- a/src/rime/dict/dict_compiler.cc +++ b/src/rime/dict/dict_compiler.cc @@ -6,6 +6,8 @@ // #include #include +#include +#include #include #include #include @@ -19,19 +21,25 @@ namespace rime { -DictCompiler::DictCompiler(Dictionary *dictionary, DictFileFinder finder) +DictCompiler::DictCompiler(Dictionary *dictionary) : dict_name_(dictionary->name()), prism_(dictionary->prism()), - table_(dictionary->table()), - dict_file_finder_(finder) { + table_(dictionary->table()) { +} + +static string LocateFile(const string& file_name) { + the resolver( + Service::instance().CreateResourceResolver({"", "", ""})); + return resolver->ResolvePath(file_name).string(); } bool DictCompiler::Compile(const string &schema_file) { LOG(INFO) << "compiling:"; bool build_table_from_source = true; DictSettings settings; - string dict_file(FindDictFile(dict_name_)); - if (dict_file.empty()) { + string dict_file = LocateFile(dict_name_ + ".dict.yaml"); + if (!boost::filesystem::exists(dict_file)) { + LOG(ERROR) << "source file '" << dict_file << "' does not exist."; build_table_from_source = false; } else { @@ -49,9 +57,12 @@ bool DictCompiler::Compile(const string &schema_file) { for(auto it = tables->begin(); it != tables->end(); ++it) { if (!Is(*it)) continue; - string dict_file(FindDictFile(As(*it)->str())); - if (dict_file.empty()) + string dict_name = As(*it)->str(); + string dict_file = LocateFile(dict_name + ".dict.yaml"); + if (!boost::filesystem::exists(dict_file)) { + LOG(ERROR) << "source file '" << dict_file << "' does not exist."; return false; + } dict_files.push_back(dict_file); } uint32_t dict_file_checksum = 0; @@ -95,7 +106,9 @@ bool DictCompiler::Compile(const string &schema_file) { << " (" << dict_file_checksum << ")"; LOG(INFO) << schema_file << " (" << schema_file_checksum << ")"; { - ReverseDb reverse_db(dict_name_); + the resolver( + Service::instance().CreateResourceResolver({"", "", ".reverse.bin"})); + ReverseDb reverse_db(resolver->ResolvePath(dict_name_).string()); if (!reverse_db.Exists() || !reverse_db.Load() || reverse_db.dict_file_checksum() != dict_file_checksum) { @@ -117,18 +130,19 @@ bool DictCompiler::Compile(const string &schema_file) { return true; } -string DictCompiler::FindDictFile(const string& dict_name) { - string dict_file(dict_name + ".dict.yaml"); - if (dict_file_finder_) { - dict_file = dict_file_finder_(dict_file); - } - return dict_file; +static string RelocateToUserDirectory(const string& file_name) { + ResourceResolver resolver(ResourceType{"", "", ""}); + resolver.set_root_path(Service::instance().deployer().user_data_dir); + auto resource_id = boost::filesystem::path(file_name).filename().string(); + return resolver.ResolvePath(resource_id).string(); } bool DictCompiler::BuildTable(DictSettings* settings, const vector& dict_files, uint32_t dict_file_checksum) { LOG(INFO) << "building table..."; + table_ = New
(RelocateToUserDirectory(table_->file_name())); + EntryCollector collector; collector.Configure(settings); collector.Collect(dict_files); @@ -172,7 +186,7 @@ bool DictCompiler::BuildTable(DictSettings* settings, } } // build .reverse.bin - ReverseDb reverse_db(dict_name_); + ReverseDb reverse_db(RelocateToUserDirectory(dict_name_ + ".reverse.bin")); if (!reverse_db.Build(settings, collector.syllabary, vocabulary, @@ -187,6 +201,8 @@ bool DictCompiler::BuildTable(DictSettings* settings, bool DictCompiler::BuildPrism(const string &schema_file, uint32_t dict_file_checksum, uint32_t schema_file_checksum) { LOG(INFO) << "building prism..."; + prism_ = New(RelocateToUserDirectory(prism_->file_name())); + // get syllabary from table Syllabary syllabary; if (!table_->Load() || !table_->GetSyllabary(&syllabary) || syllabary.empty()) diff --git a/src/rime/dict/dict_compiler.h b/src/rime/dict/dict_compiler.h index f29bb46ec..7eb9641e4 100644 --- a/src/rime/dict/dict_compiler.h +++ b/src/rime/dict/dict_compiler.h @@ -18,10 +18,6 @@ class Table; class ReverseDb; class DictSettings; -// return found dict file path, otherwize return empty string -using DictFileFinder = - function; - class DictCompiler { public: enum Options { @@ -31,14 +27,12 @@ class DictCompiler { kDump = 4, }; - RIME_API DictCompiler(Dictionary *dictionary, - DictFileFinder finder = NULL); + RIME_API DictCompiler(Dictionary *dictionary); RIME_API bool Compile(const string &schema_file); void set_options(int options) { options_ = options; } private: - string FindDictFile(const string& dict_name); bool BuildTable(DictSettings* settings, const vector& dict_files, uint32_t dict_file_checksum); @@ -50,7 +44,6 @@ class DictCompiler { an prism_; an
table_; int options_ = 0; - DictFileFinder dict_file_finder_; }; } // namespace rime diff --git a/src/rime/lever/deployment_tasks.cc b/src/rime/lever/deployment_tasks.cc index 4c66023ed..f80c73957 100644 --- a/src/rime/lever/deployment_tasks.cc +++ b/src/rime/lever/deployment_tasks.cc @@ -233,20 +233,6 @@ SchemaUpdate::SchemaUpdate(TaskInitializer arg) : verbose_(false) { } } -static string find_dict_file(const string& dict_file_name, - const fs::path& shared_data_path, - const fs::path& user_data_path) { - fs::path dict_path(user_data_path / dict_file_name); - if (!fs::exists(dict_path)) { - dict_path = shared_data_path / dict_file_name; - if (!fs::exists(dict_path)) { - LOG(ERROR) << "source file '" << dict_file_name << "' does not exist."; - return string(); - } - } - return dict_path.string(); -} - bool SchemaUpdate::Run(Deployer* deployer) { fs::path source_path(schema_file_); if (!fs::exists(source_path)) { @@ -291,9 +277,7 @@ bool SchemaUpdate::Run(Deployer* deployer) { return false; } LOG(INFO) << "preparing dictionary '" << dict_name << "'."; - DictFileFinder finder = std::bind(&find_dict_file, - _1, shared_data_path, user_data_path); - DictCompiler dict_compiler(dict.get(), finder); + DictCompiler dict_compiler(dict.get()); if (verbose_) { dict_compiler.set_options(DictCompiler::kRebuild | DictCompiler::kDump); } From 1f86413f6a69f61aac6ca3bf219c8b44ff1dddfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=85=E6=88=8E=E6=B0=8F?= Date: Wed, 4 Oct 2017 15:42:42 +0800 Subject: [PATCH 5/6] fix(levers): update deployment tasks for copy-free resource resolution --- src/rime/lever/customizer.cc | 5 - src/rime/lever/customizer.h | 2 - src/rime/lever/deployment_tasks.cc | 240 ++++++++++++++--------------- tools/rime_deployer.cc | 2 + 4 files changed, 120 insertions(+), 129 deletions(-) diff --git a/src/rime/lever/customizer.cc b/src/rime/lever/customizer.cc index 0ada600d4..349cabb8b 100644 --- a/src/rime/lever/customizer.cc +++ b/src/rime/lever/customizer.cc @@ -147,9 +147,4 @@ bool Customizer::UpdateConfigFile() { return true; } -bool Customizer::TrashCustomizedCopy() { - // TODO: unimplemented - return false; -} - } // namespace rime diff --git a/src/rime/lever/customizer.h b/src/rime/lever/customizer.h index 42d892d7b..43b85cccd 100644 --- a/src/rime/lever/customizer.h +++ b/src/rime/lever/customizer.h @@ -21,8 +21,6 @@ class Customizer { // DEPRECATED: in favor of auto-patch config compiler plugin bool UpdateConfigFile(); - bool TrashCustomizedCopy(); - protected: boost::filesystem::path source_path_; boost::filesystem::path dest_path_; diff --git a/src/rime/lever/deployment_tasks.cc b/src/rime/lever/deployment_tasks.cc index f80c73957..5e2e6d9d5 100644 --- a/src/rime/lever/deployment_tasks.cc +++ b/src/rime/lever/deployment_tasks.cc @@ -10,13 +10,14 @@ #include #include #include +#include #include +#include #include #include #include #include #include -#include #include #include #ifdef _WIN32 @@ -115,22 +116,18 @@ bool WorkspaceUpdate::Run(Deployer* deployer) { the t; t.reset(new ConfigFileUpdate("default.yaml", "config_version")); t->Run(deployer); - // since brise 0.18 t.reset(new ConfigFileUpdate("symbols.yaml", "config_version")); t->Run(deployer); t.reset(new SymlinkingPrebuiltDictionaries); t->Run(deployer); } - fs::path user_data_path(deployer->user_data_dir); - fs::path default_config_path(user_data_path / "default.yaml"); - Config config; - if (!config.LoadFromFile(default_config_path.string())) { - LOG(ERROR) << "Error loading default config from '" - << default_config_path.string() << "'."; + the config(Config::Require("config")->Create("default")); + if (!config) { + LOG(ERROR) << "Error loading default config."; return false; } - auto schema_list = config.GetList("schema_list"); + auto schema_list = config->GetList("schema_list"); if (!schema_list) { LOG(WARNING) << "schema list not defined."; return false; @@ -140,18 +137,17 @@ bool WorkspaceUpdate::Run(Deployer* deployer) { int success = 0; int failure = 0; map schemas; - for (auto it = schema_list->begin(); it != schema_list->end(); ++it) { - auto item = As(*it); - if (!item) - continue; - auto schema_property = item->GetValue("schema"); - if (!schema_property) - continue; - const string& schema_id(schema_property->str()); + the resolver( + Service::instance().CreateResourceResolver({ + "schema", "", ".schema.yaml" + })); + auto build_schema = [&](const string& schema_id) { + if (schemas.find(schema_id) != schemas.end()) // already built + return; LOG(INFO) << "schema: " << schema_id; string schema_path; if (schemas.find(schema_id) == schemas.end()) { - schema_path = GetSchemaPath(deployer, schema_id, true); + schema_path = resolver->ResolvePath(schema_id).string(); schemas[schema_id] = schema_path; } else { @@ -159,45 +155,35 @@ bool WorkspaceUpdate::Run(Deployer* deployer) { } if (schema_path.empty()) { LOG(WARNING) << "missing schema file for '" << schema_id << "'."; - continue; + return; } - // build schema the t(new SchemaUpdate(schema_path)); if (t->Run(deployer)) ++success; else ++failure; - } - // find dependencies - for (auto s = schemas.cbegin(); s != schemas.cend(); ++s) { - Config schema_config; - // user could have customized dependencies in the resulting schema - string user_schema_path = GetSchemaPath(deployer, s->first, false); - if (!schema_config.LoadFromFile(user_schema_path)) + }; + auto schema_component = Config::Require("schema"); + for (auto it = schema_list->begin(); it != schema_list->end(); ++it) { + auto item = As(*it); + if (!item) continue; - auto dependencies = schema_config.GetList("schema/dependencies"); - if (!dependencies) + auto schema_property = item->GetValue("schema"); + if (!schema_property) continue; - for (auto d = dependencies->begin(); d != dependencies->end(); ++d) { - auto dependency = As(*d); - if (!dependency) - continue; - string dependency_id(dependency->str()); - if (schemas.find(dependency_id) != schemas.end()) // already built - continue; - LOG(INFO) << "new dependency: " << dependency_id; - string dependency_path = GetSchemaPath(deployer, dependency_id, true); - schemas[dependency_id] = dependency_path; - if (dependency_path.empty()) { - LOG(WARNING) << "missing schema file for dependency '" << dependency_id << "'."; - continue; + const string& schema_id = schema_property->str(); + build_schema(schema_id); + the schema_config(schema_component->Create(schema_id)); + if (!schema_config) + continue; + if (auto dependencies = schema_config->GetList("schema/dependencies")) { + for (auto d = dependencies->begin(); d != dependencies->end(); ++d) { + auto dependency = As(*d); + if (!dependency) + continue; + const string& dependency_id = dependency->str(); + build_schema(dependency_id); } - // build dependency - the t(new SchemaUpdate(dependency_path)); - if (t->Run(deployer)) - ++success; - else - ++failure; } } LOG(INFO) << "finished updating schemas: " @@ -205,25 +191,6 @@ bool WorkspaceUpdate::Run(Deployer* deployer) { return failure == 0; } -string WorkspaceUpdate::GetSchemaPath(Deployer* deployer, - const string& schema_id, - bool prefer_shared_copy) { - fs::path schema_path; - if (prefer_shared_copy) { - fs::path shared_data_path(deployer->shared_data_dir); - schema_path = shared_data_path / (schema_id + ".schema.yaml"); - if (!fs::exists(schema_path)) - schema_path.clear(); - } - if (schema_path.empty()) { - fs::path user_data_path(deployer->user_data_dir); - schema_path = user_data_path / (schema_id + ".schema.yaml"); - if (!fs::exists(schema_path)) - schema_path.clear(); - } - return schema_path.string(); -} - SchemaUpdate::SchemaUpdate(TaskInitializer arg) : verbose_(false) { try { schema_file_ = boost::any_cast(arg); @@ -233,6 +200,43 @@ SchemaUpdate::SchemaUpdate(TaskInitializer arg) : verbose_(false) { } } +static bool IsCustomizedCopy(const string& file_name); + +static bool TrashCustomizedCopy(const fs::path& shared_copy, + const fs::path& user_copy, + const string& version_key, + const fs::path& trash) { + if (fs::equivalent(shared_copy, user_copy)) + return false; + if (IsCustomizedCopy(user_copy.string())) { + string shared_copy_version; + string user_copy_version; + Config shared_config; + if (shared_config.LoadFromFile(shared_copy.string())) { + shared_config.GetString(version_key, &shared_copy_version); + } + Config user_config; + if (user_config.LoadFromFile(user_copy.string()) && + user_config.GetString(version_key, &user_copy_version)) { + size_t custom_version_suffix = user_copy_version.find(".custom."); + if (custom_version_suffix != string::npos) { + user_copy_version.erase(custom_version_suffix); + } + } + if (CompareVersionString(shared_copy_version, user_copy_version) >= 0) { + fs::path backup = trash / user_copy.filename(); + boost::system::error_code ec; + fs::rename(user_copy, backup, ec); + if (ec) { + LOG(ERROR) << "error trashing file " << user_copy.string(); + return false; + } + return true; + } + } + return false; +} + bool SchemaUpdate::Run(Deployer* deployer) { fs::path source_path(schema_file_); if (!fs::exists(source_path)) { @@ -241,37 +245,33 @@ bool SchemaUpdate::Run(Deployer* deployer) { return false; } string schema_id; - { - Config source; - if (!source.LoadFromFile(schema_file_) || - !source.GetString("schema/schema_id", &schema_id) || - schema_id.empty()) { - LOG(ERROR) << "invalid schema definition in '" << schema_file_ << "'."; - return false; - } + the config(new Config); + if (!config->LoadFromFile(schema_file_) || + !config->GetString("schema/schema_id", &schema_id) || + schema_id.empty()) { + LOG(ERROR) << "invalid schema definition in '" << schema_file_ << "'."; + return false; } fs::path shared_data_path(deployer->shared_data_dir); fs::path user_data_path(deployer->user_data_dir); fs::path destination_path(user_data_path / (schema_id + ".schema.yaml")); - Customizer customizer(source_path, destination_path, "schema/version"); - if (customizer.TrashCustomizedCopy()) { - LOG(INFO) << "patched copy of schema '" << schema_id << "' is moved to trash"; + fs::path trash = user_data_path / "trash"; + if (TrashCustomizedCopy(source_path, + destination_path, + "schema/version", + trash)) { + LOG(INFO) << "patched copy of schema '" << schema_id + << "' is moved to trash"; } - Schema schema(schema_id, new Config); - Config* config = schema.config(); - if (!config || !config->LoadFromFile(destination_path.string())) { - LOG(ERROR) << "Error loading schema file '" - << destination_path.string() << "'."; - return false; - } string dict_name; if (!config->GetString("translator/dictionary", &dict_name)) { // not requiring a dictionary return true; } - DictionaryComponent component; - the dict(component.Create({&schema, "translator"})); + Schema schema(schema_id, config.release()); + the dict( + Dictionary::Require("dictionary")->Create({&schema, "translator"})); if (!dict) { LOG(ERROR) << "Error creating dictionary '" << dict_name << "'."; return false; @@ -281,7 +281,7 @@ bool SchemaUpdate::Run(Deployer* deployer) { if (verbose_) { dict_compiler.set_options(DictCompiler::kRebuild | DictCompiler::kDump); } - if (!dict_compiler.Compile(destination_path.string())) { + if (!dict_compiler.Compile(schema_file_)) { LOG(ERROR) << "dictionary '" << dict_name << "' failed to compile."; return false; } @@ -305,13 +305,18 @@ bool ConfigFileUpdate::Run(Deployer* deployer) { fs::path user_data_path(deployer->user_data_dir); fs::path source_config_path(shared_data_path / file_name_); fs::path dest_config_path(user_data_path / file_name_); + fs::path trash = user_data_path / "trash"; if (!fs::exists(source_config_path)) { LOG(WARNING) << "'" << file_name_ << "' is missing from shared data directory."; - source_config_path = dest_config_path; + return false; + } + if (TrashCustomizedCopy(source_config_path, + dest_config_path, + version_key_, + trash)) { + LOG(INFO) << "patched copy of '" << file_name_ << "' is moved to trash."; } - Customizer customizer(source_config_path, dest_config_path, version_key_); - customizer.TrashCustomizedCopy(); return true; } @@ -341,15 +346,16 @@ bool SymlinkingPrebuiltDictionaries::Run(Deployer* deployer) { fs::equivalent(shared_data_path, user_data_path)) return false; bool success = false; - // test existing link + // remove symlinks to shared data files created by previous version for (fs::directory_iterator test(user_data_path), end; test != end; ++test) { fs::path entry(test->path()); - if (fs::is_symlink(entry) && entry.extension().string() == ".bin") { + if (fs::is_symlink(entry)) { try { - if (!fs::exists(entry)) { - LOG(INFO) << "removing dangling symlink: " - << entry.filename().string(); + auto target_path = fs::canonical(entry); + if (target_path.has_parent_path() && + fs::equivalent(shared_data_path, target_path.parent_path())) { + LOG(INFO) << "removing symlink: " << entry.filename().string(); fs::remove(entry); } } @@ -359,24 +365,6 @@ bool SymlinkingPrebuiltDictionaries::Run(Deployer* deployer) { } } } - // create link - for (fs::directory_iterator iter(shared_data_path), end; - iter != end; ++iter) { - fs::path entry(iter->path()); - fs::path link(user_data_path / entry.filename()); - try { - if (fs::is_regular_file(entry) && - entry.extension().string() == ".bin" && - !fs::exists(link)) { - LOG(INFO) << "symlinking '" << entry.filename().string() << "'."; - fs::create_symlink(entry, link); - } - } - catch (const fs::filesystem_error& ex) { - LOG(ERROR) << ex.what(); - success = false; - } - } return success; } @@ -402,6 +390,19 @@ bool UserDictSync::Run(Deployer* deployer) { return mgr.SynchronizeAll(); } +static bool IsCustomizedCopy(const string& file_name) { + if (boost::ends_with(file_name, ".yaml") && + !boost::ends_with(file_name, ".custom.yaml")) { + Config config; + string checksum; + if (config.LoadFromFile(file_name) && + config.GetString("customization", &checksum)) { + return true; + } + } + return false; +} + bool BackupConfigFiles::Run(Deployer* deployer) { LOG(INFO) << "backing up config files."; fs::path user_data_path(deployer->user_data_dir); @@ -433,14 +434,9 @@ bool BackupConfigFiles::Run(Deployer* deployer) { ++latest; // already up-to-date continue; } - if (is_yaml_file && !boost::ends_with(entry.string(), ".custom.yaml")) { - Config config; - string checksum; - if (config.LoadFromFile(entry.string()) && - config.GetString("customization", &checksum)) { - ++skipped; // customized copy - continue; - } + if (is_yaml_file && IsCustomizedCopy(entry.string())) { + ++skipped; // customized copy + continue; } boost::system::error_code ec; fs::copy_file(entry, backup, fs::copy_option::overwrite_if_exists, ec); diff --git a/tools/rime_deployer.cc b/tools/rime_deployer.cc index 34e46aea8..3f71ea1a3 100644 --- a/tools/rime_deployer.cc +++ b/tools/rime_deployer.cc @@ -91,6 +91,7 @@ int main(int argc, char* argv[]) { if (argc >= 0 && argc <= 2 && option == "--build") { Deployer& deployer(Service::instance().deployer()); configure_deployer(&deployer, argc, argv); + LoadModules(kDeployerModules); WorkspaceUpdate update; return update.Run(&deployer) ? 0 : 1; } @@ -106,6 +107,7 @@ int main(int argc, char* argv[]) { if (argc >= 1 && option == "--compile") { Deployer& deployer(Service::instance().deployer()); configure_deployer(&deployer, argc - 1, argv + 1); + LoadModules(kDeployerModules); string schema_file(argv[0]); SchemaUpdate update(schema_file); update.set_verbose(true); From 52ae306c9b4ef9eb35913e79d8287c4baffaf827 Mon Sep 17 00:00:00 2001 From: Chen Gong Date: Sat, 21 Oct 2017 11:32:43 +0800 Subject: [PATCH 6/6] refactor(dict): use float instead of double in binaries --- src/rime/dict/prism.h | 4 +++- src/rime/dict/table.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rime/dict/prism.h b/src/rime/dict/prism.h index 05eab2362..a70d5b6c2 100644 --- a/src/rime/dict/prism.h +++ b/src/rime/dict/prism.h @@ -19,10 +19,12 @@ namespace rime { namespace prism { +using Credibility = float; + struct SpellingDescriptor { SyllableId syllable_id; int32_t type; - double credibility; + Credibility credibility; String tips; }; diff --git a/src/rime/dict/table.h b/src/rime/dict/table.h index ef177e689..f1db752d0 100644 --- a/src/rime/dict/table.h +++ b/src/rime/dict/table.h @@ -40,7 +40,7 @@ using Syllabary = Array; using Code = List; -using Weight = double; +using Weight = float; struct Entry { StringType text;