From 244f97a68e9d0f1aab34206caa43254c210ebb1b Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 30 Aug 2021 08:13:50 +0900 Subject: [PATCH 01/82] Cache entries' cover_url --- src/library/entry.cr | 12 ++++++++++-- src/library/title.cr | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 92f4defc..b5e582f6 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -81,9 +81,17 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg + + unless @book.entry_cover_url_cache + TitleInfo.new @book.dir do |info| + @book.entry_cover_url_cache = info.entry_cover_url + end + end + entry_cover_url = @book.entry_cover_url_cache + url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}" - TitleInfo.new @book.dir do |info| - info_url = info.entry_cover_url[@title]? + if entry_cover_url + info_url = entry_cover_url[@title]? unless info_url.nil? || info_url.empty? url = File.join Config.current.base_url, info_url end diff --git a/src/library/title.cr b/src/library/title.cr index 61c98132..6f2cf8c9 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -3,9 +3,12 @@ require "../archive" class Title getter dir : String, parent_id : String, title_ids : Array(String), entries : Array(Entry), title : String, id : String, - encoded_title : String, mtime : Time, signature : UInt64 + encoded_title : String, mtime : Time, signature : UInt64, + entry_cover_url_cache : Hash(String, String)? + setter entry_cover_url_cache : Hash(String, String)? @entry_display_name_cache : Hash(String, String)? + @entry_cover_url_cache : Hash(String, String)? def initialize(@dir : String, @parent_id) storage = Storage.default From 51a47b5dddec78840dd496ec03fe956a264ae1c5 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 30 Aug 2021 08:17:52 +0900 Subject: [PATCH 02/82] Cache display_name --- src/library/title.cr | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/library/title.cr b/src/library/title.cr index 6f2cf8c9..2983a69d 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -9,6 +9,7 @@ class Title @entry_display_name_cache : Hash(String, String)? @entry_cover_url_cache : Hash(String, String)? + @cached_display_name : String? def initialize(@dir : String, @parent_id) storage = Storage.default @@ -180,11 +181,15 @@ class Title end def display_name + cached_display_name = @cached_display_name + return cached_display_name unless cached_display_name.nil? + dn = @title TitleInfo.new @dir do |info| info_dn = info.display_name dn = info_dn unless info_dn.empty? end + @cached_display_name = dn dn end @@ -208,6 +213,7 @@ class Title end def set_display_name(dn) + @cached_display_name = nil TitleInfo.new @dir do |info| info.display_name = dn info.save @@ -217,6 +223,7 @@ class Title def set_display_name(entry_name : String, dn) TitleInfo.new @dir do |info| info.entry_display_name[entry_name] = dn + @entry_display_name_cache = info.entry_display_name info.save end end From 00c9cc1fcdffa1b41dddfbb21bf348252c450706 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 30 Aug 2021 08:19:15 +0900 Subject: [PATCH 03/82] Prevent saving a sort opt unnecessarily --- src/library/library.cr | 5 ----- src/library/title.cr | 6 +----- src/routes/main.cr | 6 +++--- src/util/web.cr | 20 ++++++++++++++++++++ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index a5a4a80d..30e93b26 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -51,11 +51,6 @@ class Library def sorted_titles(username, opt : SortOptions? = nil) if opt.nil? opt = SortOptions.from_info_json @dir, username - else - TitleInfo.new @dir do |info| - info.sort_by[username] = opt.to_tuple - info.save - end end # Helper function from src/util/util.cr diff --git a/src/library/title.cr b/src/library/title.cr index 2983a69d..aa4a479c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -1,3 +1,4 @@ +require "digest" require "../archive" class Title @@ -329,11 +330,6 @@ class Title def sorted_entries(username, opt : SortOptions? = nil) if opt.nil? opt = SortOptions.from_info_json @dir, username - else - TitleInfo.new @dir do |info| - info.sort_by[username] = opt.to_tuple - info.save - end end case opt.not_nil!.method diff --git a/src/routes/main.cr b/src/routes/main.cr index 57917bb7..4aa7da63 100644 --- a/src/routes/main.cr +++ b/src/routes/main.cr @@ -41,7 +41,7 @@ struct MainRouter username = get_username env sort_opt = SortOptions.from_info_json Library.default.dir, username - get_sort_opt + get_and_save_sort_opt Library.default.dir titles = Library.default.sorted_titles username, sort_opt percentage = titles.map &.load_percentage username @@ -59,12 +59,12 @@ struct MainRouter username = get_username env sort_opt = SortOptions.from_info_json title.dir, username - get_sort_opt + get_and_save_sort_opt title.dir entries = title.sorted_entries username, sort_opt - percentage = title.load_percentage_for_all_entries username, sort_opt title_percentage = title.titles.map &.load_percentage username + layout "title" rescue e Logger.error e diff --git a/src/util/web.cr b/src/util/web.cr index 12459e53..5704ea88 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -107,6 +107,26 @@ macro get_sort_opt end end +macro get_and_save_sort_opt(dir) + sort_method = env.params.query["sort"]? + + if sort_method + is_ascending = true + + ascend = env.params.query["ascend"]? + if ascend && ascend.to_i? == 0 + is_ascending = false + end + + sort_opt = SortOptions.new sort_method, is_ascending + + TitleInfo.new {{dir}} do |info| + info.sort_by[username] = sort_opt.to_tuple + info.save + end + end +end + module HTTP class Client private def self.exec(uri : URI, tls : TLSContext = nil) From 4a09aee1771b8aca11b666a83c7ad906355c43e2 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 22 Aug 2021 23:36:28 +0900 Subject: [PATCH 04/82] Implement library caching TitleInfo * Cache sum of entry progress * Cache cover_url * Cache display_name * Cache sort_opt --- src/library/cache.cr | 159 +++++++++++++++++++++++++++++++++++++++++ src/library/entry.cr | 10 +++ src/library/library.cr | 4 ++ src/library/title.cr | 25 ++++++- src/library/types.cr | 3 + src/util/web.cr | 1 + 6 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 src/library/cache.cr diff --git a/src/library/cache.cr b/src/library/cache.cr new file mode 100644 index 00000000..00c65bab --- /dev/null +++ b/src/library/cache.cr @@ -0,0 +1,159 @@ +require "digest" + +class InfoCache + alias ProgressCache = Tuple(String, Int32) + + def self.clear + clear_cover_url + clear_progress_cache + clear_sort_opt + end + + def self.clean + clean_cover_url + clean_progress_cache + clean_sort_opt + end + + # item id => cover_url + @@cached_cover_url = {} of String => String + @@cached_cover_url_previous = {} of String => String # item id => cover_url + + def self.set_cover_url(id : String, cover_url : String) + @@cached_cover_url[id] = cover_url + end + + def self.get_cover_url(id : String) + @@cached_cover_url[id]? + end + + def self.invalidate_cover_url(id : String) + @@cached_cover_url.delete id + end + + def self.move_cover_url(id : String) + if @@cached_cover_url_previous[id]? + @@cached_cover_url[id] = @@cached_cover_url_previous[id] + end + end + + private def self.clear_cover_url + @@cached_cover_url_previous = @@cached_cover_url + @@cached_cover_url = {} of String => String + end + + private def self.clean_cover_url + @@cached_cover_url_previous = {} of String => String + end + + # book.id:username => {signature, sum} + @@progress_cache = {} of String => ProgressCache + # book.id => username => {signature, sum} + @@progress_cache_previous = {} of String => Hash(String, ProgressCache) + + def self.set_progress_cache(book_id : String, username : String, + entries : Array(Entry), sum : Int32) + progress_cache_id = "#{book_id}:#{username}" + progress_cache_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + @@progress_cache[progress_cache_id] = {progress_cache_sig, sum} + Logger.debug "Progress Cached #{progress_cache_id}" + end + + def self.get_progress_cache(book_id : String, username : String, + entries : Array(Entry)) + progress_cache_id = "#{book_id}:#{username}" + progress_cache_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + cached = @@progress_cache[progress_cache_id]? + if cached && cached[0] == progress_cache_sig + Logger.debug "Progress Cache Hit! #{progress_cache_id}" + return cached[1] + end + end + + def self.invalidate_progress_cache(book_id : String, username : String) + progress_cache_id = "#{book_id}:#{username}" + if @@progress_cache[progress_cache_id]? + @@progress_cache.delete progress_cache_id + Logger.debug "Progress Invalidate Cache #{progress_cache_id}" + end + end + + def self.move_progress_cache(book_id : String) + if @@progress_cache_previous[book_id]? + @@progress_cache_previous[book_id].each do |username, cached| + id = "#{book_id}:#{username}" + unless @@progress_cache[id]? + # It would be invalidated when entries changed + @@progress_cache[id] = cached + end + end + end + end + + private def self.clear_progress_cache + @@progress_cache_previous = {} of String => Hash(String, ProgressCache) + @@progress_cache.each do |id, cached| + splitted = id.split(':', 2) + book_id = splitted[0] + username = splitted[1] + unless @@progress_cache_previous[book_id]? + @@progress_cache_previous[book_id] = {} of String => ProgressCache + end + + @@progress_cache_previous[book_id][username] = cached + end + @@progress_cache = {} of String => ProgressCache + end + + private def self.clean_progress_cache + @@progress_cache_previous = {} of String => Hash(String, ProgressCache) + end + + # book.dir:username => SortOptions + @@cached_sort_opt = {} of String => SortOptions + @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) + + def self.set_sort_opt(dir : String, username : String, sort_opt : SortOptions) + id = "#{dir}:#{username}" + @@cached_sort_opt[id] = sort_opt + end + + def self.get_sort_opt(dir : String, username : String) + id = "#{dir}:#{username}" + @@cached_sort_opt[id]? + end + + def self.invalidate_sort_opt(dir : String, username : String) + id = "#{dir}:#{username}" + @@cached_sort_opt.delete id + end + + def self.move_sort_opt(dir : String) + if @@cached_sort_opt_previous[dir]? + @@cached_sort_opt_previous[dir].each do |username, cached| + id = "#{dir}:#{username}" + unless @@cached_sort_opt[id]? + @@cached_sort_opt[id] = cached + end + end + end + end + + private def self.clear_sort_opt + @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) + @@cached_sort_opt.each do |id, cached| + splitted = id.split(':', 2) + book_dir = splitted[0] + username = splitted[1] + unless @@cached_sort_opt_previous[book_dir]? + @@cached_sort_opt_previous[book_dir] = {} of String => SortOptions + end + @@cached_sort_opt_previous[book_dir][username] = cached + end + @@cached_sort_opt = {} of String => SortOptions + end + + private def self.clean_sort_opt + @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) + end +end diff --git a/src/library/entry.cr b/src/library/entry.cr index b5e582f6..cbebf7f4 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -44,6 +44,8 @@ class Entry MIME.from_filename? e.filename end file.close + + InfoCache.move_cover_url @id end def to_slim_json : String @@ -81,6 +83,8 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg + cached_cover_url = InfoCache.get_cover_url @id + return cached_cover_url if cached_cover_url unless @book.entry_cover_url_cache TitleInfo.new @book.dir do |info| @@ -96,6 +100,7 @@ class Entry url = File.join Config.current.base_url, info_url end end + InfoCache.set_cover_url @id, url url end @@ -178,6 +183,11 @@ class Entry # For backward backward compatibility with v0.1.0, we save entry titles # instead of IDs in info.json def save_progress(username, page) + InfoCache.invalidate_progress_cache @book.id, username + @book.parents.each do |parent| + InfoCache.invalidate_progress_cache parent.id, username + end + TitleInfo.new @book.dir do |info| if info.progress[username]?.nil? info.progress[username] = {@title => page} diff --git a/src/library/library.cr b/src/library/library.cr index 30e93b26..2638ffd8 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -102,6 +102,8 @@ class Library storage = Storage.new auto_close: false + InfoCache.clear + (Dir.entries @dir) .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } @@ -115,6 +117,8 @@ class Library @title_ids << title.id end + InfoCache.clean + storage.bulk_insert_ids storage.close diff --git a/src/library/title.cr b/src/library/title.cr index aa4a479c..14b57544 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -60,6 +60,10 @@ class Title @entries.sort! do |a, b| sorter.compare a.title, b.title end + + InfoCache.move_cover_url @id + InfoCache.move_progress_cache @id + InfoCache.move_sort_opt @dir end def to_slim_json : String @@ -230,6 +234,9 @@ class Title end def cover_url + cached_cover_url = InfoCache.get_cover_url @id + return cached_cover_url if cached_cover_url + url = "#{Config.current.base_url}img/icon.png" readable_entries = @entries.select &.err_msg.nil? if readable_entries.size > 0 @@ -241,10 +248,12 @@ class Title url = File.join Config.current.base_url, info_url end end + InfoCache.set_cover_url @id, url url end def set_cover_url(url : String) + InfoCache.invalidate_cover_url @id TitleInfo.new @dir do |info| info.cover_url = url info.save @@ -252,6 +261,8 @@ class Title end def set_cover_url(entry_name : String, url : String) + selected_entry = @entries.find { |entry| entry.display_name == entry_name } + InfoCache.invalidate_cover_url selected_entry.id if selected_entry TitleInfo.new @dir do |info| info.entry_cover_url[entry_name] = url info.save @@ -273,8 +284,13 @@ class Title end def deep_read_page_count(username) : Int32 - load_progress_for_all_entries(username).sum + - titles.flat_map(&.deep_read_page_count username).sum + # CACHE HERE + cached_sum = InfoCache.get_progress_cache @id, username, @entries + return cached_sum unless cached_sum.nil? + sum = load_progress_for_all_entries(username).sum + + titles.flat_map(&.deep_read_page_count username).sum + InfoCache.set_progress_cache @id, username, @entries, sum + sum end def deep_total_page_count : Int32 @@ -422,6 +438,11 @@ class Title end def bulk_progress(action, ids : Array(String), username) + InfoCache.invalidate_progress_cache @id, username + parents.each do |parent| + InfoCache.invalidate_progress_cache parent.id, username + end + selected_entries = ids .map { |id| @entries.find &.id.==(id) diff --git a/src/library/types.cr b/src/library/types.cr index 4e831355..094cb643 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -35,12 +35,15 @@ class SortOptions end def self.from_info_json(dir, username) + cached_opt = InfoCache.get_sort_opt dir, username + return cached_opt if cached_opt opt = SortOptions.new TitleInfo.new dir do |info| if info.sort_by.has_key? username opt = SortOptions.from_tuple info.sort_by[username] end end + InfoCache.set_sort_opt dir, username, opt opt end diff --git a/src/util/web.cr b/src/util/web.cr index 5704ea88..5e873cae 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -120,6 +120,7 @@ macro get_and_save_sort_opt(dir) sort_opt = SortOptions.new sort_method, is_ascending + InfoCache.set_sort_opt {{dir}}, username, sort_opt TitleInfo.new {{dir}} do |info| info.sort_by[username] = sort_opt.to_tuple info.save From bf81a4e48b5eca75a6fc43ba011593f681dfac3a Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 22 Aug 2021 23:51:23 +0900 Subject: [PATCH 05/82] Implement sorted entries cache sorted_entries cached --- src/library/cache.cr | 87 ++++++++++++++++++++++++++++++++++++++++++ src/library/entry.cr | 5 +++ src/library/library.cr | 4 ++ src/library/title.cr | 11 ++++++ 4 files changed, 107 insertions(+) diff --git a/src/library/cache.cr b/src/library/cache.cr index 00c65bab..1d2dd538 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -157,3 +157,90 @@ class InfoCache @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) end end + +private class SortedEntriesCacheEntry + getter key : String, atime : Time + + def initialize(@ctime : Time, @key : String, @value : Array(String)) + @atime = @ctime + end + + def value + @atime = Time.utc + @value + end + + def instance_size + @value.size * (instance_sizeof(String) + sizeof(String)) + + @value.sum(&.size) + instance_sizeof(SortedEntriesCacheEntry) + end +end + +# LRU Cache +class SortedEntriesCache + @@limit : Int128 = Int128.new 1024 * 1024 * 50 # 50MB + # key => entry + @@cache = {} of String => SortedEntriesCacheEntry + + def self.gen_key(book_id : String, username : String, + entries : Array(Entry), opt : SortOptions?) + sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + user_context = opt && opt.method == SortMethod::Progress ? username : "" + Digest::SHA1.hexdigest (book_id + sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) + end + + def self.get(key : String) + entry = @@cache[key]? + Logger.debug "SortedEntries Cache Hit! #{key}" unless entry.nil? + Logger.debug "SortedEntries Cache Miss #{key}" if entry.nil? + return ids2entries entry.value unless entry.nil? + end + + def self.set(key : String, value : Array(Entry)) + @@cache[key] = SortedEntriesCacheEntry.new Time.utc, key, value.map &.id + Logger.debug "SortedEntries Cached #{key}" + remove_victim_cache + end + + def self.invalidate(key : String) + @@cache.delete key + end + + def self.print + sum = @@cache.sum { |_, entry| entry.instance_size } + Logger.debug "---- Sorted Entries Cache ----" + Logger.debug "Size: #{sum} Bytes" + Logger.debug "List:" + @@cache.each { |k, v| Logger.debug "#{k} | #{v.atime}" } + Logger.debug "------------------------------" + end + + private def self.ids2entries(ids : Array(String)) + e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} } + entries = [] of Entry + begin + ids.each do |id| + entries << e_map[id] + end + return entries if ids.size == entries.size + rescue + end + end + + private def self.is_cache_full + sum = @@cache.sum { |_, entry| entry.instance_size } + sum > @@limit + end + + private def self.remove_victim_cache + while is_cache_full && @@cache.size > 0 + Logger.debug "SortedEntries Cache Full! Remove LRU" + min = @@cache.min_by? { |_, entry| entry.atime } + Logger.debug "Target: #{min[0]}, Last Access Time: #{min[1].atime}" if min + invalidate min[0] if min + + print if Logger.get_severity == Log::Severity::Debug + end + end +end diff --git a/src/library/entry.cr b/src/library/entry.cr index cbebf7f4..5b1f3ced 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -187,6 +187,11 @@ class Entry @book.parents.each do |parent| InfoCache.invalidate_progress_cache parent.id, username end + [false, true].each do |ascend| + sorted_entries_cache_key = SortedEntriesCache.gen_key @book.id, username, + @book.entries, SortOptions.new(SortMethod::Progress, ascend) + SortedEntriesCache.invalidate sorted_entries_cache_key + end TitleInfo.new @book.dir do |info| if info.progress[username]?.nil? diff --git a/src/library/library.cr b/src/library/library.cr index 2638ffd8..21e5c8b1 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -61,6 +61,10 @@ class Library titles + titles.flat_map &.deep_titles end + def deep_entries + titles.flat_map &.deep_entries + end + def to_slim_json : String JSON.build do |json| json.object do diff --git a/src/library/title.cr b/src/library/title.cr index 14b57544..6a78ea5e 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -344,6 +344,10 @@ class Title # use the default (auto, ascending) # When `opt` is not nil, it saves the options to info.json def sorted_entries(username, opt : SortOptions? = nil) + cache_key = SortedEntriesCache.gen_key @id, username, @entries, opt + cached_entries = SortedEntriesCache.get cache_key + return cached_entries if cached_entries + if opt.nil? opt = SortOptions.from_info_json @dir, username end @@ -377,6 +381,7 @@ class Title ary.reverse! unless opt.not_nil!.ascend + SortedEntriesCache.set cache_key, ary ary end @@ -442,6 +447,12 @@ class Title parents.each do |parent| InfoCache.invalidate_progress_cache parent.id, username end + [false, true].each do |ascend| + sorted_entries_cache_key = + SortedEntriesCache.gen_key @id, username, @entries, + SortOptions.new(SortMethod::Progress, ascend) + SortedEntriesCache.invalidate sorted_entries_cache_key + end selected_entries = ids .map { |id| From e988a8c121c630bea3d48d4fd749fcfdd7f56043 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 30 Aug 2021 22:53:30 +0900 Subject: [PATCH 06/82] Add config for sorted entries cache optional --- src/config.cr | 2 ++ src/library/cache.cr | 8 +++++++- src/mango.cr | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/config.cr b/src/config.cr index 332a159e..0647aeee 100644 --- a/src/config.cr +++ b/src/config.cr @@ -20,6 +20,8 @@ class Config property plugin_path : String = File.expand_path "~/mango/plugins", home: true property download_timeout_seconds : Int32 = 30 + property sorted_entries_cache_enable = false + property sorted_entries_cache_capacity_kbs = 51200 property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/library/cache.cr b/src/library/cache.cr index 1d2dd538..959e0eb2 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -178,10 +178,16 @@ end # LRU Cache class SortedEntriesCache - @@limit : Int128 = Int128.new 1024 * 1024 * 50 # 50MB + @@limit : Int128 = Int128.new 0 # key => entry @@cache = {} of String => SortedEntriesCacheEntry + def self.init + enabled = Config.current.sorted_entries_cache_enable + cache_size = Config.current.sorted_entries_cache_capacity_kbs + @@limit = Int128.new cache_size * 1024 if enabled + end + def self.gen_key(book_id : String, username : String, entries : Array(Entry), opt : SortOptions?) sig = Digest::SHA1.hexdigest (entries.map &.id).to_s diff --git a/src/mango.cr b/src/mango.cr index e8d32a3c..9b58d503 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -55,6 +55,7 @@ class CLI < Clim Config.load(opts.config).set_current # Initialize main components + SortedEntriesCache.init Storage.default Queue.default Library.default From 601346b209fbc79c3a72ca6fbf7f51f79d184241 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 30 Aug 2021 23:07:59 +0900 Subject: [PATCH 07/82] Set cache if enabled --- src/library/title.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/library/title.cr b/src/library/title.cr index 6a78ea5e..54779c27 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -381,7 +381,9 @@ class Title ary.reverse! unless opt.not_nil!.ascend - SortedEntriesCache.set cache_key, ary + if Config.current.sorted_entries_cache_enable + SortedEntriesCache.set cache_key, ary + end ary end From 365f71cd1d2f2c760f90f2296e80c1cd0278c3b4 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 30 Aug 2021 23:09:36 +0900 Subject: [PATCH 08/82] Change kbs to mbs --- src/config.cr | 2 +- src/library/cache.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config.cr b/src/config.cr index 0647aeee..69feccdd 100644 --- a/src/config.cr +++ b/src/config.cr @@ -21,7 +21,7 @@ class Config home: true property download_timeout_seconds : Int32 = 30 property sorted_entries_cache_enable = false - property sorted_entries_cache_capacity_kbs = 51200 + property sorted_entries_cache_size_mbs = 50 property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/library/cache.cr b/src/library/cache.cr index 959e0eb2..0165496a 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -184,8 +184,8 @@ class SortedEntriesCache def self.init enabled = Config.current.sorted_entries_cache_enable - cache_size = Config.current.sorted_entries_cache_capacity_kbs - @@limit = Int128.new cache_size * 1024 if enabled + cache_size = Config.current.sorted_entries_cache_size_mbs + @@limit = Int128.new cache_size * 1024 * 1024 if enabled end def self.gen_key(book_id : String, username : String, From 0a8fd993e524fb36a7249d753992594811e0f3f9 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 3 Sep 2021 11:11:28 +0900 Subject: [PATCH 09/82] Use bytesize and add comments --- src/library/cache.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 0165496a..7870df54 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -171,8 +171,10 @@ private class SortedEntriesCacheEntry end def instance_size - @value.size * (instance_sizeof(String) + sizeof(String)) + - @value.sum(&.size) + instance_sizeof(SortedEntriesCacheEntry) + instance_sizeof(SortedEntriesCacheEntry) + # sizeof itself + instance_sizeof(String) + @key.bytesize + # allocated memory for @key + @value.size * (instance_sizeof(String) + sizeof(String)) + + @value.sum(&.bytesize) # elements in Array(String) end end From 9e90aa17b934baa923ce979af3571bcb4b3650bd Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 4 Sep 2021 14:13:33 +0900 Subject: [PATCH 10/82] Move entry specific method --- src/library/cache.cr | 72 +++++++++++++++++++++++--------------------- src/library/entry.cr | 4 +-- src/library/title.cr | 4 +-- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 7870df54..c6e93749 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -159,23 +159,47 @@ class InfoCache end private class SortedEntriesCacheEntry + @value : Array(String) + getter key : String, atime : Time - def initialize(@ctime : Time, @key : String, @value : Array(String)) + def initialize(@ctime : Time, @key : String, value : Array(Entry)) @atime = @ctime + @value = value.map &.id end def value @atime = Time.utc - @value + SortedEntriesCacheEntry.ids2entries @value + end + + # private? + def self.ids2entries(ids : Array(String)) + e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} } + entries = [] of Entry + begin + ids.each do |id| + entries << e_map[id] + end + return entries if ids.size == entries.size + rescue + end end def instance_size - instance_sizeof(SortedEntriesCacheEntry) + # sizeof itself + instance_sizeof(SortedEntriesCacheEntry) + # sizeof itself instance_sizeof(String) + @key.bytesize + # allocated memory for @key @value.size * (instance_sizeof(String) + sizeof(String)) + @value.sum(&.bytesize) # elements in Array(String) end + + def self.gen_key(book_id : String, username : String, + entries : Array(Entry), opt : SortOptions?) + sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + user_context = opt && opt.method == SortMethod::Progress ? username : "" + Digest::SHA1.hexdigest (book_id + sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) + end end # LRU Cache @@ -190,24 +214,16 @@ class SortedEntriesCache @@limit = Int128.new cache_size * 1024 * 1024 if enabled end - def self.gen_key(book_id : String, username : String, - entries : Array(Entry), opt : SortOptions?) - sig = Digest::SHA1.hexdigest (entries.map &.id).to_s - user_context = opt && opt.method == SortMethod::Progress ? username : "" - Digest::SHA1.hexdigest (book_id + sig + user_context + - (opt ? opt.to_tuple.to_s : "nil")) - end - def self.get(key : String) entry = @@cache[key]? - Logger.debug "SortedEntries Cache Hit! #{key}" unless entry.nil? - Logger.debug "SortedEntries Cache Miss #{key}" if entry.nil? - return ids2entries entry.value unless entry.nil? + Logger.debug "LRUCache Cache Hit! #{key}" unless entry.nil? + Logger.debug "LRUCache Cache Miss #{key}" if entry.nil? + return entry.value unless entry.nil? end def self.set(key : String, value : Array(Entry)) - @@cache[key] = SortedEntriesCacheEntry.new Time.utc, key, value.map &.id - Logger.debug "SortedEntries Cached #{key}" + @@cache[key] = SortedEntriesCacheEntry.new Time.utc, key, value + Logger.debug "LRUCache Cached #{key}" remove_victim_cache end @@ -217,23 +233,11 @@ class SortedEntriesCache def self.print sum = @@cache.sum { |_, entry| entry.instance_size } - Logger.debug "---- Sorted Entries Cache ----" + Logger.debug "---- LRU Cache ----" Logger.debug "Size: #{sum} Bytes" Logger.debug "List:" @@cache.each { |k, v| Logger.debug "#{k} | #{v.atime}" } - Logger.debug "------------------------------" - end - - private def self.ids2entries(ids : Array(String)) - e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} } - entries = [] of Entry - begin - ids.each do |id| - entries << e_map[id] - end - return entries if ids.size == entries.size - rescue - end + Logger.debug "-------------------" end private def self.is_cache_full @@ -243,12 +247,12 @@ class SortedEntriesCache private def self.remove_victim_cache while is_cache_full && @@cache.size > 0 - Logger.debug "SortedEntries Cache Full! Remove LRU" + Logger.debug "LRUCache Cache Full! Remove LRU" min = @@cache.min_by? { |_, entry| entry.atime } - Logger.debug "Target: #{min[0]}, Last Access Time: #{min[1].atime}" if min + Logger.debug " \ + Target: #{min[0]}, \ + Last Access Time: #{min[1].atime}" if min invalidate min[0] if min - - print if Logger.get_severity == Log::Severity::Debug end end end diff --git a/src/library/entry.cr b/src/library/entry.cr index 5b1f3ced..c7599fff 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -188,8 +188,8 @@ class Entry InfoCache.invalidate_progress_cache parent.id, username end [false, true].each do |ascend| - sorted_entries_cache_key = SortedEntriesCache.gen_key @book.id, username, - @book.entries, SortOptions.new(SortMethod::Progress, ascend) + sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, + username, @book.entries, SortOptions.new(SortMethod::Progress, ascend) SortedEntriesCache.invalidate sorted_entries_cache_key end diff --git a/src/library/title.cr b/src/library/title.cr index 54779c27..ed546ca7 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -344,7 +344,7 @@ class Title # use the default (auto, ascending) # When `opt` is not nil, it saves the options to info.json def sorted_entries(username, opt : SortOptions? = nil) - cache_key = SortedEntriesCache.gen_key @id, username, @entries, opt + cache_key = SortedEntriesCacheEntry.gen_key @id, username, @entries, opt cached_entries = SortedEntriesCache.get cache_key return cached_entries if cached_entries @@ -451,7 +451,7 @@ class Title end [false, true].each do |ascend| sorted_entries_cache_key = - SortedEntriesCache.gen_key @id, username, @entries, + SortedEntriesCacheEntry.gen_key @id, username, @entries, SortOptions.new(SortMethod::Progress, ascend) SortedEntriesCache.invalidate sorted_entries_cache_key end From 5e919d3e19c802084ce4076341c41c21e19d8f4d Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 4 Sep 2021 14:37:29 +0900 Subject: [PATCH 11/82] Make entry generic --- src/library/cache.cr | 60 +++++++++++++++++++++++++++++++++++--------- src/library/title.cr | 4 +-- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index c6e93749..6136dd3c 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -1,5 +1,7 @@ require "digest" +require "./entry" + class InfoCache alias ProgressCache = Tuple(String, Int32) @@ -158,23 +160,46 @@ class InfoCache end end -private class SortedEntriesCacheEntry - @value : Array(String) - +private class CacheEntry(SaveT, ReturnT) getter key : String, atime : Time - def initialize(@ctime : Time, @key : String, value : Array(Entry)) - @atime = @ctime - @value = value.map &.id + @value : SaveT + + def initialize(@key : String, value : ReturnT) + @atime = @ctime = Time.utc + @value = self.class.to_save_t value end def value @atime = Time.utc - SortedEntriesCacheEntry.ids2entries @value + self.class.to_return_t @value + end + + def self.to_save_t(value : ReturnT) + value end - # private? - def self.ids2entries(ids : Array(String)) + def self.to_return_t(value : SaveT) + value + end + + def instance_size + instance_sizeof(CacheEntry(SaveT, ReturnT)) + # sizeof itself + instance_sizeof(String) + @key.bytesize + # allocated memory for @key + @value.instance_size + end +end + +class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) + def self.to_save_t(value : Array(Entry)) + value.map &.id + end + + def self.to_return_t(value : Array(String)) + ids2entries value + end + + private def self.ids2entries(ids : Array(String)) e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} } entries = [] of Entry begin @@ -202,11 +227,21 @@ private class SortedEntriesCacheEntry end end +alias CacheEntryType = SortedEntriesCacheEntry + +def generate_cache_entry(key : String, value : Array(Entry) | Int32 | String) + if value.is_a? Array(Entry) + SortedEntriesCacheEntry.new key, value + else + CacheEntry(typeof(value), typeof(value)).new key, value + end +end + # LRU Cache class SortedEntriesCache @@limit : Int128 = Int128.new 0 # key => entry - @@cache = {} of String => SortedEntriesCacheEntry + @@cache = {} of String => CacheEntryType def self.init enabled = Config.current.sorted_entries_cache_enable @@ -221,8 +256,9 @@ class SortedEntriesCache return entry.value unless entry.nil? end - def self.set(key : String, value : Array(Entry)) - @@cache[key] = SortedEntriesCacheEntry.new Time.utc, key, value + def self.set(cache_entry : CacheEntryType) + key = cache_entry.key + @@cache[key] = cache_entry Logger.debug "LRUCache Cached #{key}" remove_victim_cache end diff --git a/src/library/title.cr b/src/library/title.cr index ed546ca7..c3777388 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -346,7 +346,7 @@ class Title def sorted_entries(username, opt : SortOptions? = nil) cache_key = SortedEntriesCacheEntry.gen_key @id, username, @entries, opt cached_entries = SortedEntriesCache.get cache_key - return cached_entries if cached_entries + return cached_entries if cached_entries.is_a? Array(Entry) if opt.nil? opt = SortOptions.from_info_json @dir, username @@ -382,7 +382,7 @@ class Title ary.reverse! unless opt.not_nil!.ascend if Config.current.sorted_entries_cache_enable - SortedEntriesCache.set cache_key, ary + SortedEntriesCache.set generate_cache_entry cache_key, ary end ary end From 0fd7caef4be10d480b58a3c0c217f0b65e8a1728 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 5 Sep 2021 00:02:05 +0900 Subject: [PATCH 12/82] Rename --- src/library/cache.cr | 2 +- src/library/entry.cr | 2 +- src/library/title.cr | 6 +++--- src/mango.cr | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 6136dd3c..b399632d 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -238,7 +238,7 @@ def generate_cache_entry(key : String, value : Array(Entry) | Int32 | String) end # LRU Cache -class SortedEntriesCache +class LRUCache @@limit : Int128 = Int128.new 0 # key => entry @@cache = {} of String => CacheEntryType diff --git a/src/library/entry.cr b/src/library/entry.cr index c7599fff..176efe19 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -190,7 +190,7 @@ class Entry [false, true].each do |ascend| sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, username, @book.entries, SortOptions.new(SortMethod::Progress, ascend) - SortedEntriesCache.invalidate sorted_entries_cache_key + LRUCache.invalidate sorted_entries_cache_key end TitleInfo.new @book.dir do |info| diff --git a/src/library/title.cr b/src/library/title.cr index c3777388..a5a68508 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -345,7 +345,7 @@ class Title # When `opt` is not nil, it saves the options to info.json def sorted_entries(username, opt : SortOptions? = nil) cache_key = SortedEntriesCacheEntry.gen_key @id, username, @entries, opt - cached_entries = SortedEntriesCache.get cache_key + cached_entries = LRUCache.get cache_key return cached_entries if cached_entries.is_a? Array(Entry) if opt.nil? @@ -382,7 +382,7 @@ class Title ary.reverse! unless opt.not_nil!.ascend if Config.current.sorted_entries_cache_enable - SortedEntriesCache.set generate_cache_entry cache_key, ary + LRUCache.set generate_cache_entry cache_key, ary end ary end @@ -453,7 +453,7 @@ class Title sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @id, username, @entries, SortOptions.new(SortMethod::Progress, ascend) - SortedEntriesCache.invalidate sorted_entries_cache_key + LRUCache.invalidate sorted_entries_cache_key end selected_entries = ids diff --git a/src/mango.cr b/src/mango.cr index 9b58d503..f27165ea 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -55,7 +55,7 @@ class CLI < Clim Config.load(opts.config).set_current # Initialize main components - SortedEntriesCache.init + LRUCache.init Storage.default Queue.default Library.default From de410f42b8cd67769aa1778d89837cdbde3966a9 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 5 Sep 2021 01:54:19 +0900 Subject: [PATCH 13/82] Replace InfoCache to LRUCache --- src/library/cache.cr | 197 ++++++++--------------------------------- src/library/entry.cr | 12 ++- src/library/library.cr | 4 - src/library/title.cr | 28 +++--- src/library/types.cr | 7 +- src/util/web.cr | 3 +- 6 files changed, 61 insertions(+), 190 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index b399632d..f772c66e 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -1,164 +1,7 @@ require "digest" require "./entry" - -class InfoCache - alias ProgressCache = Tuple(String, Int32) - - def self.clear - clear_cover_url - clear_progress_cache - clear_sort_opt - end - - def self.clean - clean_cover_url - clean_progress_cache - clean_sort_opt - end - - # item id => cover_url - @@cached_cover_url = {} of String => String - @@cached_cover_url_previous = {} of String => String # item id => cover_url - - def self.set_cover_url(id : String, cover_url : String) - @@cached_cover_url[id] = cover_url - end - - def self.get_cover_url(id : String) - @@cached_cover_url[id]? - end - - def self.invalidate_cover_url(id : String) - @@cached_cover_url.delete id - end - - def self.move_cover_url(id : String) - if @@cached_cover_url_previous[id]? - @@cached_cover_url[id] = @@cached_cover_url_previous[id] - end - end - - private def self.clear_cover_url - @@cached_cover_url_previous = @@cached_cover_url - @@cached_cover_url = {} of String => String - end - - private def self.clean_cover_url - @@cached_cover_url_previous = {} of String => String - end - - # book.id:username => {signature, sum} - @@progress_cache = {} of String => ProgressCache - # book.id => username => {signature, sum} - @@progress_cache_previous = {} of String => Hash(String, ProgressCache) - - def self.set_progress_cache(book_id : String, username : String, - entries : Array(Entry), sum : Int32) - progress_cache_id = "#{book_id}:#{username}" - progress_cache_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s - @@progress_cache[progress_cache_id] = {progress_cache_sig, sum} - Logger.debug "Progress Cached #{progress_cache_id}" - end - - def self.get_progress_cache(book_id : String, username : String, - entries : Array(Entry)) - progress_cache_id = "#{book_id}:#{username}" - progress_cache_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s - cached = @@progress_cache[progress_cache_id]? - if cached && cached[0] == progress_cache_sig - Logger.debug "Progress Cache Hit! #{progress_cache_id}" - return cached[1] - end - end - - def self.invalidate_progress_cache(book_id : String, username : String) - progress_cache_id = "#{book_id}:#{username}" - if @@progress_cache[progress_cache_id]? - @@progress_cache.delete progress_cache_id - Logger.debug "Progress Invalidate Cache #{progress_cache_id}" - end - end - - def self.move_progress_cache(book_id : String) - if @@progress_cache_previous[book_id]? - @@progress_cache_previous[book_id].each do |username, cached| - id = "#{book_id}:#{username}" - unless @@progress_cache[id]? - # It would be invalidated when entries changed - @@progress_cache[id] = cached - end - end - end - end - - private def self.clear_progress_cache - @@progress_cache_previous = {} of String => Hash(String, ProgressCache) - @@progress_cache.each do |id, cached| - splitted = id.split(':', 2) - book_id = splitted[0] - username = splitted[1] - unless @@progress_cache_previous[book_id]? - @@progress_cache_previous[book_id] = {} of String => ProgressCache - end - - @@progress_cache_previous[book_id][username] = cached - end - @@progress_cache = {} of String => ProgressCache - end - - private def self.clean_progress_cache - @@progress_cache_previous = {} of String => Hash(String, ProgressCache) - end - - # book.dir:username => SortOptions - @@cached_sort_opt = {} of String => SortOptions - @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) - - def self.set_sort_opt(dir : String, username : String, sort_opt : SortOptions) - id = "#{dir}:#{username}" - @@cached_sort_opt[id] = sort_opt - end - - def self.get_sort_opt(dir : String, username : String) - id = "#{dir}:#{username}" - @@cached_sort_opt[id]? - end - - def self.invalidate_sort_opt(dir : String, username : String) - id = "#{dir}:#{username}" - @@cached_sort_opt.delete id - end - - def self.move_sort_opt(dir : String) - if @@cached_sort_opt_previous[dir]? - @@cached_sort_opt_previous[dir].each do |username, cached| - id = "#{dir}:#{username}" - unless @@cached_sort_opt[id]? - @@cached_sort_opt[id] = cached - end - end - end - end - - private def self.clear_sort_opt - @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) - @@cached_sort_opt.each do |id, cached| - splitted = id.split(':', 2) - book_dir = splitted[0] - username = splitted[1] - unless @@cached_sort_opt_previous[book_dir]? - @@cached_sort_opt_previous[book_dir] = {} of String => SortOptions - end - @@cached_sort_opt_previous[book_dir][username] = cached - end - @@cached_sort_opt = {} of String => SortOptions - end - - private def self.clean_sort_opt - @@cached_sort_opt_previous = {} of String => Hash(String, SortOptions) - end -end +require "./types" private class CacheEntry(SaveT, ReturnT) getter key : String, atime : Time @@ -227,11 +70,45 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) end end -alias CacheEntryType = SortedEntriesCacheEntry +class SortOptionsCacheEntry < CacheEntry(Tuple(String, Bool), SortOptions) + def self.to_save_t(value : SortOptions) + value.to_tuple + end + + def self.to_return_t(value : Tuple(String, Bool)) + SortOptions.from_tuple value + end + + def instance_size + instance_sizeof(SortOptionsCacheEntry) + + @value[0].instance_size + end +end + +class String + def instance_size + instance_sizeof(String) + bytesize + end +end + +struct Tuple(*T) + def instance_size + sizeof(T) # iterate T and add instance_size of that + end +end + +alias CacheableType = Array(Entry) | String | Tuple(String, Int32) | + SortOptions +alias CacheEntryType = SortedEntriesCacheEntry | + SortOptionsCacheEntry | + CacheEntry(String, String) | + CacheEntry(Tuple(String, Int32), Tuple(String, Int32)) -def generate_cache_entry(key : String, value : Array(Entry) | Int32 | String) +def generate_cache_entry(key : String, value : CacheableType) if value.is_a? Array(Entry) SortedEntriesCacheEntry.new key, value + elsif value.is_a? SortOptions + SortOptionsCacheEntry.new key, value else CacheEntry(typeof(value), typeof(value)).new key, value end diff --git a/src/library/entry.cr b/src/library/entry.cr index 176efe19..2c1ae346 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -44,8 +44,6 @@ class Entry MIME.from_filename? e.filename end file.close - - InfoCache.move_cover_url @id end def to_slim_json : String @@ -83,8 +81,8 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg - cached_cover_url = InfoCache.get_cover_url @id - return cached_cover_url if cached_cover_url + cached_cover_url = LRUCache.get "#{@id}:cover_url" + return cached_cover_url if cached_cover_url.is_a? String unless @book.entry_cover_url_cache TitleInfo.new @book.dir do |info| @@ -100,7 +98,7 @@ class Entry url = File.join Config.current.base_url, info_url end end - InfoCache.set_cover_url @id, url + LRUCache.set generate_cache_entry "#{@id}:cover_url", url url end @@ -183,9 +181,9 @@ class Entry # For backward backward compatibility with v0.1.0, we save entry titles # instead of IDs in info.json def save_progress(username, page) - InfoCache.invalidate_progress_cache @book.id, username + LRUCache.invalidate "#{@book.id}:#{username}:progress_sum" @book.parents.each do |parent| - InfoCache.invalidate_progress_cache parent.id, username + LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end [false, true].each do |ascend| sorted_entries_cache_key = SortedEntriesCacheEntry.gen_key @book.id, diff --git a/src/library/library.cr b/src/library/library.cr index 21e5c8b1..9351e60d 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -106,8 +106,6 @@ class Library storage = Storage.new auto_close: false - InfoCache.clear - (Dir.entries @dir) .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } @@ -121,8 +119,6 @@ class Library @title_ids << title.id end - InfoCache.clean - storage.bulk_insert_ids storage.close diff --git a/src/library/title.cr b/src/library/title.cr index a5a68508..cdf78fd2 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -60,10 +60,6 @@ class Title @entries.sort! do |a, b| sorter.compare a.title, b.title end - - InfoCache.move_cover_url @id - InfoCache.move_progress_cache @id - InfoCache.move_sort_opt @dir end def to_slim_json : String @@ -234,8 +230,8 @@ class Title end def cover_url - cached_cover_url = InfoCache.get_cover_url @id - return cached_cover_url if cached_cover_url + cached_cover_url = LRUCache.get "#{@id}:cover_url" + return cached_cover_url if cached_cover_url.is_a? String url = "#{Config.current.base_url}img/icon.png" readable_entries = @entries.select &.err_msg.nil? @@ -248,12 +244,12 @@ class Title url = File.join Config.current.base_url, info_url end end - InfoCache.set_cover_url @id, url + LRUCache.set generate_cache_entry "#{@id}:cover_url", url url end def set_cover_url(url : String) - InfoCache.invalidate_cover_url @id + LRUCache.invalidate "#{@id}:cover_url" TitleInfo.new @dir do |info| info.cover_url = url info.save @@ -262,7 +258,7 @@ class Title def set_cover_url(entry_name : String, url : String) selected_entry = @entries.find { |entry| entry.display_name == entry_name } - InfoCache.invalidate_cover_url selected_entry.id if selected_entry + LRUCache.invalidate "#{selected_entry.id}:cover_url" if selected_entry TitleInfo.new @dir do |info| info.entry_cover_url[entry_name] = url info.save @@ -284,12 +280,14 @@ class Title end def deep_read_page_count(username) : Int32 - # CACHE HERE - cached_sum = InfoCache.get_progress_cache @id, username, @entries - return cached_sum unless cached_sum.nil? + key = "#{@id}:#{username}:progress_sum" + sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + cached_sum = LRUCache.get key + return cached_sum[1] if cached_sum.is_a? Tuple(String, Int32) && + cached_sum[0] == sig sum = load_progress_for_all_entries(username).sum + titles.flat_map(&.deep_read_page_count username).sum - InfoCache.set_progress_cache @id, username, @entries, sum + LRUCache.set generate_cache_entry key, {sig, sum} sum end @@ -445,9 +443,9 @@ class Title end def bulk_progress(action, ids : Array(String), username) - InfoCache.invalidate_progress_cache @id, username + LRUCache.invalidate "#{@id}:#{username}:progress_sum" parents.each do |parent| - InfoCache.invalidate_progress_cache parent.id, username + LRUCache.invalidate "#{parent.id}:#{username}:progress_sum" end [false, true].each do |ascend| sorted_entries_cache_key = diff --git a/src/library/types.cr b/src/library/types.cr index 094cb643..891ee08e 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -35,15 +35,16 @@ class SortOptions end def self.from_info_json(dir, username) - cached_opt = InfoCache.get_sort_opt dir, username - return cached_opt if cached_opt + key = "#{dir}:#{username}:sort_opt" + cached_opt = LRUCache.get key + return cached_opt if cached_opt.is_a? SortOptions opt = SortOptions.new TitleInfo.new dir do |info| if info.sort_by.has_key? username opt = SortOptions.from_tuple info.sort_by[username] end end - InfoCache.set_sort_opt dir, username, opt + LRUCache.set generate_cache_entry key, opt opt end diff --git a/src/util/web.cr b/src/util/web.cr index 5e873cae..9b967da6 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -120,7 +120,8 @@ macro get_and_save_sort_opt(dir) sort_opt = SortOptions.new sort_method, is_ascending - InfoCache.set_sort_opt {{dir}}, username, sort_opt + key = "#{{{dir}}}:#{username}:sort_opt" + LRUCache.set generate_cache_entry key, sort_opt TitleInfo.new {{dir}} do |info| info.sort_by[username] = sort_opt.to_tuple info.save From 847f516a65c9dac283b11e4f31de8d9ee449df54 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 5 Sep 2021 02:35:44 +0900 Subject: [PATCH 14/82] Cache TitleInfo using LRUCache --- src/library/types.cr | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/library/types.cr b/src/library/types.cr index 891ee08e..46d2a7c2 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -92,6 +92,18 @@ class TitleInfo @@mutex_hash = {} of String => Mutex def self.new(dir, &) + key = "#{dir}:info.json" + info = LRUCache.get key + if info.is_a? String + begin + instance = TitleInfo.from_json info + instance.dir = dir + yield instance + return + rescue + end + end + if @@mutex_hash[dir]? mutex = @@mutex_hash[dir] else @@ -105,6 +117,7 @@ class TitleInfo instance = TitleInfo.from_json File.read json_path end instance.dir = dir + LRUCache.set generate_cache_entry key, instance.to_json yield instance end end @@ -112,5 +125,7 @@ class TitleInfo def save json_path = File.join @dir, "info.json" File.write json_path, self.to_pretty_json + key = "#{@dir}:info.json" + LRUCache.set generate_cache_entry key, self.to_json end end From 11976b15f9f66b92df78bf00fbecdf45a8850905 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 5 Sep 2021 03:02:20 +0900 Subject: [PATCH 15/82] Make LRUCache togglable --- src/library/cache.cr | 10 ++++++++-- src/library/title.cr | 4 +--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index f772c66e..5b3d2bdd 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -120,13 +120,17 @@ class LRUCache # key => entry @@cache = {} of String => CacheEntryType + def self.enabled + Config.current.sorted_entries_cache_enable + end + def self.init - enabled = Config.current.sorted_entries_cache_enable cache_size = Config.current.sorted_entries_cache_size_mbs @@limit = Int128.new cache_size * 1024 * 1024 if enabled end def self.get(key : String) + return unless enabled entry = @@cache[key]? Logger.debug "LRUCache Cache Hit! #{key}" unless entry.nil? Logger.debug "LRUCache Cache Miss #{key}" if entry.nil? @@ -134,6 +138,7 @@ class LRUCache end def self.set(cache_entry : CacheEntryType) + return unless enabled key = cache_entry.key @@cache[key] = cache_entry Logger.debug "LRUCache Cached #{key}" @@ -141,6 +146,7 @@ class LRUCache end def self.invalidate(key : String) + return unless enabled @@cache.delete key end @@ -149,7 +155,7 @@ class LRUCache Logger.debug "---- LRU Cache ----" Logger.debug "Size: #{sum} Bytes" Logger.debug "List:" - @@cache.each { |k, v| Logger.debug "#{k} | #{v.atime}" } + @@cache.each { |k, v| Logger.debug "#{k} | #{v.atime} | #{v.instance_size}" } Logger.debug "-------------------" end diff --git a/src/library/title.cr b/src/library/title.cr index cdf78fd2..83336422 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -379,9 +379,7 @@ class Title ary.reverse! unless opt.not_nil!.ascend - if Config.current.sorted_entries_cache_enable - LRUCache.set generate_cache_entry cache_key, ary - end + LRUCache.set generate_cache_entry cache_key, ary ary end From c75c71709ff64bfb97a10a4f0d577f0f2f6ff764 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 5 Sep 2021 11:21:53 +0900 Subject: [PATCH 16/82] make check --- src/library/cache.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 5b3d2bdd..ca6af53a 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -155,7 +155,9 @@ class LRUCache Logger.debug "---- LRU Cache ----" Logger.debug "Size: #{sum} Bytes" Logger.debug "List:" - @@cache.each { |k, v| Logger.debug "#{k} | #{v.atime} | #{v.instance_size}" } + @@cache.each do |k, v| + Logger.debug "#{k} | #{v.atime} | #{v.instance_size}" + end Logger.debug "-------------------" end From c5b6a8b5b950d685f2490633c8914e53a6640c05 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 5 Sep 2021 13:57:20 +0000 Subject: [PATCH 17/82] Improve instance_size for Tuple --- src/library/cache.cr | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index ca6af53a..612345c8 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -93,7 +93,15 @@ end struct Tuple(*T) def instance_size - sizeof(T) # iterate T and add instance_size of that + sizeof(T) + # total size of non-reference types + self.sum do |e| + next 0 unless e.is_a? Reference + if e.responds_to? :instance_size + e.instance_size + else + instance_sizeof(typeof(e)) + end + end end end From 565a535d22f16d230d2048aa7e6d9ed1cd2b8fe4 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 6 Sep 2021 02:23:02 +0900 Subject: [PATCH 18/82] Remove caching verbosely, add cached_cover_url --- src/library/cache.cr | 21 +-------------------- src/library/entry.cr | 3 --- src/library/title.cr | 10 ++++------ src/library/types.cr | 4 ---- src/util/web.cr | 1 - 5 files changed, 5 insertions(+), 34 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 612345c8..5d3797ea 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -70,21 +70,6 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) end end -class SortOptionsCacheEntry < CacheEntry(Tuple(String, Bool), SortOptions) - def self.to_save_t(value : SortOptions) - value.to_tuple - end - - def self.to_return_t(value : Tuple(String, Bool)) - SortOptions.from_tuple value - end - - def instance_size - instance_sizeof(SortOptionsCacheEntry) + - @value[0].instance_size - end -end - class String def instance_size instance_sizeof(String) + bytesize @@ -105,18 +90,14 @@ struct Tuple(*T) end end -alias CacheableType = Array(Entry) | String | Tuple(String, Int32) | - SortOptions +alias CacheableType = Array(Entry) | String | Tuple(String, Int32) alias CacheEntryType = SortedEntriesCacheEntry | - SortOptionsCacheEntry | CacheEntry(String, String) | CacheEntry(Tuple(String, Int32), Tuple(String, Int32)) def generate_cache_entry(key : String, value : CacheableType) if value.is_a? Array(Entry) SortedEntriesCacheEntry.new key, value - elsif value.is_a? SortOptions - SortOptionsCacheEntry.new key, value else CacheEntry(typeof(value), typeof(value)).new key, value end diff --git a/src/library/entry.cr b/src/library/entry.cr index 2c1ae346..28b7122e 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -81,8 +81,6 @@ class Entry def cover_url return "#{Config.current.base_url}img/icon.png" if @err_msg - cached_cover_url = LRUCache.get "#{@id}:cover_url" - return cached_cover_url if cached_cover_url.is_a? String unless @book.entry_cover_url_cache TitleInfo.new @book.dir do |info| @@ -98,7 +96,6 @@ class Entry url = File.join Config.current.base_url, info_url end end - LRUCache.set generate_cache_entry "#{@id}:cover_url", url url end diff --git a/src/library/title.cr b/src/library/title.cr index 83336422..8d973877 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -11,6 +11,7 @@ class Title @entry_display_name_cache : Hash(String, String)? @entry_cover_url_cache : Hash(String, String)? @cached_display_name : String? + @cached_cover_url : String? def initialize(@dir : String, @parent_id) storage = Storage.default @@ -230,8 +231,8 @@ class Title end def cover_url - cached_cover_url = LRUCache.get "#{@id}:cover_url" - return cached_cover_url if cached_cover_url.is_a? String + cached_cover_url = @cached_cover_url + return cached_cover_url unless cached_cover_url.nil? url = "#{Config.current.base_url}img/icon.png" readable_entries = @entries.select &.err_msg.nil? @@ -244,12 +245,11 @@ class Title url = File.join Config.current.base_url, info_url end end - LRUCache.set generate_cache_entry "#{@id}:cover_url", url + @cached_cover_url = url url end def set_cover_url(url : String) - LRUCache.invalidate "#{@id}:cover_url" TitleInfo.new @dir do |info| info.cover_url = url info.save @@ -257,8 +257,6 @@ class Title end def set_cover_url(entry_name : String, url : String) - selected_entry = @entries.find { |entry| entry.display_name == entry_name } - LRUCache.invalidate "#{selected_entry.id}:cover_url" if selected_entry TitleInfo.new @dir do |info| info.entry_cover_url[entry_name] = url info.save diff --git a/src/library/types.cr b/src/library/types.cr index 46d2a7c2..a4de0075 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -35,16 +35,12 @@ class SortOptions end def self.from_info_json(dir, username) - key = "#{dir}:#{username}:sort_opt" - cached_opt = LRUCache.get key - return cached_opt if cached_opt.is_a? SortOptions opt = SortOptions.new TitleInfo.new dir do |info| if info.sort_by.has_key? username opt = SortOptions.from_tuple info.sort_by[username] end end - LRUCache.set generate_cache_entry key, opt opt end diff --git a/src/util/web.cr b/src/util/web.cr index 9b967da6..1b3a42ce 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -121,7 +121,6 @@ macro get_and_save_sort_opt(dir) sort_opt = SortOptions.new sort_method, is_ascending key = "#{{{dir}}}:#{username}:sort_opt" - LRUCache.set generate_cache_entry key, sort_opt TitleInfo.new {{dir}} do |info| info.sort_by[username] = sort_opt.to_tuple info.save From 9807db6ac02682d20dd9ce1021725c5257b749d2 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 6 Sep 2021 02:29:31 +0900 Subject: [PATCH 19/82] Fix bug on entry_cover_url_cache --- src/library/title.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/library/title.cr b/src/library/title.cr index 8d973877..f93bf3c6 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -259,6 +259,7 @@ class Title def set_cover_url(entry_name : String, url : String) TitleInfo.new @dir do |info| info.entry_cover_url[entry_name] = url + @entry_cover_url_cache = info.entry_cover_url info.save end end From 5cb85ea8577c606e6aeceeab49294a8c10e3e9bb Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Mon, 6 Sep 2021 09:40:53 +0900 Subject: [PATCH 20/82] Set cached data when changed --- src/library/title.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/library/title.cr b/src/library/title.cr index f93bf3c6..f1915d4a 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -215,7 +215,7 @@ class Title end def set_display_name(dn) - @cached_display_name = nil + @cached_display_name = dn TitleInfo.new @dir do |info| info.display_name = dn info.save @@ -250,6 +250,7 @@ class Title end def set_cover_url(url : String) + @cached_cover_url = url TitleInfo.new @dir do |info| info.cover_url = url info.save From 79ef7bcd1cc592b0fced32dc992295e8f90ee990 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 03:01:21 +0000 Subject: [PATCH 21/82] Remove unused variable --- src/util/web.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/web.cr b/src/util/web.cr index 1b3a42ce..5704ea88 100644 --- a/src/util/web.cr +++ b/src/util/web.cr @@ -120,7 +120,6 @@ macro get_and_save_sort_opt(dir) sort_opt = SortOptions.new sort_method, is_ascending - key = "#{{{dir}}}:#{username}:sort_opt" TitleInfo.new {{dir}} do |info| info.sort_by[username] = sort_opt.to_tuple info.save From 51806f18db114b3878616d293f9930edb7b0906d Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 03:35:46 +0000 Subject: [PATCH 22/82] Rename config fields and improve logging --- src/config.cr | 5 +++-- src/library/cache.cr | 31 +++++++++++++++++++------------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/config.cr b/src/config.cr index 69feccdd..99fb5b99 100644 --- a/src/config.cr +++ b/src/config.cr @@ -20,8 +20,9 @@ class Config property plugin_path : String = File.expand_path "~/mango/plugins", home: true property download_timeout_seconds : Int32 = 30 - property sorted_entries_cache_enable = false - property sorted_entries_cache_size_mbs = 50 + property cache_enabled = false + property cache_size_mbs = 50 + property cache_log_enabled = true property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/library/cache.cr b/src/library/cache.cr index 5d3797ea..496442f1 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -106,23 +106,26 @@ end # LRU Cache class LRUCache @@limit : Int128 = Int128.new 0 + @@should_log = true # key => entry @@cache = {} of String => CacheEntryType def self.enabled - Config.current.sorted_entries_cache_enable + Config.current.cache_enabled end def self.init - cache_size = Config.current.sorted_entries_cache_size_mbs + cache_size = Config.current.cache_size_mbs @@limit = Int128.new cache_size * 1024 * 1024 if enabled + @@should_log = Config.current.cache_log_enabled end def self.get(key : String) return unless enabled entry = @@cache[key]? - Logger.debug "LRUCache Cache Hit! #{key}" unless entry.nil? - Logger.debug "LRUCache Cache Miss #{key}" if entry.nil? + if @@should_log + Logger.debug "LRUCache #{entry.nil? ? "miss" : "hit"} #{key}" + end return entry.value unless entry.nil? end @@ -130,8 +133,8 @@ class LRUCache return unless enabled key = cache_entry.key @@cache[key] = cache_entry - Logger.debug "LRUCache Cached #{key}" - remove_victim_cache + Logger.debug "LRUCache cached #{key}" if @@should_log + remove_least_recent_access end def self.invalidate(key : String) @@ -140,6 +143,7 @@ class LRUCache end def self.print + return unless @@should_log sum = @@cache.sum { |_, entry| entry.instance_size } Logger.debug "---- LRU Cache ----" Logger.debug "Size: #{sum} Bytes" @@ -155,14 +159,17 @@ class LRUCache sum > @@limit end - private def self.remove_victim_cache + private def self.remove_least_recent_access + Logger.debug "Removing entries from LRUCache" if @@should_log while is_cache_full && @@cache.size > 0 - Logger.debug "LRUCache Cache Full! Remove LRU" - min = @@cache.min_by? { |_, entry| entry.atime } + min_tuple = @@cache.min_by { |_, entry| entry.atime } + min_key = min_tuple[0] + min_entry = min_tuple[1] + Logger.debug " \ - Target: #{min[0]}, \ - Last Access Time: #{min[1].atime}" if min - invalidate min[0] if min + Target: #{min_key}, \ + Last Access Time: #{min_entry.atime}" if @@should_log + invalidate min_key end end end From 15a54f4f23038ce16cf58b607a7042d1ec6efe41 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 08:10:13 +0000 Subject: [PATCH 23/82] Add `:sorted_entries` suffix to `gen_key` --- src/library/cache.cr | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 496442f1..37359f20 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -63,10 +63,11 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) def self.gen_key(book_id : String, username : String, entries : Array(Entry), opt : SortOptions?) - sig = Digest::SHA1.hexdigest (entries.map &.id).to_s + entries_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s user_context = opt && opt.method == SortMethod::Progress ? username : "" - Digest::SHA1.hexdigest (book_id + sig + user_context + - (opt ? opt.to_tuple.to_s : "nil")) + sig = Digest::SHA1.hexdigest (book_id + entries_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) + "#{sig}:sorted_entries" end end From 44d9c51ff9e86da4b45e552797d4e379556c5dc4 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 08:10:42 +0000 Subject: [PATCH 24/82] Fix logging --- src/library/cache.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 37359f20..0784363c 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -161,7 +161,9 @@ class LRUCache end private def self.remove_least_recent_access - Logger.debug "Removing entries from LRUCache" if @@should_log + if @@should_log && is_cache_full + Logger.debug "Removing entries from LRUCache" + end while is_cache_full && @@cache.size > 0 min_tuple = @@cache.min_by { |_, entry| entry.atime } min_key = min_tuple[0] From ca1e221b10459775569d93cd1816efe5e462e880 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 08:23:31 +0000 Subject: [PATCH 25/82] Rename `ids2entries` -> `ids_to_entries` --- src/library/cache.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 0784363c..05529fbf 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -39,10 +39,10 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) end def self.to_return_t(value : Array(String)) - ids2entries value + ids_to_entries value end - private def self.ids2entries(ids : Array(String)) + private def self.ids_to_entries(ids : Array(String)) e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} } entries = [] of Entry begin From d809c21ee13429872950c7f5f299da236feb7ba7 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 08:23:54 +0000 Subject: [PATCH 26/82] Document `CacheEntry` --- src/library/cache.cr | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/library/cache.cr b/src/library/cache.cr index 05529fbf..d0d3f016 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -3,6 +3,16 @@ require "digest" require "./entry" require "./types" +# Base class for an entry in the LRU cache. +# There are two ways to use it: +# 1. Use it as it is by instantiating with the appropriate `SaveT` and +# `ReturnT`. Note that in this case, `SaveT` and `ReturnT` must be the +# same type. That is, the input value will be stored as it is without +# any transformation. +# 2. You can also subclass it and provide custom implementations for +# `to_save_t` and `to_return_t`. This allows you to transform and store +# the input value to a different type. See `SortedEntriesCacheEntry` as +# an example. private class CacheEntry(SaveT, ReturnT) getter key : String, atime : Time From 60a126024c8f4fa383fa08f5ecdcbfc843df6278 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 12:58:48 +0000 Subject: [PATCH 27/82] Stricter sanitization rules for download filenames Fixes #212 --- src/plugin/downloader.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugin/downloader.cr b/src/plugin/downloader.cr index 054698e4..d826bac7 100644 --- a/src/plugin/downloader.cr +++ b/src/plugin/downloader.cr @@ -24,8 +24,9 @@ class Plugin end private def process_filename(str) - return "_" if str == ".." - str.gsub "/", "_" + str + .gsub(/[\/\s\.\177\000-\031]/, "_") + .gsub(/__+/, "_") end private def download(job : Queue::Job) From f67e4e6cb9994fd86cafe8b9d8c0541be3b894cb Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 6 Sep 2021 13:32:10 +0000 Subject: [PATCH 28/82] Support all image types (resolves #192) --- src/library/entry.cr | 6 ++---- src/library/types.cr | 2 -- src/routes/api.cr | 5 ++--- src/util/util.cr | 4 ++++ src/views/title.html.ecr | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 28b7122e..301c3ba6 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -40,8 +40,7 @@ class Entry file = ArchiveFile.new @zip_path @pages = file.entries.count do |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename + filename_is_image e.filename end file.close end @@ -103,8 +102,7 @@ class Entry ArchiveFile.open @zip_path do |file| entries = file.entries .select { |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename + filename_is_image e.filename } .sort! { |a, b| compare_numerically a.filename, b.filename diff --git a/src/library/types.cr b/src/library/types.cr index a4de0075..a91ca86a 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -1,5 +1,3 @@ -SUPPORTED_IMG_TYPES = ["image/jpeg", "image/png", "image/webp"] - enum SortMethod Auto Title diff --git a/src/routes/api.cr b/src/routes/api.cr index b1fb3b35..80cb2782 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -456,9 +456,8 @@ struct APIRouter entry_id = env.params.query["eid"]? title = Library.default.get_title(title_id).not_nil! - unless SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? filename - raise "The uploaded image must be either JPEG or PNG" + unless filename_is_image filename + raise "The uploaded file must be an image" end ext = File.extname filename diff --git a/src/util/util.cr b/src/util/util.cr index c4e168a7..2db7457b 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -120,3 +120,7 @@ class String match / s.size end end + +def filename_is_image(fn : String) : Bool + MIME.from_filename?(fn).try(&.starts_with?("image/")) || false +end diff --git a/src/views/title.html.ecr b/src/views/title.html.ecr index 78edf988..dc0dc3dd 100644 --- a/src/views/title.html.ecr +++ b/src/views/title.html.ecr @@ -101,7 +101,7 @@ Upload a cover image by dropping it here or
- "> + selecting one
From d9adb49c2712003cb7371198b133b3518e1c6b09 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 7 Sep 2021 10:45:59 +0000 Subject: [PATCH 29/82] Revert "Support all image types (resolves #192)" This reverts commit f67e4e6cb9994fd86cafe8b9d8c0541be3b894cb. --- src/library/entry.cr | 6 ++++-- src/library/types.cr | 2 ++ src/routes/api.cr | 5 +++-- src/util/util.cr | 4 ---- src/views/title.html.ecr | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 301c3ba6..28b7122e 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -40,7 +40,8 @@ class Entry file = ArchiveFile.new @zip_path @pages = file.entries.count do |e| - filename_is_image e.filename + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename end file.close end @@ -102,7 +103,8 @@ class Entry ArchiveFile.open @zip_path do |file| entries = file.entries .select { |e| - filename_is_image e.filename + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename } .sort! { |a, b| compare_numerically a.filename, b.filename diff --git a/src/library/types.cr b/src/library/types.cr index a91ca86a..a4de0075 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -1,3 +1,5 @@ +SUPPORTED_IMG_TYPES = ["image/jpeg", "image/png", "image/webp"] + enum SortMethod Auto Title diff --git a/src/routes/api.cr b/src/routes/api.cr index 80cb2782..b1fb3b35 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -456,8 +456,9 @@ struct APIRouter entry_id = env.params.query["eid"]? title = Library.default.get_title(title_id).not_nil! - unless filename_is_image filename - raise "The uploaded file must be an image" + unless SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? filename + raise "The uploaded image must be either JPEG or PNG" end ext = File.extname filename diff --git a/src/util/util.cr b/src/util/util.cr index 2db7457b..c4e168a7 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -120,7 +120,3 @@ class String match / s.size end end - -def filename_is_image(fn : String) : Bool - MIME.from_filename?(fn).try(&.starts_with?("image/")) || false -end diff --git a/src/views/title.html.ecr b/src/views/title.html.ecr index dc0dc3dd..78edf988 100644 --- a/src/views/title.html.ecr +++ b/src/views/title.html.ecr @@ -101,7 +101,7 @@ Upload a cover image by dropping it here or
- + "> selecting one
From 371796cce94a4858ae1eedde9a2547b2addd6304 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 7 Sep 2021 11:04:05 +0000 Subject: [PATCH 30/82] Support additional image formats: - APNG - AVIF - GIF - SVG --- src/library/types.cr | 10 +++++++++- src/util/util.cr | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/library/types.cr b/src/library/types.cr index a4de0075..05451841 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -1,4 +1,12 @@ -SUPPORTED_IMG_TYPES = ["image/jpeg", "image/png", "image/webp"] +SUPPORTED_IMG_TYPES = %w( + image/jpeg + image/png + image/webp + image/apng + image/avif + image/gif + image/svg+xml +) enum SortMethod Auto diff --git a/src/util/util.cr b/src/util/util.cr index c4e168a7..bc710bdd 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -35,6 +35,11 @@ def register_mime_types # FontAwesome fonts ".woff" => "font/woff", ".woff2" => "font/woff2", + + # Supported image formats. JPG, PNG, GIF, WebP, and SVG are already + # defiend by Crystal in `MIME.DEFAULT_TYPES` + ".apng" => "image/apng", + ".avif" => "image/avif", }.each do |k, v| MIME.register k, v end From d2cad6c49643111fa284bf48f0254a2cd7088ad6 Mon Sep 17 00:00:00 2001 From: i use arch btw <41193328+lincolnthedev@users.noreply.github.com> Date: Tue, 7 Sep 2021 21:12:51 -0400 Subject: [PATCH 31/82] Update .dockerignore --- .dockerignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.dockerignore b/.dockerignore index 491fc359..b333c4a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,8 @@ node_modules lib +Dockerfile +Dockerfile.arm32v7 +Dockerfile.arm64v8 +README.md +.all-contributorsrc +env.example From ccf558eaa7657a96ebe7277828ad7fdf6397fc9a Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 8 Sep 2021 10:03:05 +0000 Subject: [PATCH 32/82] Improve filename sanitization rules --- spec/util_spec.cr | 10 ++++++++++ src/plugin/downloader.cr | 12 +++--------- src/util/util.cr | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 27d97c2a..5e8b9f0e 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -61,3 +61,13 @@ describe "chapter_sort" do end.should eq ary end end + +describe "sanitize_filename" do + it "returns a random string for empty sanitized string" do + sanitize_filename("..").should_not eq sanitize_filename("..") + end + it "sanitizes correctly" do + sanitize_filename(".. \n\v.\rマンゴー/|*()<[1/2] 3.14 hello world ") + .should eq " . マンゴー_()[1_2] 3.14 hello world" + end +end diff --git a/src/plugin/downloader.cr b/src/plugin/downloader.cr index d826bac7..2800232a 100644 --- a/src/plugin/downloader.cr +++ b/src/plugin/downloader.cr @@ -23,12 +23,6 @@ class Plugin job end - private def process_filename(str) - str - .gsub(/[\/\s\.\177\000-\031]/, "_") - .gsub(/__+/, "_") - end - private def download(job : Queue::Job) @downloading = true @queue.set_status Queue::JobStatus::Downloading, job @@ -43,8 +37,8 @@ class Plugin pages = info["pages"].as_i - manga_title = process_filename job.manga_title - chapter_title = process_filename info["title"].as_s + manga_title = sanitize_filename job.manga_title + chapter_title = sanitize_filename info["title"].as_s @queue.set_pages pages, job lib_dir = @library_path @@ -69,7 +63,7 @@ class Plugin while page = plugin.next_page break unless @queue.exists? job - fn = process_filename page["filename"].as_s + fn = sanitize_filename page["filename"].as_s url = page["url"].as_s headers = HTTP::Headers.new diff --git a/src/util/util.cr b/src/util/util.cr index c4e168a7..50aa7fe0 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -120,3 +120,22 @@ class String match / s.size end end + +# Does the followings: +# - turns space-like characters into the normal whitespaces ( ) +# - strips and collapses spaces +# - removes ASCII control characters +# - replaces slashes (/) with underscores (_) +# - removes leading dots (.) +# - removes the following special characters: \:*?"<>| +# +# If the sanitized string is empty, returns a random string instead. +def sanitize_filename(str : String) : String + sanitized = str + .gsub(/\s+/, " ") + .strip + .gsub(/\//, "_") + .gsub(/^\.+/, "") + .gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "") + sanitized.size > 0 ? sanitized : random_str +end From 53226eab61537bfe64ee13374c8d54147f59f4bf Mon Sep 17 00:00:00 2001 From: i use arch btw <41193328+lincolnthedev@users.noreply.github.com> Date: Wed, 8 Sep 2021 07:58:58 -0400 Subject: [PATCH 33/82] Forgot .github --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index b333c4a4..996cb5c2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ Dockerfile.arm64v8 README.md .all-contributorsrc env.example +.github/ From 566cebfcdda40835fbac6274dbeb5153250f4075 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Thu, 9 Sep 2021 00:13:58 +0000 Subject: [PATCH 34/82] Remove all leading dots and spaces --- spec/util_spec.cr | 2 +- src/util/util.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/util_spec.cr b/spec/util_spec.cr index 5e8b9f0e..fb4efd74 100644 --- a/spec/util_spec.cr +++ b/spec/util_spec.cr @@ -68,6 +68,6 @@ describe "sanitize_filename" do end it "sanitizes correctly" do sanitize_filename(".. \n\v.\rマンゴー/|*()<[1/2] 3.14 hello world ") - .should eq " . マンゴー_()[1_2] 3.14 hello world" + .should eq "マンゴー_()[1_2] 3.14 hello world" end end diff --git a/src/util/util.cr b/src/util/util.cr index 50aa7fe0..833f1ca1 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -135,7 +135,7 @@ def sanitize_filename(str : String) : String .gsub(/\s+/, " ") .strip .gsub(/\//, "_") - .gsub(/^\.+/, "") + .gsub(/^[\.\s]+/, "") .gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "") sanitized.size > 0 ? sanitized : random_str end From 0667f01471f8f1032f778e90bfee09af42e63ed9 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 02:01:43 +0900 Subject: [PATCH 35/82] Measure scan only --- src/library/library.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index 9351e60d..32cc1619 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -98,6 +98,7 @@ class Library end def scan + start = Time.local unless Dir.exists? @dir Logger.info "The library directory #{@dir} does not exist. " \ "Attempting to create it" @@ -122,7 +123,8 @@ class Library storage.bulk_insert_ids storage.close - Logger.debug "Scan completed" + ms = (Time.local - start).total_milliseconds + Logger.debug "Scan completed. #{ms}ms" Storage.default.mark_unavailable end From 291a340cdd48b068e71cde78b9cf22227658ecf6 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Thu, 9 Sep 2021 20:45:47 +0900 Subject: [PATCH 36/82] Add yaml serializer to Library, Title, Entry --- src/library/entry.cr | 2 ++ src/library/library.cr | 2 ++ src/library/title.cr | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/library/entry.cr b/src/library/entry.cr index 28b7122e..bb0aa1b7 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,6 +1,8 @@ require "image_size" class Entry + include YAML::Serializable + getter zip_path : String, book : Title, title : String, size : String, pages : Int32, id : String, encoded_path : String, encoded_title : String, mtime : Time, err_msg : String? diff --git a/src/library/library.cr b/src/library/library.cr index 32cc1619..9c23d68d 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -1,4 +1,6 @@ class Library + include YAML::Serializable + getter dir : String, title_ids : Array(String), title_hash : Hash(String, Title) diff --git a/src/library/title.cr b/src/library/title.cr index f1915d4a..899e21f6 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -2,6 +2,8 @@ require "digest" require "../archive" class Title + include YAML::Serializable + getter dir : String, parent_id : String, title_ids : Array(String), entries : Array(Entry), title : String, id : String, encoded_title : String, mtime : Time, signature : UInt64, From 4409ed8f455d9e44705974ca1896e884a5957fc2 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Thu, 9 Sep 2021 21:14:11 +0900 Subject: [PATCH 37/82] Implement save_instance, load_instance --- src/library/library.cr | 37 +++++++++++++++++++++++++++++++++++++ src/mango.cr | 1 + 2 files changed, 38 insertions(+) diff --git a/src/library/library.cr b/src/library/library.cr index 9c23d68d..b8528f34 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -6,6 +6,41 @@ class Library use_default + def save_instance + path = Config.current.library_path + instance_file_path = File.join path, "library.yml.zip" + + writer = Compress::Zip::Writer.new instance_file_path + writer.add "instance.yml", self.to_yaml + writer.close + end + + def self.load_instance + dir = Config.current.library_path + return unless Dir.exists? dir + instance_file_path = File.join path, "library.yml.zip" + return unless File.exists? instance_file_path + + zip_file = Compress::Zip::File.new instance_file_path + instance_file = zip_file.entries.find { |entry| entry.filename == "instance.yml" } + + if instance_file.nil? + zip_file.close + return + end + begin + instance_file.open do |content| + @@default = Library.from_yaml content + end + rescue e + Logger.error e + end + + zip_file.close + + scan + end + def initialize register_mime_types @@ -128,6 +163,8 @@ class Library ms = (Time.local - start).total_milliseconds Logger.debug "Scan completed. #{ms}ms" Storage.default.mark_unavailable + + save_instance end def get_continue_reading_entries(username) diff --git a/src/mango.cr b/src/mango.cr index f27165ea..39b13525 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -58,6 +58,7 @@ class CLI < Clim LRUCache.init Storage.default Queue.default + Library.load_instance Library.default Plugin::Downloader.default From 0a90e3b3334d2be89d6517b8e576db7a546b9f34 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 10 Sep 2021 20:24:54 +0900 Subject: [PATCH 38/82] Ignore caches --- src/library/library.cr | 4 ++-- src/library/title.cr | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index b8528f34..a3f2baa6 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -18,7 +18,7 @@ class Library def self.load_instance dir = Config.current.library_path return unless Dir.exists? dir - instance_file_path = File.join path, "library.yml.zip" + instance_file_path = File.join dir, "library.yml.zip" return unless File.exists? instance_file_path zip_file = Compress::Zip::File.new instance_file_path @@ -38,7 +38,7 @@ class Library zip_file.close - scan + Library.default.scan end def initialize diff --git a/src/library/title.cr b/src/library/title.cr index 899e21f6..6c873b85 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -10,9 +10,13 @@ class Title entry_cover_url_cache : Hash(String, String)? setter entry_cover_url_cache : Hash(String, String)? + @[YAML::Field(ignore: true)] @entry_display_name_cache : Hash(String, String)? + @[YAML::Field(ignore: true)] @entry_cover_url_cache : Hash(String, String)? + @[YAML::Field(ignore: true)] @cached_display_name : String? + @[YAML::Field(ignore: true)] @cached_cover_url : String? def initialize(@dir : String, @parent_id) From eb3e37b95097abf11b91c2a1c210fd71e0890126 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 11 Sep 2021 13:30:07 +0900 Subject: [PATCH 39/82] Examine titles and recycle them --- src/library/library.cr | 8 +++++++- src/library/title.cr | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index a3f2baa6..86375be7 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -144,14 +144,20 @@ class Library storage = Storage.new auto_close: false + @title_ids.select! do |title_id| + title = @title_hash[title_id] + title.examine + end + remained_title_dirs = @title_ids.map { |id| title_hash[id].dir } + (Dir.entries @dir) .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } + .select { |path| !(remained_title_dirs.includes? path) } .select { |path| File.directory? path } .map { |path| Title.new path, "" } .select { |title| !(title.entries.empty? && title.titles.empty?) } .sort! { |a, b| a.title <=> b.title } - .tap { |_| @title_ids.clear } .each do |title| @title_hash[title.id] = title @title_ids << title.id diff --git a/src/library/title.cr b/src/library/title.cr index 6c873b85..44464c94 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -69,6 +69,12 @@ class Title end end + def examine : Bool + return false unless Dir.exists? @dir + signature = Dir.signature @dir + return @signature == signature + end + def to_slim_json : String JSON.build do |json| json.object do From fb43abb950137416043d0b923a75a10a073c395c Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 11 Sep 2021 21:16:33 +0900 Subject: [PATCH 40/82] Enhance the examine method --- src/library/title.cr | 66 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 44464c94..51b82c96 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -70,9 +70,71 @@ class Title end def examine : Bool - return false unless Dir.exists? @dir + return false unless Dir.exists? @dir # no title, should be removed signature = Dir.signature @dir - return @signature == signature + # `signature` doesn't reflect movings, renames in nested titles + # return true if @signature == signature # not changed, preserve + + # fix title + @signature = signature + storage = Storage.default + id = storage.get_title_id dir, signature + if id.nil? + id = random_str + storage.insert_title_id({ + path: dir, + id: id, + signature: signature.to_s, + }) + end + @id = id + @mtime = File.info(@dir).modification_time + + @title_ids.select! do |title_id| + title = Library.default.get_title! title_id + title.examine + end + remained_title_dirs = @title_ids.map do |id| + title = Library.default.get_title! id + title.dir + end + + @entries.select! { |entry| File.exists? entry.zip_path } + remained_entry_zip_paths = @entries.map &.zip_path + + Dir.entries(dir).each do |fn| + next if fn.starts_with? "." + path = File.join dir, fn + if File.directory? path + next if remained_title_dirs.includes? path + title = Title.new path, @id + next if title.entries.size == 0 && title.titles.size == 0 + Library.default.title_hash[title.id] = title + @title_ids << title.id + next + end + if is_supported_file path + next if remained_entry_zip_paths.includes? path + entry = Entry.new path, self + @entries << entry if entry.pages > 0 || entry.err_msg + end + end + + mtimes = [@mtime] + mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime } + mtimes += @entries.map &.mtime + @mtime = mtimes.max + + @title_ids.sort! do |a, b| + compare_numerically Library.default.title_hash[a].title, + Library.default.title_hash[b].title + end + sorter = ChapterSorter.new @entries.map &.title + @entries.sort! do |a, b| + sorter.compare a.title, b.title + end + + return true # this could be recycled end def to_slim_json : String From 80e13abc4a9d5dd53346e58aaf72b0a94a9230b4 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 11 Sep 2021 22:44:17 +0900 Subject: [PATCH 41/82] Spawn scan job --- src/library/library.cr | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index 86375be7..9ee26d1f 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -21,6 +21,7 @@ class Library instance_file_path = File.join dir, "library.yml.zip" return unless File.exists? instance_file_path + Logger.debug "Load library instance" zip_file = Compress::Zip::File.new instance_file_path instance_file = zip_file.entries.find { |entry| entry.filename == "instance.yml" } @@ -38,7 +39,13 @@ class Library zip_file.close - Library.default.scan + spawn do + start = Time.local + Library.default.scan + ms = (Time.local - start).total_milliseconds + Logger.info "Re-scanned #{Library.default.title_ids.size} titles \ + in #{ms}ms" + end end def initialize From e6214ddc5d0a752ed4abaaa268090235b27c1450 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 01:26:02 +0900 Subject: [PATCH 42/82] Rescan only if instance loaded --- src/library/library.cr | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 9ee26d1f..15674cf1 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -29,22 +29,26 @@ class Library zip_file.close return end + is_loaded = false begin instance_file.open do |content| @@default = Library.from_yaml content end + is_loaded = true rescue e Logger.error e end zip_file.close - spawn do - start = Time.local - Library.default.scan - ms = (Time.local - start).total_milliseconds - Logger.info "Re-scanned #{Library.default.title_ids.size} titles \ - in #{ms}ms" + if is_loaded + spawn do + start = Time.local + Library.default.scan + ms = (Time.local - start).total_milliseconds + Logger.info "Re-scanned #{Library.default.title_ids.size} titles \ + in #{ms}ms" + end end end From 4e8b561f70111c33a280dfa6487a382fd8fdec24 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 00:37:48 +0900 Subject: [PATCH 43/82] Apply contents signature of directories --- src/library/title.cr | 10 ++++++---- src/util/signature.cr | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 51b82c96..5caefa06 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -32,6 +32,7 @@ class Title }) end @id = id + @contents_signature = Dir.contents_signature dir @title = File.basename dir @encoded_title = URI.encode @title @title_ids = [] of String @@ -71,12 +72,13 @@ class Title def examine : Bool return false unless Dir.exists? @dir # no title, should be removed - signature = Dir.signature @dir - # `signature` doesn't reflect movings, renames in nested titles - # return true if @signature == signature # not changed, preserve + contents_signature = Dir.contents_signature @dir + # not changed, preserve + return true if @contents_signature == contents_signature # fix title - @signature = signature + @contents_signature = contents_signature + @signature = Dir.signature @dir storage = Storage.default id = storage.get_title_id dir, signature if id.nil? diff --git a/src/util/signature.cr b/src/util/signature.cr index d1a0040c..f2bf103a 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -48,4 +48,27 @@ class Dir end Digest::CRC32.checksum(signatures.sort.join).to_u64 end + + # Returns the contents signature of the directory at dirname for checking + # to rescan. + # Rescan conditions: + # - When a file added, moved, removed, renamed (including which in nested + # directories) + def self.contents_signature(dirname) : String + signatures = [] of String + self.open dirname do |dir| + dir.entries.sort.each do |fn| + next if fn.starts_with? "." + path = File.join dirname, fn + if File.directory? path + signatures << Dir.contents_signature path + else + # Only add its signature value to `signatures` when it is a + # supported file + signatures << fn if is_supported_file fn + end + end + end + Digest::SHA1.hexdigest(signatures.sort.join) + end end From a8f729f5c1021b2accfa15748f85f2f114181d61 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 00:48:23 +0900 Subject: [PATCH 44/82] Sort entries and titles when they needed --- src/library/title.cr | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 5caefa06..8260be90 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -92,6 +92,7 @@ class Title @id = id @mtime = File.info(@dir).modification_time + previous_titles_size = @title_ids.size @title_ids.select! do |title_id| title = Library.default.get_title! title_id title.examine @@ -101,9 +102,12 @@ class Title title.dir end + previous_entries_size = @entries.size @entries.select! { |entry| File.exists? entry.zip_path } remained_entry_zip_paths = @entries.map &.zip_path + is_titles_added = false + is_entries_added = false Dir.entries(dir).each do |fn| next if fn.starts_with? "." path = File.join dir, fn @@ -113,12 +117,16 @@ class Title next if title.entries.size == 0 && title.titles.size == 0 Library.default.title_hash[title.id] = title @title_ids << title.id + is_titles_added = true next end if is_supported_file path next if remained_entry_zip_paths.includes? path entry = Entry.new path, self - @entries << entry if entry.pages > 0 || entry.err_msg + if entry.pages > 0 || entry.err_msg + @entries << entry + is_entries_added = true + end end end @@ -127,13 +135,17 @@ class Title mtimes += @entries.map &.mtime @mtime = mtimes.max - @title_ids.sort! do |a, b| - compare_numerically Library.default.title_hash[a].title, - Library.default.title_hash[b].title + if is_titles_added || previous_titles_size != @title_ids.size + @title_ids.sort! do |a, b| + compare_numerically Library.default.title_hash[a].title, + Library.default.title_hash[b].title + end end - sorter = ChapterSorter.new @entries.map &.title - @entries.sort! do |a, b| - sorter.compare a.title, b.title + if is_entries_added || previous_entries_size != @entries.size + sorter = ChapterSorter.new @entries.map &.title + @entries.sort! do |a, b| + sorter.compare a.title, b.title + end end return true # this could be recycled From 9309f51df634ae3a9f498c30ed31f0f8066dec6f Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 02:03:54 +0900 Subject: [PATCH 45/82] Memoization on dir contents_signature --- src/library/title.cr | 7 ++++--- src/util/signature.cr | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 8260be90..2e9db265 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -70,11 +70,12 @@ class Title end end - def examine : Bool + def examine(cache = {} of String => String) : Bool return false unless Dir.exists? @dir # no title, should be removed - contents_signature = Dir.contents_signature @dir + contents_signature = Dir.contents_signature @dir, cache # not changed, preserve return true if @contents_signature == contents_signature + puts "Contents changed in #{@dir}" # fix title @contents_signature = contents_signature @@ -95,7 +96,7 @@ class Title previous_titles_size = @title_ids.size @title_ids.select! do |title_id| title = Library.default.get_title! title_id - title.examine + title.examine cache end remained_title_dirs = @title_ids.map do |id| title = Library.default.get_title! id diff --git a/src/util/signature.cr b/src/util/signature.cr index f2bf103a..e56aa06d 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -54,7 +54,8 @@ class Dir # Rescan conditions: # - When a file added, moved, removed, renamed (including which in nested # directories) - def self.contents_signature(dirname) : String + def self.contents_signature(dirname, cache = {} of String => String) : String + return cache[dirname] if cache[dirname]? signatures = [] of String self.open dirname do |dir| dir.entries.sort.each do |fn| @@ -69,6 +70,8 @@ class Dir end end end - Digest::SHA1.hexdigest(signatures.sort.join) + hash = Digest::SHA1.hexdigest(signatures.sort.join) + cache[dirname] = hash + hash end end From 7e36c91ea7000606aac0ca050a25d0cdb14871d7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 10:47:15 +0900 Subject: [PATCH 46/82] Remove debug print --- src/library/title.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/library/title.cr b/src/library/title.cr index 2e9db265..46de5b9e 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -75,7 +75,6 @@ class Title contents_signature = Dir.contents_signature @dir, cache # not changed, preserve return true if @contents_signature == contents_signature - puts "Contents changed in #{@dir}" # fix title @contents_signature = contents_signature From bdbdf9c94be29c62bf425c5f2349905b5c93c162 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 11:09:48 +0900 Subject: [PATCH 47/82] Fix to pass 'make check', fix comments --- src/library/library.cr | 4 +++- src/library/title.cr | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 15674cf1..4764b59a 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -23,7 +23,9 @@ class Library Logger.debug "Load library instance" zip_file = Compress::Zip::File.new instance_file_path - instance_file = zip_file.entries.find { |entry| entry.filename == "instance.yml" } + instance_file = zip_file.entries.find do |entry| + entry.filename == "instance.yml" + end if instance_file.nil? zip_file.close diff --git a/src/library/title.cr b/src/library/title.cr index 46de5b9e..ed634718 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -71,12 +71,12 @@ class Title end def examine(cache = {} of String => String) : Bool - return false unless Dir.exists? @dir # no title, should be removed + return false unless Dir.exists? @dir # No title, Remove this contents_signature = Dir.contents_signature @dir, cache - # not changed, preserve + # Not changed. Reuse this return true if @contents_signature == contents_signature - # fix title + # Fix title @contents_signature = contents_signature @signature = Dir.signature @dir storage = Storage.default @@ -97,8 +97,8 @@ class Title title = Library.default.get_title! title_id title.examine cache end - remained_title_dirs = @title_ids.map do |id| - title = Library.default.get_title! id + remained_title_dirs = @title_ids.map do |title_id| + title = Library.default.get_title! title_id title.dir end @@ -148,7 +148,7 @@ class Title end end - return true # this could be recycled + true # Fixed, reuse this end def to_slim_json : String From cd48b45f110740b1348dcbf0369caf9d9909a7db Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 12:45:24 +0900 Subject: [PATCH 48/82] Add 'require "yaml"' --- src/library/entry.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/library/entry.cr b/src/library/entry.cr index bb0aa1b7..56945513 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,4 +1,5 @@ require "image_size" +require "yaml" class Entry include YAML::Serializable From 8c90b46114f637155f2b2f9efcc651092c2ef3eb Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 13:39:28 +0900 Subject: [PATCH 49/82] Remove removed titles from title_hash --- src/library/library.cr | 4 +++- src/library/title.cr | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 4764b59a..a7b43e90 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -159,7 +159,9 @@ class Library @title_ids.select! do |title_id| title = @title_hash[title_id] - title.examine + existence = title.examine + @title_hash.delete title_id unless existence + existence end remained_title_dirs = @title_ids.map { |id| title_hash[id].dir } diff --git a/src/library/title.cr b/src/library/title.cr index ed634718..def2b66e 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -95,7 +95,9 @@ class Title previous_titles_size = @title_ids.size @title_ids.select! do |title_id| title = Library.default.get_title! title_id - title.examine cache + existence = title.examine cache + Library.default.title_hash.delete title_id unless existence + existence end remained_title_dirs = @title_ids.map do |title_id| title = Library.default.get_title! title_id From 7734dae138c009f3e4c14ba9efc3a5984817e47e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 14:36:17 +0900 Subject: [PATCH 50/82] Remove unnecessary sort --- src/util/signature.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index e56aa06d..b883e7c8 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -70,7 +70,7 @@ class Dir end end end - hash = Digest::SHA1.hexdigest(signatures.sort.join) + hash = Digest::SHA1.hexdigest(signatures.join) cache[dirname] = hash hash end From f5933a48d9ce4d7987b22524774a9efecdee8170 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 17:40:40 +0900 Subject: [PATCH 51/82] Register mime_type scan, thumbnails when loading instance --- src/library/library.cr | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index a7b43e90..f9edb7e8 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -44,19 +44,11 @@ class Library zip_file.close if is_loaded - spawn do - start = Time.local - Library.default.scan - ms = (Time.local - start).total_milliseconds - Logger.info "Re-scanned #{Library.default.title_ids.size} titles \ - in #{ms}ms" - end + Library.default.register_jobs end end def initialize - register_mime_types - @dir = Config.current.library_path # explicitly initialize @titles to bypass the compiler check. it will # be filled with actual Titles in the `scan` call below @@ -66,6 +58,12 @@ class Library @entries_count = 0 @thumbnails_count = 0 + register_jobs + end + + protected def register_jobs + register_mime_types + scan_interval = Config.current.scan_interval_minutes if scan_interval < 1 scan From 8f1383a818a43788c37e67d6af041dc71df0581a Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 18:01:16 +0900 Subject: [PATCH 52/82] Use Gzip instead of Zip --- src/library/library.cr | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index f9edb7e8..5b61827c 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -10,8 +10,9 @@ class Library path = Config.current.library_path instance_file_path = File.join path, "library.yml.zip" - writer = Compress::Zip::Writer.new instance_file_path - writer.add "instance.yml", self.to_yaml + writer = Compress::Gzip::Writer.new instance_file_path, + Compress::Gzip::BEST_COMPRESSION + writer.write self.to_yaml.to_slice writer.close end @@ -22,18 +23,10 @@ class Library return unless File.exists? instance_file_path Logger.debug "Load library instance" - zip_file = Compress::Zip::File.new instance_file_path - instance_file = zip_file.entries.find do |entry| - entry.filename == "instance.yml" - end - if instance_file.nil? - zip_file.close - return - end is_loaded = false begin - instance_file.open do |content| + Compress::Gzip::Reader.open instance_file_path do |content| @@default = Library.from_yaml content end is_loaded = true @@ -41,8 +34,6 @@ class Library Logger.error e end - zip_file.close - if is_loaded Library.default.register_jobs end From a151ec486da746e078ac96e49015b43e9d701c4e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 12 Sep 2021 18:04:41 +0900 Subject: [PATCH 53/82] Fix file extension of gzip file --- src/library/library.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 5b61827c..7ab10df8 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -8,7 +8,7 @@ class Library def save_instance path = Config.current.library_path - instance_file_path = File.join path, "library.yml.zip" + instance_file_path = File.join path, "library.yml.gz" writer = Compress::Gzip::Writer.new instance_file_path, Compress::Gzip::BEST_COMPRESSION @@ -19,7 +19,7 @@ class Library def self.load_instance dir = Config.current.library_path return unless Dir.exists? dir - instance_file_path = File.join dir, "library.yml.zip" + instance_file_path = File.join dir, "library.yml.gz" return unless File.exists? instance_file_path Logger.debug "Load library instance" From a9520d6f26702bf851482af5ca0e6bcc45ed9000 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 13 Sep 2021 10:18:07 +0000 Subject: [PATCH 54/82] Add shallow option to library API endpoints --- src/library/entry.cr | 22 +++++------------- src/library/library.cr | 26 +++++++-------------- src/library/title.cr | 53 ++++++++++++++---------------------------- src/routes/api.cr | 27 +++++++++++++-------- 4 files changed, 50 insertions(+), 78 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 28b7122e..d592e438 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -46,28 +46,18 @@ class Entry file.close end - def to_slim_json : String - JSON.build do |json| - json.object do - {% for str in ["zip_path", "title", "size", "id"] %} - json.field {{str}}, @{{str.id}} - {% end %} - json.field "title_id", @book.id - json.field "pages" { json.number @pages } - end - end - end - - def to_json(json : JSON::Builder) + def build_json(json : JSON::Builder, *, slim = false) json.object do {% for str in ["zip_path", "title", "size", "id"] %} json.field {{str}}, @{{str.id}} {% end %} json.field "title_id", @book.id - json.field "display_name", @book.display_name @title - json.field "cover_url", cover_url json.field "pages" { json.number @pages } - json.field "mtime" { json.number @mtime.to_unix } + unless slim + json.field "display_name", @book.display_name @title + json.field "cover_url", cover_url + json.field "mtime" { json.number @mtime.to_unix } + end end end diff --git a/src/library/library.cr b/src/library/library.cr index 9351e60d..06b1e0f3 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -65,26 +65,18 @@ class Library titles.flat_map &.deep_entries end - def to_slim_json : String - JSON.build do |json| - json.object do - json.field "dir", @dir - json.field "titles" do - json.array do - self.titles.each do |title| - json.raw title.to_slim_json - end - end - end - end - end - end - - def to_json(json : JSON::Builder) + def build_json(json : JSON::Builder, *, slim = false, shallow = false) json.object do json.field "dir", @dir json.field "titles" do - json.raw self.titles.to_json + json.array do + self.titles.each do |title| + raw = JSON.build do |j| + title.build_json j, slim: slim, shallow: shallow + end + json.raw raw + end + end end end end diff --git a/src/library/title.cr b/src/library/title.cr index f1915d4a..2d4e3478 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -63,24 +63,35 @@ class Title end end - def to_slim_json : String - JSON.build do |json| - json.object do - {% for str in ["dir", "title", "id"] %} + def build_json(json : JSON::Builder, *, slim = false, shallow = false) + json.object do + {% for str in ["dir", "title", "id"] %} json.field {{str}}, @{{str.id}} {% end %} - json.field "signature" { json.number @signature } + json.field "signature" { json.number @signature } + unless slim + json.field "display_name", display_name + json.field "cover_url", cover_url + json.field "mtime" { json.number @mtime.to_unix } + end + unless shallow json.field "titles" do json.array do self.titles.each do |title| - json.raw title.to_slim_json + raw = JSON.build do |j| + title.build_json j, slim: slim, shallow: shallow + end + json.raw raw end end end json.field "entries" do json.array do @entries.each do |entry| - json.raw entry.to_slim_json + raw = JSON.build do |j| + entry.build_json j, slim: slim + end + json.raw raw end end end @@ -98,34 +109,6 @@ class Title end end - def to_json(json : JSON::Builder) - json.object do - {% for str in ["dir", "title", "id"] %} - json.field {{str}}, @{{str.id}} - {% end %} - json.field "signature" { json.number @signature } - json.field "display_name", display_name - json.field "cover_url", cover_url - json.field "mtime" { json.number @mtime.to_unix } - json.field "titles" do - json.raw self.titles.to_json - end - json.field "entries" do - json.raw @entries.to_json - end - json.field "parents" do - json.array do - self.parents.each do |title| - json.object do - json.field "title", title.title - json.field "id", title.id - end - end - end - end - end - end - def titles @title_ids.map { |tid| Library.default.get_title! tid } end diff --git a/src/routes/api.cr b/src/routes/api.cr index b1fb3b35..924085b7 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -133,10 +133,12 @@ struct APIRouter end Koa.describe "Returns the book with title `tid`", <<-MD - Supply the `tid` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time + - Supply the `slim` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time + - Supply the `shallow` query parameter to get only the title information, without the list of entries and nested titles MD Koa.path "tid", desc: "Title ID" Koa.query "slim" + Koa.query "shallow" Koa.response 200, schema: "title" Koa.response 404, "Title not found" Koa.tag "library" @@ -146,10 +148,11 @@ struct APIRouter title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? - if env.params.query["slim"]? - send_json env, title.to_slim_json - else - send_json env, title.to_json + slim = !env.params.query["slim"]?.nil? + shallow = !env.params.query["shallow"]?.nil? + + json = JSON.build do |j| + title.build_json j, slim: slim, shallow: shallow end rescue e Logger.error e @@ -159,20 +162,24 @@ struct APIRouter end Koa.describe "Returns the entire library with all titles and entries", <<-MD - Supply the `tid` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time + - Supply the `slim` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time + - Supply the `shallow` query parameter to get only the top-level titles, without the nested titles and entries MD Koa.query "slim" + Koa.query "shallow" Koa.response 200, schema: { "dir" => String, "titles" => ["title"], } Koa.tag "library" get "/api/library" do |env| - if env.params.query["slim"]? - send_json env, Library.default.to_slim_json - else - send_json env, Library.default.to_json + slim = !env.params.query["slim"]?.nil? + shallow = !env.params.query["shallow"]?.nil? + + json = JSON.build do |j| + Library.default.build_json j, slim: slim, shallow: shallow end + send_json env, json end Koa.describe "Triggers a library scan" From 4b464ed36186f9aaae4dc0c9fcdf7623f1127f7c Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 13 Sep 2021 10:58:07 +0000 Subject: [PATCH 55/82] Allow sorting in /api/book endpoint --- src/library/title.cr | 13 +++++++++++-- src/routes/api.cr | 10 +++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 2d4e3478..0b916036 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -63,7 +63,10 @@ class Title end end - def build_json(json : JSON::Builder, *, slim = false, shallow = false) + alias SortContext = NamedTuple(username: String, opt: SortOptions) + + def build_json(json : JSON::Builder, *, slim = false, shallow = false, + sort_context : SortContext? = nil) json.object do {% for str in ["dir", "title", "id"] %} json.field {{str}}, @{{str.id}} @@ -87,7 +90,13 @@ class Title end json.field "entries" do json.array do - @entries.each do |entry| + _entries = if sort_context + sorted_entries sort_context[:username], + sort_context[:opt] + else + @entries + end + _entries.each do |entry| raw = JSON.build do |j| entry.build_json j, slim: slim end diff --git a/src/routes/api.cr b/src/routes/api.cr index 924085b7..3dc878ee 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -139,11 +139,18 @@ struct APIRouter Koa.path "tid", desc: "Title ID" Koa.query "slim" Koa.query "shallow" + Koa.query "sort", desc: "Sorting option for entries. Can be one of 'auto', 'title', 'progress', 'time_added' and 'time_modified'" + Koa.query "ascend", desc: "Sorting direction for entries. Set to 0 for the descending order. Doesn't work without specifying 'sort'" Koa.response 200, schema: "title" Koa.response 404, "Title not found" Koa.tag "library" get "/api/book/:tid" do |env| begin + username = get_username env + + sort_opt = SortOptions.new + get_sort_opt + tid = env.params.url["tid"] title = Library.default.get_title tid raise "Title ID `#{tid}` not found" if title.nil? @@ -152,7 +159,8 @@ struct APIRouter shallow = !env.params.query["shallow"]?.nil? json = JSON.build do |j| - title.build_json j, slim: slim, shallow: shallow + title.build_json j, slim: slim, shallow: shallow, + sort_context: {username: username, opt: sort_opt} end rescue e Logger.error e From 4eaf271fa46681ca0e449b5c64f641c59e34ae66 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 14 Sep 2021 02:30:57 +0000 Subject: [PATCH 56/82] Simplify #json_build --- src/library/entry.cr | 20 +++++++------ src/library/library.cr | 17 +++++------ src/library/title.cr | 68 ++++++++++++++++++++---------------------- src/routes/api.cr | 12 +++----- 4 files changed, 55 insertions(+), 62 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index d592e438..0a082465 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -46,17 +46,19 @@ class Entry file.close end - def build_json(json : JSON::Builder, *, slim = false) - json.object do - {% for str in ["zip_path", "title", "size", "id"] %} + def build_json(*, slim = false) + JSON.build do |json| + json.object do + {% for str in ["zip_path", "title", "size", "id"] %} json.field {{str}}, @{{str.id}} {% end %} - json.field "title_id", @book.id - json.field "pages" { json.number @pages } - unless slim - json.field "display_name", @book.display_name @title - json.field "cover_url", cover_url - json.field "mtime" { json.number @mtime.to_unix } + json.field "title_id", @book.id + json.field "pages" { json.number @pages } + unless slim + json.field "display_name", @book.display_name @title + json.field "cover_url", cover_url + json.field "mtime" { json.number @mtime.to_unix } + end end end end diff --git a/src/library/library.cr b/src/library/library.cr index 06b1e0f3..508a2748 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -65,16 +65,15 @@ class Library titles.flat_map &.deep_entries end - def build_json(json : JSON::Builder, *, slim = false, shallow = false) - json.object do - json.field "dir", @dir - json.field "titles" do - json.array do - self.titles.each do |title| - raw = JSON.build do |j| - title.build_json j, slim: slim, shallow: shallow + def build_json(*, slim = false, shallow = false) + JSON.build do |json| + json.object do + json.field "dir", @dir + json.field "titles" do + json.array do + self.titles.each do |title| + json.raw title.build_json(slim: slim, shallow: shallow) end - json.raw raw end end end diff --git a/src/library/title.cr b/src/library/title.cr index 0b916036..957a6081 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -65,51 +65,47 @@ class Title alias SortContext = NamedTuple(username: String, opt: SortOptions) - def build_json(json : JSON::Builder, *, slim = false, shallow = false, + def build_json(*, slim = false, shallow = false, sort_context : SortContext? = nil) - json.object do - {% for str in ["dir", "title", "id"] %} + JSON.build do |json| + json.object do + {% for str in ["dir", "title", "id"] %} json.field {{str}}, @{{str.id}} {% end %} - json.field "signature" { json.number @signature } - unless slim - json.field "display_name", display_name - json.field "cover_url", cover_url - json.field "mtime" { json.number @mtime.to_unix } - end - unless shallow - json.field "titles" do - json.array do - self.titles.each do |title| - raw = JSON.build do |j| - title.build_json j, slim: slim, shallow: shallow + json.field "signature" { json.number @signature } + unless slim + json.field "display_name", display_name + json.field "cover_url", cover_url + json.field "mtime" { json.number @mtime.to_unix } + end + unless shallow + json.field "titles" do + json.array do + self.titles.each do |title| + json.raw title.build_json(slim: slim, shallow: shallow) end - json.raw raw end end - end - json.field "entries" do - json.array do - _entries = if sort_context - sorted_entries sort_context[:username], - sort_context[:opt] - else - @entries - end - _entries.each do |entry| - raw = JSON.build do |j| - entry.build_json j, slim: slim + json.field "entries" do + json.array do + _entries = if sort_context + sorted_entries sort_context[:username], + sort_context[:opt] + else + @entries + end + _entries.each do |entry| + json.raw entry.build_json(slim: slim) end - json.raw raw end end - end - json.field "parents" do - json.array do - self.parents.each do |title| - json.object do - json.field "title", title.title - json.field "id", title.id + json.field "parents" do + json.array do + self.parents.each do |title| + json.object do + json.field "title", title.title + json.field "id", title.id + end end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 3dc878ee..8c66e13e 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -158,10 +158,9 @@ struct APIRouter slim = !env.params.query["slim"]?.nil? shallow = !env.params.query["shallow"]?.nil? - json = JSON.build do |j| - title.build_json j, slim: slim, shallow: shallow, - sort_context: {username: username, opt: sort_opt} - end + send_json env, title.build_json(slim: slim, shallow: shallow, + sort_context: {username: username, + opt: sort_opt}) rescue e Logger.error e env.response.status_code = 404 @@ -184,10 +183,7 @@ struct APIRouter slim = !env.params.query["slim"]?.nil? shallow = !env.params.query["shallow"]?.nil? - json = JSON.build do |j| - Library.default.build_json j, slim: slim, shallow: shallow - end - send_json env, json + send_json env, Library.default.build_json(slim: slim, shallow: shallow) end Koa.describe "Triggers a library scan" From 03e044a1aa6b4d88a8d210b97c563f11204ada95 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Tue, 14 Sep 2021 07:16:14 +0000 Subject: [PATCH 57/82] Improve logging --- src/library/library.cr | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 7ab10df8..6d69563e 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -9,6 +9,7 @@ class Library def save_instance path = Config.current.library_path instance_file_path = File.join path, "library.yml.gz" + Logger.debug "Caching library to #{instance_file_path}" writer = Compress::Gzip::Writer.new instance_file_path, Compress::Gzip::BEST_COMPRESSION @@ -22,21 +23,16 @@ class Library instance_file_path = File.join dir, "library.yml.gz" return unless File.exists? instance_file_path - Logger.debug "Load library instance" + Logger.debug "Loading cached library from #{instance_file_path}" - is_loaded = false begin Compress::Gzip::Reader.open instance_file_path do |content| @@default = Library.from_yaml content end - is_loaded = true + Library.default.register_jobs rescue e Logger.error e end - - if is_loaded - Library.default.register_jobs - end end def initialize @@ -64,7 +60,7 @@ class Library start = Time.local scan ms = (Time.local - start).total_milliseconds - Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" + Logger.debug "Library initialized in #{ms}ms" sleep scan_interval.minutes end end @@ -171,10 +167,13 @@ class Library storage.close ms = (Time.local - start).total_milliseconds - Logger.debug "Scan completed. #{ms}ms" + Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" + Storage.default.mark_unavailable - save_instance + spawn do + save_instance + end end def get_continue_reading_entries(username) From be47f309b09a61a32e2862dd9ce13b3ea7d4b217 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 18:11:08 +0900 Subject: [PATCH 58/82] Use cache when calculating contents_signature --- src/util/signature.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index b883e7c8..190a4a66 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -62,7 +62,7 @@ class Dir next if fn.starts_with? "." path = File.join dirname, fn if File.directory? path - signatures << Dir.contents_signature path + signatures << Dir.contents_signature path, cache else # Only add its signature value to `signatures` when it is a # supported file From 523195d6496bfd596afcc36f95654dd88edd05ee Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 22:37:30 +0900 Subject: [PATCH 59/82] Define ExamineContext, apply it when scanning --- src/library/library.cr | 12 ++++++++++-- src/library/title.cr | 17 +++++++++++------ src/library/types.cr | 7 +++++++ src/util/signature.cr | 4 ++-- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 6d69563e..582440ac 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -142,20 +142,28 @@ class Library storage = Storage.new auto_close: false + examine_context : ExamineContext = { + file_count: 0, + cached_contents_signature: {} of String => String, + deleted_title_ids: [] of String, + deleted_entry_ids: [] of String + } + @title_ids.select! do |title_id| title = @title_hash[title_id] - existence = title.examine + existence = title.examine examine_context @title_hash.delete title_id unless existence existence end remained_title_dirs = @title_ids.map { |id| title_hash[id].dir } + cache = examine_context["cached_contents_signature"] (Dir.entries @dir) .select { |fn| !fn.starts_with? "." } .map { |fn| File.join @dir, fn } .select { |path| !(remained_title_dirs.includes? path) } .select { |path| File.directory? path } - .map { |path| Title.new path, "" } + .map { |path| Title.new path, "", cache } .select { |title| !(title.entries.empty? && title.titles.empty?) } .sort! { |a, b| a.title <=> b.title } .each do |title| diff --git a/src/library/title.cr b/src/library/title.cr index def2b66e..12c565fb 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -19,7 +19,7 @@ class Title @[YAML::Field(ignore: true)] @cached_cover_url : String? - def initialize(@dir : String, @parent_id) + def initialize(@dir : String, @parent_id, cache : Hash(String, String)?) storage = Storage.default @signature = Dir.signature dir id = storage.get_title_id dir, signature @@ -32,7 +32,7 @@ class Title }) end @id = id - @contents_signature = Dir.contents_signature dir + @contents_signature = Dir.contents_signature dir, cache @title = File.basename dir @encoded_title = URI.encode @title @title_ids = [] of String @@ -70,9 +70,14 @@ class Title end end - def examine(cache = {} of String => String) : Bool + def self.new(dir : String, parent_id) + new dir, parent_id, nil + end + + def examine(context : ExamineContext) : Bool return false unless Dir.exists? @dir # No title, Remove this - contents_signature = Dir.contents_signature @dir, cache + contents_signature = Dir.contents_signature @dir, + context["cached_contents_signature"] # Not changed. Reuse this return true if @contents_signature == contents_signature @@ -95,7 +100,7 @@ class Title previous_titles_size = @title_ids.size @title_ids.select! do |title_id| title = Library.default.get_title! title_id - existence = title.examine cache + existence = title.examine context Library.default.title_hash.delete title_id unless existence existence end @@ -115,7 +120,7 @@ class Title path = File.join dir, fn if File.directory? path next if remained_title_dirs.includes? path - title = Title.new path, @id + title = Title.new path, @id, context["cached_contents_signature"] next if title.entries.size == 0 && title.titles.size == 0 Library.default.title_hash[title.id] = title @title_ids << title.id diff --git a/src/library/types.cr b/src/library/types.cr index 05451841..eb875452 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -133,3 +133,10 @@ class TitleInfo LRUCache.set generate_cache_entry key, self.to_json end end + +alias ExamineContext = NamedTuple( + file_count: Int32, + cached_contents_signature: Hash(String, String), + deleted_title_ids: Array(String), + deleted_entry_ids: Array(String) +) diff --git a/src/util/signature.cr b/src/util/signature.cr index 190a4a66..c4104fe4 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -55,7 +55,7 @@ class Dir # - When a file added, moved, removed, renamed (including which in nested # directories) def self.contents_signature(dirname, cache = {} of String => String) : String - return cache[dirname] if cache[dirname]? + return cache[dirname] if !cache.nil? && cache[dirname]? signatures = [] of String self.open dirname do |dir| dir.entries.sort.each do |fn| @@ -71,7 +71,7 @@ class Dir end end hash = Digest::SHA1.hexdigest(signatures.join) - cache[dirname] = hash + cache[dirname] = hash unless cache.nil? hash end end From 2e09efbd6238a331253a237736915d4f01788712 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 22:51:05 +0900 Subject: [PATCH 60/82] Collect deleted ids --- src/library/library.cr | 5 ++++- src/library/title.cr | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 582440ac..012d14b4 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -152,7 +152,10 @@ class Library @title_ids.select! do |title_id| title = @title_hash[title_id] existence = title.examine examine_context - @title_hash.delete title_id unless existence + unless existence + @title_hash.delete title_id + examine_context["deleted_title_ids"] << title_id + end existence end remained_title_dirs = @title_ids.map { |id| title_hash[id].dir } diff --git a/src/library/title.cr b/src/library/title.cr index 12c565fb..fb2d2fe3 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -101,7 +101,10 @@ class Title @title_ids.select! do |title_id| title = Library.default.get_title! title_id existence = title.examine context - Library.default.title_hash.delete title_id unless existence + unless existence + Library.default.title_hash.delete title_id + context["deleted_title_ids"] << title_id + end existence end remained_title_dirs = @title_ids.map do |title_id| @@ -110,7 +113,11 @@ class Title end previous_entries_size = @entries.size - @entries.select! { |entry| File.exists? entry.zip_path } + @entries.select! do |entry| + existence = File.exists? entry.zip_path + context["deleted_entry_ids"] << entry.id unless existence + existence + end remained_entry_zip_paths = @entries.map &.zip_path is_titles_added = false From 670cf54957c744acfc94b3fe6985248792cd1806 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 22:51:37 +0900 Subject: [PATCH 61/82] Apply yield forcely --- src/library/title.cr | 1 + src/library/types.cr | 5 +++++ src/util/signature.cr | 1 + 3 files changed, 7 insertions(+) diff --git a/src/library/title.cr b/src/library/title.cr index fb2d2fe3..1a8d488f 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -115,6 +115,7 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| existence = File.exists? entry.zip_path + yield_process_file context context["deleted_entry_ids"] << entry.id unless existence existence end diff --git a/src/library/types.cr b/src/library/types.cr index eb875452..1355fcd1 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -140,3 +140,8 @@ alias ExamineContext = NamedTuple( deleted_title_ids: Array(String), deleted_entry_ids: Array(String) ) + +def yield_process_file(context : ExamineContext) + context["file_count"] += 1 + Fiber.yield if context["file_count"] % 1000 == 0 +end diff --git a/src/util/signature.cr b/src/util/signature.cr index c4104fe4..d0a55a6c 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -56,6 +56,7 @@ class Dir # directories) def self.contents_signature(dirname, cache = {} of String => String) : String return cache[dirname] if !cache.nil? && cache[dirname]? + Fiber.yield # Yield first signatures = [] of String self.open dirname do |dir| dir.entries.sort.each do |fn| From 9489d6abfdac6af1fab3a805b48639081dbe987a Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 23:07:47 +0900 Subject: [PATCH 62/82] Use reference instead of primitive --- src/library/library.cr | 6 +++--- src/library/title.cr | 2 +- src/library/types.cr | 23 +++++++++++++++-------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 012d14b4..0a04b1a3 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -143,10 +143,10 @@ class Library storage = Storage.new auto_close: false examine_context : ExamineContext = { - file_count: 0, + file_counter: (YieldCounter.new 1000), cached_contents_signature: {} of String => String, - deleted_title_ids: [] of String, - deleted_entry_ids: [] of String + deleted_title_ids: [] of String, + deleted_entry_ids: [] of String, } @title_ids.select! do |title_id| diff --git a/src/library/title.cr b/src/library/title.cr index 1a8d488f..fcc5f2f5 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -115,7 +115,7 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| existence = File.exists? entry.zip_path - yield_process_file context + context["file_counter"].count_and_yield context["deleted_entry_ids"] << entry.id unless existence existence end diff --git a/src/library/types.cr b/src/library/types.cr index 1355fcd1..7856660f 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -134,14 +134,21 @@ class TitleInfo end end +class YieldCounter + setter threshold : Int32 + + def initialize(@threshold : Int32) + @file_count = 0 + end + + def count_and_yield + @file_count += 1 + Fiber.yield if @file_count % @threshold == 0 + end +end + alias ExamineContext = NamedTuple( - file_count: Int32, + file_counter: YieldCounter, cached_contents_signature: Hash(String, String), deleted_title_ids: Array(String), - deleted_entry_ids: Array(String) -) - -def yield_process_file(context : ExamineContext) - context["file_count"] += 1 - Fiber.yield if context["file_count"] % 1000 == 0 -end + deleted_entry_ids: Array(String)) From 57b2f7c625ad83fbbc2d08012a84c46ffc27c3c8 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 23:08:07 +0900 Subject: [PATCH 63/82] Get nested ids when title removed --- src/library/library.cr | 3 ++- src/library/title.cr | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 0a04b1a3..6cf8c328 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -153,8 +153,9 @@ class Library title = @title_hash[title_id] existence = title.examine examine_context unless existence + examine_context["deleted_title_ids"].concat title.deep_titles.map &.id + examine_context["deleted_entry_ids"].concat title.deep_entries.map &.id @title_hash.delete title_id - examine_context["deleted_title_ids"] << title_id end existence end diff --git a/src/library/title.cr b/src/library/title.cr index fcc5f2f5..db2ca30a 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -102,8 +102,9 @@ class Title title = Library.default.get_title! title_id existence = title.examine context unless existence + context["deleted_title_ids"].concat title.deep_titles.map &.id + context["deleted_entry_ids"].concat title.deep_entries.map &.id Library.default.title_hash.delete title_id - context["deleted_title_ids"] << title_id end existence end From 663c0c0b38e47944e0da0d228b3593cdf317c871 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 23:15:40 +0900 Subject: [PATCH 64/82] Remove nested title including self --- src/library/library.cr | 6 ++++-- src/library/title.cr | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 6cf8c328..8029fcb3 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -153,13 +153,15 @@ class Library title = @title_hash[title_id] existence = title.examine examine_context unless existence - examine_context["deleted_title_ids"].concat title.deep_titles.map &.id + examine_context["deleted_title_ids"].concat [title_id] + title.deep_titles.map &.id examine_context["deleted_entry_ids"].concat title.deep_entries.map &.id - @title_hash.delete title_id end existence end remained_title_dirs = @title_ids.map { |id| title_hash[id].dir } + examine_context["deleted_title_ids"].each do |title_id| + @title_hash.delete title_id + end cache = examine_context["cached_contents_signature"] (Dir.entries @dir) diff --git a/src/library/title.cr b/src/library/title.cr index db2ca30a..6bae6939 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -102,9 +102,8 @@ class Title title = Library.default.get_title! title_id existence = title.examine context unless existence - context["deleted_title_ids"].concat title.deep_titles.map &.id + context["deleted_title_ids"].concat [title_id] + title.deep_titles.map &.id context["deleted_entry_ids"].concat title.deep_entries.map &.id - Library.default.title_hash.delete title_id end existence end From f4d7128b59fd229031735fac38006ba4a86ce7f8 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Tue, 14 Sep 2021 23:30:03 +0900 Subject: [PATCH 65/82] Mark unavailable only in candidates --- src/library/library.cr | 3 ++- src/storage.cr | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/library/library.cr b/src/library/library.cr index 8029fcb3..12d662f5 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -183,7 +183,8 @@ class Library ms = (Time.local - start).total_milliseconds Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms" - Storage.default.mark_unavailable + Storage.default.mark_unavailable examine_context["deleted_entry_ids"], + examine_context["deleted_title_ids"] spawn do save_instance diff --git a/src/storage.cr b/src/storage.cr index 39116b98..1f0aab70 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -466,6 +466,47 @@ class Storage end end + # Limit mark targets with given arguments + def mark_unavailable(trash_ids_candidates : Array(String), trash_titles_candidates : Array(String)) + MainFiber.run do + get_db do |db| + # Detect dangling entry IDs + trash_ids = [] of String + db.query "select path, id from ids where id in " \ + "(#{trash_ids_candidates.join "," { |i| "'#{i}'" }})" do |rs| + rs.each do + path = rs.read String + fullpath = Path.new(path).expand(Config.current.library_path).to_s + trash_ids << rs.read String unless File.exists? fullpath + end + end + + unless trash_ids.empty? + Logger.debug "Marking #{trash_ids.size} entries as unavailable" + end + db.exec "update ids set unavailable = 1 where id in " \ + "(#{trash_ids.join "," { |i| "'#{i}'" }})" + + # Detect dangling title IDs + trash_titles = [] of String + db.query "select path, id from titles where id in " \ + "(#{trash_titles_candidates.join "," { |i| "'#{i}'" }})" do |rs| + rs.each do + path = rs.read String + fullpath = Path.new(path).expand(Config.current.library_path).to_s + trash_titles << rs.read String unless Dir.exists? fullpath + end + end + + unless trash_titles.empty? + Logger.debug "Marking #{trash_titles.size} titles as unavailable" + end + db.exec "update titles set unavailable = 1 where id in " \ + "(#{trash_titles.join "," { |i| "'#{i}'" }})" + end + end + end + private def get_missing(tablename) ary = [] of IDTuple MainFiber.run do From a3b2cdd372d546612e8806143b9baddce7cad415 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Wed, 15 Sep 2021 01:17:44 +0900 Subject: [PATCH 66/82] Lint --- src/library/library.cr | 3 ++- src/library/title.cr | 3 ++- src/storage.cr | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 12d662f5..b50fb872 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -153,7 +153,8 @@ class Library title = @title_hash[title_id] existence = title.examine examine_context unless existence - examine_context["deleted_title_ids"].concat [title_id] + title.deep_titles.map &.id + examine_context["deleted_title_ids"].concat [title_id] + + title.deep_titles.map &.id examine_context["deleted_entry_ids"].concat title.deep_entries.map &.id end existence diff --git a/src/library/title.cr b/src/library/title.cr index 6bae6939..41efd7cf 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -102,7 +102,8 @@ class Title title = Library.default.get_title! title_id existence = title.examine context unless existence - context["deleted_title_ids"].concat [title_id] + title.deep_titles.map &.id + context["deleted_title_ids"].concat [title_id] + + title.deep_titles.map &.id context["deleted_entry_ids"].concat title.deep_entries.map &.id end existence diff --git a/src/storage.cr b/src/storage.cr index 1f0aab70..2f01863d 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -467,13 +467,14 @@ class Storage end # Limit mark targets with given arguments - def mark_unavailable(trash_ids_candidates : Array(String), trash_titles_candidates : Array(String)) + def mark_unavailable(ids_candidates : Array(String), + titles_candidates : Array(String)) MainFiber.run do get_db do |db| # Detect dangling entry IDs trash_ids = [] of String db.query "select path, id from ids where id in " \ - "(#{trash_ids_candidates.join "," { |i| "'#{i}'" }})" do |rs| + "(#{ids_candidates.join "," { |i| "'#{i}'" }})" do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s @@ -490,7 +491,7 @@ class Storage # Detect dangling title IDs trash_titles = [] of String db.query "select path, id from titles where id in " \ - "(#{trash_titles_candidates.join "," { |i| "'#{i}'" }})" do |rs| + "(#{titles_candidates.join "," { |i| "'#{i}'" }})" do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s From d13cfc045fd344e4557f882bdc1eb748fb8bc0a0 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Wed, 15 Sep 2021 01:27:05 +0900 Subject: [PATCH 67/82] Add a comment --- src/storage.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storage.cr b/src/storage.cr index 2f01863d..45112552 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -467,6 +467,8 @@ class Storage end # Limit mark targets with given arguments + # They should be checked again if they are really gone, + # since they would be available which are renamed or moved def mark_unavailable(ids_candidates : Array(String), titles_candidates : Array(String)) MainFiber.run do From de193906a2b3bf72ea255e330621fceea9edcac0 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Wed, 15 Sep 2021 16:54:55 +0900 Subject: [PATCH 68/82] Refactor mark_unavailable --- src/storage.cr | 55 +++++++++++++------------------------------------- 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/src/storage.cr b/src/storage.cr index 45112552..9f6f45ba 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -429,54 +429,24 @@ class Storage end def mark_unavailable - MainFiber.run do - get_db do |db| - # Detect dangling entry IDs - trash_ids = [] of String - db.query "select path, id from ids where unavailable = 0" do |rs| - rs.each do - path = rs.read String - fullpath = Path.new(path).expand(Config.current.library_path).to_s - trash_ids << rs.read String unless File.exists? fullpath - end - end - - unless trash_ids.empty? - Logger.debug "Marking #{trash_ids.size} entries as unavailable" - end - db.exec "update ids set unavailable = 1 where id in " \ - "(#{trash_ids.join "," { |i| "'#{i}'" }})" - - # Detect dangling title IDs - trash_titles = [] of String - db.query "select path, id from titles where unavailable = 0" do |rs| - rs.each do - path = rs.read String - fullpath = Path.new(path).expand(Config.current.library_path).to_s - trash_titles << rs.read String unless Dir.exists? fullpath - end - end - - unless trash_titles.empty? - Logger.debug "Marking #{trash_titles.size} titles as unavailable" - end - db.exec "update titles set unavailable = 1 where id in " \ - "(#{trash_titles.join "," { |i| "'#{i}'" }})" - end - end + mark_unavailable nil, nil end # Limit mark targets with given arguments # They should be checked again if they are really gone, # since they would be available which are renamed or moved - def mark_unavailable(ids_candidates : Array(String), - titles_candidates : Array(String)) + def mark_unavailable(ids_candidates : Array(String) | Nil, + titles_candidates : Array(String) | Nil) MainFiber.run do get_db do |db| # Detect dangling entry IDs trash_ids = [] of String - db.query "select path, id from ids where id in " \ - "(#{ids_candidates.join "," { |i| "'#{i}'" }})" do |rs| + # Use query builder instead? + query = "select path, id from ids where unavailable = 0" + unless ids_candidates.nil? + query += " and id in (#{ids_candidates.join "," { |i| "'#{i}'" }})" + end + db.query query do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s @@ -492,8 +462,11 @@ class Storage # Detect dangling title IDs trash_titles = [] of String - db.query "select path, id from titles where id in " \ - "(#{titles_candidates.join "," { |i| "'#{i}'" }})" do |rs| + query = "select path, id from titles where unavailable = 0" + unless titles_candidates.nil? + query += " and id in (#{titles_candidates.join "," { |i| "'#{i}'" }})" + end + db.query query do |rs| rs.each do path = rs.read String fullpath = Path.new(path).expand(Config.current.library_path).to_s From d330db131e52ad0c62ffbf6e0df0568662813551 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 15 Sep 2021 08:46:30 +0000 Subject: [PATCH 69/82] Simplify `mark_unavailable` --- src/storage.cr | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/storage.cr b/src/storage.cr index 9f6f45ba..762946de 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -428,15 +428,11 @@ class Storage end end - def mark_unavailable - mark_unavailable nil, nil - end - # Limit mark targets with given arguments # They should be checked again if they are really gone, # since they would be available which are renamed or moved - def mark_unavailable(ids_candidates : Array(String) | Nil, - titles_candidates : Array(String) | Nil) + def mark_unavailable(ids_candidates : Array(String)?, + titles_candidates : Array(String)?) MainFiber.run do get_db do |db| # Detect dangling entry IDs From 44a6f822cd2d69156beb359dfa26b522f47e875d Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 15 Sep 2021 09:00:30 +0000 Subject: [PATCH 70/82] Simplify Title.new --- src/library/title.cr | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 3abc38d2..ee1edd01 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -19,7 +19,7 @@ class Title @[YAML::Field(ignore: true)] @cached_cover_url : String? - def initialize(@dir : String, @parent_id, cache : Hash(String, String)?) + def initialize(@dir : String, @parent_id, cache : Hash(String, String)? = nil) storage = Storage.default @signature = Dir.signature dir id = storage.get_title_id dir, signature @@ -70,10 +70,6 @@ class Title end end - def self.new(dir : String, parent_id) - new dir, parent_id, nil - end - def examine(context : ExamineContext) : Bool return false unless Dir.exists? @dir # No title, Remove this contents_signature = Dir.contents_signature @dir, From 70ab198a33ee34bba7c1ce8f1152918a81316569 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Thu, 16 Sep 2021 00:16:26 +0900 Subject: [PATCH 71/82] Add config 'forcely_yield_count' the default value 1000 would make a fiber yield on each 4ms on SSD Apply yield counter in Dir.contents_signauture Use contents_signature cache in Title.new --- src/config.cr | 1 + src/library/library.cr | 3 ++- src/library/title.cr | 6 +++--- src/util/signature.cr | 11 +++++++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/config.cr b/src/config.cr index 99fb5b99..bc7aa181 100644 --- a/src/config.cr +++ b/src/config.cr @@ -23,6 +23,7 @@ class Config property cache_enabled = false property cache_size_mbs = 50 property cache_log_enabled = true + property forcely_yield_count = 1000 property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/library/library.cr b/src/library/library.cr index ea2a5e3c..de8d7b40 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -133,8 +133,9 @@ class Library storage = Storage.new auto_close: false + count = Config.current.forcely_yield_count examine_context : ExamineContext = { - file_counter: (YieldCounter.new 1000), + file_counter: (YieldCounter.new count), cached_contents_signature: {} of String => String, deleted_title_ids: [] of String, deleted_entry_ids: [] of String, diff --git a/src/library/title.cr b/src/library/title.cr index ee1edd01..f57024cc 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -19,7 +19,7 @@ class Title @[YAML::Field(ignore: true)] @cached_cover_url : String? - def initialize(@dir : String, @parent_id, cache : Hash(String, String)? = nil) + def initialize(@dir : String, @parent_id, cache = {} of String => String) storage = Storage.default @signature = Dir.signature dir id = storage.get_title_id dir, signature @@ -43,7 +43,7 @@ class Title next if fn.starts_with? "." path = File.join dir, fn if File.directory? path - title = Title.new path, @id + title = Title.new path, @id, cache next if title.entries.size == 0 && title.titles.size == 0 Library.default.title_hash[title.id] = title @title_ids << title.id @@ -73,7 +73,7 @@ class Title def examine(context : ExamineContext) : Bool return false unless Dir.exists? @dir # No title, Remove this contents_signature = Dir.contents_signature @dir, - context["cached_contents_signature"] + context["cached_contents_signature"], context["file_counter"] # Not changed. Reuse this return true if @contents_signature == contents_signature diff --git a/src/util/signature.cr b/src/util/signature.cr index d0a55a6c..59a83110 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -54,9 +54,11 @@ class Dir # Rescan conditions: # - When a file added, moved, removed, renamed (including which in nested # directories) - def self.contents_signature(dirname, cache = {} of String => String) : String - return cache[dirname] if !cache.nil? && cache[dirname]? - Fiber.yield # Yield first + def self.contents_signature(dirname, + cache = {} of String => String, + counter : YieldCounter? = nil) : String + return cache[dirname] if cache[dirname]? + counter.count_and_yield unless counter.nil? signatures = [] of String self.open dirname do |dir| dir.entries.sort.each do |fn| @@ -69,10 +71,11 @@ class Dir # supported file signatures << fn if is_supported_file fn end + counter.count_and_yield unless counter.nil? end end hash = Digest::SHA1.hexdigest(signatures.join) - cache[dirname] = hash unless cache.nil? + cache[dirname] = hash hash end end From 9769e760a0ad0ce9035ad350185497942b280795 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Thu, 16 Sep 2021 07:49:12 +0900 Subject: [PATCH 72/82] Pass a counter to recursive calls, Ignore negative threshold --- src/library/types.cr | 4 ++-- src/util/signature.cr | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/library/types.cr b/src/library/types.cr index 7856660f..a0e0428e 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -135,7 +135,7 @@ class TitleInfo end class YieldCounter - setter threshold : Int32 + @file_count : UInt32 def initialize(@threshold : Int32) @file_count = 0 @@ -143,7 +143,7 @@ class YieldCounter def count_and_yield @file_count += 1 - Fiber.yield if @file_count % @threshold == 0 + Fiber.yield if @threshold > 0 && @file_count % @threshold == 0 end end diff --git a/src/util/signature.cr b/src/util/signature.cr index 59a83110..76e26288 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -65,7 +65,7 @@ class Dir next if fn.starts_with? "." path = File.join dirname, fn if File.directory? path - signatures << Dir.contents_signature path, cache + signatures << Dir.contents_signature path, cache, counter else # Only add its signature value to `signatures` when it is a # supported file From b56e16e1e19bca9a654fc813d79674159c1c42ea Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 18 Sep 2021 10:59:43 +0900 Subject: [PATCH 73/82] Remove counter, yield everytime --- src/config.cr | 1 - src/library/library.cr | 2 -- src/library/title.cr | 4 ++-- src/library/types.cr | 14 -------------- src/util/signature.cr | 10 ++++------ 5 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/config.cr b/src/config.cr index bc7aa181..99fb5b99 100644 --- a/src/config.cr +++ b/src/config.cr @@ -23,7 +23,6 @@ class Config property cache_enabled = false property cache_size_mbs = 50 property cache_log_enabled = true - property forcely_yield_count = 1000 property disable_login = false property default_username = "" property auth_proxy_header_name = "" diff --git a/src/library/library.cr b/src/library/library.cr index de8d7b40..ce1a8fc5 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -133,9 +133,7 @@ class Library storage = Storage.new auto_close: false - count = Config.current.forcely_yield_count examine_context : ExamineContext = { - file_counter: (YieldCounter.new count), cached_contents_signature: {} of String => String, deleted_title_ids: [] of String, deleted_entry_ids: [] of String, diff --git a/src/library/title.cr b/src/library/title.cr index f57024cc..6a959d8c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -73,7 +73,7 @@ class Title def examine(context : ExamineContext) : Bool return false unless Dir.exists? @dir # No title, Remove this contents_signature = Dir.contents_signature @dir, - context["cached_contents_signature"], context["file_counter"] + context["cached_contents_signature"] # Not changed. Reuse this return true if @contents_signature == contents_signature @@ -112,7 +112,7 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| existence = File.exists? entry.zip_path - context["file_counter"].count_and_yield + Fiber.yield context["deleted_entry_ids"] << entry.id unless existence existence end diff --git a/src/library/types.cr b/src/library/types.cr index a0e0428e..4c9dc937 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -134,21 +134,7 @@ class TitleInfo end end -class YieldCounter - @file_count : UInt32 - - def initialize(@threshold : Int32) - @file_count = 0 - end - - def count_and_yield - @file_count += 1 - Fiber.yield if @threshold > 0 && @file_count % @threshold == 0 - end -end - alias ExamineContext = NamedTuple( - file_counter: YieldCounter, cached_contents_signature: Hash(String, String), deleted_title_ids: Array(String), deleted_entry_ids: Array(String)) diff --git a/src/util/signature.cr b/src/util/signature.cr index 76e26288..5ca3e14e 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -54,24 +54,22 @@ class Dir # Rescan conditions: # - When a file added, moved, removed, renamed (including which in nested # directories) - def self.contents_signature(dirname, - cache = {} of String => String, - counter : YieldCounter? = nil) : String + def self.contents_signature(dirname, cache = {} of String => String) : String return cache[dirname] if cache[dirname]? - counter.count_and_yield unless counter.nil? + Fiber.yield signatures = [] of String self.open dirname do |dir| dir.entries.sort.each do |fn| next if fn.starts_with? "." path = File.join dirname, fn if File.directory? path - signatures << Dir.contents_signature path, cache, counter + signatures << Dir.contents_signature path, cache else # Only add its signature value to `signatures` when it is a # supported file signatures << fn if is_supported_file fn end - counter.count_and_yield unless counter.nil? + Fiber.yield end end hash = Digest::SHA1.hexdigest(signatures.join) From 96f1ef3dde56cc4132f69a0f22be810443281440 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Sep 2021 02:00:10 +0000 Subject: [PATCH 74/82] Improve comments on examine --- src/library/title.cr | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index f57024cc..f9499c18 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -70,14 +70,21 @@ class Title end end + # Utility method used in library rescanning. + # - When the title does not exist on the file system anymore, return false + # and let it be deleted from the libaray instance + # - When the title exists, but its contents sigature is now different from + # the cache, it means some of its content (nested titles or entries) + # has been added, deleted, or renamed. In this case we update its + # contents signature and instance variables + # - When the title exists and its contents sigature is still the same, we + # return true so it can be reused without rescanning def examine(context : ExamineContext) : Bool return false unless Dir.exists? @dir # No title, Remove this contents_signature = Dir.contents_signature @dir, context["cached_contents_signature"], context["file_counter"] - # Not changed. Reuse this return true if @contents_signature == contents_signature - # Fix title @contents_signature = contents_signature @signature = Dir.signature @dir storage = Storage.default @@ -160,7 +167,7 @@ class Title end end - true # Fixed, reuse this + true end alias SortContext = NamedTuple(username: String, opt: SortOptions) From 3f73591dd44fbd705de2cc9bd4b936fa3d16ddf8 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Sep 2021 02:14:22 +0000 Subject: [PATCH 75/82] Update comments --- src/library/title.cr | 6 +++--- src/storage.cr | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 853ed1c3..1709e9b5 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -72,12 +72,12 @@ class Title # Utility method used in library rescanning. # - When the title does not exist on the file system anymore, return false - # and let it be deleted from the libaray instance - # - When the title exists, but its contents sigature is now different from + # and let it be deleted from the library instance + # - When the title exists, but its contents signature is now different from # the cache, it means some of its content (nested titles or entries) # has been added, deleted, or renamed. In this case we update its # contents signature and instance variables - # - When the title exists and its contents sigature is still the same, we + # - When the title exists and its contents signature is still the same, we # return true so it can be reused without rescanning def examine(context : ExamineContext) : Bool return false unless Dir.exists? @dir # No title, Remove this diff --git a/src/storage.cr b/src/storage.cr index 762946de..eea5927a 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -428,9 +428,10 @@ class Storage end end - # Limit mark targets with given arguments - # They should be checked again if they are really gone, - # since they would be available which are renamed or moved + # Mark titles and entries that no longer exist on the file system as + # unavailable. By supplying `id_candidates` and `titles_candidates`, it + # only checks the existence of the candidate titles/entries to speed up + # the process. def mark_unavailable(ids_candidates : Array(String)?, titles_candidates : Array(String)?) MainFiber.run do From 16397050dd7604b0ab2f21aec3a573041052cd90 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Sep 2021 02:24:50 +0000 Subject: [PATCH 76/82] Update comments --- src/library/title.cr | 2 +- src/storage.cr | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index 1709e9b5..7efee42f 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -80,7 +80,7 @@ class Title # - When the title exists and its contents signature is still the same, we # return true so it can be reused without rescanning def examine(context : ExamineContext) : Bool - return false unless Dir.exists? @dir # No title, Remove this + return false unless Dir.exists? @dir contents_signature = Dir.contents_signature @dir, context["cached_contents_signature"] return true if @contents_signature == contents_signature diff --git a/src/storage.cr b/src/storage.cr index eea5927a..32f446a3 100644 --- a/src/storage.cr +++ b/src/storage.cr @@ -438,7 +438,6 @@ class Storage get_db do |db| # Detect dangling entry IDs trash_ids = [] of String - # Use query builder instead? query = "select path, id from ids where unavailable = 0" unless ids_candidates.nil? query += " and id in (#{ids_candidates.join "," { |i| "'#{i}'" }})" From 97168b65d827e735f30574c80a38fad63fd5d7df Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Sep 2021 08:40:08 +0000 Subject: [PATCH 77/82] Make library cache path configurable --- src/config.cr | 2 ++ src/library/library.cr | 17 +++++++---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/config.cr b/src/config.cr index 99fb5b99..86f27336 100644 --- a/src/config.cr +++ b/src/config.cr @@ -11,6 +11,8 @@ class Config property session_secret : String = "mango-session-secret" property library_path : String = File.expand_path "~/mango/library", home: true + property library_cache_path = File.expand_path "~/mango/library.yml.cbz", + home: true property db_path : String = File.expand_path "~/mango/mango.db", home: true property scan_interval_minutes : Int32 = 5 property thumbnail_generation_interval_hours : Int32 = 24 diff --git a/src/library/library.cr b/src/library/library.cr index ce1a8fc5..90f3a448 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -7,26 +7,23 @@ class Library use_default def save_instance - path = Config.current.library_path - instance_file_path = File.join path, "library.yml.gz" - Logger.debug "Caching library to #{instance_file_path}" + path = Config.current.library_cache_path + Logger.debug "Caching library to #{path}" - writer = Compress::Gzip::Writer.new instance_file_path, + writer = Compress::Gzip::Writer.new path, Compress::Gzip::BEST_COMPRESSION writer.write self.to_yaml.to_slice writer.close end def self.load_instance - dir = Config.current.library_path - return unless Dir.exists? dir - instance_file_path = File.join dir, "library.yml.gz" - return unless File.exists? instance_file_path + path = Config.current.library_cache_path + return unless File.exists? path - Logger.debug "Loading cached library from #{instance_file_path}" + Logger.debug "Loading cached library from #{path}" begin - Compress::Gzip::Reader.open instance_file_path do |content| + Compress::Gzip::Reader.open path do |content| @@default = Library.from_yaml content end Library.default.register_jobs From 2c022a07e7c1dce777582fc677e8dbad33b715eb Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 18 Sep 2021 18:49:29 +0900 Subject: [PATCH 78/82] Avoid unnecessary sorts when getting deep percentage This make page loading fast --- src/library/title.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/title.cr b/src/library/title.cr index 7efee42f..8efd9c0c 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -380,7 +380,7 @@ class Title cached_sum = LRUCache.get key return cached_sum[1] if cached_sum.is_a? Tuple(String, Int32) && cached_sum[0] == sig - sum = load_progress_for_all_entries(username).sum + + sum = load_progress_for_all_entries(username, nil, true).sum + titles.flat_map(&.deep_read_page_count username).sum LRUCache.set generate_cache_entry key, {sig, sum} sum From 72fae7f5ed3278f93974da847785e4708b0b2535 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sat, 18 Sep 2021 12:41:25 +0000 Subject: [PATCH 79/82] Fix typo cbz -> gz --- src/config.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.cr b/src/config.cr index 86f27336..aa818c32 100644 --- a/src/config.cr +++ b/src/config.cr @@ -11,7 +11,7 @@ class Config property session_secret : String = "mango-session-secret" property library_path : String = File.expand_path "~/mango/library", home: true - property library_cache_path = File.expand_path "~/mango/library.yml.cbz", + property library_cache_path = File.expand_path "~/mango/library.yml.gz", home: true property db_path : String = File.expand_path "~/mango/mango.db", home: true property scan_interval_minutes : Int32 = 5 From 33e7e31fbc218896f52b7d6fc7a81add97047ed0 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 20 Sep 2021 01:11:26 +0000 Subject: [PATCH 80/82] Bump version to 0.24.0 --- README.md | 2 +- shard.yml | 2 +- src/mango.cr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a04caddd..4bb25564 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r ### CLI ``` - Mango - Manga Server and Web Reader. Version 0.23.0 + Mango - Manga Server and Web Reader. Version 0.24.0 Usage: diff --git a/shard.yml b/shard.yml index 1da553d7..0054a230 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: mango -version: 0.23.0 +version: 0.24.0 authors: - Alex Ling diff --git a/src/mango.cr b/src/mango.cr index 39b13525..8716d044 100644 --- a/src/mango.cr +++ b/src/mango.cr @@ -7,7 +7,7 @@ require "option_parser" require "clim" require "tallboy" -MANGO_VERSION = "0.23.0" +MANGO_VERSION = "0.24.0" # From http://www.network-science.de/ascii/ BANNER = %{ From 4fbe5b471cc027b64e53500c3ecb6e5a0cc87e29 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Mon, 20 Sep 2021 01:36:31 +0000 Subject: [PATCH 81/82] Update README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4bb25564..1a010ddd 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,10 @@ log_level: info upload_path: ~/mango/uploads plugin_path: ~/mango/plugins download_timeout_seconds: 30 +library_cache_path: ~/mango/library.yml.gz +cache_enabled: false +cache_size_mbs: 50 +cache_log_enabled: true disable_login: false default_username: "" auth_proxy_header_name: "" @@ -97,12 +101,12 @@ mangadex: download_queue_db_path: ~/mango/queue.db chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}' manga_rename_rule: '{title}' - subscription_update_interval_hours: 24 ``` - `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging - You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work. +- By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`. ### Library Structure From 974b6cfe9bf227be720ae9f3d921f8e853f2061b Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Wed, 22 Sep 2021 09:17:14 +0000 Subject: [PATCH 82/82] Use `depth` instead of `shallow` in API --- src/library/library.cr | 4 ++-- src/library/title.cr | 21 +++++++++++---------- src/routes/api.cr | 24 ++++++++++++++++-------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/library/library.cr b/src/library/library.cr index 90f3a448..93eac0c9 100644 --- a/src/library/library.cr +++ b/src/library/library.cr @@ -97,14 +97,14 @@ class Library titles.flat_map &.deep_entries end - def build_json(*, slim = false, shallow = false) + def build_json(*, slim = false, depth = -1) JSON.build do |json| json.object do json.field "dir", @dir json.field "titles" do json.array do self.titles.each do |title| - json.raw title.build_json(slim: slim, shallow: shallow) + json.raw title.build_json(slim: slim, depth: depth) end end end diff --git a/src/library/title.cr b/src/library/title.cr index 8efd9c0c..9b797f41 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -172,7 +172,7 @@ class Title alias SortContext = NamedTuple(username: String, opt: SortOptions) - def build_json(*, slim = false, shallow = false, + def build_json(*, slim = false, depth = -1, sort_context : SortContext? = nil) JSON.build do |json| json.object do @@ -185,11 +185,12 @@ class Title json.field "cover_url", cover_url json.field "mtime" { json.number @mtime.to_unix } end - unless shallow + unless depth == 0 json.field "titles" do json.array do self.titles.each do |title| - json.raw title.build_json(slim: slim, shallow: shallow) + json.raw title.build_json(slim: slim, + depth: depth > 0 ? depth - 1 : depth) end end end @@ -206,13 +207,13 @@ class Title end end end - json.field "parents" do - json.array do - self.parents.each do |title| - json.object do - json.field "title", title.title - json.field "id", title.id - end + end + json.field "parents" do + json.array do + self.parents.each do |title| + json.object do + json.field "title", title.title + json.field "id", title.id end end end diff --git a/src/routes/api.cr b/src/routes/api.cr index 8c66e13e..ed9bb299 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -134,11 +134,15 @@ struct APIRouter Koa.describe "Returns the book with title `tid`", <<-MD - Supply the `slim` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time - - Supply the `shallow` query parameter to get only the title information, without the list of entries and nested titles + - Supply the `depth` query parameter to control the depth of nested titles to return. + - When `depth` is 1, returns the top-level titles and sub-titles/entries one level in them + - When `depth` is 0, returns the top-level titles without their sub-titles/entries + - When `depth` is N, returns the top-level titles and sub-titles/entries N levels in them + - When `depth` is negative, returns the entire library MD Koa.path "tid", desc: "Title ID" Koa.query "slim" - Koa.query "shallow" + Koa.query "depth" Koa.query "sort", desc: "Sorting option for entries. Can be one of 'auto', 'title', 'progress', 'time_added' and 'time_modified'" Koa.query "ascend", desc: "Sorting direction for entries. Set to 0 for the descending order. Doesn't work without specifying 'sort'" Koa.response 200, schema: "title" @@ -156,9 +160,9 @@ struct APIRouter raise "Title ID `#{tid}` not found" if title.nil? slim = !env.params.query["slim"]?.nil? - shallow = !env.params.query["shallow"]?.nil? + depth = env.params.query["depth"]?.try(&.to_i?) || -1 - send_json env, title.build_json(slim: slim, shallow: shallow, + send_json env, title.build_json(slim: slim, depth: depth, sort_context: {username: username, opt: sort_opt}) rescue e @@ -170,10 +174,14 @@ struct APIRouter Koa.describe "Returns the entire library with all titles and entries", <<-MD - Supply the `slim` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time - - Supply the `shallow` query parameter to get only the top-level titles, without the nested titles and entries + - Supply the `dpeth` query parameter to control the depth of nested titles to return. + - When `depth` is 1, returns the requested title and sub-titles/entries one level in it + - When `depth` is 0, returns the requested title without its sub-titles/entries + - When `depth` is N, returns the requested title and sub-titles/entries N levels in it + - When `depth` is negative, returns the requested title and all sub-titles/entries in it MD Koa.query "slim" - Koa.query "shallow" + Koa.query "depth" Koa.response 200, schema: { "dir" => String, "titles" => ["title"], @@ -181,9 +189,9 @@ struct APIRouter Koa.tag "library" get "/api/library" do |env| slim = !env.params.query["slim"]?.nil? - shallow = !env.params.query["shallow"]?.nil? + depth = env.params.query["depth"]?.try(&.to_i?) || -1 - send_json env, Library.default.build_json(slim: slim, shallow: shallow) + send_json env, Library.default.build_json(slim: slim, depth: depth) end Koa.describe "Triggers a library scan"