From ea6cbbd9cef2d03c3712a00b668148b2c72c6f38 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 14 May 2022 22:52:19 +0900 Subject: [PATCH 01/24] Split Entry and ZippedEntry, Fix to work anyway make Entry an abstract class --- src/library/entry.cr | 271 ++++++++++++++++++-------------- src/library/title.cr | 10 +- src/routes/api.cr | 6 +- src/routes/reader.cr | 1 + src/views/opds/title.xml.ecr | 2 +- src/views/reader-error.html.ecr | 2 +- src/views/reader.html.ecr | 2 +- 7 files changed, 166 insertions(+), 128 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index dd50ed34..9fb1ef9d 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,62 +1,27 @@ require "image_size" require "yaml" -class Entry - include YAML::Serializable - - getter zip_path : String, book : Title, title : String, - size : String, pages : Int32, id : String, encoded_path : String, - encoded_title : String, mtime : Time, err_msg : String? - - @[YAML::Field(ignore: true)] - @sort_title : String? - - def initialize(@zip_path, @book) - storage = Storage.default - @encoded_path = URI.encode @zip_path - @title = File.basename @zip_path, File.extname @zip_path - @encoded_title = URI.encode @title - @size = (File.size @zip_path).humanize_bytes - id = storage.get_entry_id @zip_path, File.signature(@zip_path) - if id.nil? - id = random_str - storage.insert_entry_id({ - path: @zip_path, - id: id, - signature: File.signature(@zip_path).to_s, - }) - end - @id = id - @mtime = File.info(@zip_path).modification_time - - unless File.readable? @zip_path - @err_msg = "File #{@zip_path} is not readable." - Logger.warn "#{@err_msg} Please make sure the " \ - "file permission is configured correctly." - return - end - - archive_exception = validate_archive @zip_path - unless archive_exception.nil? - @err_msg = "Archive error: #{archive_exception}" - Logger.warn "Unable to extract archive #{@zip_path}. " \ - "Ignoring it. #{@err_msg}" - return - end +abstract class Entry + getter id : String, book : Title, title : String, + size : String, pages : Int32, mtime : Time, + encoded_path : String, encoded_title : String, err_msg : String? + + def initialize( + @id, @title, @book, + @size, @pages, @mtime, + @encoded_path, @encoded_title, @err_msg) + end - file = ArchiveFile.new @zip_path - @pages = file.entries.count do |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - end - file.close + def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) + # TODO: check node? and select proper subclass + ZippedEntry.new ctx, node end def build_json(*, slim = false) JSON.build do |json| json.object do - {% for str in %w(zip_path title size id) %} - json.field {{str}}, @{{str.id}} + {% for str in %w(path title size id) %} + json.field {{str}}, {{str.id}} {% end %} if err_msg json.field "err_msg", err_msg @@ -74,6 +39,9 @@ class Entry end end + @[YAML::Field(ignore: true)] + @sort_title : String? + def sort_title sort_title_cached = @sort_title return sort_title_cached if sort_title_cached @@ -131,58 +99,6 @@ class Entry url end - private def sorted_archive_entries - ArchiveFile.open @zip_path do |file| - entries = file.entries - .select { |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - } - .sort! { |a, b| - compare_numerically a.filename, b.filename - } - yield file, entries - end - end - - def read_page(page_num) - raise "Unreadble archive. #{@err_msg}" if @err_msg - img = nil - begin - sorted_archive_entries do |file, entries| - page = entries[page_num - 1] - data = file.read_entry page - if data - img = Image.new data, MIME.from_filename(page.filename), - page.filename, data.size - end - end - rescue e - Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" - end - img - end - - def page_dimensions - sizes = [] of Hash(String, Int32) - sorted_archive_entries do |file, entries| - entries.each_with_index do |e, i| - begin - data = file.read_entry(e).not_nil! - size = ImageSize.get data - sizes << { - "width" => size.width, - "height" => size.height, - } - rescue e - Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" - sizes << {"width" => 1000_i32, "height" => 1000_i32} - end - end - end - sizes - end - def next_entry(username) entries = @book.sorted_entries username idx = entries.index self @@ -197,20 +113,6 @@ class Entry entries[idx - 1] end - def date_added - date_added = nil - TitleInfo.new @book.dir do |info| - info_da = info.date_added[@title]? - if info_da.nil? - date_added = info.date_added[@title] = ctime @zip_path - info.save - else - date_added = info_da - end - end - date_added.not_nil! # is it ok to set not_nil! here? - end - # For backward backward compatibility with v0.1.0, we save entry titles # instead of IDs in info.json def save_progress(username, page) @@ -290,7 +192,7 @@ class Entry end Storage.default.save_thumbnail @id, img rescue e - Logger.warn "Failed to generate thumbnail for file #{@zip_path}. #{e}" + Logger.warn "Failed to generate thumbnail for file #{path}. #{e}" end img @@ -299,4 +201,139 @@ class Entry def get_thumbnail : Image? Storage.default.get_thumbnail @id end + + def date_added : Time + date_added = nil + TitleInfo.new @book.dir do |info| + info_da = info.date_added[@title]? + if info_da.nil? + date_added = info.date_added[@title] = createtime + info.save + else + date_added = info_da + end + end + date_added.not_nil! # is it ok to set not_nil! here? + end + + abstract def path : String + + abstract def createtime : Time + + abstract def read_page(page_num) + + abstract def page_dimensions + + abstract def exists? : Bool? +end + +class ZippedEntry < Entry + include YAML::Serializable + + getter zip_path : String + + def initialize(@zip_path, @book) + storage = Storage.default + @encoded_path = URI.encode @zip_path + @title = File.basename @zip_path, File.extname @zip_path + @encoded_title = URI.encode @title + @size = (File.size @zip_path).humanize_bytes + id = storage.get_entry_id @zip_path, File.signature(@zip_path) + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @zip_path, + id: id, + signature: File.signature(@zip_path).to_s, + }) + end + @id = id + @mtime = File.info(@zip_path).modification_time + + unless File.readable? @zip_path + @err_msg = "File #{@zip_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + archive_exception = validate_archive @zip_path + unless archive_exception.nil? + @err_msg = "Archive error: #{archive_exception}" + Logger.warn "Unable to extract archive #{@zip_path}. " \ + "Ignoring it. #{@err_msg}" + return + end + + file = ArchiveFile.new @zip_path + @pages = file.entries.count do |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + end + file.close + end + + def path : String + @zip_path + end + + def createtime : Time + ctime @zip_path + end + + private def sorted_archive_entries + ArchiveFile.open @zip_path do |file| + entries = file.entries + .select { |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + } + .sort! { |a, b| + compare_numerically a.filename, b.filename + } + yield file, entries + end + end + + def read_page(page_num) + raise "Unreadble archive. #{@err_msg}" if @err_msg + img = nil + begin + sorted_archive_entries do |file, entries| + page = entries[page_num - 1] + data = file.read_entry page + if data + img = Image.new data, MIME.from_filename(page.filename), + page.filename, data.size + end + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_archive_entries do |file, entries| + entries.each_with_index do |e, i| + begin + data = file.read_entry(e).not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + end + sizes + end + + def exists? : Bool + File.exists? @zip_path + end end diff --git a/src/library/title.cr b/src/library/title.cr index e3d79d55..eeef8e79 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -55,7 +55,7 @@ class Title next end if is_supported_file path - entry = Entry.new path, self + entry = ZippedEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end end @@ -127,12 +127,12 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| - existence = File.exists? entry.zip_path + existence = entry.exists? Fiber.yield context["deleted_entry_ids"] << entry.id unless existence existence end - remained_entry_zip_paths = @entries.map &.zip_path + remained_entry_zip_paths = @entries.map &.path is_titles_added = false is_entries_added = false @@ -162,7 +162,7 @@ class Title end if is_supported_file path next if remained_entry_zip_paths.includes? path - entry = Entry.new path, self + entry = ZippedEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true @@ -627,7 +627,7 @@ class Title @entries.each do |e| next if da.has_key? e.title - da[e.title] = ctime e.zip_path + da[e.title] = ctime e.path end TitleInfo.new @dir do |info| diff --git a/src/routes/api.cr b/src/routes/api.cr index e664b281..03d6e532 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -40,7 +40,7 @@ struct APIRouter Koa.schema "entry", { "pages" => Int32, "mtime" => Int64, - }.merge(s %w(zip_path title size id title_id display_name cover_url)), + }.merge(s %w(path title size id title_id display_name cover_url)), desc: "An entry in a book" Koa.schema "title", { @@ -1138,7 +1138,7 @@ struct APIRouter entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - file_hash = Digest::SHA1.hexdigest (entry.zip_path + entry.mtime.to_s) + file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) e_tag = "W/#{file_hash}" if e_tag == prev_e_tag env.response.status_code = 304 @@ -1172,7 +1172,7 @@ struct APIRouter title = (Library.default.get_title env.params.url["tid"]).not_nil! entry = (title.get_entry env.params.url["eid"]).not_nil! - send_attachment env, entry.zip_path + send_attachment env, entry.path rescue e Logger.error e env.response.status_code = 404 diff --git a/src/routes/reader.cr b/src/routes/reader.cr index 40b86aa7..052e212c 100644 --- a/src/routes/reader.cr +++ b/src/routes/reader.cr @@ -53,6 +53,7 @@ struct ReaderRouter render "src/views/reader.html.ecr" rescue e Logger.error e + puts e.backtrace? env.response.status_code = 404 end end diff --git a/src/views/opds/title.xml.ecr b/src/views/opds/title.xml.ecr index b1596879..1d824901 100644 --- a/src/views/opds/title.xml.ecr +++ b/src/views/opds/title.xml.ecr @@ -29,7 +29,7 @@ - + diff --git a/src/views/reader-error.html.ecr b/src/views/reader-error.html.ecr index 62a80fcc..ad3580f1 100644 --- a/src/views/reader-error.html.ecr +++ b/src/views/reader-error.html.ecr @@ -5,7 +5,7 @@

Error

-

<%= entry.zip_path %>

+

<%= entry.path %>

<%= entry.err_msg %>

diff --git a/src/views/reader.html.ecr b/src/views/reader.html.ecr index feac115b..19e2b19f 100644 --- a/src/views/reader.html.ecr +++ b/src/views/reader.html.ecr @@ -67,7 +67,7 @@

<%= entry.display_name %>

-

<%= entry.zip_path %>

+

<%= entry.path %>

From 10587f48cbad42cb5ad9d544167a6b1eea4c060e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 16:03:15 +0900 Subject: [PATCH 02/24] Implement is_supported_image_file --- src/library/types.cr | 10 ---------- src/util/util.cr | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/library/types.cr b/src/library/types.cr index 973aa5ea..d6a014f6 100644 --- a/src/library/types.cr +++ b/src/library/types.cr @@ -1,13 +1,3 @@ -SUPPORTED_IMG_TYPES = %w( - image/jpeg - image/png - image/webp - image/apng - image/avif - image/gif - image/svg+xml -) - enum SortMethod Auto Title diff --git a/src/util/util.cr b/src/util/util.cr index e7b1b1aa..7d834fbe 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -3,6 +3,16 @@ ENTRIES_IN_HOME_SECTIONS = 8 UPLOAD_URL_PREFIX = "/uploads" STATIC_DIRS = %w(/css /js /img /webfonts /favicon.ico /robots.txt) SUPPORTED_FILE_EXTNAMES = [".zip", ".cbz", ".rar", ".cbr"] +SUPPORTED_IMG_TYPES = %w( + image/jpeg + image/png + image/webp + image/apng + image/avif + image/gif + image/svg+xml +) + def random_str UUID.random.to_s.gsub "-", "" @@ -49,6 +59,10 @@ def is_supported_file(path) SUPPORTED_FILE_EXTNAMES.includes? File.extname(path).downcase end +def is_supported_image_file(path) + SUPPORTED_IMG_TYPES.includes? MIME.from_filename? path +end + struct Int def or(other : Int) if self == 0 From 55ccd928a2c27f6c94cb4bb5dbad12b5a2ad1ca7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 14:00:18 +0900 Subject: [PATCH 03/24] Implement DirectoryEntry --- src/library/entry.cr | 142 ++++++++++++++++++++++++++++++++++++++++++ src/util/signature.cr | 20 +++++- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 9fb1ef9d..b43dd77b 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -337,3 +337,145 @@ class ZippedEntry < Entry File.exists? @zip_path end end + +class DirectoryEntry < Entry + include YAML::Serializable + + getter dir_path : String + + @[YAML::Field(ignore: true)] + @sorted_files : Array(String)? + + @signature : String + + def initialize(@dir_path, @book) + storage = Storage.default + @encoded_path = URI.encode @dir_path + @title = File.basename @dir_path + @encoded_title = URI.encode @title + + unless File.readable? @dir_path + @err_msg = "Directory #{@dir_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + unless DirectoryEntry.validate_directory_entry @dir_path + @err_msg = "Directory #{@dir_path} is not valid directory entry." + Logger.warn "#{@err_msg} Please make sure the " \ + "directory has valid images." + return + end + + size_sum = 0 + sorted_files.each do |file_path| + size_sum += File.size file_path + end + @size = size_sum.humanize_bytes + + @signature = Dir.directory_entry_signature @dir_path + id = storage.get_entry_id @dir_path, @signature + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @dir_path, + id: id, + signature: @signature, + }) + end + @id = id + + mtimes = sorted_files.map { |file_path| File.info(file_path).modification_time } + @mtime = mtimes.max + + @pages = sorted_files.size + end + + def path : String + @dir_path + end + + def createtime : Time + ctime @dir_path + end + + def read_page(page_num) + img = nil + begin + files = sorted_files + file_path = files[page_num - 1] + data = File.read(file_path).to_slice + if data + img = Image.new data, MIME.from_filename(file_path), + File.basename(file_path), data.size + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@dir_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_files.each_with_index do |path, i| + data = File.read(path).to_slice + begin + data.not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{@dir_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + sizes + end + + def exists? : Bool + existence = File.exists? @dir_path + return false unless existence + files = DirectoryEntry.get_valid_files @dir_path + signature = Dir.directory_entry_signature @dir_path + existence = files.size > 0 && @signature == signature + @sorted_files = nil unless existence + + # For more efficient, + # Fix a directory instance with new property + # and return true + existence + end + + def sorted_files + cached_sorted_files = @sorted_files + return cached_sorted_files if cached_sorted_files + @sorted_files = DirectoryEntry.get_valid_files_sorted @dir_path + @sorted_files.not_nil! + end + + def self.validate_directory_entry(dir_path) + files = DirectoryEntry.get_valid_files dir_path + files.size > 0 + end + + def self.get_valid_files(dir_path) + files = [] of String + Dir.entries(dir_path).each do |fn| + next if fn.starts_with? "." + path = File.join dir_path, fn + next unless is_supported_image_file path + next if File.directory? path + next unless File.readable? path + files << path + end + files + end + + def self.get_valid_files_sorted(dir_path) + files = DirectoryEntry.get_valid_files dir_path + files.sort! { |a, b| compare_numerically a, b } + end +end diff --git a/src/util/signature.cr b/src/util/signature.cr index 5ca3e14e..63d31ce6 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -19,7 +19,7 @@ class File # information as long as the above changes do not happen together with # a file/folder rename, with no library scan in between. def self.signature(filename) : UInt64 - if is_supported_file filename + if is_supported_file(filename) || is_supported_image_file(filename) File.info(filename).inode else 0u64 @@ -64,6 +64,9 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache + if DirectoryEntry.validate_directory_entry path + signatures << Dir.directory_entry_signature path, cache + end else # Only add its signature value to `signatures` when it is a # supported file @@ -76,4 +79,19 @@ class Dir cache[dirname] = hash hash end + + def self.directory_entry_signature(dirname, cache = {} of String => String) + return cache[dirname + "?entry"] if cache[dirname + "?entry"]? + Fiber.yield + signatures = [] of String + image_files = DirectoryEntry.get_valid_files_sorted dirname + if image_files.size > 0 + image_files.each do |path| + signatures << File.signature(path).to_s + end + end + hash = Digest::SHA1.hexdigest(signatures.join) + cache[dirname + "?entry"] = hash + hash + end end From 3b3a0738e83d2e009500fbbb4f701480378e60b0 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 15:31:11 +0900 Subject: [PATCH 04/24] Scan DirectoryEntry when init and examine --- src/library/title.cr | 57 +++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/src/library/title.cr b/src/library/title.cr index eeef8e79..ad1672d0 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -49,9 +49,14 @@ class Title path = File.join dir, fn if File.directory? path title = Title.new path, @id, cache - next if title.entries.size == 0 && title.titles.size == 0 - Library.default.title_hash[title.id] = title - @title_ids << title.id + unless title.entries.size == 0 && title.titles.size == 0 + Library.default.title_hash[title.id] = title + @title_ids << title.id + end + if DirectoryEntry.validate_directory_entry path + entry = DirectoryEntry.new path, self + @entries << entry if entry.pages > 0 || entry.err_msg + end next end if is_supported_file path @@ -132,7 +137,7 @@ class Title context["deleted_entry_ids"] << entry.id unless existence existence end - remained_entry_zip_paths = @entries.map &.path + remained_entry_paths = @entries.map &.path is_titles_added = false is_entries_added = false @@ -140,28 +145,42 @@ class Title next if fn.starts_with? "." path = File.join dir, fn if File.directory? path + unless remained_entry_paths.includes? path + if DirectoryEntry.validate_directory_entry path + entry = DirectoryEntry.new path, self + if entry.pages > 0 || entry.err_msg + @entries << entry + is_entries_added = true + context["deleted_entry_ids"].select! do |deleted_entry_id| + entry.id != deleted_entry_id + end + end + end + end + next if remained_title_dirs.includes? path title = Title.new path, @id, context["cached_contents_signature"] - next if title.entries.size == 0 && title.titles.size == 0 - Library.default.title_hash[title.id] = title - @title_ids << title.id - is_titles_added = true - - # We think they are removed, but they are here! - # Cancel reserved jobs - revival_title_ids = [title.id] + title.deep_titles.map &.id - context["deleted_title_ids"].select! do |deleted_title_id| - !(revival_title_ids.includes? deleted_title_id) - end - revival_entry_ids = title.deep_entries.map &.id - context["deleted_entry_ids"].select! do |deleted_entry_id| - !(revival_entry_ids.includes? deleted_entry_id) + unless title.entries.size == 0 && title.titles.size == 0 + Library.default.title_hash[title.id] = title + @title_ids << title.id + is_titles_added = true + + # We think they are removed, but they are here! + # Cancel reserved jobs + revival_title_ids = [title.id] + title.deep_titles.map &.id + context["deleted_title_ids"].select! do |deleted_title_id| + !(revival_title_ids.includes? deleted_title_id) + end + revival_entry_ids = title.deep_entries.map &.id + context["deleted_entry_ids"].select! do |deleted_entry_id| + !(revival_entry_ids.includes? deleted_entry_id) + end end next end if is_supported_file path - next if remained_entry_zip_paths.includes? path + next if remained_entry_paths.includes? path entry = ZippedEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry From 137e84dfb671c1240a9d2d60ff3c9f22ef86b86c Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 15:31:33 +0900 Subject: [PATCH 05/24] Fix caching policy Before rendering it, the Mango reader should check the E-Tag of page or it renders wrong image when an image file is moved/removed/reordered --- src/routes/api.cr | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/routes/api.cr b/src/routes/api.cr index 03d6e532..840ce92c 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -142,8 +142,13 @@ struct APIRouter env.response.status_code = 304 "" else + if entry.is_a? DirectoryEntry + cache_control = "no-cache, max-age=86400" + else + cache_control = "public, max-age=86400" + end env.response.headers["ETag"] = e_tag - env.response.headers["Cache-Control"] = "public, max-age=86400" + env.response.headers["Cache-Control"] = cache_control send_img env, img end rescue e @@ -1138,15 +1143,24 @@ struct APIRouter entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) + if entry.is_a? DirectoryEntry + file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s + entry.size) + else + file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) + end e_tag = "W/#{file_hash}" if e_tag == prev_e_tag env.response.status_code = 304 send_text env, "" else sizes = entry.page_dimensions + if entry.is_a? DirectoryEntry + cache_control = "no-cache, max-age=86400" + else + cache_control = "public, max-age=86400" + end env.response.headers["ETag"] = e_tag - env.response.headers["Cache-Control"] = "public, max-age=86400" + env.response.headers["Cache-Control"] = cache_control send_json env, { "success" => true, "dimensions" => sizes, From caf4cfb6cd721827a312c54e25c17f81c0c753b7 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 16:12:43 +0900 Subject: [PATCH 06/24] Fix Entry.new in YAML::Serializable to support DirectyEntry so hacky --- src/library/entry.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index b43dd77b..94e13fbd 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -14,7 +14,11 @@ abstract class Entry def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # TODO: check node? and select proper subclass - ZippedEntry.new ctx, node + begin + ZippedEntry.new ctx, node + rescue e + DirectoryEntry.new ctx, node + end end def build_json(*, slim = false) From 9f6be70995c065f8959ddedaad1b0854d57549d2 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 16:28:53 +0900 Subject: [PATCH 07/24] Rename Entry.exists? to Entry.examine --- src/library/entry.cr | 11 +++++------ src/library/title.cr | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 94e13fbd..7399b0d1 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -228,7 +228,7 @@ abstract class Entry abstract def page_dimensions - abstract def exists? : Bool? + abstract def examine : Bool? end class ZippedEntry < Entry @@ -337,7 +337,7 @@ class ZippedEntry < Entry sizes end - def exists? : Bool + def examine : Bool File.exists? @zip_path end end @@ -439,7 +439,7 @@ class DirectoryEntry < Entry sizes end - def exists? : Bool + def examine : Bool existence = File.exists? @dir_path return false unless existence files = DirectoryEntry.get_valid_files @dir_path @@ -447,9 +447,8 @@ class DirectoryEntry < Entry existence = files.size > 0 && @signature == signature @sorted_files = nil unless existence - # For more efficient, - # Fix a directory instance with new property - # and return true + # For more efficient, update a directory entry with new property + # and return true like Title.examine existence end diff --git a/src/library/title.cr b/src/library/title.cr index ad1672d0..f3b48667 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -132,7 +132,7 @@ class Title previous_entries_size = @entries.size @entries.select! do |entry| - existence = entry.exists? + existence = entry.examine Fiber.yield context["deleted_entry_ids"] << entry.id unless existence existence From 3a60286c3aaffe46967d1e24e738f51e5892ce00 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 17:02:29 +0900 Subject: [PATCH 08/24] Run 'crystal tool format' --- src/library/cache.cr | 8 ++++---- src/library/entry.cr | 3 ++- src/routes/admin.cr | 2 +- src/routes/api.cr | 4 ++-- src/util/util.cr | 3 +-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/library/cache.cr b/src/library/cache.cr index 10e4f60f..f35af8b5 100644 --- a/src/library/cache.cr +++ b/src/library/cache.cr @@ -76,8 +76,8 @@ class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry)) entries : Array(Entry), opt : SortOptions?) entries_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s user_context = opt && opt.method == SortMethod::Progress ? username : "" - sig = Digest::SHA1.hexdigest (book_id + entries_sig + user_context + - (opt ? opt.to_tuple.to_s : "nil")) + sig = Digest::SHA1.hexdigest(book_id + entries_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) "#{sig}:sorted_entries" end end @@ -101,8 +101,8 @@ class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title)) def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?) titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s user_context = opt && opt.method == SortMethod::Progress ? username : "" - sig = Digest::SHA1.hexdigest (titles_sig + user_context + - (opt ? opt.to_tuple.to_s : "nil")) + sig = Digest::SHA1.hexdigest(titles_sig + user_context + + (opt ? opt.to_tuple.to_s : "nil")) "#{sig}:sorted_titles" end end diff --git a/src/library/entry.cr b/src/library/entry.cr index 7399b0d1..627f4fc7 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -9,7 +9,8 @@ abstract class Entry def initialize( @id, @title, @book, @size, @pages, @mtime, - @encoded_path, @encoded_title, @err_msg) + @encoded_path, @encoded_title, @err_msg + ) end def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 23481f96..6987a115 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -63,7 +63,7 @@ struct AdminRouter redirect_url = URI.new \ path: "/admin/user/edit", query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) + "admin" => admin, "error" => e.message}) redirect env, redirect_url.to_s end diff --git a/src/routes/api.cr b/src/routes/api.cr index 840ce92c..bf2c4c96 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -1144,9 +1144,9 @@ struct APIRouter raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? if entry.is_a? DirectoryEntry - file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s + entry.size) + file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s + entry.size) else - file_hash = Digest::SHA1.hexdigest (entry.path + entry.mtime.to_s) + file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s) end e_tag = "W/#{file_hash}" if e_tag == prev_e_tag diff --git a/src/util/util.cr b/src/util/util.cr index 7d834fbe..e08bd9d2 100644 --- a/src/util/util.cr +++ b/src/util/util.cr @@ -3,7 +3,7 @@ ENTRIES_IN_HOME_SECTIONS = 8 UPLOAD_URL_PREFIX = "/uploads" STATIC_DIRS = %w(/css /js /img /webfonts /favicon.ico /robots.txt) SUPPORTED_FILE_EXTNAMES = [".zip", ".cbz", ".rar", ".cbr"] -SUPPORTED_IMG_TYPES = %w( +SUPPORTED_IMG_TYPES = %w( image/jpeg image/png image/webp @@ -13,7 +13,6 @@ SUPPORTED_IMG_TYPES = %w( image/svg+xml ) - def random_str UUID.random.to_s.gsub "-", "" end From 3da5d9ba4ed56ba7e9af51c4d5080cc58647168f Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 17:36:57 +0900 Subject: [PATCH 09/24] Fix contents_signature --- src/util/signature.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index 63d31ce6..904d4e6d 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -65,7 +65,7 @@ class Dir if File.directory? path signatures << Dir.contents_signature path, cache if DirectoryEntry.validate_directory_entry path - signatures << Dir.directory_entry_signature path, cache + signatures << fn end else # Only add its signature value to `signatures` when it is a From 0ed565519b2f88f81c794c2437cbe9042d9eac0f Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sun, 15 May 2022 17:38:21 +0900 Subject: [PATCH 10/24] Rollback crystal format --- src/routes/admin.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/admin.cr b/src/routes/admin.cr index 6987a115..23481f96 100644 --- a/src/routes/admin.cr +++ b/src/routes/admin.cr @@ -63,7 +63,7 @@ struct AdminRouter redirect_url = URI.new \ path: "/admin/user/edit", query: hash_to_query({"username" => original_username, \ - "admin" => admin, "error" => e.message}) + "admin" => admin, "error" => e.message}) redirect env, redirect_url.to_s end From f18f6a5418f6c1c0d576a96551980489e4f5075d Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Thu, 19 May 2022 12:41:07 +0000 Subject: [PATCH 11/24] Fix linter issues --- src/library/entry.cr | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 627f4fc7..77af3e5a 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -15,11 +15,9 @@ abstract class Entry def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # TODO: check node? and select proper subclass - begin - ZippedEntry.new ctx, node - rescue e - DirectoryEntry.new ctx, node - end + ZippedEntry.new ctx, node + rescue e + DirectoryEntry.new ctx, node end def build_json(*, slim = false) @@ -391,9 +389,9 @@ class DirectoryEntry < Entry end @id = id - mtimes = sorted_files.map { |file_path| File.info(file_path).modification_time } - @mtime = mtimes.max - + @mtime = sorted_files.map do |file_path| + File.info(file_path).modification_time + end.max @pages = sorted_files.size end From 1f5aed64f7e42f131324c2e61d7fac774fcd0824 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 20 May 2022 09:51:56 +0900 Subject: [PATCH 12/24] Rename Entries to ArchiveEntry and DirEntry --- src/library/entry.cr | 18 +++++++++--------- src/library/title.cr | 12 ++++++------ src/routes/api.cr | 6 +++--- src/util/signature.cr | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 77af3e5a..ef0556db 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -15,9 +15,9 @@ abstract class Entry def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) # TODO: check node? and select proper subclass - ZippedEntry.new ctx, node + ArchiveEntry.new ctx, node rescue e - DirectoryEntry.new ctx, node + DirEntry.new ctx, node end def build_json(*, slim = false) @@ -230,7 +230,7 @@ abstract class Entry abstract def examine : Bool? end -class ZippedEntry < Entry +class ArchiveEntry < Entry include YAML::Serializable getter zip_path : String @@ -341,7 +341,7 @@ class ZippedEntry < Entry end end -class DirectoryEntry < Entry +class DirEntry < Entry include YAML::Serializable getter dir_path : String @@ -364,7 +364,7 @@ class DirectoryEntry < Entry return end - unless DirectoryEntry.validate_directory_entry @dir_path + unless DirEntry.validate_directory_entry @dir_path @err_msg = "Directory #{@dir_path} is not valid directory entry." Logger.warn "#{@err_msg} Please make sure the " \ "directory has valid images." @@ -441,7 +441,7 @@ class DirectoryEntry < Entry def examine : Bool existence = File.exists? @dir_path return false unless existence - files = DirectoryEntry.get_valid_files @dir_path + files = DirEntry.get_valid_files @dir_path signature = Dir.directory_entry_signature @dir_path existence = files.size > 0 && @signature == signature @sorted_files = nil unless existence @@ -454,12 +454,12 @@ class DirectoryEntry < Entry def sorted_files cached_sorted_files = @sorted_files return cached_sorted_files if cached_sorted_files - @sorted_files = DirectoryEntry.get_valid_files_sorted @dir_path + @sorted_files = DirEntry.get_valid_files_sorted @dir_path @sorted_files.not_nil! end def self.validate_directory_entry(dir_path) - files = DirectoryEntry.get_valid_files dir_path + files = DirEntry.get_valid_files dir_path files.size > 0 end @@ -477,7 +477,7 @@ class DirectoryEntry < Entry end def self.get_valid_files_sorted(dir_path) - files = DirectoryEntry.get_valid_files dir_path + files = DirEntry.get_valid_files dir_path files.sort! { |a, b| compare_numerically a, b } end end diff --git a/src/library/title.cr b/src/library/title.cr index f3b48667..3f30490a 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -53,14 +53,14 @@ class Title Library.default.title_hash[title.id] = title @title_ids << title.id end - if DirectoryEntry.validate_directory_entry path - entry = DirectoryEntry.new path, self + if DirEntry.validate_directory_entry path + entry = DirEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end next end if is_supported_file path - entry = ZippedEntry.new path, self + entry = ArchiveEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end end @@ -146,8 +146,8 @@ class Title path = File.join dir, fn if File.directory? path unless remained_entry_paths.includes? path - if DirectoryEntry.validate_directory_entry path - entry = DirectoryEntry.new path, self + if DirEntry.validate_directory_entry path + entry = DirEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true @@ -181,7 +181,7 @@ class Title end if is_supported_file path next if remained_entry_paths.includes? path - entry = ZippedEntry.new path, self + entry = ArchiveEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry is_entries_added = true diff --git a/src/routes/api.cr b/src/routes/api.cr index bf2c4c96..509f13d3 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -142,7 +142,7 @@ struct APIRouter env.response.status_code = 304 "" else - if entry.is_a? DirectoryEntry + if entry.is_a? DirEntry cache_control = "no-cache, max-age=86400" else cache_control = "public, max-age=86400" @@ -1143,7 +1143,7 @@ struct APIRouter entry = title.get_entry eid raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil? - if entry.is_a? DirectoryEntry + if entry.is_a? DirEntry file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s + entry.size) else file_hash = Digest::SHA1.hexdigest(entry.path + entry.mtime.to_s) @@ -1154,7 +1154,7 @@ struct APIRouter send_text env, "" else sizes = entry.page_dimensions - if entry.is_a? DirectoryEntry + if entry.is_a? DirEntry cache_control = "no-cache, max-age=86400" else cache_control = "public, max-age=86400" diff --git a/src/util/signature.cr b/src/util/signature.cr index 904d4e6d..b5fe781d 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -64,7 +64,7 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - if DirectoryEntry.validate_directory_entry path + if DirEntry.validate_directory_entry path signatures << fn end else @@ -84,7 +84,7 @@ class Dir return cache[dirname + "?entry"] if cache[dirname + "?entry"]? Fiber.yield signatures = [] of String - image_files = DirectoryEntry.get_valid_files_sorted dirname + image_files = DirEntry.get_valid_files_sorted dirname if image_files.size > 0 image_files.each do |path| signatures << File.signature(path).to_s From 238539c27da6a1af01e29c09354965d7169fba08 Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Fri, 20 May 2022 14:21:08 +0900 Subject: [PATCH 13/24] Split files --- src/library/archive_entry.cr | 114 ++++++++++++++++ src/library/dir_entry.cr | 144 ++++++++++++++++++++ src/library/entry.cr | 253 ----------------------------------- 3 files changed, 258 insertions(+), 253 deletions(-) create mode 100644 src/library/archive_entry.cr create mode 100644 src/library/dir_entry.cr diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr new file mode 100644 index 00000000..4075590f --- /dev/null +++ b/src/library/archive_entry.cr @@ -0,0 +1,114 @@ +require "yaml" + +require "./entry" + +class ArchiveEntry < Entry + include YAML::Serializable + + getter zip_path : String + + def initialize(@zip_path, @book) + storage = Storage.default + @encoded_path = URI.encode @zip_path + @title = File.basename @zip_path, File.extname @zip_path + @encoded_title = URI.encode @title + @size = (File.size @zip_path).humanize_bytes + id = storage.get_entry_id @zip_path, File.signature(@zip_path) + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @zip_path, + id: id, + signature: File.signature(@zip_path).to_s, + }) + end + @id = id + @mtime = File.info(@zip_path).modification_time + + unless File.readable? @zip_path + @err_msg = "File #{@zip_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + archive_exception = validate_archive @zip_path + unless archive_exception.nil? + @err_msg = "Archive error: #{archive_exception}" + Logger.warn "Unable to extract archive #{@zip_path}. " \ + "Ignoring it. #{@err_msg}" + return + end + + file = ArchiveFile.new @zip_path + @pages = file.entries.count do |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + end + file.close + end + + def path : String + @zip_path + end + + def createtime : Time + ctime @zip_path + end + + private def sorted_archive_entries + ArchiveFile.open @zip_path do |file| + entries = file.entries + .select { |e| + SUPPORTED_IMG_TYPES.includes? \ + MIME.from_filename? e.filename + } + .sort! { |a, b| + compare_numerically a.filename, b.filename + } + yield file, entries + end + end + + def read_page(page_num) + raise "Unreadble archive. #{@err_msg}" if @err_msg + img = nil + begin + sorted_archive_entries do |file, entries| + page = entries[page_num - 1] + data = file.read_entry page + if data + img = Image.new data, MIME.from_filename(page.filename), + page.filename, data.size + end + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_archive_entries do |file, entries| + entries.each_with_index do |e, i| + begin + data = file.read_entry(e).not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + end + sizes + end + + def examine : Bool + File.exists? @zip_path + end +end diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr new file mode 100644 index 00000000..a444cb0f --- /dev/null +++ b/src/library/dir_entry.cr @@ -0,0 +1,144 @@ +require "yaml" + +require "./entry" + +class DirEntry < Entry + include YAML::Serializable + + getter dir_path : String + + @[YAML::Field(ignore: true)] + @sorted_files : Array(String)? + + @signature : String + + def initialize(@dir_path, @book) + storage = Storage.default + @encoded_path = URI.encode @dir_path + @title = File.basename @dir_path + @encoded_title = URI.encode @title + + unless File.readable? @dir_path + @err_msg = "Directory #{@dir_path} is not readable." + Logger.warn "#{@err_msg} Please make sure the " \ + "file permission is configured correctly." + return + end + + unless DirEntry.validate_directory_entry @dir_path + @err_msg = "Directory #{@dir_path} is not valid directory entry." + Logger.warn "#{@err_msg} Please make sure the " \ + "directory has valid images." + return + end + + size_sum = 0 + sorted_files.each do |file_path| + size_sum += File.size file_path + end + @size = size_sum.humanize_bytes + + @signature = Dir.directory_entry_signature @dir_path + id = storage.get_entry_id @dir_path, @signature + if id.nil? + id = random_str + storage.insert_entry_id({ + path: @dir_path, + id: id, + signature: @signature, + }) + end + @id = id + + @mtime = sorted_files.map do |file_path| + File.info(file_path).modification_time + end.max + @pages = sorted_files.size + end + + def path : String + @dir_path + end + + def createtime : Time + ctime @dir_path + end + + def read_page(page_num) + img = nil + begin + files = sorted_files + file_path = files[page_num - 1] + data = File.read(file_path).to_slice + if data + img = Image.new data, MIME.from_filename(file_path), + File.basename(file_path), data.size + end + rescue e + Logger.warn "Unable to read page #{page_num} of #{@dir_path}. Error: #{e}" + end + img + end + + def page_dimensions + sizes = [] of Hash(String, Int32) + sorted_files.each_with_index do |path, i| + data = File.read(path).to_slice + begin + data.not_nil! + size = ImageSize.get data + sizes << { + "width" => size.width, + "height" => size.height, + } + rescue e + Logger.warn "Failed to read page #{i} of entry #{@dir_path}. #{e}" + sizes << {"width" => 1000_i32, "height" => 1000_i32} + end + end + sizes + end + + def examine : Bool + existence = File.exists? @dir_path + return false unless existence + files = DirEntry.get_valid_files @dir_path + signature = Dir.directory_entry_signature @dir_path + existence = files.size > 0 && @signature == signature + @sorted_files = nil unless existence + + # For more efficient, update a directory entry with new property + # and return true like Title.examine + existence + end + + def sorted_files + cached_sorted_files = @sorted_files + return cached_sorted_files if cached_sorted_files + @sorted_files = DirEntry.get_valid_files_sorted @dir_path + @sorted_files.not_nil! + end + + def self.validate_directory_entry(dir_path) + files = DirEntry.get_valid_files dir_path + files.size > 0 + end + + def self.get_valid_files(dir_path) + files = [] of String + Dir.entries(dir_path).each do |fn| + next if fn.starts_with? "." + path = File.join dir_path, fn + next unless is_supported_image_file path + next if File.directory? path + next unless File.readable? path + files << path + end + files + end + + def self.get_valid_files_sorted(dir_path) + files = DirEntry.get_valid_files dir_path + files.sort! { |a, b| compare_numerically a, b } + end +end diff --git a/src/library/entry.cr b/src/library/entry.cr index ef0556db..b718ed73 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,5 +1,4 @@ require "image_size" -require "yaml" abstract class Entry getter id : String, book : Title, title : String, @@ -229,255 +228,3 @@ abstract class Entry abstract def examine : Bool? end - -class ArchiveEntry < Entry - include YAML::Serializable - - getter zip_path : String - - def initialize(@zip_path, @book) - storage = Storage.default - @encoded_path = URI.encode @zip_path - @title = File.basename @zip_path, File.extname @zip_path - @encoded_title = URI.encode @title - @size = (File.size @zip_path).humanize_bytes - id = storage.get_entry_id @zip_path, File.signature(@zip_path) - if id.nil? - id = random_str - storage.insert_entry_id({ - path: @zip_path, - id: id, - signature: File.signature(@zip_path).to_s, - }) - end - @id = id - @mtime = File.info(@zip_path).modification_time - - unless File.readable? @zip_path - @err_msg = "File #{@zip_path} is not readable." - Logger.warn "#{@err_msg} Please make sure the " \ - "file permission is configured correctly." - return - end - - archive_exception = validate_archive @zip_path - unless archive_exception.nil? - @err_msg = "Archive error: #{archive_exception}" - Logger.warn "Unable to extract archive #{@zip_path}. " \ - "Ignoring it. #{@err_msg}" - return - end - - file = ArchiveFile.new @zip_path - @pages = file.entries.count do |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - end - file.close - end - - def path : String - @zip_path - end - - def createtime : Time - ctime @zip_path - end - - private def sorted_archive_entries - ArchiveFile.open @zip_path do |file| - entries = file.entries - .select { |e| - SUPPORTED_IMG_TYPES.includes? \ - MIME.from_filename? e.filename - } - .sort! { |a, b| - compare_numerically a.filename, b.filename - } - yield file, entries - end - end - - def read_page(page_num) - raise "Unreadble archive. #{@err_msg}" if @err_msg - img = nil - begin - sorted_archive_entries do |file, entries| - page = entries[page_num - 1] - data = file.read_entry page - if data - img = Image.new data, MIME.from_filename(page.filename), - page.filename, data.size - end - end - rescue e - Logger.warn "Unable to read page #{page_num} of #{@zip_path}. Error: #{e}" - end - img - end - - def page_dimensions - sizes = [] of Hash(String, Int32) - sorted_archive_entries do |file, entries| - entries.each_with_index do |e, i| - begin - data = file.read_entry(e).not_nil! - size = ImageSize.get data - sizes << { - "width" => size.width, - "height" => size.height, - } - rescue e - Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}" - sizes << {"width" => 1000_i32, "height" => 1000_i32} - end - end - end - sizes - end - - def examine : Bool - File.exists? @zip_path - end -end - -class DirEntry < Entry - include YAML::Serializable - - getter dir_path : String - - @[YAML::Field(ignore: true)] - @sorted_files : Array(String)? - - @signature : String - - def initialize(@dir_path, @book) - storage = Storage.default - @encoded_path = URI.encode @dir_path - @title = File.basename @dir_path - @encoded_title = URI.encode @title - - unless File.readable? @dir_path - @err_msg = "Directory #{@dir_path} is not readable." - Logger.warn "#{@err_msg} Please make sure the " \ - "file permission is configured correctly." - return - end - - unless DirEntry.validate_directory_entry @dir_path - @err_msg = "Directory #{@dir_path} is not valid directory entry." - Logger.warn "#{@err_msg} Please make sure the " \ - "directory has valid images." - return - end - - size_sum = 0 - sorted_files.each do |file_path| - size_sum += File.size file_path - end - @size = size_sum.humanize_bytes - - @signature = Dir.directory_entry_signature @dir_path - id = storage.get_entry_id @dir_path, @signature - if id.nil? - id = random_str - storage.insert_entry_id({ - path: @dir_path, - id: id, - signature: @signature, - }) - end - @id = id - - @mtime = sorted_files.map do |file_path| - File.info(file_path).modification_time - end.max - @pages = sorted_files.size - end - - def path : String - @dir_path - end - - def createtime : Time - ctime @dir_path - end - - def read_page(page_num) - img = nil - begin - files = sorted_files - file_path = files[page_num - 1] - data = File.read(file_path).to_slice - if data - img = Image.new data, MIME.from_filename(file_path), - File.basename(file_path), data.size - end - rescue e - Logger.warn "Unable to read page #{page_num} of #{@dir_path}. Error: #{e}" - end - img - end - - def page_dimensions - sizes = [] of Hash(String, Int32) - sorted_files.each_with_index do |path, i| - data = File.read(path).to_slice - begin - data.not_nil! - size = ImageSize.get data - sizes << { - "width" => size.width, - "height" => size.height, - } - rescue e - Logger.warn "Failed to read page #{i} of entry #{@dir_path}. #{e}" - sizes << {"width" => 1000_i32, "height" => 1000_i32} - end - end - sizes - end - - def examine : Bool - existence = File.exists? @dir_path - return false unless existence - files = DirEntry.get_valid_files @dir_path - signature = Dir.directory_entry_signature @dir_path - existence = files.size > 0 && @signature == signature - @sorted_files = nil unless existence - - # For more efficient, update a directory entry with new property - # and return true like Title.examine - existence - end - - def sorted_files - cached_sorted_files = @sorted_files - return cached_sorted_files if cached_sorted_files - @sorted_files = DirEntry.get_valid_files_sorted @dir_path - @sorted_files.not_nil! - end - - def self.validate_directory_entry(dir_path) - files = DirEntry.get_valid_files dir_path - files.size > 0 - end - - def self.get_valid_files(dir_path) - files = [] of String - Dir.entries(dir_path).each do |fn| - next if fn.starts_with? "." - path = File.join dir_path, fn - next unless is_supported_image_file path - next if File.directory? path - next unless File.readable? path - files << path - end - files - end - - def self.get_valid_files_sorted(dir_path) - files = DirEntry.get_valid_files dir_path - files.sort! { |a, b| compare_numerically a, b } - end -end From 648cdd772ccb9b279400b3cef66368ee69db70f8 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 02:48:06 +0000 Subject: [PATCH 14/24] Add back `zip_path` for backward compatibility --- src/library/entry.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/library/entry.cr b/src/library/entry.cr index b718ed73..596d34df 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -28,6 +28,8 @@ abstract class Entry if err_msg json.field "err_msg", err_msg end + # for API backward compatability + json.field "zip_path", path json.field "title_id", @book.id json.field "title_title", @book.title json.field "sort_title", sort_title From ae503ae099545c23ff636d971068f8aacee9e337 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 02:54:05 +0000 Subject: [PATCH 15/24] Remove unnecessary `createtime` method --- src/library/archive_entry.cr | 4 ---- src/library/dir_entry.cr | 4 ---- src/library/entry.cr | 4 +--- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr index 4075590f..ab229afd 100644 --- a/src/library/archive_entry.cr +++ b/src/library/archive_entry.cr @@ -52,10 +52,6 @@ class ArchiveEntry < Entry @zip_path end - def createtime : Time - ctime @zip_path - end - private def sorted_archive_entries ArchiveFile.open @zip_path do |file| entries = file.entries diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index a444cb0f..601431bc 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -60,10 +60,6 @@ class DirEntry < Entry @dir_path end - def createtime : Time - ctime @dir_path - end - def read_page(page_num) img = nil begin diff --git a/src/library/entry.cr b/src/library/entry.cr index 596d34df..5f19d778 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -211,7 +211,7 @@ abstract class Entry TitleInfo.new @book.dir do |info| info_da = info.date_added[@title]? if info_da.nil? - date_added = info.date_added[@title] = createtime + date_added = info.date_added[@title] = ctime path info.save else date_added = info_da @@ -222,8 +222,6 @@ abstract class Entry abstract def path : String - abstract def createtime : Time - abstract def read_page(page_num) abstract def page_dimensions From 82c60ccc1d1430467a61a6b688b9d4d777736876 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 04:04:40 +0000 Subject: [PATCH 16/24] Replace puts with Logger.debug --- src/routes/reader.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/reader.cr b/src/routes/reader.cr index 052e212c..f76dc2d7 100644 --- a/src/routes/reader.cr +++ b/src/routes/reader.cr @@ -53,7 +53,7 @@ struct ReaderRouter render "src/views/reader.html.ecr" rescue e Logger.error e - puts e.backtrace? + Logger.debug e.backtrace? env.response.status_code = 404 end end From 872e6dc6d6730a3ebad0c5bfef05bf93e50f0b41 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 04:20:14 +0000 Subject: [PATCH 17/24] Better method naming in DirEntry --- src/library/dir_entry.cr | 33 ++++++++++++++------------------- src/util/signature.cr | 2 +- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index 601431bc..59657aff 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -98,7 +98,7 @@ class DirEntry < Entry def examine : Bool existence = File.exists? @dir_path return false unless existence - files = DirEntry.get_valid_files @dir_path + files = DirEntry.image_files @dir_path signature = Dir.directory_entry_signature @dir_path existence = files.size > 0 && @signature == signature @sorted_files = nil unless existence @@ -111,30 +111,25 @@ class DirEntry < Entry def sorted_files cached_sorted_files = @sorted_files return cached_sorted_files if cached_sorted_files - @sorted_files = DirEntry.get_valid_files_sorted @dir_path + @sorted_files = DirEntry.sorted_image_files @dir_path @sorted_files.not_nil! end - def self.validate_directory_entry(dir_path) - files = DirEntry.get_valid_files dir_path - files.size > 0 + def self.image_files(dir_path) + Dir.entries(dir_path) + .reject(&.starts_with? ".") + .map { |fn| File.join dir_path, fn } + .select { |fn| is_supported_image_file fn } + .reject { |fn| File.directory? fn } + .select { |fn| File.readable? fn } end - def self.get_valid_files(dir_path) - files = [] of String - Dir.entries(dir_path).each do |fn| - next if fn.starts_with? "." - path = File.join dir_path, fn - next unless is_supported_image_file path - next if File.directory? path - next unless File.readable? path - files << path - end - files + def self.sorted_image_files(dir_path) + self.image_files(dir_path) + .sort { |a, b| compare_numerically a, b } end - def self.get_valid_files_sorted(dir_path) - files = DirEntry.get_valid_files dir_path - files.sort! { |a, b| compare_numerically a, b } + def self.validate_directory_entry(dir_path) + image_files(dir_path).size > 0 end end diff --git a/src/util/signature.cr b/src/util/signature.cr index b5fe781d..6f67a581 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -84,7 +84,7 @@ class Dir return cache[dirname + "?entry"] if cache[dirname + "?entry"]? Fiber.yield signatures = [] of String - image_files = DirEntry.get_valid_files_sorted dirname + image_files = DirEntry.sorted_image_files dirname if image_files.size > 0 image_files.each do |path| signatures << File.signature(path).to_s From e6dbeb623b8e4e7c6580ae0b32d4f1837349b001 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 05:12:43 +0000 Subject: [PATCH 18/24] Use `is_valid?` --- src/library/archive_entry.cr | 4 ++++ src/library/dir_entry.cr | 6 +++--- src/library/entry.cr | 13 +++++++++++-- src/library/title.cr | 4 ++-- src/util/signature.cr | 2 +- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr index ab229afd..da313cd0 100644 --- a/src/library/archive_entry.cr +++ b/src/library/archive_entry.cr @@ -107,4 +107,8 @@ class ArchiveEntry < Entry def examine : Bool File.exists? @zip_path end + + def self.is_valid?(path : String) : Bool + is_supported_file path + end end diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index 59657aff..82b7ccc4 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -25,7 +25,7 @@ class DirEntry < Entry return end - unless DirEntry.validate_directory_entry @dir_path + unless DirEntry.is_valid? @dir_path @err_msg = "Directory #{@dir_path} is not valid directory entry." Logger.warn "#{@err_msg} Please make sure the " \ "directory has valid images." @@ -129,7 +129,7 @@ class DirEntry < Entry .sort { |a, b| compare_numerically a, b } end - def self.validate_directory_entry(dir_path) - image_files(dir_path).size > 0 + def self.is_valid?(path : String) : Bool + image_files(path).size > 0 end end diff --git a/src/library/entry.cr b/src/library/entry.cr index 5f19d778..131e4891 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -28,8 +28,7 @@ abstract class Entry if err_msg json.field "err_msg", err_msg end - # for API backward compatability - json.field "zip_path", path + json.field "zip_path", path # for API backward compatability json.field "title_id", @book.id json.field "title_title", @book.title json.field "sort_title", sort_title @@ -220,6 +219,16 @@ abstract class Entry date_added.not_nil! # is it ok to set not_nil! here? end + # Hack to have abstract class methods + # https://github.com/crystal-lang/crystal/issues/5956 + private module ClassMethods + abstract def is_valid?(path : String) : Bool + end + + macro inherited + extend ClassMethods + end + abstract def path : String abstract def read_page(page_num) diff --git a/src/library/title.cr b/src/library/title.cr index 3f30490a..e9873f27 100644 --- a/src/library/title.cr +++ b/src/library/title.cr @@ -53,7 +53,7 @@ class Title Library.default.title_hash[title.id] = title @title_ids << title.id end - if DirEntry.validate_directory_entry path + if DirEntry.is_valid? path entry = DirEntry.new path, self @entries << entry if entry.pages > 0 || entry.err_msg end @@ -146,7 +146,7 @@ class Title path = File.join dir, fn if File.directory? path unless remained_entry_paths.includes? path - if DirEntry.validate_directory_entry path + if DirEntry.is_valid? path entry = DirEntry.new path, self if entry.pages > 0 || entry.err_msg @entries << entry diff --git a/src/util/signature.cr b/src/util/signature.cr index 6f67a581..f0f68bd0 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -64,7 +64,7 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - if DirEntry.validate_directory_entry path + if DirEntry.is_valid? path signatures << fn end else From 5b23a112b2ebbb236fed2826b08d67cf935c22f9 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 22 May 2022 05:17:05 +0000 Subject: [PATCH 19/24] Remove unnecessary `path` method --- src/library/archive_entry.cr | 5 +---- src/library/dir_entry.cr | 5 +---- src/library/entry.cr | 6 ++---- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/library/archive_entry.cr b/src/library/archive_entry.cr index da313cd0..cd63f17c 100644 --- a/src/library/archive_entry.cr +++ b/src/library/archive_entry.cr @@ -9,6 +9,7 @@ class ArchiveEntry < Entry def initialize(@zip_path, @book) storage = Storage.default + @path = @zip_path @encoded_path = URI.encode @zip_path @title = File.basename @zip_path, File.extname @zip_path @encoded_title = URI.encode @title @@ -48,10 +49,6 @@ class ArchiveEntry < Entry file.close end - def path : String - @zip_path - end - private def sorted_archive_entries ArchiveFile.open @zip_path do |file| entries = file.entries diff --git a/src/library/dir_entry.cr b/src/library/dir_entry.cr index 82b7ccc4..0ce4e71b 100644 --- a/src/library/dir_entry.cr +++ b/src/library/dir_entry.cr @@ -14,6 +14,7 @@ class DirEntry < Entry def initialize(@dir_path, @book) storage = Storage.default + @path = @dir_path @encoded_path = URI.encode @dir_path @title = File.basename @dir_path @encoded_title = URI.encode @title @@ -56,10 +57,6 @@ class DirEntry < Entry @pages = sorted_files.size end - def path : String - @dir_path - end - def read_page(page_num) img = nil begin diff --git a/src/library/entry.cr b/src/library/entry.cr index 131e4891..2cb4562e 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,12 +1,12 @@ require "image_size" abstract class Entry - getter id : String, book : Title, title : String, + getter id : String, book : Title, title : String, path : String, size : String, pages : Int32, mtime : Time, encoded_path : String, encoded_title : String, err_msg : String? def initialize( - @id, @title, @book, + @id, @title, @book, @path, @size, @pages, @mtime, @encoded_path, @encoded_title, @err_msg ) @@ -229,8 +229,6 @@ abstract class Entry extend ClassMethods end - abstract def path : String - abstract def read_page(page_num) abstract def page_dimensions From 2fb620211d3db39465b73e6a81634af767015c2c Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 29 May 2022 05:24:41 +0000 Subject: [PATCH 20/24] Choose correct subclass based on YAML node --- src/library/entry.cr | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 2cb4562e..58d1b26c 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -1,5 +1,15 @@ require "image_size" +private def node_has_key(node : YAML::Nodes::Mapping, key : String) + node.nodes + .map_with_index { |n, i| {n, i} } + .select(&.[1].even?) + .map(&.[0]) + .select(&.is_a?(YAML::Nodes::Scalar)) + .map(&.as(YAML::Nodes::Scalar).value) + .includes? key +end + abstract class Entry getter id : String, book : Title, title : String, path : String, size : String, pages : Int32, mtime : Time, @@ -13,10 +23,20 @@ abstract class Entry end def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node) - # TODO: check node? and select proper subclass - ArchiveEntry.new ctx, node - rescue e - DirEntry.new ctx, node + unless node.is_a? YAML::Nodes::Mapping + raise "Unexpected node type in YAML" + end + # Doing YAML::Any.new(ctx, node) here causes a weird error, so + # instead we are using a more hacky approach (see `node_has_key`). + # TODO: Use a more elegant approach + if node_has_key node, "zip_path" + ArchiveEntry.new ctx, node + elsif node_has_key node, "dir_path" + DirEntry.new ctx, node + else + raise "Unknown entry found in YAML cache. Try deleting the " \ + "`library.yml.gz` file" + end end def build_json(*, slim = false) From df618704ea50b9c450f1cfa89043acdda60f77d6 Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 29 May 2022 05:28:50 +0000 Subject: [PATCH 21/24] Fix linter --- src/library/entry.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 58d1b26c..82c1e04c 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -5,7 +5,7 @@ private def node_has_key(node : YAML::Nodes::Mapping, key : String) .map_with_index { |n, i| {n, i} } .select(&.[1].even?) .map(&.[0]) - .select(&.is_a?(YAML::Nodes::Scalar)) + .select(YAML::Nodes::Scalar) .map(&.as(YAML::Nodes::Scalar).value) .includes? key end From 39a331c87937358986caaeb478e33e46f9fe3fcc Mon Sep 17 00:00:00 2001 From: Alex Ling Date: Sun, 29 May 2022 05:44:11 +0000 Subject: [PATCH 22/24] Avoid not_nil in date_added --- src/library/entry.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 82c1e04c..3a4748ca 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -226,7 +226,7 @@ abstract class Entry end def date_added : Time - date_added = nil + date_added = Time::UNIX_EPOCH TitleInfo.new @book.dir do |info| info_da = info.date_added[@title]? if info_da.nil? @@ -236,7 +236,7 @@ abstract class Entry date_added = info_da end end - date_added.not_nil! # is it ok to set not_nil! here? + date_added end # Hack to have abstract class methods From 8e4bb995d316513c6f8b6fdcbe523c2d49dbdf0b Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 4 Jun 2022 00:18:45 +0900 Subject: [PATCH 23/24] Add zip_path to API document, add path property --- src/library/entry.cr | 1 + src/routes/api.cr | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/library/entry.cr b/src/library/entry.cr index 3a4748ca..16666eaf 100644 --- a/src/library/entry.cr +++ b/src/library/entry.cr @@ -49,6 +49,7 @@ abstract class Entry json.field "err_msg", err_msg end json.field "zip_path", path # for API backward compatability + json.field "path", path json.field "title_id", @book.id json.field "title_title", @book.title json.field "sort_title", sort_title diff --git a/src/routes/api.cr b/src/routes/api.cr index 509f13d3..89b4a308 100644 --- a/src/routes/api.cr +++ b/src/routes/api.cr @@ -40,7 +40,7 @@ struct APIRouter Koa.schema "entry", { "pages" => Int32, "mtime" => Int64, - }.merge(s %w(path title size id title_id display_name cover_url)), + }.merge(s %w(zip_path path title size id title_id display_name cover_url)), desc: "An entry in a book" Koa.schema "title", { From 9ce8e918f0f255f21a51fa0a19e188811800992e Mon Sep 17 00:00:00 2001 From: Leeingnyo Date: Sat, 4 Jun 2022 00:19:14 +0900 Subject: [PATCH 24/24] Replace to is_valid? --- src/util/signature.cr | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/util/signature.cr b/src/util/signature.cr index f0f68bd0..8d2b961e 100644 --- a/src/util/signature.cr +++ b/src/util/signature.cr @@ -19,7 +19,7 @@ class File # information as long as the above changes do not happen together with # a file/folder rename, with no library scan in between. def self.signature(filename) : UInt64 - if is_supported_file(filename) || is_supported_image_file(filename) + if ArchiveEntry.is_valid?(filename) || is_supported_image_file(filename) File.info(filename).inode else 0u64 @@ -64,13 +64,11 @@ class Dir path = File.join dirname, fn if File.directory? path signatures << Dir.contents_signature path, cache - if DirEntry.is_valid? path - signatures << fn - end + signatures << fn if DirEntry.is_valid? path else # Only add its signature value to `signatures` when it is a # supported file - signatures << fn if is_supported_file fn + signatures << fn if ArchiveEntry.is_valid? fn end Fiber.yield end