Bump version to v0.7.2

Kemal is having some issues in 0.35.0: https://github.com/kemalcr/kemal/issues/575

.github/workflows/build.yml

@@ -12,7 +12,7 @@ jobs:
 
     runs-on: ubuntu-latest
     container:
-      image: crystallang/crystal:0.35.1-alpine
+      image: crystallang/crystal:0.34.0-alpine
 
     steps:
     - uses: actions/checkout@v2

Dockerfile

@@ -1,4 +1,4 @@
-FROM crystallang/crystal:0.35.1-alpine AS builder
+FROM crystallang/crystal:0.34.0-alpine AS builder
 
 WORKDIR /Mango
 

README.md

@@ -50,7 +50,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
 ### CLI
 
 ```
-  Mango - Manga Server and Web Reader. Version 0.7.0
+  Mango - Manga Server and Web Reader. Version 0.7.2
 
   Usage:
 

shard.lock

@@ -1,46 +1,46 @@
-version: 2.0
+version: 1.0
 shards:
   ameba:
-    git: https://github.com/crystal-ameba/ameba.git
+    github: crystal-ameba/ameba
     version: 0.12.1
 
   archive:
-    git: https://github.com/hkalexling/archive.cr.git
+    github: hkalexling/archive.cr
     version: 0.2.0
 
   baked_file_system:
-    git: https://github.com/schovi/baked_file_system.git
+    github: schovi/baked_file_system
     version: 0.9.8
 
   clim:
-    git: https://github.com/at-grandpa/clim.git
+    github: at-grandpa/clim
     version: 0.12.0
 
   db:
-    git: https://github.com/crystal-lang/crystal-db.git
+    github: crystal-lang/crystal-db
     version: 0.9.0
 
   exception_page:
-    git: https://github.com/crystal-loot/exception_page.git
+    github: crystal-loot/exception_page
     version: 0.1.4
 
   kemal:
-    git: https://github.com/kemalcr/kemal.git
-    version: 0.26.1+git.commit.a8c0f09b858162bd13c96663febef5527b322a32
+    github: kemalcr/kemal
+    version: 0.26.1
 
   kemal-session:
-    git: https://github.com/kemalcr/kemal-session.git
+    github: kemalcr/kemal-session
     version: 0.12.1
 
   kilt:
-    git: https://github.com/jeromegn/kilt.git
+    github: jeromegn/kilt
     version: 0.4.0
 
   radix:
-    git: https://github.com/luislavena/radix.git
+    github: luislavena/radix
     version: 0.3.9
 
   sqlite3:
-    git: https://github.com/crystal-lang/crystal-sqlite3.git
+    github: crystal-lang/crystal-sqlite3
     version: 0.16.0
 

shard.yml

@@ -1,5 +1,5 @@
 name: mango
-version: 0.7.0
+version: 0.7.2
 
 authors:
   - Alex Ling <hkalexling@gmail.com>
@@ -8,14 +8,13 @@ targets:
   mango:
     main: src/mango.cr
 
-crystal: 0.35.0
+crystal: 0.34.0
 
 license: MIT
 
 dependencies:
   kemal:
     github: kemalcr/kemal
-    commit: a8c0f09b858162bd13c96663febef5527b322a32
   kemal-session:
     github: kemalcr/kemal-session
   sqlite3:

src/archive.cr

@@ -1,13 +1,13 @@
-require "compress/zip"
+require "zip"
 require "archive"
 
-# A unified class to handle all supported archive formats. It uses the
-#   Compress::Zip module in crystal standard library if the target file is a
-#   zip archive. Otherwise it uses `archive.cr`.
+# A unified class to handle all supported archive formats. It uses the ::Zip
+#   module in crystal standard library if the target file is a zip archive.
+#   Otherwise it uses `archive.cr`.
 class ArchiveFile
   def initialize(@filename : String)
     if [".cbz", ".zip"].includes? File.extname filename
-      @archive_file = Compress::Zip::File.new filename
+      @archive_file = Zip::File.new filename
     else
       @archive_file = Archive::File.new filename
     end
@@ -20,16 +20,16 @@ class ArchiveFile
   end
 
   def close
-    if @archive_file.is_a? Compress::Zip::File
-      @archive_file.as(Compress::Zip::File).close
+    if @archive_file.is_a? Zip::File
+      @archive_file.as(Zip::File).close
     end
   end
 
   # Lists all file entries
   def entries
-    ary = [] of Compress::Zip::File::Entry | Archive::Entry
+    ary = [] of Zip::File::Entry | Archive::Entry
     @archive_file.entries.map do |e|
-      if (e.is_a? Compress::Zip::File::Entry && e.file?) ||
+      if (e.is_a? Zip::File::Entry && e.file?) ||
          (e.is_a? Archive::Entry && e.info.file?)
         ary.push e
       end
@@ -37,8 +37,8 @@ class ArchiveFile
     ary
   end
 
-  def read_entry(e : Compress::Zip::File::Entry | Archive::Entry) : Bytes?
-    if e.is_a? Compress::Zip::File::Entry
+  def read_entry(e : Zip::File::Entry | Archive::Entry) : Bytes?
+    if e.is_a? Zip::File::Entry
       data = nil
       e.open do |io|
         slice = Bytes.new e.uncompressed_size

src/library.cr

@@ -33,7 +33,16 @@ class Entry
         MIME.from_filename? e.filename
     end
     file.close
-    @id = storage.get_id @zip_path, false
+    id = storage.get_id @zip_path, false
+    if id.nil?
+      id = random_str
+      storage.insert_id({
+        path:     @zip_path,
+        id:       id,
+        is_title: false,
+      })
+    end
+    @id = id
     @mtime = File.info(@zip_path).modification_time
   end
 
@@ -178,7 +187,16 @@ class Title
 
   def initialize(@dir : String, @parent_id, storage,
                  @library : Library)
-    @id = storage.get_id @dir, true
+    id = storage.get_id @dir, true
+    if id.nil?
+      id = random_str
+      storage.insert_id({
+        path:     @dir,
+        id:       id,
+        is_title: true,
+      })
+    end
+    @id = id
     @title = File.basename dir
     @encoded_title = URI.encode @title
     @title_ids = [] of String
@@ -374,7 +392,7 @@ class Title
   end
 
   def deep_read_page_count(username) : Int32
-    entries.map { |e| e.load_progress username }.sum +
+    load_progress_for_all_entries(username).sum +
       titles.map { |t| t.deep_read_page_count username }.flatten.sum
   end
 
@@ -401,6 +419,85 @@ class Title
       latest_read_entry
     end
   end
+
+  def load_progress_for_all_entries(username)
+    progress = {} of String => Int32
+    TitleInfo.new @dir do |info|
+      progress = info.progress[username]?
+    end
+
+    @entries.map do |e|
+      info_progress = 0
+      if progress && progress.has_key? e.title
+        info_progress = [progress[e.title], e.pages].min
+      end
+      info_progress
+    end
+  end
+
+  def load_percentage_for_all_entries(username)
+    progress = load_progress_for_all_entries username
+    @entries.map_with_index do |e, i|
+      progress[i] / e.pages
+    end
+  end
+
+  # === helper methods ===
+
+  # Gets the last read entry in the title. If the entry has been completed,
+  #   returns the next entry. Returns nil when no entry has been read yet,
+  #   or when all entries are completed
+  def get_last_read_entry(username) : Entry?
+    progress = {} of String => Int32
+    TitleInfo.new @dir do |info|
+      progress = info.progress[username]?
+    end
+    return if progress.nil?
+
+    last_read_entry = nil
+
+    @entries.reverse_each do |e|
+      if progress.has_key? e.title
+        last_read_entry = e
+        break
+      end
+    end
+
+    if last_read_entry && last_read_entry.finished? username
+      last_read_entry = last_read_entry.next_entry
+    end
+
+    last_read_entry
+  end
+
+  # Equivalent to `@entries.map &. date_added`, but much more efficient
+  def get_date_added_for_all_entries
+    da = {} of String => Time
+    TitleInfo.new @dir do |info|
+      da = info.date_added
+    end
+
+    @entries.each do |e|
+      next if da.has_key? e.title
+      da[e.title] = ctime e.zip_path
+    end
+
+    TitleInfo.new @dir do |info|
+      info.date_added = da
+      info.save
+    end
+
+    @entries.map { |e| da[e.title] }
+  end
+
+  def deep_entries_with_date_added
+    da_ary = get_date_added_for_all_entries
+    zip = @entries.map_with_index do |e, i|
+      {entry: e, date_added: da_ary[i]}
+    end
+    return zip if title_ids.empty?
+    zip + titles.map { |t| t.deep_entries_with_date_added }.flatten
+  end
 end
 
 class TitleInfo
@@ -446,7 +543,7 @@ end
 
 class Library
   property dir : String, title_ids : Array(String), scan_interval : Int32,
-    storage : Storage, title_hash : Hash(String, Title)
+    title_hash : Hash(String, Title)
 
   def self.default : self
     unless @@default
@@ -456,7 +553,8 @@ class Library
   end
 
   def initialize
-    @storage = Storage.default
+    register_mime_types
+
     @dir = Config.current.library_path
     @scan_interval = Config.current.scan_interval
     # explicitly initialize @titles to bypass the compiler check. it will
@@ -508,35 +606,32 @@ class Library
       Dir.mkdir_p @dir
     end
     @title_ids.clear
+
+    storage = Storage.new auto_close: false
+
     (Dir.entries @dir)
       .select { |fn| !fn.starts_with? "." }
       .map { |fn| File.join @dir, fn }
       .select { |path| File.directory? path }
-      .map { |path| Title.new path, "", @storage, self }
+      .map { |path| Title.new path, "", storage, self }
       .select { |title| !(title.entries.empty? && title.titles.empty?) }
       .sort { |a, b| a.title <=> b.title }
       .each do |title|
         @title_hash[title.id] = title
         @title_ids << title.id
       end
+
+    storage.bulk_insert_ids
+    storage.close
+
     Logger.debug "Scan completed"
   end
 
   def get_continue_reading_entries(username)
     cr_entries = deep_titles
-      # For each Title, get the last read entry. If the user has finished
-      #   reading this entry, get the next entry
-      .map { |t|
-        last_read_entry = t.entries.reverse_each.find do |e|
-          e.started? username
-        end
-        if last_read_entry && last_read_entry.finished? username
-          last_read_entry = last_read_entry.next_entry
-        end
-        last_read_entry
-      }
+      .map { |t| t.get_last_read_entry username }
       # Select elements with type `Entry` from the array and ignore all `Nil`s
-      .select(Entry)
+      .select(Entry)[0..11]
       .map { |e|
         # Get the last read time of the entry. If it hasn't been started, get
         #   the last read time of the previous entry
@@ -558,7 +653,7 @@ class Library
       next 1 if a[:last_read].nil?
       next -1 if b[:last_read].nil?
       b[:last_read].not_nil! <=> a[:last_read].not_nil!
-    }[0..11]
+    }
   end
 
   alias RA = NamedTuple(
@@ -568,15 +663,16 @@ class Library
 
   def get_recently_added_entries(username)
     recently_added = [] of RA
+    last_date_added = nil
 
-    titles.map { |t| t.deep_entries }
-      .flatten
-      .select { |e| e.date_added > 1.month.ago }
-      .sort { |a, b| b.date_added <=> a.date_added }
+    titles.map { |t| t.deep_entries_with_date_added }.flatten
+      .select { |e| e[:date_added] > 1.month.ago }
+      .sort { |a, b| b[:date_added] <=> a[:date_added] }
       .each do |e|
+        break if recently_added.size > 12
         last = recently_added.last?
-        if last && e.title_id == last[:entry].title_id &&
-           (e.date_added - last[:entry].date_added).duration < 1.day
+        if last && e[:entry].title_id == last[:entry].title_id &&
+           (e[:date_added] - last_date_added.not_nil!).duration < 1.day
           # A NamedTuple is immutable, so we have to cast it to a Hash first
           last_hash = last.to_h
           count = last_hash[:grouped_count].as(Int32)
@@ -586,9 +682,10 @@ class Library
           last_hash[:percentage] = -1.0
           recently_added[recently_added.size - 1] = RA.from last_hash
         else
+          last_date_added = e[:date_added]
           recently_added << {
-            entry:         e,
-            percentage:    e.load_percentage(username),
+            entry:         e[:entry],
+            percentage:    e[:entry].load_percentage(username),
             grouped_count: 1,
           }
         end

src/logger.cr

@@ -31,9 +31,9 @@ class Logger
     {% end %}
 
     @log = Log.for("")
-    @backend = Log::IOBackend.new
 
-    format_proc = ->(entry : Log::Entry, io : IO) do
+    @backend = Log::IOBackend.new
+    @backend.formatter = ->(entry : Log::Entry, io : IO) do
       color = :default
       {% begin %}
         case entry.severity.label.to_s().downcase
@@ -50,14 +50,12 @@ class Logger
       io << entry.message
     end
 
-    @backend.formatter = Log::Formatter.new &format_proc
-    Log.setup @@severity, @backend
+    Log.builder.bind "*", @@severity, @backend
   end
 
   # Ignores @@severity and always log msg
   def log(msg)
-    @backend.write Log::Entry.new "", Log::Severity::None, msg,
-      Log::Metadata.empty, nil
+    @backend.write Log::Entry.new "", Log::Severity::None, msg, nil
   end
 
   def self.log(msg)

src/mangadex/downloader.cr

@@ -1,13 +1,13 @@
 require "./api"
 require "sqlite3"
-require "compress/zip"
+require "zip"
 
 module MangaDex
   class PageJob
     property success = false
     property url : String
     property filename : String
-    property writer : Compress::Zip::Writer
+    property writer : Zip::Writer
     property tries_remaning : Int32
 
     def initialize(@url, @filename, @writer, @tries_remaning)
@@ -324,7 +324,7 @@ module MangaDex
       # Find the number of digits needed to store the number of pages
       len = Math.log10(chapter.pages.size).to_i + 1
 
-      writer = Compress::Zip::Writer.new zip_path
+      writer = Zip::Writer.new zip_path
       # Create a buffered channel. It works as an FIFO queue
       channel = Channel(PageJob).new chapter.pages.size
       spawn do

src/mango.cr

@@ -4,7 +4,7 @@ require "./mangadex/*"
 require "option_parser"
 require "clim"
 
-MANGO_VERSION = "0.7.0"
+MANGO_VERSION = "0.7.2"
 
 macro common_option
   option "-c PATH", "--config=PATH", type: String,

src/routes/main.cr

@@ -53,7 +53,7 @@ class MainRouter < Router
       begin
         title = (@context.library.get_title env.params.url["title"]).not_nil!
         username = get_username env
-        percentage = title.entries.map &.load_percentage username
+        percentage = title.load_percentage_for_all_entries username
         title_percentage = title.titles.map &.load_percentage username
         layout "title"
       rescue e

src/storage.cr

@@ -14,6 +14,12 @@ end
 
 class Storage
   @path : String
+  @db : DB::Database?
+  @insert_ids = [] of IDTuple
+
+  alias IDTuple = NamedTuple(path: String,
+    id: String,
+    is_title: Bool)
 
   def self.default : self
     unless @@default
@@ -22,7 +28,8 @@ class Storage
     @@default.not_nil!
   end
 
-  def initialize(db_path : String? = nil, init_user = true)
+  def initialize(db_path : String? = nil, init_user = true, *,
+                 @auto_close = true)
     @path = db_path || Config.current.db_path
     dir = File.dirname @path
     unless Dir.exists? dir
@@ -60,6 +67,9 @@ class Storage
         init_admin if init_user
       end
     end
+    unless @auto_close
+      @db = DB.open "sqlite3://#{@path}"
+    end
   end
 
   macro init_admin
@@ -71,8 +81,18 @@ class Storage
                "#{{"username" => "admin", "password" => random_pw}}"
   end
 
+  private def get_db(&block : DB::Database ->)
+    if @db.nil?
+      DB.open "sqlite3://#{@path}" do |db|
+        yield db
+      end
+    else
+      yield @db.not_nil!
+    end
+  end
+
   def verify_user(username, password)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         hash, token = db.query_one "select password, token from " \
                                    "users where username = (?)",
@@ -97,7 +117,7 @@ class Storage
 
   def verify_token(token)
     username = nil
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         username = db.query_one "select username from users where " \
                                 "token = (?)", token, as: String
@@ -110,7 +130,7 @@ class Storage
 
   def verify_admin(token)
     is_admin = false
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         is_admin = db.query_one "select admin from users where " \
                                 "token = (?)", token, as: Bool
@@ -123,7 +143,7 @@ class Storage
 
   def list_users
     results = Array(Tuple(String, Bool)).new
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       db.query "select username, admin from users" do |rs|
         rs.each do
           results << {rs.read(String), rs.read(Bool)}
@@ -137,7 +157,7 @@ class Storage
     validate_username username
     validate_password password
     admin = (admin ? 1 : 0)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       hash = hash_password password
       db.exec "insert into users values (?, ?, ?, ?)",
         username, hash, nil, admin
@@ -148,7 +168,7 @@ class Storage
     admin = (admin ? 1 : 0)
     validate_username username
     validate_password password unless password.empty?
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       if password.empty?
         db.exec "update users set username = (?), admin = (?) " \
                 "where username = (?)",
@@ -163,13 +183,13 @@ class Storage
   end
 
   def delete_user(username)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       db.exec "delete from users where username = (?)", username
     end
   end
 
   def logout(token)
-    DB.open "sqlite3://#{@path}" do |db|
+    get_db do |db|
       begin
         db.exec "update users set token = (?) where token = (?)", nil, token
       rescue
@@ -178,18 +198,36 @@ class Storage
   end
 
   def get_id(path, is_title)
-    id = random_str
-    DB.open "sqlite3://#{@path}" do |db|
-      begin
-        id = db.query_one "select id from ids where path = (?)", path,
-          as: {String}
-      rescue
-        db.exec "insert into ids values (?, ?, ?)", path, id, is_title ? 1 : 0
-      end
+    id = nil
+    get_db do |db|
+      id = db.query_one? "select id from ids where path = (?)", path,
+        as: {String}
     end
     id
   end
 
+  def insert_id(tp : IDTuple)
+    @insert_ids << tp
+  end
+
+  def bulk_insert_ids
+    get_db do |db|
+      db.transaction do |tx|
+        @insert_ids.each do |tp|
+          tx.connection.exec "insert into ids values (?, ?, ?)", tp[:path],
+            tp[:id], tp[:is_title] ? 1 : 0
+        end
+      end
+    end
+    @insert_ids.clear
+  end
+
+  def close
+    unless @db.nil?
+      @db.not_nil!.close
+    end
+  end
+
   def to_json(json : JSON::Builder)
     json.string self
   end

src/util.cr

@@ -41,8 +41,6 @@ def send_json(env, json)
 end
 
 def send_attachment(env, path)
-  MIME.register ".cbz", "application/vnd.comicbook+zip"
-  MIME.register ".cbr", "application/vnd.comicbook-rar"
   send_file env, path, filename: File.basename(path), disposition: "attachment"
 end
 
@@ -154,3 +152,14 @@ def ctime(file_path : String) : Time
     Time.new stat.st_ctim, Time::Location::UTC
   {% end %}
 end
+
+def register_mime_types
+  {
+    ".zip" => "application/zip",
+    ".rar" => "application/x-rar-compressed",
+    ".cbz" => "application/vnd.comicbook+zip",
+    ".cbr" => "application/vnd.comicbook-rar",
+  }.each do |k, v|
+    MIME.register k, v
+  end
+end

src/views/opds/index.xml.ecr

@@ -14,7 +14,7 @@
 
   <% titles.each do |t| %>
     <entry>
-      <title><%= t.display_name %></title>
+      <title><%= HTML.escape(t.display_name) %></title>
       <id>urn:mango:<%= t.id %></id>
       <link type="application/atom+xml;profile=opds-catalog;kind=navigation" rel="subsection" href="<%= base_url %>opds/book/<%= t.id %>" />
     </entry>

src/views/opds/title.xml.ecr

@@ -5,7 +5,7 @@
   <link rel="self" href="<%= base_url %>opds/book/<%= title.id %>" type="application/atom+xml;profile=opds-catalog;kind=navigation" />
   <link rel="start" href="<%= base_url %>opds/" type="application/atom+xml;profile=opds-catalog;kind=navigation" />
 
-  <title><%= title.display_name %></title>
+  <title><%= HTML.escape(title.display_name) %></title>
 
   <author>
     <name>Mango</name>
@@ -14,7 +14,7 @@
 
   <% title.titles.each do |t| %>
     <entry>
-      <title><%= t.display_name %></title>
+      <title><%= HTML.escape(t.display_name) %></title>
       <id>urn:mango:<%= t.id %></id>
       <link type="application/atom+xml;profile=opds-catalog;kind=navigation" rel="subsection" href="<%= base_url %>opds/book/<%= t.id %>" />
     </entry>
@@ -22,7 +22,7 @@
 
   <% title.entries.each do |e| %>
     <entry>
-      <title><%= e.display_name %></title>
+      <title><%= HTML.escape(e.display_name) %></title>
       <id>urn:mango:<%= e.id %></id>
 
       <link rel="http://opds-spec.org/image" href="<%= e.cover_url %>" />