Compare commits

..

2 Commits

Author SHA1 Message Date
Alex Ling
bfd5ee2a6e Link to Discord [skip ci] 2021-08-28 09:47:31 +08:00
Alex Ling
35dbe1bed8 Add Discord badge [skip ci] 2021-08-28 09:45:40 +08:00
33 changed files with 241 additions and 1303 deletions

View File

@@ -104,33 +104,6 @@
"contributions": [
"infra"
]
},
{
"login": "lincolnthedev",
"name": "i use arch btw",
"avatar_url": "https://avatars.githubusercontent.com/u/41193328?v=4",
"profile": "https://lncn.dev",
"contributions": [
"infra"
]
},
{
"login": "BradleyDS2",
"name": "BradleyDS2",
"avatar_url": "https://avatars.githubusercontent.com/u/2174921?v=4",
"profile": "https://github.com/BradleyDS2",
"contributions": [
"doc"
]
},
{
"login": "nduja",
"name": "Robbo",
"avatar_url": "https://avatars.githubusercontent.com/u/69299134?v=4",
"profile": "https://github.com/nduja",
"contributions": [
"code"
]
}
],
"contributorsPerLine": 7,

View File

@@ -1,9 +1,2 @@
node_modules
lib
Dockerfile
Dockerfile.arm32v7
Dockerfile.arm64v8
README.md
.all-contributorsrc
env.example
.github/

View File

@@ -13,7 +13,7 @@ Mango is a self-hosted manga server and reader. Its features include
- Supports nested folders in library
- Automatically stores reading progress
- Thumbnail generation
- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from third-party sites
- Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites
- The web reader is responsive and works well on mobile, so there is no need for a mobile app
- All the static files are embedded in the binary, so the deployment process is easy and painless
@@ -51,7 +51,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
### CLI
```
Mango - Manga Server and Web Reader. Version 0.25.0
Mango - Manga Server and Web Reader. Version 0.23.0
Usage:
@@ -80,26 +80,29 @@ base_url: /
session_secret: mango-session-secret
library_path: ~/mango/library
db_path: ~/mango/mango.db
queue_db_path: ~/mango/queue.db
scan_interval_minutes: 5
thumbnail_generation_interval_hours: 24
log_level: info
upload_path: ~/mango/uploads
plugin_path: ~/mango/plugins
download_timeout_seconds: 30
library_cache_path: ~/mango/library.yml.gz
cache_enabled: true
cache_size_mbs: 50
cache_log_enabled: true
disable_login: false
default_username: ""
auth_proxy_header_name: ""
mangadex:
base_url: https://mangadex.org
api_url: https://api.mangadex.org/v2
download_wait_seconds: 5
download_retries: 4
download_queue_db_path: ~/mango/queue.db
chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}'
manga_rename_rule: '{title}'
subscription_update_interval_hours: 24
```
- `scan_interval_minutes`, `thumbnail_generation_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
- `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging
- You can disable authentication by setting `disable_login` to true. Note that `default_username` must be set to an existing username for this to work.
- By setting `cache_enabled` to `true`, you can enable an experimental feature where Mango caches library metadata to improve page load time. You can further fine-tune the feature with `cache_size_mbs` and `cache_log_enabled`.
### Library Structure
@@ -171,9 +174,6 @@ Please check the [development guideline](https://github.com/hkalexling/Mango/wik
<td align="center"><a href="https://github.com/Leeingnyo"><img src="https://avatars0.githubusercontent.com/u/6760150?v=4?s=100" width="100px;" alt=""/><br /><sub><b>이인용</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=Leeingnyo" title="Code">💻</a></td>
<td align="center"><a href="http://h45h74x.eu.org"><img src="https://avatars1.githubusercontent.com/u/27204033?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Simon</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=h45h74x" title="Code">💻</a></td>
<td align="center"><a href="https://github.com/davidkna"><img src="https://avatars.githubusercontent.com/u/835177?v=4?s=100" width="100px;" alt=""/><br /><sub><b>David Knaack</b></sub></a><br /><a href="#infra-davidkna" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
<td align="center"><a href="https://lncn.dev"><img src="https://avatars.githubusercontent.com/u/41193328?v=4?s=100" width="100px;" alt=""/><br /><sub><b>i use arch btw</b></sub></a><br /><a href="#infra-lincolnthedev" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
<td align="center"><a href="https://github.com/BradleyDS2"><img src="https://avatars.githubusercontent.com/u/2174921?v=4?s=100" width="100px;" alt=""/><br /><sub><b>BradleyDS2</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=BradleyDS2" title="Documentation">📖</a></td>
<td align="center"><a href="https://github.com/nduja"><img src="https://avatars.githubusercontent.com/u/69299134?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Robbo</b></sub></a><br /><a href="https://github.com/hkalexling/Mango/commits?author=nduja" title="Code">💻</a></td>
</tr>
</table>

View File

@@ -1,94 +0,0 @@
class SortTitle < MG::Base
def up : String
<<-SQL
-- add sort_title column to ids and titles
ALTER TABLE ids ADD COLUMN sort_title TEXT;
ALTER TABLE titles ADD COLUMN sort_title TEXT;
SQL
end
def down : String
<<-SQL
-- remove sort_title column from ids
ALTER TABLE ids RENAME TO tmp;
CREATE TABLE ids (
path TEXT NOT NULL,
id TEXT NOT NULL,
signature TEXT,
unavailable INTEGER NOT NULL DEFAULT 0
);
INSERT INTO ids
SELECT path, id, signature, unavailable
FROM tmp;
DROP TABLE tmp;
-- recreate the indices
CREATE UNIQUE INDEX path_idx ON ids (path);
CREATE UNIQUE INDEX id_idx ON ids (id);
-- recreate the foreign key constraint on thumbnails
ALTER TABLE thumbnails RENAME TO tmp;
CREATE TABLE thumbnails (
id TEXT NOT NULL,
data BLOB NOT NULL,
filename TEXT NOT NULL,
mime TEXT NOT NULL,
size INTEGER NOT NULL,
FOREIGN KEY (id) REFERENCES ids (id)
ON UPDATE CASCADE
ON DELETE CASCADE
);
INSERT INTO thumbnails
SELECT * FROM tmp;
DROP TABLE tmp;
CREATE UNIQUE INDEX tn_index ON thumbnails (id);
-- remove sort_title column from titles
ALTER TABLE titles RENAME TO tmp;
CREATE TABLE titles (
id TEXT NOT NULL,
path TEXT NOT NULL,
signature TEXT,
unavailable INTEGER NOT NULL DEFAULT 0
);
INSERT INTO titles
SELECT id, path, signature, unavailable
FROM tmp;
DROP TABLE tmp;
-- recreate the indices
CREATE UNIQUE INDEX titles_id_idx on titles (id);
CREATE UNIQUE INDEX titles_path_idx on titles (path);
-- recreate the foreign key constraint on tags
ALTER TABLE tags RENAME TO tmp;
CREATE TABLE tags (
id TEXT NOT NULL,
tag TEXT NOT NULL,
UNIQUE (id, tag),
FOREIGN KEY (id) REFERENCES titles (id)
ON UPDATE CASCADE
ON DELETE CASCADE
);
INSERT INTO tags
SELECT * FROM tmp;
DROP TABLE tmp;
CREATE INDEX tags_id_idx ON tags (id);
CREATE INDEX tags_tag_idx ON tags (tag);
SQL
end
end

View File

@@ -55,7 +55,7 @@ const component = () => {
jobAction(action, event) {
let url = `${base_url}api/admin/mangadex/queue/${action}`;
if (event) {
const id = event.currentTarget.closest('tr').id.split('-').slice(1).join('-');
const id = event.currentTarget.closest('tr').id.split('-')[1];
url = `${url}?${$.param({
id: id
})}`;

View File

@@ -68,12 +68,7 @@ const buildTable = (chapters) => {
$('table').append(thead);
const rows = chapters.map(ch => {
const tds = Object.values(ch).map(v => {
const maxLength = 40;
const shouldShrink = v && v.length > maxLength;
const content = shouldShrink ? `<span title="${v}">${v.substring(0, maxLength)}...</span><div uk-dropdown><span>${v}</span></div>` : v;
return `<td>${content}</td>`
}).join('');
const tds = Object.values(ch).map(v => `<td>${v}</td>`).join('');
return `<tr data-id="${ch.id}" data-title="${ch.title}">${tds}</tr>`;
});
const tbody = `<tbody id="selectable">${rows}</tbody>`;

View File

@@ -13,7 +13,6 @@ const readerComponent = () => {
selectedIndex: 0, // 0: not selected; 1: the first page
margin: 30,
preloadLookahead: 3,
enableRightToLeft: false,
/**
* Initialize the component by fetching the page dimensions
@@ -65,13 +64,6 @@ const readerComponent = () => {
const savedFlipAnimation = localStorage.getItem('enableFlipAnimation');
this.enableFlipAnimation = savedFlipAnimation === null || savedFlipAnimation === 'true';
const savedRightToLeft = localStorage.getItem('enableRightToLeft');
if (savedRightToLeft === null) {
this.enableRightToLeft = false;
} else {
this.enableRightToLeft = (savedRightToLeft === 'true');
}
})
.catch(e => {
const errMsg = `Failed to get the page dimensions. ${e}`;
@@ -122,9 +114,9 @@ const readerComponent = () => {
if (this.mode === 'continuous') return;
if (event.key === 'ArrowLeft' || event.key === 'k')
this.flipPage(false ^ this.enableRightToLeft);
this.flipPage(false);
if (event.key === 'ArrowRight' || event.key === 'j')
this.flipPage(true ^ this.enableRightToLeft);
this.flipPage(true);
},
/**
* Flips to the next or the previous page
@@ -144,7 +136,7 @@ const readerComponent = () => {
this.toPage(newIdx);
if (this.enableFlipAnimation) {
if (isNext ^ this.enableRightToLeft)
if (isNext)
this.flipAnimation = 'right';
else
this.flipAnimation = 'left';
@@ -328,9 +320,5 @@ const readerComponent = () => {
enableFlipAnimationChanged() {
localStorage.setItem('enableFlipAnimation', this.enableFlipAnimation);
},
enableRightToLeftChanged() {
localStorage.setItem('enableRightToLeft', this.enableRightToLeft);
},
};
}

View File

@@ -60,11 +60,6 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi
UIkit.modal($('#modal')).show();
}
UIkit.util.on(document, 'hidden', '#modal', () => {
$('#read-btn').off('click');
$('#unread-btn').off('click');
});
const updateProgress = (tid, eid, page) => {
let url = `${base_url}api/progress/${tid}/${page}`
const query = $.param({
@@ -95,6 +90,8 @@ const renameSubmit = (name, eid) => {
const upload = $('.upload-field');
const titleId = upload.attr('data-title-id');
console.log(name);
if (name.length === 0) {
alert('danger', 'The display name should not be empty');
return;
@@ -125,47 +122,15 @@ const renameSubmit = (name, eid) => {
});
};
const renameSortNameSubmit = (name, eid) => {
const upload = $('.upload-field');
const titleId = upload.attr('data-title-id');
const params = {};
if (eid) params.eid = eid;
if (name) params.name = name;
const query = $.param(params);
let url = `${base_url}api/admin/sort_title/${titleId}?${query}`;
$.ajax({
type: 'PUT',
url,
contentType: 'application/json',
dataType: 'json'
})
.done(data => {
if (data.error) {
alert('danger', `Failed to update sort title. Error: ${data.error}`);
return;
}
location.reload();
})
.fail((jqXHR, status) => {
alert('danger', `Failed to update sort title. Error: [${jqXHR.status}] ${jqXHR.statusText}`);
});
};
const edit = (eid) => {
const cover = $('#edit-modal #cover');
let url = cover.attr('data-title-cover');
let displayName = $('h2.uk-title > span').text();
let fileTitle = $('h2.uk-title').attr('data-file-title');
let sortTitle = $('h2.uk-title').attr('data-sort-title');
if (eid) {
const item = $(`#${eid}`);
url = item.find('img').attr('data-src');
displayName = item.find('.uk-card-title').attr('data-title');
fileTitle = item.find('.uk-card-title').attr('data-file-title');
sortTitle = item.find('.uk-card-title').attr('data-sort-title');
$('#title-progress-control').attr('hidden', '');
} else {
$('#title-progress-control').removeAttr('hidden');
@@ -175,26 +140,14 @@ const edit = (eid) => {
const displayNameField = $('#display-name-field');
displayNameField.attr('value', displayName);
displayNameField.attr('placeholder', fileTitle);
console.log(displayNameField);
displayNameField.keyup(event => {
if (event.keyCode === 13) {
renameSubmit(displayNameField.val() || fileTitle, eid);
renameSubmit(displayNameField.val(), eid);
}
});
displayNameField.siblings('a.uk-form-icon').click(() => {
renameSubmit(displayNameField.val() || fileTitle, eid);
});
const sortTitleField = $('#sort-title-field');
sortTitleField.val(sortTitle);
sortTitleField.attr('placeholder', fileTitle);
sortTitleField.keyup(event => {
if (event.keyCode === 13) {
renameSortNameSubmit(sortTitleField.val(), eid);
}
});
sortTitleField.siblings('a.uk-form-icon').click(() => {
renameSortNameSubmit(sortTitleField.val(), eid);
renameSubmit(displayNameField.val(), eid);
});
setupUpload(eid);
@@ -202,16 +155,6 @@ const edit = (eid) => {
UIkit.modal($('#edit-modal')).show();
};
UIkit.util.on(document, 'hidden', '#edit-modal', () => {
const displayNameField = $('#display-name-field');
displayNameField.off('keyup');
displayNameField.off('click');
const sortTitleField = $('#sort-title-field');
sortTitleField.off('keyup');
sortTitleField.off('click');
});
const setupUpload = (eid) => {
const upload = $('.upload-field');
const bar = $('#upload-progress').get(0);
@@ -223,6 +166,7 @@ const setupUpload = (eid) => {
queryObj['eid'] = eid;
const query = $.param(queryObj);
const url = `${base_url}api/admin/upload/cover?${query}`;
console.log(url);
UIkit.upload('.upload-field', {
url: url,
name: 'file',

View File

@@ -1,5 +1,5 @@
name: mango
version: 0.25.0
version: 0.23.0
authors:
- Alex Ling <hkalexling@gmail.com>

View File

@@ -61,13 +61,3 @@ describe "chapter_sort" do
end.should eq ary
end
end
describe "sanitize_filename" do
it "returns a random string for empty sanitized string" do
sanitize_filename("..").should_not eq sanitize_filename("..")
end
it "sanitizes correctly" do
sanitize_filename(".. \n\v.\rマンゴー/|*()<[1/2] 3.14 hello world ")
.should eq "マンゴー_()[1_2] 3.14 hello world"
end
end

View File

@@ -4,27 +4,38 @@ class Config
include YAML::Serializable
@[YAML::Field(ignore: true)]
property path = ""
property host = "0.0.0.0"
property path : String = ""
property host : String = "0.0.0.0"
property port : Int32 = 9000
property base_url = "/"
property session_secret = "mango-session-secret"
property library_path = "~/mango/library"
property library_cache_path = "~/mango/library.yml.gz"
property db_path = "~/mango/mango.db"
property queue_db_path = "~/mango/queue.db"
property base_url : String = "/"
property session_secret : String = "mango-session-secret"
property library_path : String = File.expand_path "~/mango/library",
home: true
property db_path : String = File.expand_path "~/mango/mango.db", home: true
property scan_interval_minutes : Int32 = 5
property thumbnail_generation_interval_hours : Int32 = 24
property log_level = "info"
property upload_path = "~/mango/uploads"
property plugin_path = "~/mango/plugins"
property log_level : String = "info"
property upload_path : String = File.expand_path "~/mango/uploads",
home: true
property plugin_path : String = File.expand_path "~/mango/plugins",
home: true
property download_timeout_seconds : Int32 = 30
property cache_enabled = true
property cache_size_mbs = 50
property cache_log_enabled = true
property disable_login = false
property default_username = ""
property auth_proxy_header_name = ""
property mangadex = Hash(String, String | Int32).new
@[YAML::Field(ignore: true)]
@mangadex_defaults = {
"base_url" => "https://mangadex.org",
"api_url" => "https://api.mangadex.org/v2",
"download_wait_seconds" => 5,
"download_retries" => 4,
"download_queue_db_path" => File.expand_path("~/mango/queue.db",
home: true),
"chapter_rename_rule" => "[Vol.{volume} ][Ch.{chapter} ]{title|id}",
"manga_rename_rule" => "{title}",
}
@@singlet : Config?
@@ -42,7 +53,7 @@ class Config
if File.exists? cfg_path
config = self.from_yaml File.read cfg_path
config.path = path
config.expand_paths
config.fill_defaults
config.preprocess
return config
end
@@ -50,7 +61,7 @@ class Config
"Dumping the default config there."
default = self.allocate
default.path = path
default.expand_paths
default.fill_defaults
cfg_dir = File.dirname cfg_path
unless Dir.exists? cfg_dir
Dir.mkdir_p cfg_dir
@@ -60,9 +71,13 @@ class Config
default
end
def expand_paths
{% for p in %w(library library_cache db queue_db upload plugin) %}
@{{p.id}}_path = File.expand_path @{{p.id}}_path, home: true
def fill_defaults
{% for hash_name in ["mangadex"] %}
@{{hash_name.id}}_defaults.map do |k, v|
if @{{hash_name.id}}[k]?.nil?
@{{hash_name.id}}[k] = v
end
end
{% end %}
end
@@ -77,5 +92,24 @@ class Config
raise "Login is disabled, but default username is not set. " \
"Please set a default username"
end
# `Logger.default` is not available yet
Log.setup :debug
unless mangadex["api_url"] =~ /\/v2/
Log.warn { "It looks like you are using the deprecated MangaDex API " \
"v1 in your config file. Please update it to " \
"https://api.mangadex.org/v2 to suppress this warning." }
mangadex["api_url"] = "https://api.mangadex.org/v2"
end
if mangadex["api_url"] =~ /\/api\/v2/
Log.warn { "It looks like you are using the outdated MangaDex API " \
"url (mangadex.org/api/v2) in your config file. Please " \
"update it to https://api.mangadex.org/v2 to suppress this " \
"warning." }
mangadex["api_url"] = "https://api.mangadex.org/v2"
end
mangadex["api_url"] = mangadex["api_url"].to_s.rstrip "/"
mangadex["base_url"] = mangadex["base_url"].to_s.rstrip "/"
end
end

View File

@@ -54,9 +54,8 @@ class AuthHandler < Kemal::Handler
end
def call(env)
# Skip all authentication if requesting /login, /logout, /api/login,
# or a static file
if request_path_startswith(env, ["/login", "/logout", "/api/login"]) ||
# Skip all authentication if requesting /login, /logout, or a static file
if request_path_startswith(env, ["/login", "/logout"]) ||
requesting_static_file env
return call_next(env)
end

View File

@@ -1,218 +0,0 @@
require "digest"
require "./entry"
require "./title"
require "./types"
# Base class for an entry in the LRU cache.
# There are two ways to use it:
# 1. Use it as it is by instantiating with the appropriate `SaveT` and
# `ReturnT`. Note that in this case, `SaveT` and `ReturnT` must be the
# same type. That is, the input value will be stored as it is without
# any transformation.
# 2. You can also subclass it and provide custom implementations for
# `to_save_t` and `to_return_t`. This allows you to transform and store
# the input value to a different type. See `SortedEntriesCacheEntry` as
# an example.
private class CacheEntry(SaveT, ReturnT)
getter key : String, atime : Time
@value : SaveT
def initialize(@key : String, value : ReturnT)
@atime = @ctime = Time.utc
@value = self.class.to_save_t value
end
def value
@atime = Time.utc
self.class.to_return_t @value
end
def self.to_save_t(value : ReturnT)
value
end
def self.to_return_t(value : SaveT)
value
end
def instance_size
instance_sizeof(CacheEntry(SaveT, ReturnT)) + # sizeof itself
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
@value.instance_size
end
end
class SortedEntriesCacheEntry < CacheEntry(Array(String), Array(Entry))
def self.to_save_t(value : Array(Entry))
value.map &.id
end
def self.to_return_t(value : Array(String))
ids_to_entries value
end
private def self.ids_to_entries(ids : Array(String))
e_map = Library.default.deep_entries.to_h { |entry| {entry.id, entry} }
entries = [] of Entry
begin
ids.each do |id|
entries << e_map[id]
end
return entries if ids.size == entries.size
rescue
end
end
def instance_size
instance_sizeof(SortedEntriesCacheEntry) + # sizeof itself
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
@value.size * (instance_sizeof(String) + sizeof(String)) +
@value.sum(&.bytesize) # elements in Array(String)
end
def self.gen_key(book_id : String, username : String,
entries : Array(Entry), opt : SortOptions?)
entries_sig = Digest::SHA1.hexdigest (entries.map &.id).to_s
user_context = opt && opt.method == SortMethod::Progress ? username : ""
sig = Digest::SHA1.hexdigest (book_id + entries_sig + user_context +
(opt ? opt.to_tuple.to_s : "nil"))
"#{sig}:sorted_entries"
end
end
class SortedTitlesCacheEntry < CacheEntry(Array(String), Array(Title))
def self.to_save_t(value : Array(Title))
value.map &.id
end
def self.to_return_t(value : Array(String))
value.map { |title_id| Library.default.title_hash[title_id].not_nil! }
end
def instance_size
instance_sizeof(SortedTitlesCacheEntry) + # sizeof itself
instance_sizeof(String) + @key.bytesize + # allocated memory for @key
@value.size * (instance_sizeof(String) + sizeof(String)) +
@value.sum(&.bytesize) # elements in Array(String)
end
def self.gen_key(username : String, titles : Array(Title), opt : SortOptions?)
titles_sig = Digest::SHA1.hexdigest (titles.map &.id).to_s
user_context = opt && opt.method == SortMethod::Progress ? username : ""
sig = Digest::SHA1.hexdigest (titles_sig + user_context +
(opt ? opt.to_tuple.to_s : "nil"))
"#{sig}:sorted_titles"
end
end
class String
def instance_size
instance_sizeof(String) + bytesize
end
end
struct Tuple(*T)
def instance_size
sizeof(T) + # total size of non-reference types
self.sum do |e|
next 0 unless e.is_a? Reference
if e.responds_to? :instance_size
e.instance_size
else
instance_sizeof(typeof(e))
end
end
end
end
alias CacheableType = Array(Entry) | Array(Title) | String |
Tuple(String, Int32)
alias CacheEntryType = SortedEntriesCacheEntry |
SortedTitlesCacheEntry |
CacheEntry(String, String) |
CacheEntry(Tuple(String, Int32), Tuple(String, Int32))
def generate_cache_entry(key : String, value : CacheableType)
if value.is_a? Array(Entry)
SortedEntriesCacheEntry.new key, value
elsif value.is_a? Array(Title)
SortedTitlesCacheEntry.new key, value
else
CacheEntry(typeof(value), typeof(value)).new key, value
end
end
# LRU Cache
class LRUCache
@@limit : Int128 = Int128.new 0
@@should_log = true
# key => entry
@@cache = {} of String => CacheEntryType
def self.enabled
Config.current.cache_enabled
end
def self.init
cache_size = Config.current.cache_size_mbs
@@limit = Int128.new cache_size * 1024 * 1024 if enabled
@@should_log = Config.current.cache_log_enabled
end
def self.get(key : String)
return unless enabled
entry = @@cache[key]?
if @@should_log
Logger.debug "LRUCache #{entry.nil? ? "miss" : "hit"} #{key}"
end
return entry.value unless entry.nil?
end
def self.set(cache_entry : CacheEntryType)
return unless enabled
key = cache_entry.key
@@cache[key] = cache_entry
Logger.debug "LRUCache cached #{key}" if @@should_log
remove_least_recent_access
end
def self.invalidate(key : String)
return unless enabled
@@cache.delete key
end
def self.print
return unless @@should_log
sum = @@cache.sum { |_, entry| entry.instance_size }
Logger.debug "---- LRU Cache ----"
Logger.debug "Size: #{sum} Bytes"
Logger.debug "List:"
@@cache.each do |k, v|
Logger.debug "#{k} | #{v.atime} | #{v.instance_size}"
end
Logger.debug "-------------------"
end
private def self.is_cache_full
sum = @@cache.sum { |_, entry| entry.instance_size }
sum > @@limit
end
private def self.remove_least_recent_access
if @@should_log && is_cache_full
Logger.debug "Removing entries from LRUCache"
end
while is_cache_full && @@cache.size > 0
min_tuple = @@cache.min_by { |_, entry| entry.atime }
min_key = min_tuple[0]
min_entry = min_tuple[1]
Logger.debug " \
Target: #{min_key}, \
Last Access Time: #{min_entry.atime}" if @@should_log
invalidate min_key
end
end
end

View File

@@ -1,16 +1,10 @@
require "image_size"
require "yaml"
class Entry
include YAML::Serializable
getter zip_path : String, book : Title, title : String,
size : String, pages : Int32, id : String, encoded_path : String,
encoded_title : String, mtime : Time, err_msg : String?
@[YAML::Field(ignore: true)]
@sort_title : String?
def initialize(@zip_path, @book)
storage = Storage.default
@encoded_path = URI.encode @zip_path
@@ -52,51 +46,29 @@ class Entry
file.close
end
def build_json(*, slim = false)
def to_slim_json : String
JSON.build do |json|
json.object do
{% for str in ["zip_path", "title", "size", "id"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "title_id", @book.id
json.field "sort_title", sort_title
json.field "pages" { json.number @pages }
unless slim
json.field "display_name", @book.display_name @title
json.field "cover_url", cover_url
json.field "mtime" { json.number @mtime.to_unix }
end
end
end
end
def sort_title
sort_title_cached = @sort_title
return sort_title_cached if sort_title_cached
sort_title = @book.entry_sort_title_db id
if sort_title
@sort_title = sort_title
return sort_title
def to_json(json : JSON::Builder)
json.object do
{% for str in ["zip_path", "title", "size", "id"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "title_id", @book.id
json.field "display_name", @book.display_name @title
json.field "cover_url", cover_url
json.field "pages" { json.number @pages }
json.field "mtime" { json.number @mtime.to_unix }
end
@sort_title = @title
@title
end
def set_sort_title(sort_title : String | Nil, username : String)
Storage.default.set_entry_sort_title id, sort_title
if sort_title == "" || sort_title.nil?
@sort_title = nil
else
@sort_title = sort_title
end
@book.entry_sort_title_cache = nil
@book.remove_sorted_entries_cache [SortMethod::Auto, SortMethod::Title],
username
end
def sort_title_db
@book.entry_sort_title_db @id
end
def display_name
@@ -109,17 +81,9 @@ class Entry
def cover_url
return "#{Config.current.base_url}img/icon.png" if @err_msg
unless @book.entry_cover_url_cache
TitleInfo.new @book.dir do |info|
@book.entry_cover_url_cache = info.entry_cover_url
end
end
entry_cover_url = @book.entry_cover_url_cache
url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}"
if entry_cover_url
info_url = entry_cover_url[@title]?
TitleInfo.new @book.dir do |info|
info_url = info.entry_cover_url[@title]?
unless info_url.nil? || info_url.empty?
url = File.join Config.current.base_url, info_url
end
@@ -206,12 +170,6 @@ class Entry
# For backward backward compatibility with v0.1.0, we save entry titles
# instead of IDs in info.json
def save_progress(username, page)
LRUCache.invalidate "#{@book.id}:#{username}:progress_sum"
@book.parents.each do |parent|
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
end
@book.remove_sorted_caches [SortMethod::Progress], username
TitleInfo.new @book.dir do |info|
if info.progress[username]?.nil?
info.progress[username] = {@title => page}

View File

@@ -1,94 +1,20 @@
class Library
struct ThumbnailContext
property current : Int32, total : Int32
def initialize
@current = 0
@total = 0
end
def progress
if total == 0
0
else
current / total
end
end
def reset
@current = 0
@total = 0
end
def increment
@current += 1
end
end
include YAML::Serializable
getter dir : String, title_ids : Array(String),
title_hash : Hash(String, Title)
@[YAML::Field(ignore: true)]
getter thumbnail_ctx = ThumbnailContext.new
use_default
def save_instance
path = Config.current.library_cache_path
Logger.debug "Caching library to #{path}"
writer = Compress::Gzip::Writer.new path,
Compress::Gzip::BEST_COMPRESSION
writer.write self.to_yaml.to_slice
writer.close
end
def self.load_instance
path = Config.current.library_cache_path
return unless File.exists? path
Logger.debug "Loading cached library from #{path}"
begin
Compress::Gzip::Reader.open path do |content|
loaded = Library.from_yaml content
# We will have to do a full restart in these cases. Otherwise having
# two instances of the library will cause some weirdness.
if loaded.dir != Config.current.library_path
Logger.fatal "Cached library dir #{loaded.dir} does not match " \
"current library dir #{Config.current.library_path}. " \
"Deleting cache"
delete_cache_and_exit path
end
if loaded.title_ids.size > 0 &&
Storage.default.count_titles == 0
Logger.fatal "The library cache is inconsistent with the DB. " \
"Deleting cache"
delete_cache_and_exit path
end
@@default = loaded
Logger.debug "Library cache loaded"
end
Library.default.register_jobs
rescue e
Logger.error e
end
end
def initialize
register_mime_types
@dir = Config.current.library_path
# explicitly initialize @titles to bypass the compiler check. it will
# be filled with actual Titles in the `scan` call below
@title_ids = [] of String
@title_hash = {} of String => Title
register_jobs
end
protected def register_jobs
register_mime_types
@entries_count = 0
@thumbnails_count = 0
scan_interval = Config.current.scan_interval_minutes
if scan_interval < 1
@@ -99,7 +25,7 @@ class Library
start = Time.local
scan
ms = (Time.local - start).total_milliseconds
Logger.debug "Library initialized in #{ms}ms"
Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
sleep scan_interval.minutes
end
end
@@ -125,6 +51,11 @@ class Library
def sorted_titles(username, opt : SortOptions? = nil)
if opt.nil?
opt = SortOptions.from_info_json @dir, username
else
TitleInfo.new @dir do |info|
info.sort_by[username] = opt.to_tuple
info.save
end
end
# Helper function from src/util/util.cr
@@ -135,18 +66,14 @@ class Library
titles + titles.flat_map &.deep_titles
end
def deep_entries
titles.flat_map &.deep_entries
end
def build_json(*, slim = false, depth = -1)
def to_slim_json : String
JSON.build do |json|
json.object do
json.field "dir", @dir
json.field "titles" do
json.array do
self.titles.each do |title|
json.raw title.build_json(slim: slim, depth: depth)
json.raw title.to_slim_json
end
end
end
@@ -154,6 +81,15 @@ class Library
end
end
def to_json(json : JSON::Builder)
json.object do
json.field "dir", @dir
json.field "titles" do
json.raw self.titles.to_json
end
end
end
def get_title(tid)
@title_hash[tid]?
end
@@ -163,7 +99,6 @@ class Library
end
def scan
start = Time.local
unless Dir.exists? @dir
Logger.info "The library directory #{@dir} does not exist. " \
"Attempting to create it"
@@ -172,38 +107,14 @@ class Library
storage = Storage.new auto_close: false
examine_context : ExamineContext = {
cached_contents_signature: {} of String => String,
deleted_title_ids: [] of String,
deleted_entry_ids: [] of String,
}
library_paths = (Dir.entries @dir)
(Dir.entries @dir)
.select { |fn| !fn.starts_with? "." }
.map { |fn| File.join @dir, fn }
@title_ids.select! do |title_id|
title = @title_hash[title_id]
next false unless library_paths.includes? title.dir
existence = title.examine examine_context
unless existence
examine_context["deleted_title_ids"].concat [title_id] +
title.deep_titles.map &.id
examine_context["deleted_entry_ids"].concat title.deep_entries.map &.id
end
existence
end
remained_title_dirs = @title_ids.map { |id| title_hash[id].dir }
examine_context["deleted_title_ids"].each do |title_id|
@title_hash.delete title_id
end
cache = examine_context["cached_contents_signature"]
library_paths
.select { |path| !(remained_title_dirs.includes? path) }
.select { |path| File.directory? path }
.map { |path| Title.new path, "", cache }
.map { |path| Title.new path, "" }
.select { |title| !(title.entries.empty? && title.titles.empty?) }
.sort! { |a, b| a.sort_title <=> b.sort_title }
.sort! { |a, b| a.title <=> b.title }
.tap { |_| @title_ids.clear }
.each do |title|
@title_hash[title.id] = title
@title_ids << title.id
@@ -212,15 +123,8 @@ class Library
storage.bulk_insert_ids
storage.close
ms = (Time.local - start).total_milliseconds
Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
Storage.default.mark_unavailable examine_context["deleted_entry_ids"],
examine_context["deleted_title_ids"]
spawn do
save_instance
end
Logger.debug "Scan completed"
Storage.default.mark_unavailable
end
def get_continue_reading_entries(username)
@@ -304,29 +208,34 @@ class Library
.shuffle!
end
def thumbnail_generation_progress
return 0 if @entries_count == 0
@thumbnails_count / @entries_count
end
def generate_thumbnails
if thumbnail_ctx.current > 0
if @thumbnails_count > 0
Logger.debug "Thumbnail generation in progress"
return
end
Logger.info "Starting thumbnail generation"
entries = deep_titles.flat_map(&.deep_entries).reject &.err_msg
thumbnail_ctx.total = entries.size
thumbnail_ctx.current = 0
@entries_count = entries.size
@thumbnails_count = 0
# Report generation progress regularly
spawn do
loop do
unless thumbnail_ctx.current == 0
unless @thumbnails_count == 0
Logger.debug "Thumbnail generation progress: " \
"#{(thumbnail_ctx.progress * 100).round 1}%"
"#{(thumbnail_generation_progress * 100).round 1}%"
end
# Generation is completed. We reset the count to 0 to allow subsequent
# calls to the function, and break from the loop to stop the progress
# report fiber
if thumbnail_ctx.progress.to_i == 1
thumbnail_ctx.reset
if thumbnail_generation_progress.to_i == 1
@thumbnails_count = 0
break
end
sleep 10.seconds
@@ -340,7 +249,7 @@ class Library
# and CPU
sleep 1.seconds
end
thumbnail_ctx.increment
@thumbnails_count += 1
end
Logger.info "Thumbnail generation finished"
end

View File

@@ -1,30 +1,13 @@
require "digest"
require "../archive"
class Title
include YAML::Serializable
getter dir : String, parent_id : String, title_ids : Array(String),
entries : Array(Entry), title : String, id : String,
encoded_title : String, mtime : Time, signature : UInt64,
entry_cover_url_cache : Hash(String, String)?
setter entry_cover_url_cache : Hash(String, String)?,
entry_sort_title_cache : Hash(String, String | Nil)?
encoded_title : String, mtime : Time, signature : UInt64
@[YAML::Field(ignore: true)]
@sort_title : String?
@[YAML::Field(ignore: true)]
@entry_sort_title_cache : Hash(String, String | Nil)?
@[YAML::Field(ignore: true)]
@entry_display_name_cache : Hash(String, String)?
@[YAML::Field(ignore: true)]
@entry_cover_url_cache : Hash(String, String)?
@[YAML::Field(ignore: true)]
@cached_display_name : String?
@[YAML::Field(ignore: true)]
@cached_cover_url : String?
def initialize(@dir : String, @parent_id, cache = {} of String => String)
def initialize(@dir : String, @parent_id)
storage = Storage.default
@signature = Dir.signature dir
id = storage.get_title_id dir, signature
@@ -37,7 +20,6 @@ class Title
})
end
@id = id
@contents_signature = Dir.contents_signature dir, cache
@title = File.basename dir
@encoded_title = URI.encode @title
@title_ids = [] of String
@@ -48,7 +30,7 @@ class Title
next if fn.starts_with? "."
path = File.join dir, fn
if File.directory? path
title = Title.new path, @id, cache
title = Title.new path, @id
next if title.entries.size == 0 && title.titles.size == 0
Library.default.title_hash[title.id] = title
@title_ids << title.id
@@ -71,170 +53,28 @@ class Title
end
sorter = ChapterSorter.new @entries.map &.title
@entries.sort! do |a, b|
sorter.compare a.sort_title, b.sort_title
sorter.compare a.title, b.title
end
end
# Utility method used in library rescanning.
# - When the title does not exist on the file system anymore, return false
# and let it be deleted from the library instance
# - When the title exists, but its contents signature is now different from
# the cache, it means some of its content (nested titles or entries)
# has been added, deleted, or renamed. In this case we update its
# contents signature and instance variables
# - When the title exists and its contents signature is still the same, we
# return true so it can be reused without rescanning
def examine(context : ExamineContext) : Bool
return false unless Dir.exists? @dir
contents_signature = Dir.contents_signature @dir,
context["cached_contents_signature"]
return true if @contents_signature == contents_signature
@contents_signature = contents_signature
@signature = Dir.signature @dir
storage = Storage.default
id = storage.get_title_id dir, signature
if id.nil?
id = random_str
storage.insert_title_id({
path: dir,
id: id,
signature: signature.to_s,
})
end
@id = id
@mtime = File.info(@dir).modification_time
previous_titles_size = @title_ids.size
@title_ids.select! do |title_id|
title = Library.default.get_title title_id
unless title # for if data consistency broken
context["deleted_title_ids"].concat [title_id]
next false
end
existence = title.examine context
unless existence
context["deleted_title_ids"].concat [title_id] +
title.deep_titles.map &.id
context["deleted_entry_ids"].concat title.deep_entries.map &.id
end
existence
end
remained_title_dirs = @title_ids.map do |title_id|
title = Library.default.get_title! title_id
title.dir
end
previous_entries_size = @entries.size
@entries.select! do |entry|
existence = File.exists? entry.zip_path
Fiber.yield
context["deleted_entry_ids"] << entry.id unless existence
existence
end
remained_entry_zip_paths = @entries.map &.zip_path
is_titles_added = false
is_entries_added = false
Dir.entries(dir).each do |fn|
next if fn.starts_with? "."
path = File.join dir, fn
if File.directory? path
next if remained_title_dirs.includes? path
title = Title.new path, @id, context["cached_contents_signature"]
next if title.entries.size == 0 && title.titles.size == 0
Library.default.title_hash[title.id] = title
@title_ids << title.id
is_titles_added = true
# We think they are removed, but they are here!
# Cancel reserved jobs
revival_title_ids = [title.id] + title.deep_titles.map &.id
context["deleted_title_ids"].select! do |deleted_title_id|
!(revival_title_ids.includes? deleted_title_id)
end
revival_entry_ids = title.deep_entries.map &.id
context["deleted_entry_ids"].select! do |deleted_entry_id|
!(revival_entry_ids.includes? deleted_entry_id)
end
next
end
if is_supported_file path
next if remained_entry_zip_paths.includes? path
entry = Entry.new path, self
if entry.pages > 0 || entry.err_msg
@entries << entry
is_entries_added = true
context["deleted_entry_ids"].select! do |deleted_entry_id|
entry.id != deleted_entry_id
end
end
end
end
mtimes = [@mtime]
mtimes += @title_ids.map { |e| Library.default.title_hash[e].mtime }
mtimes += @entries.map &.mtime
@mtime = mtimes.max
if is_titles_added || previous_titles_size != @title_ids.size
@title_ids.sort! do |a, b|
compare_numerically Library.default.title_hash[a].title,
Library.default.title_hash[b].title
end
end
if is_entries_added || previous_entries_size != @entries.size
sorter = ChapterSorter.new @entries.map &.sort_title
@entries.sort! do |a, b|
sorter.compare a.sort_title, b.sort_title
end
end
if @title_ids.size > 0 || @entries.size > 0
true
else
context["deleted_title_ids"].concat [@id]
false
end
end
alias SortContext = NamedTuple(username: String, opt: SortOptions)
def build_json(*, slim = false, depth = -1,
sort_context : SortContext? = nil)
def to_slim_json : String
JSON.build do |json|
json.object do
{% for str in ["dir", "title", "id"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "signature" { json.number @signature }
json.field "sort_title", sort_title
unless slim
json.field "display_name", display_name
json.field "cover_url", cover_url
json.field "mtime" { json.number @mtime.to_unix }
end
unless depth == 0
json.field "titles" do
json.array do
self.titles.each do |title|
json.raw title.build_json(slim: slim,
depth: depth > 0 ? depth - 1 : depth)
end
json.field "titles" do
json.array do
self.titles.each do |title|
json.raw title.to_slim_json
end
end
json.field "entries" do
json.array do
_entries = if sort_context
sorted_entries sort_context[:username],
sort_context[:opt]
else
@entries
end
_entries.each do |entry|
json.raw entry.build_json(slim: slim)
end
end
json.field "entries" do
json.array do
@entries.each do |entry|
json.raw entry.to_slim_json
end
end
end
@@ -252,17 +92,36 @@ class Title
end
end
def titles
@title_ids.map { |tid| Library.default.get_title! tid }
def to_json(json : JSON::Builder)
json.object do
{% for str in ["dir", "title", "id"] %}
json.field {{str}}, @{{str.id}}
{% end %}
json.field "signature" { json.number @signature }
json.field "display_name", display_name
json.field "cover_url", cover_url
json.field "mtime" { json.number @mtime.to_unix }
json.field "titles" do
json.raw self.titles.to_json
end
json.field "entries" do
json.raw @entries.to_json
end
json.field "parents" do
json.array do
self.parents.each do |title|
json.object do
json.field "title", title.title
json.field "id", title.id
end
end
end
end
end
end
def sorted_titles(username, opt : SortOptions? = nil)
if opt.nil?
opt = SortOptions.from_info_json @dir, username
end
# Helper function from src/util/util.cr
sort_titles titles, opt.not_nil!, username
def titles
@title_ids.map { |tid| Library.default.get_title! tid }
end
# Get all entries, including entries in nested titles
@@ -301,48 +160,6 @@ class Title
ary.join " and "
end
def sort_title
sort_title_cached = @sort_title
return sort_title_cached if sort_title_cached
sort_title = Storage.default.get_title_sort_title id
if sort_title
@sort_title = sort_title
return sort_title
end
@sort_title = @title
@title
end
def set_sort_title(sort_title : String | Nil, username : String)
Storage.default.set_title_sort_title id, sort_title
if sort_title == "" || sort_title.nil?
@sort_title = nil
else
@sort_title = sort_title
end
if parents.size > 0
target = parents[-1].titles
else
target = Library.default.titles
end
remove_sorted_titles_cache target,
[SortMethod::Auto, SortMethod::Title], username
end
def sort_title_db
Storage.default.get_title_sort_title id
end
def entry_sort_title_db(entry_id)
unless @entry_sort_title_cache
@entry_sort_title_cache =
Storage.default.get_entries_sort_title @entries.map &.id
end
@entry_sort_title_cache.not_nil![entry_id]?
end
def tags
Storage.default.get_title_tags @id
end
@@ -360,15 +177,11 @@ class Title
end
def display_name
cached_display_name = @cached_display_name
return cached_display_name unless cached_display_name.nil?
dn = @title
TitleInfo.new @dir do |info|
info_dn = info.display_name
dn = info_dn unless info_dn.empty?
end
@cached_display_name = dn
dn
end
@@ -392,7 +205,6 @@ class Title
end
def set_display_name(dn)
@cached_display_name = dn
TitleInfo.new @dir do |info|
info.display_name = dn
info.save
@@ -402,15 +214,11 @@ class Title
def set_display_name(entry_name : String, dn)
TitleInfo.new @dir do |info|
info.entry_display_name[entry_name] = dn
@entry_display_name_cache = info.entry_display_name
info.save
end
end
def cover_url
cached_cover_url = @cached_cover_url
return cached_cover_url unless cached_cover_url.nil?
url = "#{Config.current.base_url}img/icon.png"
readable_entries = @entries.select &.err_msg.nil?
if readable_entries.size > 0
@@ -422,12 +230,10 @@ class Title
url = File.join Config.current.base_url, info_url
end
end
@cached_cover_url = url
url
end
def set_cover_url(url : String)
@cached_cover_url = url
TitleInfo.new @dir do |info|
info.cover_url = url
info.save
@@ -437,7 +243,6 @@ class Title
def set_cover_url(entry_name : String, url : String)
TitleInfo.new @dir do |info|
info.entry_cover_url[entry_name] = url
@entry_cover_url_cache = info.entry_cover_url
info.save
end
end
@@ -457,15 +262,8 @@ class Title
end
def deep_read_page_count(username) : Int32
key = "#{@id}:#{username}:progress_sum"
sig = Digest::SHA1.hexdigest (entries.map &.id).to_s
cached_sum = LRUCache.get key
return cached_sum[1] if cached_sum.is_a? Tuple(String, Int32) &&
cached_sum[0] == sig
sum = load_progress_for_all_entries(username, nil, true).sum +
titles.flat_map(&.deep_read_page_count username).sum
LRUCache.set generate_cache_entry key, {sig, sum}
sum
load_progress_for_all_entries(username).sum +
titles.flat_map(&.deep_read_page_count username).sum
end
def deep_total_page_count : Int32
@@ -519,46 +317,44 @@ class Title
# use the default (auto, ascending)
# When `opt` is not nil, it saves the options to info.json
def sorted_entries(username, opt : SortOptions? = nil)
cache_key = SortedEntriesCacheEntry.gen_key @id, username, @entries, opt
cached_entries = LRUCache.get cache_key
return cached_entries if cached_entries.is_a? Array(Entry)
if opt.nil?
opt = SortOptions.from_info_json @dir, username
else
TitleInfo.new @dir do |info|
info.sort_by[username] = opt.to_tuple
info.save
end
end
case opt.not_nil!.method
when .title?
ary = @entries.sort do |a, b|
compare_numerically a.sort_title, b.sort_title
end
ary = @entries.sort { |a, b| compare_numerically a.title, b.title }
when .time_modified?
ary = @entries.sort { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.sort_title, b.sort_title }
compare_numerically a.title, b.title }
when .time_added?
ary = @entries.sort { |a, b| (a.date_added <=> b.date_added).or \
compare_numerically a.sort_title, b.sort_title }
compare_numerically a.title, b.title }
when .progress?
percentage_ary = load_percentage_for_all_entries username, opt, true
ary = @entries.zip(percentage_ary)
.sort { |a_tp, b_tp| (a_tp[1] <=> b_tp[1]).or \
compare_numerically a_tp[0].sort_title, b_tp[0].sort_title }
compare_numerically a_tp[0].title, b_tp[0].title }
.map &.[0]
else
unless opt.method.auto?
Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \
"Auto instead"
end
sorter = ChapterSorter.new @entries.map &.sort_title
sorter = ChapterSorter.new @entries.map &.title
ary = @entries.sort do |a, b|
sorter.compare(a.sort_title, b.sort_title).or \
compare_numerically a.sort_title, b.sort_title
sorter.compare(a.title, b.title).or \
compare_numerically a.title, b.title
end
end
ary.reverse! unless opt.not_nil!.ascend
LRUCache.set generate_cache_entry cache_key, ary
ary
end
@@ -619,33 +415,7 @@ class Title
zip + titles.flat_map &.deep_entries_with_date_added
end
def remove_sorted_entries_cache(sort_methods : Array(SortMethod),
username : String)
[false, true].each do |ascend|
sort_methods.each do |sort_method|
sorted_entries_cache_key =
SortedEntriesCacheEntry.gen_key @id, username, @entries,
SortOptions.new(sort_method, ascend)
LRUCache.invalidate sorted_entries_cache_key
end
end
end
def remove_sorted_caches(sort_methods : Array(SortMethod), username : String)
remove_sorted_entries_cache sort_methods, username
parents.each do |parent|
remove_sorted_titles_cache parent.titles, sort_methods, username
end
remove_sorted_titles_cache Library.default.titles, sort_methods, username
end
def bulk_progress(action, ids : Array(String), username)
LRUCache.invalidate "#{@id}:#{username}:progress_sum"
parents.each do |parent|
LRUCache.invalidate "#{parent.id}:#{username}:progress_sum"
end
remove_sorted_caches [SortMethod::Progress], username
selected_entries = ids
.map { |id|
@entries.find &.id.==(id)

View File

@@ -1,12 +1,4 @@
SUPPORTED_IMG_TYPES = %w(
image/jpeg
image/png
image/webp
image/apng
image/avif
image/gif
image/svg+xml
)
SUPPORTED_IMG_TYPES = ["image/jpeg", "image/png", "image/webp"]
enum SortMethod
Auto
@@ -96,18 +88,6 @@ class TitleInfo
@@mutex_hash = {} of String => Mutex
def self.new(dir, &)
key = "#{dir}:info.json"
info = LRUCache.get key
if info.is_a? String
begin
instance = TitleInfo.from_json info
instance.dir = dir
yield instance
return
rescue
end
end
if @@mutex_hash[dir]?
mutex = @@mutex_hash[dir]
else
@@ -121,7 +101,6 @@ class TitleInfo
instance = TitleInfo.from_json File.read json_path
end
instance.dir = dir
LRUCache.set generate_cache_entry key, instance.to_json
yield instance
end
end
@@ -129,12 +108,5 @@ class TitleInfo
def save
json_path = File.join @dir, "info.json"
File.write json_path, self.to_pretty_json
key = "#{@dir}:info.json"
LRUCache.set generate_cache_entry key, self.to_json
end
end
alias ExamineContext = NamedTuple(
cached_contents_signature: Hash(String, String),
deleted_title_ids: Array(String),
deleted_entry_ids: Array(String))

View File

@@ -7,7 +7,7 @@ require "option_parser"
require "clim"
require "tallboy"
MANGO_VERSION = "0.25.0"
MANGO_VERSION = "0.23.0"
# From http://www.network-science.de/ascii/
BANNER = %{
@@ -55,10 +55,8 @@ class CLI < Clim
Config.load(opts.config).set_current
# Initialize main components
LRUCache.init
Storage.default
Queue.default
Library.load_instance
Library.default
Plugin::Downloader.default

View File

@@ -23,6 +23,11 @@ class Plugin
job
end
private def process_filename(str)
return "_" if str == ".."
str.gsub "/", "_"
end
private def download(job : Queue::Job)
@downloading = true
@queue.set_status Queue::JobStatus::Downloading, job
@@ -37,8 +42,8 @@ class Plugin
pages = info["pages"].as_i
manga_title = sanitize_filename job.manga_title
chapter_title = sanitize_filename info["title"].as_s
manga_title = process_filename job.manga_title
chapter_title = process_filename info["title"].as_s
@queue.set_pages pages, job
lib_dir = @library_path
@@ -63,7 +68,7 @@ class Plugin
while page = plugin.next_page
break unless @queue.exists? job
fn = sanitize_filename page["filename"].as_s
fn = process_filename page["filename"].as_s
url = page["url"].as_s
headers = HTTP::Headers.new

View File

@@ -112,7 +112,7 @@ class Queue
use_default
def initialize(db_path : String? = nil)
@path = db_path || Config.current.queue_db_path.to_s
@path = db_path || Config.current.mangadex["download_queue_db_path"].to_s
dir = File.dirname @path
unless Dir.exists? dir
Logger.info "The queue DB directory #{dir} does not exist. " \

View File

@@ -66,6 +66,7 @@ struct AdminRouter
end
get "/admin/downloads" do |env|
mangadex_base_url = Config.current.mangadex["base_url"]
layout "download-manager"
end

View File

@@ -23,7 +23,7 @@ struct APIRouter
# Authentication
All endpoints except `/api/login` require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access.
All endpoints require authentication. After logging in, your session ID would be stored as a cookie named `mango-sessid-#{Config.current.port}`, which can be used to authenticate the API access. Note that all admin API endpoints (`/api/admin/...`) require the logged-in user to have admin access.
# Terminologies
@@ -56,29 +56,6 @@ struct APIRouter
"error" => String?,
}
Koa.describe "Authenticates a user", <<-MD
After successful login, the cookie `mango-sessid-#{Config.current.port}` will contain a valid session ID that can be used for subsequent requests
MD
Koa.body schema: {
"username" => String,
"password" => String,
}
Koa.tag "users"
post "/api/login" do |env|
begin
username = env.params.json["username"].as String
password = env.params.json["password"].as String
token = Storage.default.verify_user(username, password).not_nil!
env.session.string "token", token
"Authenticated"
rescue e
Logger.error e
env.response.status_code = 403
e.message
end
end
Koa.describe "Returns a page in a manga entry"
Koa.path "tid", desc: "Title ID"
Koa.path "eid", desc: "Entry ID"
@@ -156,38 +133,24 @@ struct APIRouter
end
Koa.describe "Returns the book with title `tid`", <<-MD
- Supply the `slim` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time
- Supply the `depth` query parameter to control the depth of nested titles to return.
- When `depth` is 1, returns the top-level titles and sub-titles/entries one level in them
- When `depth` is 0, returns the top-level titles without their sub-titles/entries
- When `depth` is N, returns the top-level titles and sub-titles/entries N levels in them
- When `depth` is negative, returns the entire library
Supply the `tid` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time
MD
Koa.path "tid", desc: "Title ID"
Koa.query "slim"
Koa.query "depth"
Koa.query "sort", desc: "Sorting option for entries. Can be one of 'auto', 'title', 'progress', 'time_added' and 'time_modified'"
Koa.query "ascend", desc: "Sorting direction for entries. Set to 0 for the descending order. Doesn't work without specifying 'sort'"
Koa.response 200, schema: "title"
Koa.response 404, "Title not found"
Koa.tag "library"
get "/api/book/:tid" do |env|
begin
username = get_username env
sort_opt = SortOptions.new
get_sort_opt
tid = env.params.url["tid"]
title = Library.default.get_title tid
raise "Title ID `#{tid}` not found" if title.nil?
slim = !env.params.query["slim"]?.nil?
depth = env.params.query["depth"]?.try(&.to_i?) || -1
send_json env, title.build_json(slim: slim, depth: depth,
sort_context: {username: username,
opt: sort_opt})
if env.params.query["slim"]?
send_json env, title.to_slim_json
else
send_json env, title.to_json
end
rescue e
Logger.error e
env.response.status_code = 404
@@ -196,25 +159,20 @@ struct APIRouter
end
Koa.describe "Returns the entire library with all titles and entries", <<-MD
- Supply the `slim` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time
- Supply the `dpeth` query parameter to control the depth of nested titles to return.
- When `depth` is 1, returns the requested title and sub-titles/entries one level in it
- When `depth` is 0, returns the requested title without its sub-titles/entries
- When `depth` is N, returns the requested title and sub-titles/entries N levels in it
- When `depth` is negative, returns the requested title and all sub-titles/entries in it
Supply the `tid` query parameter to strip away "display_name", "cover_url", and "mtime" from the returned object to speed up the loading time
MD
Koa.query "slim"
Koa.query "depth"
Koa.response 200, schema: {
"dir" => String,
"titles" => ["title"],
}
Koa.tag "library"
get "/api/library" do |env|
slim = !env.params.query["slim"]?.nil?
depth = env.params.query["depth"]?.try(&.to_i?) || -1
send_json env, Library.default.build_json(slim: slim, depth: depth)
if env.params.query["slim"]?
send_json env, Library.default.to_slim_json
else
send_json env, Library.default.to_json
end
end
Koa.describe "Triggers a library scan"
@@ -240,7 +198,7 @@ struct APIRouter
}
get "/api/admin/thumbnail_progress" do |env|
send_json env, {
"progress" => Library.default.thumbnail_ctx.progress,
"progress" => Library.default.thumbnail_generation_progress,
}.to_json
end
@@ -371,38 +329,6 @@ struct APIRouter
end
end
Koa.describe "Sets the sort title of a title or an entry", <<-MD
When `eid` is provided, apply the sort title to the entry. Otherwise, apply the sort title to the title identified by `tid`.
MD
Koa.tags ["admin", "library"]
Koa.path "tid", desc: "Title ID"
Koa.query "eid", desc: "Entry ID", required: false
Koa.query "name", desc: "The new sort title"
Koa.response 200, schema: "result"
put "/api/admin/sort_title/:tid" do |env|
username = get_username env
begin
title = (Library.default.get_title env.params.url["tid"])
.not_nil!
name = env.params.query["name"]?
entry = env.params.query["eid"]?
if entry.nil?
title.set_sort_title name, username
else
eobj = title.get_entry entry
eobj.set_sort_title name, username unless eobj.nil?
end
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
else
send_json env, {"success" => true}.to_json
end
end
ws "/api/admin/mangadex/queue" do |socket, env|
interval_raw = env.params.query["interval"]?
interval = (interval_raw.to_i? if interval_raw) || 5

View File

@@ -41,7 +41,7 @@ struct MainRouter
username = get_username env
sort_opt = SortOptions.from_info_json Library.default.dir, username
get_and_save_sort_opt Library.default.dir
get_sort_opt
titles = Library.default.sorted_titles username, sort_opt
percentage = titles.map &.load_percentage username
@@ -59,18 +59,12 @@ struct MainRouter
username = get_username env
sort_opt = SortOptions.from_info_json title.dir, username
get_and_save_sort_opt title.dir
get_sort_opt
sorted_titles = title.sorted_titles username, sort_opt
entries = title.sorted_entries username, sort_opt
percentage = title.load_percentage_for_all_entries username, sort_opt
title_percentage = title.titles.map &.load_percentage username
title_percentage_map = {} of String => Float64
title_percentage.each_with_index do |tp, i|
t = title.titles[i]
title_percentage_map[t.id] = tp
end
layout "title"
rescue e
Logger.error e

View File

@@ -342,67 +342,6 @@ class Storage
end
end
def get_title_sort_title(title_id : String)
sort_title = nil
MainFiber.run do
get_db do |db|
sort_title =
db.query_one? "Select sort_title from titles where id = (?)",
title_id, as: String | Nil
end
end
sort_title
end
def set_title_sort_title(title_id : String, sort_title : String | Nil)
sort_title = nil if sort_title == ""
MainFiber.run do
get_db do |db|
db.exec "update titles set sort_title = (?) where id = (?)",
sort_title, title_id
end
end
end
def get_entry_sort_title(entry_id : String)
sort_title = nil
MainFiber.run do
get_db do |db|
sort_title =
db.query_one? "Select sort_title from ids where id = (?)",
entry_id, as: String | Nil
end
end
sort_title
end
def get_entries_sort_title(ids : Array(String))
results = Hash(String, String | Nil).new
MainFiber.run do
get_db do |db|
db.query "select id, sort_title from ids where id in " \
"(#{ids.join "," { |id| "'#{id}'" }})" do |rs|
rs.each do
id = rs.read String
sort_title = rs.read String | Nil
results[id] = sort_title
end
end
end
end
results
end
def set_entry_sort_title(entry_id : String, sort_title : String | Nil)
sort_title = nil if sort_title == ""
MainFiber.run do
get_db do |db|
db.exec "update ids set sort_title = (?) where id = (?)",
sort_title, entry_id
end
end
end
def save_thumbnail(id : String, img : Image)
MainFiber.run do
get_db do |db|
@@ -489,21 +428,12 @@ class Storage
end
end
# Mark titles and entries that no longer exist on the file system as
# unavailable. By supplying `id_candidates` and `titles_candidates`, it
# only checks the existence of the candidate titles/entries to speed up
# the process.
def mark_unavailable(ids_candidates : Array(String)?,
titles_candidates : Array(String)?)
def mark_unavailable
MainFiber.run do
get_db do |db|
# Detect dangling entry IDs
trash_ids = [] of String
query = "select path, id from ids where unavailable = 0"
unless ids_candidates.nil?
query += " and id in (#{ids_candidates.join "," { |i| "'#{i}'" }})"
end
db.query query do |rs|
db.query "select path, id from ids where unavailable = 0" do |rs|
rs.each do
path = rs.read String
fullpath = Path.new(path).expand(Config.current.library_path).to_s
@@ -519,11 +449,7 @@ class Storage
# Detect dangling title IDs
trash_titles = [] of String
query = "select path, id from titles where unavailable = 0"
unless titles_candidates.nil?
query += " and id in (#{titles_candidates.join "," { |i| "'#{i}'" }})"
end
db.query query do |rs|
db.query "select path, id from titles where unavailable = 0" do |rs|
rs.each do
path = rs.read String
fullpath = Path.new(path).expand(Config.current.library_path).to_s
@@ -619,20 +545,6 @@ class Storage
{token, expires}
end
def count_titles : Int32
count = 0
MainFiber.run do
get_db do |db|
db.query "select count(*) from titles" do |rs|
rs.each do
count = rs.read Int32
end
end
end
end
count
end
def close
MainFiber.run do
unless @db.nil?

View File

@@ -48,32 +48,4 @@ class Dir
end
Digest::CRC32.checksum(signatures.sort.join).to_u64
end
# Returns the contents signature of the directory at dirname for checking
# to rescan.
# Rescan conditions:
# - When a file added, moved, removed, renamed (including which in nested
# directories)
def self.contents_signature(dirname, cache = {} of String => String) : String
return cache[dirname] if cache[dirname]?
Fiber.yield
signatures = [] of String
self.open dirname do |dir|
dir.entries.sort.each do |fn|
next if fn.starts_with? "."
path = File.join dirname, fn
if File.directory? path
signatures << Dir.contents_signature path, cache
else
# Only add its signature value to `signatures` when it is a
# supported file
signatures << fn if is_supported_file fn
end
Fiber.yield
end
end
hash = Digest::SHA1.hexdigest(signatures.join)
cache[dirname] = hash
hash
end
end

View File

@@ -35,11 +35,6 @@ def register_mime_types
# FontAwesome fonts
".woff" => "font/woff",
".woff2" => "font/woff2",
# Supported image formats. JPG, PNG, GIF, WebP, and SVG are already
# defiend by Crystal in `MIME.DEFAULT_TYPES`
".apng" => "image/apng",
".avif" => "image/avif",
}.each do |k, v|
MIME.register k, v
end
@@ -87,49 +82,30 @@ def env_is_true?(key : String) : Bool
end
def sort_titles(titles : Array(Title), opt : SortOptions, username : String)
cache_key = SortedTitlesCacheEntry.gen_key username, titles, opt
cached_titles = LRUCache.get cache_key
return cached_titles if cached_titles.is_a? Array(Title)
ary = titles
case opt.method
when .time_modified?
ary = titles.sort { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.sort_title, b.sort_title }
ary.sort! { |a, b| (a.mtime <=> b.mtime).or \
compare_numerically a.title, b.title }
when .progress?
ary = titles.sort do |a, b|
ary.sort! do |a, b|
(a.load_percentage(username) <=> b.load_percentage(username)).or \
compare_numerically a.sort_title, b.sort_title
end
when .title?
ary = titles.sort do |a, b|
compare_numerically a.sort_title, b.sort_title
compare_numerically a.title, b.title
end
else
unless opt.method.auto?
Logger.warn "Unknown sorting method #{opt.not_nil!.method}. Using " \
"Auto instead"
end
ary = titles.sort { |a, b| compare_numerically a.sort_title, b.sort_title }
ary.sort! { |a, b| compare_numerically a.title, b.title }
end
ary.reverse! unless opt.not_nil!.ascend
LRUCache.set generate_cache_entry cache_key, ary
ary
end
def remove_sorted_titles_cache(titles : Array(Title),
sort_methods : Array(SortMethod),
username : String)
[false, true].each do |ascend|
sort_methods.each do |sort_method|
sorted_titles_cache_key = SortedTitlesCacheEntry.gen_key username,
titles, SortOptions.new(sort_method, ascend)
LRUCache.invalidate sorted_titles_cache_key
end
end
end
class String
# Returns the similarity (in [0, 1]) of two paths.
# For the two paths, separate them into arrays of components, count the
@@ -144,31 +120,3 @@ class String
match / s.size
end
end
# Does the followings:
# - turns space-like characters into the normal whitespaces ( )
# - strips and collapses spaces
# - removes ASCII control characters
# - replaces slashes (/) with underscores (_)
# - removes leading dots (.)
# - removes the following special characters: \:*?"<>|
#
# If the sanitized string is empty, returns a random string instead.
def sanitize_filename(str : String) : String
sanitized = str
.gsub(/\s+/, " ")
.strip
.gsub(/\//, "_")
.gsub(/^[\.\s]+/, "")
.gsub(/[\177\000-\031\\:\*\?\"<>\|]/, "")
sanitized.size > 0 ? sanitized : random_str
end
def delete_cache_and_exit(path : String)
File.delete path
Logger.fatal "Invalid library cache deleted. Mango needs to " \
"perform a full reset to recover from this. " \
"Pleae restart Mango. This is NOT a bug."
Logger.fatal "Exiting"
exit 1
end

View File

@@ -107,26 +107,6 @@ macro get_sort_opt
end
end
macro get_and_save_sort_opt(dir)
sort_method = env.params.query["sort"]?
if sort_method
is_ascending = true
ascend = env.params.query["ascend"]?
if ascend && ascend.to_i? == 0
is_ascending = false
end
sort_opt = SortOptions.new sort_method, is_ascending
TitleInfo.new {{dir}} do |info|
info.sort_by[username] = sort_opt.to_tuple
info.save
end
end
end
module HTTP
class Client
private def self.exec(uri : URI, tls : TLSContext = nil)

View File

@@ -61,9 +61,7 @@
<% if page == "home" && item.is_a? Entry %>
<%= "uk-margin-remove-bottom" %>
<% end %>
" data-title="<%= HTML.escape(item.display_name) %>"
data-file-title="<%= HTML.escape(item.title || "") %>"
data-sort-title="<%= HTML.escape(item.sort_title_db || "") %>"><%= HTML.escape(item.display_name) %>
" data-title="<%= HTML.escape(item.display_name) %>"><%= HTML.escape(item.display_name) %>
</h3>
<% if page == "home" && item.is_a? Entry %>
<a class="uk-card-title break-word uk-margin-remove-top uk-text-meta uk-display-inline-block no-modal" data-title="<%= HTML.escape(item.book.display_name) %>" href="<%= base_url %>book/<%= item.book.id %>"><%= HTML.escape(item.book.display_name) %></a>

View File

@@ -24,10 +24,16 @@
<template x-if="job.plugin_id">
<td x-text="job.title"></td>
</template>
<template x-if="!job.plugin_id">
<td><a :href="`<%= mangadex_base_url %>/chapter/${job.id}`" x-text="job.title"></td>
</template>
<template x-if="job.plugin_id">
<td x-text="job.manga_title"></td>
</template>
<template x-if="!job.plugin_id">
<td><a :href="`<%= mangadex_base_url %>/manga/${job.manga_id}`" x-text="job.manga_title"></td>
</template>
<td x-text="`${job.success_count}/${job.pages}`"></td>
<td x-text="`${moment(job.time).fromNow()}`"></td>

View File

@@ -32,10 +32,10 @@
</div>
<div class="uk-position-top">
<div class="uk-navbar-container uk-navbar-transparent" uk-navbar="uk-navbar">
<div class="uk-navbar-left uk-hidden@m">
<div class="uk-navbar-left uk-hidden@s">
<div class="uk-navbar-toggle" uk-navbar-toggle-icon="uk-navbar-toggle-icon" uk-toggle="target: #mobile-nav"></div>
</div>
<div class="uk-navbar-left uk-visible@m">
<div class="uk-navbar-left uk-visible@s">
<a class="uk-navbar-item uk-logo" href="<%= base_url %>"><img src="<%= base_url %>img/icon.png" style="width:90px;height:90px;"></a>
<ul class="uk-navbar-nav">
<li><a href="<%= base_url %>">Home</a></li>
@@ -57,7 +57,7 @@
<% end %>
</ul>
</div>
<div class="uk-navbar-right uk-visible@m">
<div class="uk-navbar-right uk-visible@s">
<ul class="uk-navbar-nav">
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="<%= base_url %>logout">Logout</a></li>

View File

@@ -10,7 +10,6 @@
<div class="uk-margin-bottom uk-width-1-4@s">
<% hash = {
"auto" => "Auto",
"title" => "Name",
"time_modified" => "Date Modified",
"progress" => "Progress"
} %>

View File

@@ -55,8 +55,8 @@
object-fit: contain;
`" />
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false ^ enableRightToLeft)"></div>
<div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true ^ enableRightToLeft)"></div>
<div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false)"></div>
<div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true)"></div>
</div>
</div>
@@ -114,12 +114,6 @@
</div>
</div>
<div class="uk-margin uk-form-horizontal" x-show="mode !== 'continuous'">
<label class="uk-form-label" for="enable-right-to-left">Right to Left</label>
<div class="uk-form-controls">
<input id="enable-right-to-left" class="uk-checkbox" type="checkbox" x-model="enableRightToLeft" @change="enableRightToLeftChanged()">
</div>
</div>
<hr class="uk-divider-icon">
<div class="uk-margin">

View File

@@ -18,8 +18,7 @@
</div>
</div>
</div>
<h2 class=uk-title data-file-title="<%= HTML.escape(title.title) %>" data-sort-title="<%= HTML.escape(title.sort_title_db || "") %>">
<span><%= title.display_name %></span>
<h2 class=uk-title><span><%= title.display_name %></span>
&nbsp;
<% if is_admin %>
<a onclick="edit()" class="uk-icon-button" uk-icon="icon:pencil"></a>
@@ -60,8 +59,8 @@
</div>
<div class="uk-child-width-1-4@m uk-child-width-1-2" uk-grid>
<% sorted_titles.each do |item| %>
<% progress = title_percentage_map[item.id] %>
<% title.titles.each_with_index do |item, i| %>
<% progress = title_percentage[i] %>
<%= render_component "card" %>
<% end %>
</div>
@@ -90,13 +89,6 @@
<input class="uk-input" type="text" name="display-name" id="display-name-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label" for="sort-title">Sort Title</label>
<div class="uk-inline">
<a class="uk-form-icon uk-form-icon-flip" uk-icon="icon:check"></a>
<input class="uk-input" type="text" name="sort-title" id="sort-title-field">
</div>
</div>
<div class="uk-margin">
<label class="uk-form-label">Cover Image</label>
<div class="uk-grid">