Remove MangaDex integration

This commit is contained in:
Alex Ling
2021-08-20 00:25:21 +00:00
parent 1973564272
commit d9565718a4
12 changed files with 77 additions and 850 deletions

View File

@@ -1,172 +0,0 @@
require "mangadex"
require "compress/zip"
require "../rename"
require "./ext"
module MangaDex
class PageJob
property success = false
property url : String
property filename : String
property writer : Compress::Zip::Writer
property tries_remaning : Int32
def initialize(@url, @filename, @writer, @tries_remaning)
end
end
class Downloader < Queue::Downloader
@wait_seconds : Int32 = Config.current.mangadex["download_wait_seconds"]
.to_i32
@retries : Int32 = Config.current.mangadex["download_retries"].to_i32
use_default
def initialize
@client = Client.from_config
super
end
def pop : Queue::Job?
job = nil
MainFiber.run do
DB.open "sqlite3://#{@queue.path}" do |db|
begin
db.query_one "select * from queue where id not like '%-%' " \
"and (status = 0 or status = 1) " \
"order by time limit 1" do |res|
job = Queue::Job.from_query_result res
end
rescue
end
end
end
job
end
private def download(job : Queue::Job)
@downloading = true
@queue.set_status Queue::JobStatus::Downloading, job
begin
chapter = @client.chapter job.id
# We must put the `.pages` call in a rescue block to handle external
# chapters.
pages = chapter.pages
rescue e
Logger.error e
@queue.set_status Queue::JobStatus::Error, job
unless e.message.nil?
@queue.add_message e.message.not_nil!, job
end
@downloading = false
return
end
@queue.set_pages pages.size, job
lib_dir = @library_path
rename_rule = Rename::Rule.new \
Config.current.mangadex["manga_rename_rule"].to_s
manga_dir = File.join lib_dir, chapter.manga.rename rename_rule
unless File.exists? manga_dir
Dir.mkdir_p manga_dir
end
zip_path = File.join manga_dir, "#{job.title}.cbz.part"
# Find the number of digits needed to store the number of pages
len = Math.log10(pages.size).to_i + 1
writer = Compress::Zip::Writer.new zip_path
# Create a buffered channel. It works as an FIFO queue
channel = Channel(PageJob).new pages.size
spawn do
pages.each_with_index do |url, i|
fn = Path.new(URI.parse(url).path).basename
ext = File.extname fn
fn = "#{i.to_s.rjust len, '0'}#{ext}"
page_job = PageJob.new url, fn, writer, @retries
Logger.debug "Downloading #{url}"
loop do
sleep @wait_seconds.seconds
download_page page_job
break if page_job.success ||
page_job.tries_remaning <= 0
page_job.tries_remaning -= 1
Logger.warn "Failed to download page #{url}. " \
"Retrying... Remaining retries: " \
"#{page_job.tries_remaning}"
end
channel.send page_job
break unless @queue.exists? job
end
end
spawn do
page_jobs = [] of PageJob
pages.size.times do
page_job = channel.receive
break unless @queue.exists? job
Logger.debug "[#{page_job.success ? "success" : "failed"}] " \
"#{page_job.url}"
page_jobs << page_job
if page_job.success
@queue.add_success job
else
@queue.add_fail job
msg = "Failed to download page #{page_job.url}"
@queue.add_message msg, job
Logger.error msg
end
end
unless @queue.exists? job
Logger.debug "Download cancelled"
@downloading = false
next
end
fail_count = page_jobs.count { |j| !j.success }
Logger.debug "Download completed. " \
"#{fail_count}/#{page_jobs.size} failed"
writer.close
filename = File.join File.dirname(zip_path), File.basename(zip_path,
".part")
File.rename zip_path, filename
Logger.debug "cbz File created at #{filename}"
zip_exception = validate_archive filename
if !zip_exception.nil?
@queue.add_message "The downloaded archive is corrupted. " \
"Error: #{zip_exception}", job
@queue.set_status Queue::JobStatus::Error, job
elsif fail_count > 0
@queue.set_status Queue::JobStatus::MissingPages, job
else
@queue.set_status Queue::JobStatus::Completed, job
end
@downloading = false
end
end
private def download_page(job : PageJob)
Logger.debug "downloading #{job.url}"
headers = HTTP::Headers{
"User-agent" => "Mangadex.cr",
}
begin
HTTP::Client.get job.url, headers do |res|
unless res.success?
raise "Failed to download page #{job.url}. " \
"[#{res.status_code}] #{res.status_message}"
end
job.writer.add job.filename, res.body_io
end
job.success = true
rescue e
Logger.error e
job.success = false
end
end
end
end

View File

@@ -1,60 +0,0 @@
private macro properties_to_hash(names)
{
{% for name in names %}
"{{name.id}}" => {{name.id}}.to_s,
{% end %}
}
end
# Monkey-patch the structures in the `mangadex` shard to suit our needs
module MangaDex
struct Client
@@group_cache = {} of String => Group
def self.from_config : Client
self.new base_url: Config.current.mangadex["base_url"].to_s,
api_url: Config.current.mangadex["api_url"].to_s
end
end
struct Manga
def rename(rule : Rename::Rule)
rule.render properties_to_hash %w(id title author artist)
end
def to_info_json
hash = JSON.parse(to_json).as_h
_chapters = chapters.map do |c|
JSON.parse c.to_info_json
end
hash["chapters"] = JSON::Any.new _chapters
hash.to_json
end
end
struct Chapter
def rename(rule : Rename::Rule)
hash = properties_to_hash %w(id title volume chapter lang_code language)
hash["groups"] = groups.join(",", &.name)
rule.render hash
end
def full_title
rule = Rename::Rule.new \
Config.current.mangadex["chapter_rename_rule"].to_s
rename rule
end
def to_info_json
hash = JSON.parse(to_json).as_h
hash["language"] = JSON::Any.new language
_groups = {} of String => JSON::Any
groups.each do |g|
_groups[g.name] = JSON::Any.new g.id
end
hash["groups"] = JSON::Any.new _groups
hash["full_title"] = JSON::Any.new full_title
hash.to_json
end
end
end

View File

@@ -2,7 +2,6 @@ require "./config"
require "./queue"
require "./server"
require "./main_fiber"
require "./mangadex/*"
require "./plugin/*"
require "option_parser"
require "clim"
@@ -59,7 +58,6 @@ class CLI < Clim
Storage.default
Queue.default
Library.default
MangaDex::Downloader.default
Plugin::Downloader.default
spawn do

View File

@@ -73,9 +73,5 @@ struct AdminRouter
get "/admin/missing" do |env|
layout "missing-items"
end
get "/admin/mangadex" do |env|
layout "mangadex"
end
end
end

View File

@@ -1,4 +1,3 @@
require "../mangadex/*"
require "../upload"
require "koa"
@@ -56,19 +55,6 @@ struct APIRouter
"error" => String?,
}
Koa.schema("mdChapter", {
"id" => Int64,
"group" => {} of String => String,
}.merge(s %w(title volume chapter language full_title time
manga_title manga_id)),
desc: "A MangaDex chapter")
Koa.schema "mdManga", {
"id" => Int64,
"chapters" => ["mdChapter"],
}.merge(s %w(title description author artist cover_url)),
desc: "A MangaDex manga"
Koa.describe "Returns a page in a manga entry"
Koa.path "tid", desc: "Title ID"
Koa.path "eid", desc: "Entry ID"
@@ -323,58 +309,6 @@ struct APIRouter
end
end
Koa.describe "Returns a MangaDex manga identified by `id`", <<-MD
On error, returns a JSON that contains the error message in the `error` field.
MD
Koa.tags ["admin", "mangadex"]
Koa.path "id", desc: "A MangaDex manga ID"
Koa.response 200, schema: "mdManga"
get "/api/admin/mangadex/manga/:id" do |env|
begin
id = env.params.url["id"]
manga = MangaDex::Client.from_config.manga id
send_json env, manga.to_info_json
rescue e
Logger.error e
send_json env, {"error" => e.message}.to_json
end
end
Koa.describe "Adds a list of MangaDex chapters to the download queue", <<-MD
On error, returns a JSON that contains the error message in the `error` field.
MD
Koa.tags ["admin", "mangadex", "downloader"]
Koa.body schema: {
"chapters" => ["mdChapter"],
}
Koa.response 200, schema: {
"success" => Int32,
"fail" => Int32,
}
post "/api/admin/mangadex/download" do |env|
begin
chapters = env.params.json["chapters"].as(Array).map &.as_h
jobs = chapters.map { |chapter|
Queue::Job.new(
chapter["id"].as_i64.to_s,
chapter["mangaId"].as_i64.to_s,
chapter["full_title"].as_s,
chapter["mangaTitle"].as_s,
Queue::JobStatus::Pending,
Time.unix chapter["timestamp"].as_i64
)
}
inserted_count = Queue.default.push jobs
send_json env, {
"success": inserted_count,
"fail": jobs.size - inserted_count,
}.to_json
rescue e
Logger.error e
send_json env, {"error" => e.message}.to_json
end
end
ws "/api/admin/mangadex/queue" do |socket, env|
interval_raw = env.params.query["interval"]?
interval = (interval_raw.to_i? if interval_raw) || 5
@@ -904,115 +838,6 @@ struct APIRouter
end
end
Koa.describe "Logs the current user into their MangaDex account", <<-MD
If successful, returns the expiration date (as a unix timestamp) of the newly created token.
MD
Koa.body schema: {
"username" => String,
"password" => String,
}
Koa.response 200, schema: {
"success" => Bool,
"error" => String?,
"expires" => Int64?,
}
Koa.tags ["admin", "mangadex", "users"]
post "/api/admin/mangadex/login" do |env|
begin
username = env.params.json["username"].as String
password = env.params.json["password"].as String
mango_username = get_username env
client = MangaDex::Client.from_config
client.auth username, password
Storage.default.save_md_token mango_username, client.token.not_nil!,
client.token_expires
send_json env, {
"success" => true,
"error" => nil,
"expires" => client.token_expires.to_unix,
}.to_json
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
Koa.describe "Returns the expiration date (as a unix timestamp) of the mangadex token if it exists"
Koa.response 200, schema: {
"success" => Bool,
"error" => String?,
"expires" => Int64?,
}
Koa.tags ["admin", "mangadex", "users"]
get "/api/admin/mangadex/expires" do |env|
begin
username = get_username env
_, expires = Storage.default.get_md_token username
send_json env, {
"success" => true,
"error" => nil,
"expires" => expires.try &.to_unix,
}.to_json
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
Koa.describe "Searches MangaDex for manga matching `query`", <<-MD
Returns an empty list if the current user hasn't logged in to MangaDex.
MD
Koa.query "query"
Koa.response 200, schema: {
"success" => Bool,
"error" => String?,
"manga?" => [{
"id" => Int64,
"title" => String,
"description" => String,
"mainCover" => String,
}],
}
Koa.tags ["admin", "mangadex"]
get "/api/admin/mangadex/search" do |env|
begin
username = get_username env
token, expires = Storage.default.get_md_token username
unless expires && token
raise "No token found for user #{username}"
end
client = MangaDex::Client.from_config
client.token = token
client.token_expires = expires
query = env.params.query["query"]
send_json env, {
"success" => true,
"error" => nil,
"manga" => client.partial_search query,
}.to_json
rescue e
Logger.error e
send_json env, {
"success" => false,
"error" => e.message,
}.to_json
end
end
doc = Koa.generate
@@api_json = doc.to_json if doc

View File

@@ -72,11 +72,6 @@ struct MainRouter
end
end
get "/download" do |env|
mangadex_base_url = Config.current.mangadex["base_url"]
layout "download"
end
get "/download/plugins" do |env|
begin
id = env.params.query["plugin"]?

View File

@@ -33,7 +33,6 @@
<option>System</option>
</select>
</li>
<li><a class="uk-link-reset" href="<%= base_url %>admin/mangadex">Connect to MangaDex</a></li>
</ul>
<hr class="uk-divider-icon">

View File

@@ -1,89 +1,87 @@
<!DOCTYPE html>
<html>
<%= render_component "head" %>
<%= render_component "head" %>
<body>
<div class="uk-offcanvas-content">
<div class="uk-navbar-container uk-navbar-transparent" uk-navbar="uk-navbar">
<div id="mobile-nav" uk-offcanvas="overlay: true">
<div class="uk-offcanvas-bar uk-flex uk-flex-column">
<ul class="uk-nav-parent-icon uk-nav-primary uk-nav-center uk-margin-auto-vertical" uk-nav>
<li><a href="<%= base_url %>">Home</a></li>
<li><a href="<%= base_url %>library">Library</a></li>
<li><a href="<%= base_url %>tags">Tags</a></li>
<% if is_admin %>
<li><a href="<%= base_url %>admin">Admin</a></li>
<li class="uk-parent">
<a href="#">Download</a>
<ul class="uk-nav-sub">
<li><a href="<%= base_url %>download">MangaDex</a></li>
<li><a href="<%= base_url %>download/plugins">Plugins</a></li>
<li><a href="<%= base_url %>admin/downloads">Download Manager</a></li>
</ul>
</li>
<% end %>
<hr uk-divider>
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="<%= base_url %>logout">Logout</a></li>
</ul>
</div>
</div>
</div>
<body>
<div class="uk-offcanvas-content">
<div class="uk-navbar-container uk-navbar-transparent" uk-navbar="uk-navbar">
<div id="mobile-nav" uk-offcanvas="overlay: true">
<div class="uk-offcanvas-bar uk-flex uk-flex-column">
<ul class="uk-nav-parent-icon uk-nav-primary uk-nav-center uk-margin-auto-vertical" uk-nav>
<li><a href="<%= base_url %>">Home</a></li>
<li><a href="<%= base_url %>library">Library</a></li>
<li><a href="<%= base_url %>tags">Tags</a></li>
<% if is_admin %>
<li><a href="<%= base_url %>admin">Admin</a></li>
<li class="uk-parent">
<a href="#">Download</a>
<ul class="uk-nav-sub">
<li><a href="<%= base_url %>download/plugins">Plugins</a></li>
<li><a href="<%= base_url %>admin/downloads">Download Manager</a></li>
</ul>
</li>
<% end %>
<hr uk-divider>
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="<%= base_url %>logout">Logout</a></li>
</ul>
</div>
</div>
<div class="uk-position-top">
<div class="uk-navbar-container uk-navbar-transparent" uk-navbar="uk-navbar">
<div class="uk-navbar-left uk-hidden@s">
<div class="uk-navbar-toggle" uk-navbar-toggle-icon="uk-navbar-toggle-icon" uk-toggle="target: #mobile-nav"></div>
</div>
<div class="uk-navbar-left uk-visible@s">
<a class="uk-navbar-item uk-logo" href="<%= base_url %>"><img src="<%= base_url %>img/icon.png" style="width:90px;height:90px;"></a>
<ul class="uk-navbar-nav">
<li><a href="<%= base_url %>">Home</a></li>
<li><a href="<%= base_url %>library">Library</a></li>
<li><a href="<%= base_url %>tags">Tags</a></li>
<% if is_admin %>
<li><a href="<%= base_url %>admin">Admin</a></li>
<li>
<a href="#">Download</a>
<div class="uk-navbar-dropdown">
<ul class="uk-nav uk-navbar-dropdown-nav">
<li class="uk-nav-header">Source</li>
<li><a href="<%= base_url %>download">MangaDex</a></li>
<li><a href="<%= base_url %>download/plugins">Plugins</a></li>
<li class="uk-nav-divider"></li>
<li><a href="<%= base_url %>admin/downloads">Download Manager</a></li>
</ul>
</div>
</li>
<% end %>
</ul>
</div>
<div class="uk-navbar-right uk-visible@s">
<ul class="uk-navbar-nav">
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="<%= base_url %>logout">Logout</a></li>
</ul>
</div>
</div>
</div>
</div>
<div class="uk-position-top">
<div class="uk-navbar-container uk-navbar-transparent" uk-navbar="uk-navbar">
<div class="uk-navbar-left uk-hidden@s">
<div class="uk-navbar-toggle" uk-navbar-toggle-icon="uk-navbar-toggle-icon" uk-toggle="target: #mobile-nav"></div>
</div>
<div class="uk-section uk-section-small">
</div>
<div class="uk-section uk-section-small" style="position:relative;">
<div class="uk-container uk-container-small">
<div id="alert"></div>
<%= content %>
<div class="uk-visible@m" id="totop-wrapper" x-data="{}" x-show="$('body').height() > 1.5 * $(window).height()">
<a href="#" uk-totop uk-scroll></a>
<div class="uk-navbar-left uk-visible@s">
<a class="uk-navbar-item uk-logo" href="<%= base_url %>"><img src="<%= base_url %>img/icon.png" style="width:90px;height:90px;"></a>
<ul class="uk-navbar-nav">
<li><a href="<%= base_url %>">Home</a></li>
<li><a href="<%= base_url %>library">Library</a></li>
<li><a href="<%= base_url %>tags">Tags</a></li>
<% if is_admin %>
<li><a href="<%= base_url %>admin">Admin</a></li>
<li>
<a href="#">Download</a>
<div class="uk-navbar-dropdown">
<ul class="uk-nav uk-navbar-dropdown-nav">
<li class="uk-nav-header">Source</li>
<li><a href="<%= base_url %>download/plugins">Plugins</a></li>
<li class="uk-nav-divider"></li>
<li><a href="<%= base_url %>admin/downloads">Download Manager</a></li>
</ul>
</div>
</div>
</li>
<% end %>
</ul>
</div>
<script>
setTheme();
const base_url = "<%= base_url %>";
</script>
<%= render_component "uikit" %>
<%= yield_content "script" %>
</body>
<div class="uk-navbar-right uk-visible@s">
<ul class="uk-navbar-nav">
<li><a onclick="toggleTheme()"><i class="fas fa-adjust"></i></a></li>
<li><a href="<%= base_url %>logout">Logout</a></li>
</ul>
</div>
</div>
</div>
<div class="uk-section uk-section-small">
</div>
<div class="uk-section uk-section-small" style="position:relative;">
<div class="uk-container uk-container-small">
<div id="alert"></div>
<%= content %>
<div class="uk-visible@m" id="totop-wrapper" x-data="{}" x-show="$('body').height() > 1.5 * $(window).height()">
<a href="#" uk-totop uk-scroll></a>
</div>
</div>
</div>
<script>
setTheme();
const base_url = "<%= base_url %>";
</script>
<%= render_component "uikit" %>
<%= yield_content "script" %>
</body>
</html>