Merge branch 'dev'

(#93)

Dockerfile.arm32v7

@@ -5,7 +5,7 @@ RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ lib
 RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.34.0 && make deps && cd ..
 RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd ..
 RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd ..
-RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr/ext/libwebp && git checkout v0.1.1 && make && cd ../stbi && make
+RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd ..
 
 COPY mango-arm32v7.o .
 

Dockerfile.arm64v8

@@ -5,10 +5,10 @@ RUN apt-get update && apt-get install -y wget git make llvm-8 llvm-8-dev g++ lib
 RUN git clone https://github.com/crystal-lang/crystal && cd crystal && git checkout 0.34.0 && make deps && cd ..
 RUN git clone https://github.com/kostya/myhtml && cd myhtml/src/ext && git checkout v1.5.0 && make && cd ..
 RUN git clone https://github.com/jessedoyle/duktape.cr && cd duktape.cr/ext && git checkout v0.20.0 && make && cd ..
-RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr/ext/libwebp && git checkout v0.1.1 && make && cd ../stbi && make
+RUN git clone https://github.com/hkalexling/image_size.cr && cd image_size.cr && git checkout v0.2.0 && make && cd ..
 
 COPY mango-arm64v8.o .
 
-RUN cc 'mango-arm64v8.o' -o 'mango' -rdynamic -lxml2 -L/image_size.cr/ext/libwebp -lwebp -L/image_size.cr/ext/stbi -lstbi /myhtml/src/ext/modest-c/lib/libmodest_static.a -L/duktape.cr/src/.build/lib -L/duktape.cr/src/.build/include -lduktape -lm `pkg-config libarchive --libs` -lz `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libssl || printf %s '-lssl -lcrypto'` `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libcrypto || printf %s '-lcrypto'` -lgmp -lsqlite3 -lyaml -lpcre -lm /usr/lib/arm-linux-gnueabihf/libgc.so -lpthread /crystal/src/ext/libcrystal.a -levent -lrt -ldl -L/usr/bin/../lib/crystal/lib -L/usr/bin/../lib/crystal/lib
+RUN cc 'mango-arm64v8.o' -o 'mango' -rdynamic -lxml2 -L/image_size.cr/ext/libwebp -lwebp -L/image_size.cr/ext/stbi -lstbi /myhtml/src/ext/modest-c/lib/libmodest_static.a -L/duktape.cr/src/.build/lib -L/duktape.cr/src/.build/include -lduktape -lm `pkg-config libarchive --libs` -lz `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libssl || printf %s '-lssl -lcrypto'` `command -v pkg-config > /dev/null && pkg-config --libs --silence-errors libcrypto || printf %s '-lcrypto'` -lgmp -lsqlite3 -lyaml -lpcre -lm /usr/lib/aarch64-linux-gnu/libgc.so -lpthread /crystal/src/ext/libcrystal.a -levent -lrt -ldl -L/usr/bin/../lib/crystal/lib -L/usr/bin/../lib/crystal/lib
 
 CMD ["./mango"]

README.md

@@ -12,6 +12,7 @@ Mango is a self-hosted manga server and reader. Its features include
 - Supported formats: `.cbz`, `.zip`, `.cbr` and `.rar`
 - Supports nested folders in library
 - Automatically stores reading progress
+- Thumbnail generation
 - Built-in [MangaDex](https://mangadex.org/) downloader
 - Supports [plugins](https://github.com/hkalexling/mango-plugins) to download from thrid-party sites
 - The web reader is responsive and works well on mobile, so there is no need for a mobile app
@@ -51,7 +52,7 @@ The official docker images are available on [Dockerhub](https://hub.docker.com/r
 ### CLI
 
 ```
-  Mango - Manga Server and Web Reader. Version 0.12.0
+  Mango - Manga Server and Web Reader. Version 0.16.0
 
   Usage:
 
@@ -76,22 +77,27 @@ The default config file location is `~/.config/mango/config.yml`. It might be di
 ---
 port: 9000
 base_url: /
+session_secret: mango-session-secret
 library_path: ~/mango/library
 db_path: ~/mango/mango.db
 scan_interval_minutes: 5
+thumbnail_generation_interval_hours: 24
+db_optimization_interval_hours: 24
 log_level: info
 upload_path: ~/mango/uploads
+plugin_path: ~/mango/plugins
+download_timeout_seconds: 30
 mangadex:
   base_url: https://mangadex.org
   api_url: https://mangadex.org/api
   download_wait_seconds: 5
   download_retries: 4
-  download_queue_db_path: ~/mango/queue.db
+  download_queue_db_path: /home/alex_ling/mango/queue.db
   chapter_rename_rule: '[Vol.{volume} ][Ch.{chapter} ]{title|id}'
   manga_rename_rule: '{title}'
 ```
 
-- `scan_interval_minutes` can be any non-negative integer. Setting it to `0` disables the periodic scan
+- `scan_interval_minutes`, `thumbnail_generation_interval_hours` and `db_optimization_interval_hours` can be any non-negative integer. Setting them to `0` disables the periodic tasks
 - `log_level` can be `debug`, `info`, `warn`, `error`, `fatal` or `off`. Setting it to `off` disables the logging
 
 ### Library Structure
@@ -142,6 +148,7 @@ Mobile UI:
 ## Sponsors
 
 <a href="https://casinoshunter.com/online-casinos/"><img src="https://i.imgur.com/EJb3wBo.png" width="150" height="auto"></a>
+<a href="https://www.browserstack.com/open-source"><img src="https://i.imgur.com/hGJUJXD.png" width="150" height="auto"></a>
 
 ## Contributors
 

gulpfile.js

@@ -1,15 +1,43 @@
 const gulp = require('gulp');
-const minify = require("gulp-babel-minify");
+const babel = require('gulp-babel');
+const minify = require('gulp-babel-minify');
 const minifyCss = require('gulp-minify-css');
 const less = require('gulp-less');
 
-gulp.task('copy-uikit-js', () => {
-	return gulp.src('node_modules/uikit/dist/js/*.min.js')
+// Copy libraries from node_moduels to public/js
+gulp.task('copy-js', () => {
+	return gulp.src([
+			'node_modules/@fortawesome/fontawesome-free/js/fontawesome.min.js',
+			'node_modules/@fortawesome/fontawesome-free/js/solid.min.js',
+			'node_modules/uikit/dist/js/uikit.min.js',
+			'node_modules/uikit/dist/js/uikit-icons.min.js'
+		])
 		.pipe(gulp.dest('public/js'));
 });
 
-gulp.task('minify-js', () => {
-	return gulp.src('public/js/*.js')
+// Copy UIKit SVG icons to public/img
+gulp.task('copy-uikit-icons', () => {
+	return gulp.src('node_modules/uikit/src/images/backgrounds/*.svg')
+		.pipe(gulp.dest('public/img'));
+});
+
+// Compile less
+gulp.task('less', () => {
+	return gulp.src('public/css/*.less')
+		.pipe(less())
+		.pipe(gulp.dest('public/css'));
+});
+
+// Transpile and minify JS files and output to dist
+gulp.task('babel', () => {
+	return gulp.src(['public/js/*.js', '!public/js/*.min.js'])
+		.pipe(babel({
+			presets: [
+				['@babel/preset-env', {
+					targets: '>0.25%, not dead, ios>=9'
+				}]
+			],
+		}))
 		.pipe(minify({
 			removeConsole: true,
 			builtIns: false
@@ -17,40 +45,26 @@ gulp.task('minify-js', () => {
 		.pipe(gulp.dest('dist/js'));
 });
 
-gulp.task('less', () => {
-	return gulp.src('public/css/*.less')
-		.pipe(less())
-		.pipe(gulp.dest('public/css'));
-});
-
+// Minify CSS and output to dist
 gulp.task('minify-css', () => {
 	return gulp.src('public/css/*.css')
 		.pipe(minifyCss())
 		.pipe(gulp.dest('dist/css'));
 });
 
-gulp.task('copy-uikit-icons', () => {
-	return gulp.src('node_modules/uikit/src/images/backgrounds/*.svg')
-		.pipe(gulp.dest('public/img'));
-});
-
-gulp.task('img', () => {
-	return gulp.src('public/img/*')
-		.pipe(gulp.dest('dist/img'));
-});
-
+// Copy static files (includeing images) to dist
 gulp.task('copy-files', () => {
-	return gulp.src('public/*.*')
+	return gulp.src(['public/img/*', 'public/*.*', 'public/js/*.min.js'], {
+			base: 'public'
+		})
 		.pipe(gulp.dest('dist'));
 });
 
-gulp.task('default', gulp.parallel(
-	gulp.series('copy-uikit-js', 'minify-js'),
-	gulp.series('less', 'minify-css'),
-	gulp.series('copy-uikit-icons', 'img'),
-	'copy-files'
-));
+// Set up the public folder for development
+gulp.task('dev', gulp.parallel('copy-js', 'copy-uikit-icons', 'less'));
 
-gulp.task('dev', gulp.parallel(
-	'copy-uikit-js', 'less', 'copy-uikit-icons'
-));
+// Set up the dist folder for deployment
+gulp.task('deploy', gulp.parallel('babel', 'minify-css', 'copy-files'));
+
+// Default task
+gulp.task('default', gulp.series('dev', 'deploy'));

package.json

@@ -1,21 +1,24 @@
 {
-  "name": "mango",
-  "version": "1.0.0",
-  "main": "index.js",
-  "repository": "https://github.com/hkalexling/Mango.git",
-  "author": "Alex Ling <hkalexling@gmail.com>",
-  "license": "MIT",
-  "devDependencies": {
-    "gulp": "^4.0.2",
-    "gulp-babel-minify": "^0.5.1",
-    "gulp-less": "^4.0.1",
-    "gulp-minify-css": "^1.2.4",
-    "less": "^3.11.3"
-  },
-  "scripts": {
-    "uglify": "gulp"
-  },
-  "dependencies": {
-    "uikit": "^3.5.4"
-  }
+	"name": "mango",
+	"version": "1.0.0",
+	"main": "index.js",
+	"repository": "https://github.com/hkalexling/Mango.git",
+	"author": "Alex Ling <hkalexling@gmail.com>",
+	"license": "MIT",
+	"devDependencies": {
+		"@babel/preset-env": "^7.11.5",
+		"gulp": "^4.0.2",
+		"gulp-babel": "^8.0.0",
+		"gulp-babel-minify": "^0.5.1",
+		"gulp-less": "^4.0.1",
+		"gulp-minify-css": "^1.2.4",
+		"less": "^3.11.3"
+	},
+	"scripts": {
+		"uglify": "gulp"
+	},
+	"dependencies": {
+		"@fortawesome/fontawesome-free": "^5.14.0",
+		"uikit": "^3.5.4"
+	}
 }

public/js/admin.js

@@ -1,40 +1,90 @@
-let scanning = false;
-
-const scan = () => {
-	scanning = true;
-	$('#scan-status > div').removeAttr('hidden');
-	$('#scan-status > span').attr('hidden', '');
-	const color = $('#scan').css('color');
-	$('#scan').css('color', 'gray');
-	$.post(base_url + 'api/admin/scan', (data) => {
-		const ms = data.milliseconds;
-		const titles = data.titles;
-		$('#scan-status > span').text('Scanned ' + titles + ' titles in ' + ms + 'ms');
-		$('#scan-status > span').removeAttr('hidden');
-		$('#scan').css('color', color);
-		$('#scan-status > div').attr('hidden', '');
-		scanning = false;
-	});
-}
-
-String.prototype.capitalize = function() {
-	return this.charAt(0).toUpperCase() + this.slice(1);
-}
-
 $(() => {
-	$('li').click((e) => {
-		const url = $(e.currentTarget).attr('data-url');
-		if (url) {
-			$(location).attr('href', url);
-		}
-	});
-
 	const setting = loadThemeSetting();
-	$('#theme-select').val(setting.capitalize());
-
+	$('#theme-select').val(capitalize(setting));
 	$('#theme-select').change((e) => {
 		const newSetting = $(e.currentTarget).val().toLowerCase();
 		saveThemeSetting(newSetting);
 		setTheme();
 	});
+
+	getProgress();
+	setInterval(getProgress, 5000);
 });
+
+/**
+ * Capitalize String
+ *
+ * @function capitalize
+ * @param {string} str - The string to be capitalized
+ * @return {string} The capitalized string
+ */
+const capitalize = (str) => {
+	return str.charAt(0).toUpperCase() + str.slice(1);
+};
+
+/**
+ * Set an alpine.js property
+ *
+ * @function setProp
+ * @param {string} key - Key of the data property
+ * @param {*} prop - The data property
+ */
+const setProp = (key, prop) => {
+	$('#root').get(0).__x.$data[key] = prop;
+};
+
+/**
+ * Get an alpine.js property
+ *
+ * @function getProp
+ * @param {string} key - Key of the data property
+ * @return {*} The data property
+ */
+const getProp = (key) => {
+	return $('#root').get(0).__x.$data[key];
+};
+
+/**
+ * Get the thumbnail generation progress from the API
+ *
+ * @function getProgress
+ */
+const getProgress = () => {
+	$.get(`${base_url}api/admin/thumbnail_progress`)
+		.then(data => {
+			setProp('progress', data.progress);
+			const generating = data.progress > 0
+			setProp('generating', generating);
+		});
+};
+
+/**
+ * Trigger the thumbnail generation
+ *
+ * @function generateThumbnails
+ */
+const generateThumbnails = () => {
+	setProp('generating', true);
+	setProp('progress', 0.0);
+	$.post(`${base_url}api/admin/generate_thumbnails`)
+		.then(getProgress);
+};
+
+/**
+ * Trigger the scan
+ *
+ * @function scan
+ */
+const scan = () => {
+	setProp('scanning', true);
+	setProp('scanMs', -1);
+	setProp('scanTitles', 0);
+	$.post(`${base_url}api/admin/scan`)
+		.then(data => {
+			setProp('scanMs', data.milliseconds);
+			setProp('scanTitles', data.titles);
+		})
+		.always(() => {
+			setProp('scanning', false);
+		});
+}

public/js/dots.js

@@ -1,17 +1,26 @@
-const truncate = () => {
-	$('.uk-card-title').each((i, e) => {
-		$(e).dotdotdot({
-			truncate: 'letter',
-			watch: true,
-			callback: (truncated) => {
-				if (truncated) {
-					$(e).attr('uk-tooltip', $(e).attr('data-title'));
-				} else {
-					$(e).removeAttr('uk-tooltip');
-				}
+/**
+ * Truncate a .uk-card-title element
+ *
+ * @function truncate
+ * @param {object} e - The title element to truncate
+ */
+const truncate = (e) => {
+	$(e).dotdotdot({
+		truncate: 'letter',
+		watch: true,
+		callback: (truncated) => {
+			if (truncated) {
+				$(e).attr('uk-tooltip', $(e).attr('data-title'));
+			} else {
+				$(e).removeAttr('uk-tooltip');
 			}
-		});
+		}
 	});
 };
 
-truncate();
+$('.uk-card-title').each((i, e) => {
+	// Truncate the title when it first enters the view
+	$(e).one('inview', () => {
+		truncate(e);
+	});
+});

public/js/download.js

@@ -95,7 +95,7 @@ const search = () => {
 
 	try {
 		const path = new URL(input).pathname;
-		const match = /\/title\/([0-9]+)/.exec(path);
+		const match = /\/(?:title|manga)\/([0-9]+)/.exec(path);
 		int_id = parseInt(match[1]);
 	} catch (e) {
 		int_id = parseInt(input);

public/js/reader.js

@@ -1,3 +1,7 @@
+let lastSavedPage = page;
+let items = [];
+let longPages = false;
+
 $(() => {
 	getPages();
 
@@ -5,8 +9,58 @@ $(() => {
 		const p = parseInt($('#page-select').val());
 		toPage(p);
 	});
+
+	$('#mode-select').change(() => {
+		const mode = $('#mode-select').val();
+		const curIdx = parseInt($('#page-select').val());
+
+		updateMode(mode, curIdx);
+	});
 });
 
+$(window).resize(() => {
+	const mode = getProp('mode');
+	if (mode === 'continuous') return;
+
+	const wideScreen = $(window).width() > $(window).height();
+	const propMode = wideScreen ? 'height' : 'width';
+	setProp('mode', propMode);
+});
+
+/**
+ * Update the reader mode
+ *
+ * @function updateMode
+ * @param {string} mode - The mode. Can be one of the followings:
+ * 		{'continuous', 'paged', 'height', 'width'}
+ * @param {number} targetPage - The one-based index of the target page
+ */
+const updateMode = (mode, targetPage) => {
+	localStorage.setItem('mode', mode);
+
+	// The mode to be put into the `mode` prop. It can't be `screen`
+	let propMode = mode;
+
+	if (mode === 'paged') {
+		const wideScreen = $(window).width() > $(window).height();
+		propMode = wideScreen ? 'height' : 'width';
+	}
+
+	setProp('mode', propMode);
+
+	if (mode === 'continuous') {
+		waitForPage(items.length, () => {
+			setupScroller();
+		});
+	}
+
+	waitForPage(targetPage, () => {
+		setTimeout(() => {
+			toPage(targetPage);
+		}, 100);
+	});
+};
+
 /**
  * Set an alpine.js property
  *
@@ -18,6 +72,17 @@ const setProp = (key, prop) => {
 	$('#root').get(0).__x.$data[key] = prop;
 };
 
+/**
+ * Get an alpine.js property
+ *
+ * @function getProp
+ * @param {string} key - Key of the data property
+ * @return {*} The data property
+ */
+const getProp = (key) => {
+	return $('#root').get(0).__x.$data[key];
+};
+
 /**
  * Get dimension of the pages in the entry from the API and update the view
  */
@@ -28,7 +93,7 @@ const getPages = () => {
 				throw new Error(resp.error);
 			const dimensions = data.dimensions;
 
-			const items = dimensions.map((d, i) => {
+			items = dimensions.map((d, i) => {
 				return {
 					id: i + 1,
 					url: `${base_url}api/page/${tid}/${eid}/${i+1}`,
@@ -37,13 +102,21 @@ const getPages = () => {
 				};
 			});
 
+			const avgRatio = items.reduce((acc, cur) => {
+				return acc + cur.height / cur.width
+			}, 0) / items.length;
+
+			console.log(avgRatio);
+			longPages = avgRatio > 2;
+
 			setProp('items', items);
 			setProp('loading', false);
 
-			waitForPage(items.length, () => {
-				toPage(page);
-				setupScroller();
-			});
+			const storedMode = localStorage.getItem('mode') || 'continuous';
+
+			setProp('mode', storedMode);
+			updateMode(storedMode, page);
+			$('#mode-select').val(storedMode);
 		})
 		.catch(e => {
 			const errMsg = `Failed to get the page dimensions. ${e}`;
@@ -60,7 +133,15 @@ const getPages = () => {
  * @param {number} idx - One-based index of the page
  */
 const toPage = (idx) => {
-	$(`#${idx}`).get(0).scrollIntoView(true);
+	const mode = getProp('mode');
+	if (mode === 'continuous') {
+		$(`#${idx}`).get(0).scrollIntoView(true);
+	} else {
+		if (idx >= 1 && idx <= items.length) {
+			setProp('curItem', items[idx - 1]);
+		}
+	}
+	replaceHistory(idx);
 	UIkit.modal($('#modal-sections')).hide();
 };
 
@@ -126,6 +207,8 @@ const replaceHistory = (idx) => {
  * @function setupScroller
  */
 const setupScroller = () => {
+	const mode = getProp('mode');
+	if (mode !== 'continuous') return;
 	$('#root img').each((idx, el) => {
 		$(el).on('inview', (event, inView) => {
 			if (inView) {
@@ -136,26 +219,92 @@ const setupScroller = () => {
 	});
 };
 
-let lastSavedPage = page;
-
 /**
- * Update the backend reading progress if the current page is more than
- * 		five pages away from the last saved page
+ * Update the backend reading progress if:
+ * 		1) the current page is more than five pages away from the last 
+ * 			saved page, or
+ * 		2) the average height/width ratio of the pages is over 2, or
+ * 		3) the current page is the first page, or
+ * 		4) the current page is the last page
  *
  * @function saveProgress
  * @param {number} idx - One-based index of the page
+ * @param {function} cb - Callback
  */
-const saveProgress = (idx) => {
-	if (Math.abs(idx - lastSavedPage) < 5) return;
-	lastSavedPage = idx;
+const saveProgress = (idx, cb) => {
+	idx = parseInt(idx);
+	if (Math.abs(idx - lastSavedPage) >= 5 ||
+		longPages ||
+		idx === 1 || idx === items.length
+	) {
+		lastSavedPage = idx;
+		console.log('saving progress', idx);
 
-	const url = `${base_url}api/progress/${tid}/${idx}?${$.param({entry: eid})}`;
-	$.post(url)
-		.then(data => {
-			if (data.error) throw new Error(data.error);
-		})
-		.catch(e => {
-			console.error(e);
-			alert('danger', e);
-		});
+		const url = `${base_url}api/progress/${tid}/${idx}?${$.param({entry: eid})}`;
+		$.post(url)
+			.then(data => {
+				if (data.error) throw new Error(data.error);
+				if (cb) cb();
+			})
+			.catch(e => {
+				console.error(e);
+				alert('danger', e);
+			});
+	}
+};
+
+/**
+ * Mark progress to 100% and redirect to the next entry
+ * 	Used as the onclick handler for the "Next Entry" button
+ *
+ * @function nextEntry
+ * @param {string} nextUrl - URL of the next entry
+ */
+const nextEntry = (nextUrl) => {
+	saveProgress(items.length, () => {
+		redirect(nextUrl);
+	});
+};
+
+/**
+ * Show the next or the previous page
+ *
+ * @function flipPage
+ * @param {bool} isNext - Whether we are going to the next page
+ */
+const flipPage = (isNext) => {
+	const curItem = getProp('curItem');
+	const idx = parseInt(curItem.id);
+	const delta = isNext ? 1 : -1;
+	const newIdx = idx + delta;
+
+	toPage(newIdx);
+
+	if (isNext)
+		setProp('flipAnimation', 'right');
+	else
+		setProp('flipAnimation', 'left');
+
+	setTimeout(() => {
+		setProp('flipAnimation', null);
+	}, 500);
+
+	replaceHistory(newIdx);
+	saveProgress(newIdx);
+};
+
+/**
+ * Handle the global keydown events
+ *
+ * @function keyHandler
+ * @param {event} event - The $event object
+ */
+const keyHandler = (event) => {
+	const mode = getProp('mode');
+	if (mode === 'continuous') return;
+
+	if (event.key === 'ArrowLeft' || event.key === 'k')
+		flipPage(false);
+	if (event.key === 'ArrowRight' || event.key === 'j')
+		flipPage(true);
 };

public/js/title.js

@@ -55,7 +55,7 @@ function showModal(encodedPath, pages, percentage, encodedeTitle, encodedEntryTi
 
 	$('#modal-edit-btn').attr('onclick', `edit("${entryID}")`);
 
-	$('#modal-download-btn').attr('href', `/opds/download/${titleID}/${entryID}`);
+	$('#modal-download-btn').attr('href', `${base_url}opds/download/${titleID}/${entryID}`);
 
 	UIkit.modal($('#modal')).show();
 }

shard.lock

@@ -34,7 +34,7 @@ shards:
 
   image_size:
     github: hkalexling/image_size.cr
-    version: 0.1.1
+    version: 0.4.0
 
   kemal:
     github: kemalcr/kemal

shard.yml

@@ -1,5 +1,5 @@
 name: mango
-version: 0.12.0
+version: 0.16.0
 
 authors:
   - Alex Ling <hkalexling@gmail.com>

src/config.cr

@@ -11,13 +11,15 @@ class Config
   property library_path : String = File.expand_path "~/mango/library",
     home: true
   property db_path : String = File.expand_path "~/mango/mango.db", home: true
-  @[YAML::Field(key: "scan_interval_minutes")]
-  property scan_interval : Int32 = 5
+  property scan_interval_minutes : Int32 = 5
+  property thumbnail_generation_interval_hours : Int32 = 24
+  property db_optimization_interval_hours : Int32 = 24
   property log_level : String = "info"
   property upload_path : String = File.expand_path "~/mango/uploads",
     home: true
   property plugin_path : String = File.expand_path "~/mango/plugins",
     home: true
+  property download_timeout_seconds : Int32 = 30
   property mangadex = Hash(String, String | Int32).new
 
   @[YAML::Field(ignore: true)]

src/library/entry.cr

@@ -69,7 +69,7 @@ class Entry
 
   def cover_url
     return "#{Config.current.base_url}img/icon.png" if @err_msg
-    url = "#{Config.current.base_url}api/page/#{@book.id}/#{@id}/1"
+    url = "#{Config.current.base_url}api/cover/#{@book.id}/#{@id}"
     TitleInfo.new @book.dir do |info|
       info_url = info.entry_cover_url[@title]?
       unless info_url.nil? || info_url.empty?
@@ -118,8 +118,8 @@ class Entry
             "width"  => size.width,
             "height" => size.height,
           }
-        rescue
-          Logger.warn "Failed to read page #{i} of entry #{@id}"
+        rescue e
+          Logger.warn "Failed to read page #{i} of entry #{zip_path}. #{e}"
           sizes << {"width" => 1000_i32, "height" => 1000_i32}
         end
       end
@@ -207,4 +207,33 @@ class Entry
   def started?(username)
     load_progress(username) > 0
   end
+
+  def generate_thumbnail : Image?
+    return if @err_msg
+
+    img = read_page(1).not_nil!
+    begin
+      size = ImageSize.get img.data
+      if size.height > size.width
+        thumbnail = ImageSize.resize img.data, width: 200
+      else
+        thumbnail = ImageSize.resize img.data, height: 300
+      end
+      img.data = thumbnail
+      img.size = thumbnail.size
+      unless img.mime == "image/webp"
+        # image_size.cr resizes non-webp images to jpg
+        img.mime = "image/jpeg"
+      end
+      Storage.default.save_thumbnail @id, img
+    rescue e
+      Logger.warn "Failed to generate thumbnail for file #{@zip_path}. #{e}"
+    end
+
+    img
+  end
+
+  def get_thumbnail : Image?
+    Storage.default.get_thumbnail @id
+  end
 end

src/library/library.cr

@@ -1,5 +1,5 @@
 class Library
-  property dir : String, title_ids : Array(String), scan_interval : Int32,
+  property dir : String, title_ids : Array(String),
     title_hash : Hash(String, Title)
 
   use_default
@@ -8,20 +8,48 @@ class Library
     register_mime_types
 
     @dir = Config.current.library_path
-    @scan_interval = Config.current.scan_interval
     # explicitly initialize @titles to bypass the compiler check. it will
     #   be filled with actual Titles in the `scan` call below
     @title_ids = [] of String
     @title_hash = {} of String => Title
 
-    return scan if @scan_interval < 1
-    spawn do
-      loop do
-        start = Time.local
-        scan
-        ms = (Time.local - start).total_milliseconds
-        Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
-        sleep @scan_interval * 60
+    @entries_count = 0
+    @thumbnails_count = 0
+
+    scan_interval = Config.current.scan_interval_minutes
+    if scan_interval < 1
+      scan
+    else
+      spawn do
+        loop do
+          start = Time.local
+          scan
+          ms = (Time.local - start).total_milliseconds
+          Logger.info "Scanned #{@title_ids.size} titles in #{ms}ms"
+          sleep scan_interval.minutes
+        end
+      end
+    end
+
+    thumbnail_interval = Config.current.thumbnail_generation_interval_hours
+    unless thumbnail_interval < 1
+      spawn do
+        loop do
+          # Wait for scan to complete (in most cases)
+          sleep 1.minutes
+          generate_thumbnails
+          sleep thumbnail_interval.hours
+        end
+      end
+    end
+
+    db_interval = Config.current.db_optimization_interval_hours
+    unless db_interval < 1
+      spawn do
+        loop do
+          Storage.default.optimize
+          sleep db_interval.hours
+        end
       end
     end
   end
@@ -194,4 +222,50 @@ class Library
       .sample(ENTRIES_IN_HOME_SECTIONS)
       .shuffle
   end
+
+  def thumbnail_generation_progress
+    return 0 if @entries_count == 0
+    @thumbnails_count / @entries_count
+  end
+
+  def generate_thumbnails
+    if @thumbnails_count > 0
+      Logger.debug "Thumbnail generation in progress"
+      return
+    end
+
+    Logger.info "Starting thumbnail generation"
+    entries = deep_titles.map(&.deep_entries).flatten.reject &.err_msg
+    @entries_count = entries.size
+    @thumbnails_count = 0
+
+    # Report generation progress regularly
+    spawn do
+      loop do
+        unless @thumbnails_count == 0
+          Logger.debug "Thumbnail generation progress: " \
+                       "#{(thumbnail_generation_progress * 100).round 1}%"
+        end
+        # Generation is completed. We reset the count to 0 to allow subsequent
+        #   calls to the function, and break from the loop to stop the progress
+        #   report fiber
+        if thumbnail_generation_progress.to_i == 1
+          @thumbnails_count = 0
+          break
+        end
+        sleep 10.seconds
+      end
+    end
+
+    entries.each do |e|
+      unless e.get_thumbnail
+        e.generate_thumbnail
+        # Sleep after each generation to minimize the impact on disk IO
+        #   and CPU
+        sleep 0.5.seconds
+      end
+      @thumbnails_count += 1
+    end
+    Logger.info "Thumbnail generation finished"
+  end
 end

src/library/types.cr

@@ -57,6 +57,16 @@ struct Image
 
   def initialize(@data, @mime, @filename, @size)
   end
+
+  def self.from_db(res : DB::ResultSet)
+    img = Image.allocate
+    res.read String
+    img.data = res.read Bytes
+    img.filename = res.read String
+    img.mime = res.read String
+    img.size = res.read Int32
+    img
+  end
 end
 
 class TitleInfo

src/mango.cr

@@ -7,7 +7,7 @@ require "option_parser"
 require "clim"
 require "./plugin/*"
 
-MANGO_VERSION = "0.12.0"
+MANGO_VERSION = "0.16.0"
 
 # From http://www.network-science.de/ascii/
 BANNER = %{

src/routes/api.cr

@@ -26,6 +26,28 @@ class APIRouter < Router
       end
     end
 
+    get "/api/cover/:tid/:eid" do |env|
+      begin
+        tid = env.params.url["tid"]
+        eid = env.params.url["eid"]
+
+        title = @context.library.get_title tid
+        raise "Title ID `#{tid}` not found" if title.nil?
+        entry = title.get_entry eid
+        raise "Entry ID `#{eid}` of `#{title.title}` not found" if entry.nil?
+
+        img = entry.get_thumbnail || entry.read_page 1
+        raise "Failed to get cover of `#{title.title}/#{entry.title}`" \
+           if img.nil?
+
+        send_img env, img
+      rescue e
+        @context.error e
+        env.response.status_code = 500
+        e.message
+      end
+    end
+
     get "/api/book/:tid" do |env|
       begin
         tid = env.params.url["tid"]
@@ -54,6 +76,18 @@ class APIRouter < Router
       }.to_json
     end
 
+    get "/api/admin/thumbnail_progress" do |env|
+      send_json env, {
+        "progress" => Library.default.thumbnail_generation_progress,
+      }.to_json
+    end
+
+    post "/api/admin/generate_thumbnails" do |env|
+      spawn do
+        Library.default.generate_thumbnails
+      end
+    end
+
     post "/api/admin/user/delete/:username" do |env|
       begin
         username = env.params.url["username"]

src/routes/reader.cr

@@ -12,7 +12,7 @@ class ReaderRouter < Router
         next layout "reader-error" if entry.err_msg
 
         # load progress
-        page = entry.load_progress username
+        page = [1, entry.load_progress username].max
 
         # start from page 1 if the user has finished reading the entry
         page = 1 if entry.finished? username

src/storage.cr

@@ -35,9 +35,11 @@ class Storage
     MainFiber.run do
       DB.open "sqlite3://#{@path}" do |db|
         begin
-          # We create the `ids` table first. even if the uses has an
-          #   early version installed and has the `user` table only,
-          #   we will still be able to create `ids`
+          db.exec "create table thumbnails " \
+                  "(id text, data blob, filename text, " \
+                  "mime text, size integer)"
+          db.exec "create unique index tn_index on thumbnails (id)"
+
           db.exec "create table ids" \
                   "(path text, id text, is_title integer)"
           db.exec "create unique index path_idx on ids (path)"
@@ -243,6 +245,58 @@ class Storage
     end
   end
 
+  def save_thumbnail(id : String, img : Image)
+    MainFiber.run do
+      get_db do |db|
+        db.exec "insert into thumbnails values (?, ?, ?, ?, ?)", id, img.data,
+          img.filename, img.mime, img.size
+      end
+    end
+  end
+
+  def get_thumbnail(id : String) : Image?
+    img = nil
+    MainFiber.run do
+      get_db do |db|
+        db.query_one? "select * from thumbnails where id = (?)", id do |res|
+          img = Image.from_db res
+        end
+      end
+    end
+    img
+  end
+
+  def optimize
+    MainFiber.run do
+      Logger.info "Starting DB optimization"
+      get_db do |db|
+        trash_ids = [] of String
+        db.query "select path, id from ids" do |rs|
+          rs.each do
+            path = rs.read String
+            trash_ids << rs.read String unless File.exists? path
+          end
+        end
+
+        # Delete dangling IDs
+        db.exec "delete from ids where id in " \
+                "(#{trash_ids.map { |i| "'#{i}'" }.join ","})"
+        Logger.debug "#{trash_ids.size} dangling IDs deleted" \
+           if trash_ids.size > 0
+
+        # Delete dangling thumbnails
+        trash_thumbnails_count = db.query_one "select count(*) from " \
+                                              "thumbnails where id not in " \
+                                              "(select id from ids)", as: Int32
+        if trash_thumbnails_count > 0
+          db.exec "delete from thumbnails where id not in (select id from ids)"
+          Logger.info "#{trash_thumbnails_count} dangling thumbnails deleted"
+        end
+      end
+      Logger.info "DB optimization finished"
+    end
+  end
+
   def close
     MainFiber.run do
       unless @db.nil?

src/util/proxy.cr

@@ -5,7 +5,7 @@ require "http_proxy"
 module HTTP
   class Client
     private def self.exec(uri : URI, tls : TLSContext = nil)
-      Logger.debug "Using monkey-patched HTTP::Client"
+      Logger.debug "Setting proxy"
       previous_def uri, tls do |client, path|
         client.set_proxy get_proxy uri
         yield client, path

src/util/web.cr

@@ -81,3 +81,15 @@ macro get_sort_opt
     sort_opt = SortOptions.new sort_method, is_ascending
   end
 end
+
+module HTTP
+  class Client
+    private def self.exec(uri : URI, tls : TLSContext = nil)
+      Logger.debug "Setting read timeout"
+      previous_def uri, tls do |client, path|
+        client.read_timeout = Config.current.download_timeout_seconds.seconds
+        yield client, path
+      end
+    end
+  end
+end

src/views/admin.html.ecr

@@ -1,11 +1,17 @@
-<ul class="uk-list uk-list-large uk-list-divider">
-  <li data-url="<%= base_url %>admin/user">User Managerment</li>
-  <li onclick="if(!scanning){scan()}">
-    <span id="scan">Scan Library Files</span>
-    <span id="scan-status" class="uk-align-right">
-      <div uk-spinner hidden></div>
-      <span hidden></span>
-    </span>
+<ul class="uk-list uk-list-large uk-list-divider" id="root" x-data="{progress : 1.0, generating : false, scanTitles: 0, scanMs: -1, scanning : false}">
+  <li @click="location.href = '<%= base_url %>admin/user'">User Managerment</li>
+  <li :class="{'nopointer' : scanning}" @click="scan()">
+    <span :style="`${scanning ? 'color:grey' : ''}`">Scan Library Files</span>
+    <div class="uk-align-right">
+      <div uk-spinner x-show="scanning"></div>
+      <span x-show="!scanning && scanMs > 0" x-text="`Scan ${scanTitles} titles in ${scanMs}ms`"></span>
+    </div>
+  </li>
+  <li :class="{'nopointer' : generating}" @click="generateThumbnails()">
+    <span :style="`${generating ? 'color:grey' : ''}`">Generate Thumbnails</span>
+    <div class="uk-align-right">
+      <span x-show="generating && progress > 0" x-text="`${(progress * 100).toFixed(2)}%`"></span>
+    </div>
   </li>
   <li class="nopointer">
     <span>Theme</span>

src/views/components/dots-scripts.html.ecr

@@ -0,0 +1,3 @@
+<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/protonet-jquery.inview/1.1.2/jquery.inview.min.js"></script>
+<script src="<%= base_url %>js/dots.js"></script>

src/views/components/head.html.ecr

@@ -7,9 +7,12 @@
   <link rel="stylesheet" href="<%= base_url %>css/uikit.css" />
   <link rel="stylesheet" href="<%= base_url %>css/mango.css" />
   <link rel="icon" href="<%= base_url %>favicon.ico">
+
+  <script src="https://polyfill.io/v3/polyfill.min.js?features=matchMedia%2Cdefault&flags=gated"></script>
   <script defer src="<%= base_url %>js/fontawesome.min.js"></script>
   <script defer src="<%= base_url %>js/solid.min.js"></script>
   <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
-  <script src="https://cdn.jsdelivr.net/gh/alpinejs/alpine@v2.5.0/dist/alpine.min.js" defer></script>
+  <script type="module" src="https://cdn.jsdelivr.net/gh/alpinejs/alpine@v2.5.0/dist/alpine.min.js"></script>
+  <script nomodule src="https://cdn.jsdelivr.net/gh/alpinejs/alpine@v2.5.0/dist/alpine-ie11.min.js" defer></script>
   <script src="<%= base_url %>js/theme.js"></script>
 </head>

src/views/home.html.ecr

@@ -11,7 +11,7 @@
       <dd>Update <code>config.yml</code> located at: <code><%= Config.current.path %></code></dd>
       <dt style="font-weight: 500;">Can't see your files yet?</dt>
       <dd>
-      You must wait <%= Config.current.scan_interval %> minutes for the library scan to complete
+      You must wait <%= Config.current.scan_interval_minutes %> minutes for the library scan to complete
       <% if is_admin %>
         , or manually re-scan from <a href="<%= base_url %>admin">Admin</a>
       <% end %>.
@@ -77,8 +77,7 @@
 <%- end -%>
 
 <% content_for "script" do %>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
-  <script src="<%= base_url %>js/dots.js"></script>
+  <%= render_component "dots-scripts" %>
   <script src="<%= base_url %>js/alert.js"></script>
   <script src="<%= base_url %>js/title.js"></script>
 <% end %>

src/views/library.html.ecr

@@ -24,8 +24,7 @@
 </div>
 
 <% content_for "script" do %>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
-  <script src="<%= base_url %>js/dots.js"></script>
+  <%= render_component "dots-scripts" %>
   <script src="<%= base_url %>js/search.js"></script>
   <script src="<%= base_url %>js/sort-items.js"></script>
 <% end %>

src/views/reader.html.ecr

@@ -3,41 +3,69 @@
 
   <%= render_component "head" %>
 
-  <body>
-    <div class="uk-section uk-section-default uk-section-small reader-bg">
+  <body style="position:relative;">
+    <div class="uk-section uk-section-default uk-section-small reader-bg"
+         id="root" 
+         :style="mode === 'continuous' ? '' : 'padding:0'"
+         x-data="{
+         loading: true, 
+         mode: 'continuous', // can be 'continuous', 'height' or 'width'
+         msg: 'Loading the web reader. Please wait...',
+         alertClass: 'uk-alert-primary',
+         items: [],
+         curItem: {},
+         flipAnimation: null
+         }">
+
+      <div @keydown.window.debounce="keyHandler($event)"></div>
+
       <div class="uk-container uk-container-small">
         <div id="alert"></div>
-        <div id="root" x-data="{
-        loading: true, 
-        msg: 'Loading the web reader. Please wait...',
-        alertClass: 'uk-alert-primary',
-        items: []
-        }">
-          <div x-show="loading">
-            <div :class="alertClass" x-show="msg" uk-alert>
-              <p x-text="msg"></p>
-            </div>
-          </div>
-          <div x-show="!loading" x-cloak>
-            <template x-for="item in items">
-              <img
-                      uk-img
-                      class="uk-align-center"
-                      :data-src="item.url"
-                      :width="item.width"
-                      :height="item.height"
-                      :id="item.id"
-                      @click="showControl($event)"
-                      />
-            </template>
-            <%- if next_entry_url -%>
-              <button id="next-btn" class="uk-align-center uk-button uk-button-primary" @click="redirect('<%= next_entry_url %>')">Next Entry</button>
-            <%- else -%>
-              <button id="next-btn" class="uk-align-center uk-button uk-button-primary" @click="redirect('<%= exit_url %>')">Exit Reader</button>
-            <%- end -%>
+        <div x-show="loading">
+          <div :class="alertClass" x-show="msg" uk-alert>
+            <p x-text="msg"></p>
           </div>
         </div>
       </div>
+
+      <div
+               :class="{'uk-container': true, 'uk-container-small': mode === 'continuous', 'uk-container-expand': mode !== 'continuous'}">
+        <div x-show="!loading && mode === 'continuous'" x-cloak>
+          <template x-for="item in items">
+            <img
+                    uk-img
+                    class="uk-align-center"
+                    :data-src="item.url"
+                    :width="item.width"
+                    :height="item.height"
+                    :id="item.id"
+                    @click="showControl($event)"
+                    />
+          </template>
+          <%- if next_entry_url -%>
+            <button id="next-btn" class="uk-align-center uk-button uk-button-primary" @click="nextEntry('<%= next_entry_url %>')">Next Entry</button>
+          <%- else -%>
+            <button id="next-btn" class="uk-align-center uk-button uk-button-primary" @click="redirect('<%= exit_url %>')">Exit Reader</button>
+          <%- end -%>
+        </div>
+
+        <div x-cloak x-show="!loading && mode !== 'continuous'" class="uk-flex uk-flex-middle" style="height:100vh">
+
+          <img uk-img :class="{
+          'uk-align-center': true, 
+          'uk-animation-slide-left': flipAnimation === 'left', 
+          'uk-animation-slide-right': flipAnimation === 'right'
+          }" :data-src="curItem.url" :width="curItem.width" :height="curItem.height" :id="curItem.id" @click="showControl($event)" :style="`
+          width:${mode === 'width' ? '100vw' : 'auto'}; 
+          height:${mode === 'height' ? '100vh' : 'auto'}; 
+          margin-bottom:0;
+          `" />
+
+          <div style="position:absolute;z-index:1; top:0;left:0; width:30%;height:100%;" @click="flipPage(false)"></div>
+          <div style="position:absolute;z-index:1; top:0;right:0; width:30%;height:100%;" @click="flipPage(true)"></div>
+        </div>
+
+      </div>
     </div>
 
     <div id="modal-sections" class="uk-flex-top" uk-modal>
@@ -52,7 +80,7 @@
             <p id="progress-label"></p>
           </div>
           <div class="uk-margin">
-            <label class="uk-form-label" for="form-stacked-select">Jump to page</label>
+            <label class="uk-form-label" for="page-select">Jump to page</label>
             <div class="uk-form-controls">
               <select id="page-select" class="uk-select">
                 <%- (1..entry.pages).each do |p| -%>
@@ -61,6 +89,15 @@
               </select>
             </div>
           </div>
+          <div class="uk-margin">
+            <label class="uk-form-label" for="mode-select">Mode</label>
+            <div class="uk-form-controls">
+              <select id="mode-select" class="uk-select">
+                <option value="continuous">Continuous</option>
+                <option value="paged">Paged</option>
+              </select>
+            </div>
+          </div>
         </div>
         <div class="uk-modal-footer uk-text-right">
           <button class="uk-button uk-button-danger" type="button" onclick="redirect('<%= exit_url %>')">Exit Reader</button>

src/views/title.html.ecr

@@ -117,8 +117,7 @@
 </div>
 
 <% content_for "script" do %>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery.dotdotdot/4.0.11/dotdotdot.js"></script>
-  <script src="<%= base_url %>js/dots.js"></script>
+  <%= render_component "dots-scripts" %>
   <script src="<%= base_url %>js/alert.js"></script>
   <script src="<%= base_url %>js/title.js"></script>
   <script src="<%= base_url %>js/search.js"></script>