diff --git a/config.example.jsonc b/config.example.jsonc index 41bcd45..1e3bfa2 100644 --- a/config.example.jsonc +++ b/config.example.jsonc @@ -10,5 +10,8 @@ // Fetch page data from Multi-Page Viewer "mpv": false, // The folder that to store database. - "db_path": "./downloads" + "db_path": "./downloads", + "max_retry_count": 3, + "max_download_img_count": 3, + "download_original_img": true } diff --git a/config.ts b/config.ts index 32224fc..bbb57d1 100644 --- a/config.ts +++ b/config.ts @@ -62,6 +62,15 @@ export class Config { get mpv() { return this._return_bool("mpv") || false; } + get max_retry_count() { + return this._return_number("max_retry_count") || 3; + } + get max_download_img_count() { + return this._return_number("max_download_img_count") || 3; + } + get download_original_img() { + return this._return_bool("download_original_img") || false; + } } export async function load_settings(path: string) { diff --git a/db.ts b/db.ts index 22ecc0e..3219ecc 100644 --- a/db.ts +++ b/db.ts @@ -303,7 +303,7 @@ export class EhDb { this.db.query("INSERT INTO gtag VALUES (?, ?);", [gid, id]); } } - add_file(f: EhFile, overwrite = true) { + add_file(f: EhFile, overwrite = true): EhFile { if (overwrite) { const ofiles = this.get_files(f.gid, f.token); if (ofiles.length) { diff --git a/page/MPVPage.ts b/page/MPVPage.ts index b65e076..006025c 100644 --- a/page/MPVPage.ts +++ b/page/MPVPage.ts @@ -1,7 +1,7 @@ import { DOMParser } from "deno_dom/deno-dom-wasm-noinit.ts"; import { Client } from "../client.ts"; import { initDOMParser } from "../utils.ts"; -import { PMeta } from "../db.ts"; +import { EhFile, PMeta } from "../db.ts"; export type MPVRawImage = { /**Image name*/ @@ -49,6 +49,38 @@ class MPVImage { this.index = index; this.#mpv = mpv; } + get_file(path: string): EhFile | undefined { + const width = this.xres; + if (width === undefined) return undefined; + const height = this.yres; + if (height === undefined) return undefined; + const is_original = this.is_original; + if (is_original === undefined) return undefined; + return { + id: 0, + gid: this.#mpv.gid, + token: this.page_token, + path, + width, + height, + is_original, + }; + } + get_original_file(path: string): EhFile | undefined { + const width = this.origin_xres; + if (width === undefined) return undefined; + const height = this.origin_yres; + if (height === undefined) return undefined; + return { + id: 0, + gid: this.#mpv.gid, + token: this.page_token, + path, + width, + height, + is_original: true, + }; + } get is_original() { const t = this.data?.o; if (!t) return undefined; @@ -130,6 +162,17 @@ class MPVImage { } return re; } + async load_image(reload = true) { + const src = this.src; + if (src) { + const re = await this.#load_image(src); + if (re) return re; + } + if (!reload) return; + await this.load(); + const src2 = this.src; + if (src2) return await this.#load_image(src2); + } async load_original_image() { if (this.redirected_url) { const re = await this.#load_image(this.redirected_url); diff --git a/page/MPVPage_test.ts b/page/MPVPage_test.ts index 523afdb..c4ee7fe 100644 --- a/page/MPVPage_test.ts +++ b/page/MPVPage_test.ts @@ -1,7 +1,6 @@ -import { assertEquals } from "https://deno.land/std@0.188.0/testing/asserts.ts"; +import { assert, assertEquals } from "std/testing/asserts.ts"; import { Client } from "../client.ts"; import { load_settings } from "../config.ts"; -import { assert } from "https://deno.land/std@0.188.0/_util/asserts.ts"; Deno.test("MPVPage_test", async () => { const cfg = await load_settings("./config.json"); diff --git a/tasks/download.ts b/tasks/download.ts index b9a9fd5..574d60f 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -1,7 +1,60 @@ +import { assert } from "std/testing/asserts.ts"; import { Client } from "../client.ts"; import { Config } from "../config.ts"; import { EhDb } from "../db.ts"; import { Task } from "../task.ts"; +import { + asyncFilter, + promiseState, + PromiseStatus, + sleep, + sure_dir, +} from "../utils.ts"; +import { join } from "std/path/mod.ts"; +import { exists } from "std/fs/exists.ts"; + +class DownloadManager { + #max_download_count; + #running_tasks: Promise[]; + #has_failed_task = false; + constructor(cfg: Config) { + this.#max_download_count = cfg.max_download_img_count; + this.#running_tasks = []; + } + async #check_tasks() { + this.#running_tasks = await asyncFilter( + this.#running_tasks, + async (t) => { + const s = await promiseState(t); + if (s.status === PromiseStatus.Rejected) { + console.log(s.reason); + this.#has_failed_task = true; + } + return s.status === PromiseStatus.Pending; + }, + ); + } + async add_new_task(f: () => Promise) { + while (1) { + await this.#check_tasks(); + if (this.#running_tasks.length < this.#max_download_count) { + this.#running_tasks.push(f()); + break; + } + await sleep(10); + } + } + get has_failed_task() { + return this.#has_failed_task; + } + async join() { + while (1) { + await this.#check_tasks(); + if (!this.#running_tasks.length) break; + await sleep(10); + } + } +} export async function download_task( task: Task, @@ -20,8 +73,93 @@ export async function download_task( const gmeta = gdatas.convert(gdata); db.add_gmeta(gmeta); await db.add_gtag(task.gid, new Set(gdata.tags)); + const base_path = join(cfg.base, task.gid.toString()); + await sure_dir(base_path); + const m = new DownloadManager(cfg); if (cfg.mpv) { const mpv = await client.fetchMPVPage(task.gid, task.token); + for (const i of mpv.imagelist) { + await m.add_new_task(async () => { + const ofiles = db.get_files(task.gid, i.page_token); + if (ofiles.length) { + const t = ofiles[0]; + if ( + (t.is_original || !cfg.download_original_img) && + (await exists(t.path)) + ) { + console.log("Already download page", i.index); + return; + } + } + function load() { + return new Promise((resolve, reject) => { + const errors: unknown[] = []; + function try_load(a: number) { + if (a >= cfg.max_retry_count) reject(errors); + i.load().then(resolve).catch((e) => { + errors.push(e); + try_load(a + 1); + }); + } + try_load(0); + }); + } + await load(); + assert(i.data); + const pmeta = i.to_pmeta(); + if (pmeta) db.add_pmeta(pmeta); + const download_original = cfg.download_original_img && + !i.is_original; + if (download_original) console.log(i.index, i.data.o); + const path = join(base_path, i.name); + function download_img() { + return new Promise((resolve, reject) => { + async function download() { + const re = await (download_original + ? i.load_original_image() + : i.load_image()); + if (re === undefined) { + throw Error("Failed to fetch image."); + } + if (re.body === null) { + throw Error("Response don't have a body."); + } + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, + }); + try { + await re.body.pipeTo(f.writable, { + preventClose: true, + }); + } finally { + f.close(); + } + } + const errors: unknown[] = []; + function try_download(a: number) { + if (a >= cfg.max_retry_count) { + reject(errors); + } + download().then(resolve).catch((e) => { + errors.push(e); + try_download(a + 1); + }); + } + try_download(0); + }); + } + await download_img(); + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + db.add_file(f); + return; + }); + } } + await m.join(); return task; } diff --git a/utils.ts b/utils.ts index b018eea..2b39e75 100644 --- a/utils.ts +++ b/utils.ts @@ -77,3 +77,16 @@ export function try_remove_sync( return; } } + +export async function asyncFilter( + arr: T[], + callback: (element: T, index: number, array: T[]) => Promise, +): Promise { + const fail = Symbol(); + const t = <[T][]> (await Promise.all( + arr.map(async (item, index, array) => + (await callback(item, index, array)) ? [item] : fail + ), + )).filter((i) => i !== fail); + return t.map((t) => t[0]); +} diff --git a/utils_test.ts b/utils_test.ts index df5a462..d0d4f12 100644 --- a/utils_test.ts +++ b/utils_test.ts @@ -1,6 +1,6 @@ import { assertEquals } from "std/testing/asserts.ts"; import { check_running } from "./pid_check.ts"; -import { promiseState, sleep } from "./utils.ts"; +import { asyncFilter, promiseState, PromiseStatus, sleep } from "./utils.ts"; Deno.test("promiseState_test", async () => { const p1 = new Promise((res) => setTimeout(() => res(100), 100)); @@ -26,3 +26,19 @@ Deno.test("Pid_Test", async () => { assertEquals(await check_running(Deno.pid), true); } }); + +Deno.test("asyncFilter_test", async () => { + const t = [3, 5]; + const r = await asyncFilter(t, async (t) => { + await sleep(10); + return t === 3; + }); + assertEquals(r, [3]); + const e = [new Promise((res) => setTimeout(() => res(100), 100))]; + const v = await asyncFilter(e, async (t) => { + const s = await promiseState(t); + return s.status === PromiseStatus.Pending; + }); + assertEquals(v, e); + await Promise.allSettled(e); +});