Update

config.example.jsonc

@@ -10,5 +10,8 @@
     // Fetch page data from Multi-Page Viewer
     "mpv": false,
     // The folder that to store database.
-    "db_path": "./downloads"
+    "db_path": "./downloads",
+    "max_retry_count": 3,
+    "max_download_img_count": 3,
+    "download_original_img": true
 }

config.ts

@@ -62,6 +62,15 @@ export class Config {
     get mpv() {
         return this._return_bool("mpv") || false;
     }
+    get max_retry_count() {
+        return this._return_number("max_retry_count") || 3;
+    }
+    get max_download_img_count() {
+        return this._return_number("max_download_img_count") || 3;
+    }
+    get download_original_img() {
+        return this._return_bool("download_original_img") || false;
+    }
 }
 
 export async function load_settings(path: string) {

db.ts

@@ -303,7 +303,7 @@ export class EhDb {
             this.db.query("INSERT INTO gtag VALUES (?, ?);", [gid, id]);
         }
     }
-    add_file(f: EhFile, overwrite = true) {
+    add_file(f: EhFile, overwrite = true): EhFile {
         if (overwrite) {
             const ofiles = this.get_files(f.gid, f.token);
             if (ofiles.length) {

page/MPVPage.ts

@@ -1,7 +1,7 @@
 import { DOMParser } from "deno_dom/deno-dom-wasm-noinit.ts";
 import { Client } from "../client.ts";
 import { initDOMParser } from "../utils.ts";
-import { PMeta } from "../db.ts";
+import { EhFile, PMeta } from "../db.ts";
 
 export type MPVRawImage = {
     /**Image name*/
@@ -49,6 +49,38 @@ class MPVImage {
         this.index = index;
         this.#mpv = mpv;
     }
+    get_file(path: string): EhFile | undefined {
+        const width = this.xres;
+        if (width === undefined) return undefined;
+        const height = this.yres;
+        if (height === undefined) return undefined;
+        const is_original = this.is_original;
+        if (is_original === undefined) return undefined;
+        return {
+            id: 0,
+            gid: this.#mpv.gid,
+            token: this.page_token,
+            path,
+            width,
+            height,
+            is_original,
+        };
+    }
+    get_original_file(path: string): EhFile | undefined {
+        const width = this.origin_xres;
+        if (width === undefined) return undefined;
+        const height = this.origin_yres;
+        if (height === undefined) return undefined;
+        return {
+            id: 0,
+            gid: this.#mpv.gid,
+            token: this.page_token,
+            path,
+            width,
+            height,
+            is_original: true,
+        };
+    }
     get is_original() {
         const t = this.data?.o;
         if (!t) return undefined;
@@ -130,6 +162,17 @@ class MPVImage {
         }
         return re;
     }
+    async load_image(reload = true) {
+        const src = this.src;
+        if (src) {
+            const re = await this.#load_image(src);
+            if (re) return re;
+        }
+        if (!reload) return;
+        await this.load();
+        const src2 = this.src;
+        if (src2) return await this.#load_image(src2);
+    }
     async load_original_image() {
         if (this.redirected_url) {
             const re = await this.#load_image(this.redirected_url);

page/MPVPage_test.ts

@@ -1,7 +1,6 @@
-import { assertEquals } from "https://deno.land/[email protected]/testing/asserts.ts";
+import { assert, assertEquals } from "std/testing/asserts.ts";
 import { Client } from "../client.ts";
 import { load_settings } from "../config.ts";
-import { assert } from "https://deno.land/[email protected]/_util/asserts.ts";
 
 Deno.test("MPVPage_test", async () => {
     const cfg = await load_settings("./config.json");

tasks/download.ts

@@ -1,7 +1,60 @@
+import { assert } from "std/testing/asserts.ts";
 import { Client } from "../client.ts";
 import { Config } from "../config.ts";
 import { EhDb } from "../db.ts";
 import { Task } from "../task.ts";
+import {
+    asyncFilter,
+    promiseState,
+    PromiseStatus,
+    sleep,
+    sure_dir,
+} from "../utils.ts";
+import { join } from "std/path/mod.ts";
+import { exists } from "std/fs/exists.ts";
+
+class DownloadManager {
+    #max_download_count;
+    #running_tasks: Promise<unknown>[];
+    #has_failed_task = false;
+    constructor(cfg: Config) {
+        this.#max_download_count = cfg.max_download_img_count;
+        this.#running_tasks = [];
+    }
+    async #check_tasks() {
+        this.#running_tasks = await asyncFilter(
+            this.#running_tasks,
+            async (t) => {
+                const s = await promiseState(t);
+                if (s.status === PromiseStatus.Rejected) {
+                    console.log(s.reason);
+                    this.#has_failed_task = true;
+                }
+                return s.status === PromiseStatus.Pending;
+            },
+        );
+    }
+    async add_new_task(f: () => Promise<unknown>) {
+        while (1) {
+            await this.#check_tasks();
+            if (this.#running_tasks.length < this.#max_download_count) {
+                this.#running_tasks.push(f());
+                break;
+            }
+            await sleep(10);
+        }
+    }
+    get has_failed_task() {
+        return this.#has_failed_task;
+    }
+    async join() {
+        while (1) {
+            await this.#check_tasks();
+            if (!this.#running_tasks.length) break;
+            await sleep(10);
+        }
+    }
+}
 
 export async function download_task(
     task: Task,
@@ -20,8 +73,93 @@ export async function download_task(
     const gmeta = gdatas.convert(gdata);
     db.add_gmeta(gmeta);
     await db.add_gtag(task.gid, new Set(gdata.tags));
+    const base_path = join(cfg.base, task.gid.toString());
+    await sure_dir(base_path);
+    const m = new DownloadManager(cfg);
     if (cfg.mpv) {
         const mpv = await client.fetchMPVPage(task.gid, task.token);
+        for (const i of mpv.imagelist) {
+            await m.add_new_task(async () => {
+                const ofiles = db.get_files(task.gid, i.page_token);
+                if (ofiles.length) {
+                    const t = ofiles[0];
+                    if (
+                        (t.is_original || !cfg.download_original_img) &&
+                        (await exists(t.path))
+                    ) {
+                        console.log("Already download page", i.index);
+                        return;
+                    }
+                }
+                function load() {
+                    return new Promise<void>((resolve, reject) => {
+                        const errors: unknown[] = [];
+                        function try_load(a: number) {
+                            if (a >= cfg.max_retry_count) reject(errors);
+                            i.load().then(resolve).catch((e) => {
+                                errors.push(e);
+                                try_load(a + 1);
+                            });
+                        }
+                        try_load(0);
+                    });
+                }
+                await load();
+                assert(i.data);
+                const pmeta = i.to_pmeta();
+                if (pmeta) db.add_pmeta(pmeta);
+                const download_original = cfg.download_original_img &&
+                    !i.is_original;
+                if (download_original) console.log(i.index, i.data.o);
+                const path = join(base_path, i.name);
+                function download_img() {
+                    return new Promise<void>((resolve, reject) => {
+                        async function download() {
+                            const re = await (download_original
+                                ? i.load_original_image()
+                                : i.load_image());
+                            if (re === undefined) {
+                                throw Error("Failed to fetch image.");
+                            }
+                            if (re.body === null) {
+                                throw Error("Response don't have a body.");
+                            }
+                            const f = await Deno.open(path, {
+                                create: true,
+                                write: true,
+                                truncate: true,
+                            });
+                            try {
+                                await re.body.pipeTo(f.writable, {
+                                    preventClose: true,
+                                });
+                            } finally {
+                                f.close();
+                            }
+                        }
+                        const errors: unknown[] = [];
+                        function try_download(a: number) {
+                            if (a >= cfg.max_retry_count) {
+                                reject(errors);
+                            }
+                            download().then(resolve).catch((e) => {
+                                errors.push(e);
+                                try_download(a + 1);
+                            });
+                        }
+                        try_download(0);
+                    });
+                }
+                await download_img();
+                const f = download_original
+                    ? i.get_original_file(path)
+                    : i.get_file(path);
+                if (f === undefined) throw Error("Failed to get file.");
+                db.add_file(f);
+                return;
+            });
+        }
     }
+    await m.join();
     return task;
 }

utils.ts

@@ -77,3 +77,16 @@ export function try_remove_sync(
         return;
     }
 }
+
+export async function asyncFilter<T>(
+    arr: T[],
+    callback: (element: T, index: number, array: T[]) => Promise<boolean>,
+): Promise<T[]> {
+    const fail = Symbol();
+    const t = <[T][]> (await Promise.all(
+        arr.map(async (item, index, array) =>
+            (await callback(item, index, array)) ? [item] : fail
+        ),
+    )).filter((i) => i !== fail);
+    return t.map((t) => t[0]);
+}

utils_test.ts

@@ -1,6 +1,6 @@
 import { assertEquals } from "std/testing/asserts.ts";
 import { check_running } from "./pid_check.ts";
-import { promiseState, sleep } from "./utils.ts";
+import { asyncFilter, promiseState, PromiseStatus, sleep } from "./utils.ts";
 
 Deno.test("promiseState_test", async () => {
     const p1 = new Promise((res) => setTimeout(() => res(100), 100));
@@ -26,3 +26,19 @@ Deno.test("Pid_Test", async () => {
         assertEquals(await check_running(Deno.pid), true);
     }
 });
+
+Deno.test("asyncFilter_test", async () => {
+    const t = [3, 5];
+    const r = await asyncFilter(t, async (t) => {
+        await sleep(10);
+        return t === 3;
+    });
+    assertEquals(r, [3]);
+    const e = [new Promise<number>((res) => setTimeout(() => res(100), 100))];
+    const v = await asyncFilter(e, async (t) => {
+        const s = await promiseState(t);
+        return s.status === PromiseStatus.Pending;
+    });
+    assertEquals(v, e);
+    await Promise.allSettled(e);
+});