diff --git a/tasks/download.ts b/tasks/download.ts index 9417c5c..5e89bde 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -1,7 +1,7 @@ import { assert } from "std/testing/asserts.ts"; import { Client } from "../client.ts"; import { Config } from "../config.ts"; -import { EhDb } from "../db.ts"; +import { EhDb, EhFile, PMeta } from "../db.ts"; import { Task, TaskDownloadProgess, TaskType } from "../task.ts"; import { TaskManager } from "../task_manager.ts"; import { @@ -73,7 +73,7 @@ class DownloadManager { this.#progress, ); } - async add_new_task(f: () => Promise) { + async add_new_task(f: () => Promise) { while (1) { if (this.#abort.aborted) break; await this.#check_tasks(); @@ -100,6 +100,20 @@ class DownloadManager { } } +interface Image { + data: unknown; + index: number; + is_original: boolean | undefined; + name: string; + page_token: string; + get_file(path: string): EhFile | undefined; + get_original_file(path: string): EhFile | undefined; + load(): Promise; + load_image(reload?: boolean): Promise; + load_original_image(): Promise; + to_pmeta(): PMeta | undefined; +} + export async function download_task( task: Task, client: Client, @@ -149,6 +163,124 @@ export async function download_task( ? dcfg.remove_previous_gallery : cfg.remove_previous_gallery; const g = await client.fetchGalleryPage(task.gid, task.token); + async function download_task(names: Record, i: Image) { + const ofiles = db.get_files(i.page_token); + if (ofiles.length) { + const t = ofiles[0]; + if ( + (t.is_original || !download_original_img) && + (await exists(t.path)) + ) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, + i.page_token, + ); + if (op) { + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } else { + const ops = db.get_pmeta_by_token_only( + i.page_token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } + } + } + console.log("Already download page", i.index); + return; + } + } + function load() { + return new Promise((resolve, reject) => { + const errors: unknown[] = []; + function try_load(a: number) { + if (a >= max_retry_count) reject(errors); + i.load().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_load(a + 1); + }); + } + try_load(0); + }); + } + await load(); + assert(i.data); + const pmeta = i.to_pmeta(); + if (pmeta) db.add_pmeta(pmeta); + const download_original = download_original_img && + !i.is_original; + let path = resolve(join(base_path, i.name)); + if (names[i.name] > 1) { + path = add_suffix_to_path(path, i.page_token); + console.log("Changed path to", path); + } + function download_img() { + return new Promise((resolve, reject) => { + async function download() { + const re = await (download_original + ? i.load_original_image() + : i.load_image()); + if (re === undefined) { + throw Error("Failed to fetch image."); + } + if (re.body === null) { + throw Error("Response don't have a body."); + } + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, + }); + try { + await re.body.pipeTo(f.writable, { + signal: force_abort, + preventClose: true, + }); + } finally { + try { + f.close(); + } catch (_) { + null; + } + } + } + const errors: unknown[] = []; + function try_download(a: number) { + if (a >= max_retry_count) { + reject(errors); + } + download().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_download(a + 1); + }); + } + try_download(0); + }); + } + await download_img(); + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + db.add_file(f); + return; + } if (mpv_enabled || g.mpv_enabled) { const mpv = await client.fetchMPVPage(task.gid, task.token); m.set_total_page(mpv.pagecount); @@ -162,123 +294,7 @@ export async function download_task( for (const i of mpv.imagelist) { if (abort.aborted) break; await m.add_new_task(async () => { - const ofiles = db.get_files(i.page_token); - if (ofiles.length) { - const t = ofiles[0]; - if ( - (t.is_original || !download_original_img) && - (await exists(t.path)) - ) { - const p = db.get_pmeta_by_index(task.gid, i.index); - if (!p) { - const op = db.get_pmeta_by_token( - task.gid, - i.page_token, - ); - if (op) { - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } else { - const ops = db.get_pmeta_by_token_only( - i.page_token, - ); - if (ops.length) { - const op = ops[0]; - op.gid = task.gid; - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } - } - } - console.log("Already download page", i.index); - return; - } - } - function load() { - return new Promise((resolve, reject) => { - const errors: unknown[] = []; - function try_load(a: number) { - if (a >= max_retry_count) reject(errors); - i.load().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_load(a + 1); - }); - } - try_load(0); - }); - } - await load(); - assert(i.data); - const pmeta = i.to_pmeta(); - if (pmeta) db.add_pmeta(pmeta); - const download_original = download_original_img && - !i.is_original; - if (download_original) console.log(i.index, i.data.o); - let path = resolve(join(base_path, i.name)); - if (names[i.name] > 1) { - path = add_suffix_to_path(path, i.page_token); - console.log("Changed path to", path); - } - function download_img() { - return new Promise((resolve, reject) => { - async function download() { - const re = await (download_original - ? i.load_original_image() - : i.load_image()); - if (re === undefined) { - throw Error("Failed to fetch image."); - } - if (re.body === null) { - throw Error("Response don't have a body."); - } - const f = await Deno.open(path, { - create: true, - write: true, - truncate: true, - }); - try { - await re.body.pipeTo(f.writable, { - signal: force_abort, - preventClose: true, - }); - } finally { - try { - f.close(); - } catch (_) { - null; - } - } - } - const errors: unknown[] = []; - function try_download(a: number) { - if (a >= max_retry_count) { - reject(errors); - } - download().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_download(a + 1); - }); - } - try_download(0); - }); - } - await download_img(); - const f = download_original - ? i.get_original_file(path) - : i.get_file(path); - if (f === undefined) throw Error("Failed to get file."); - db.add_file(f); - return; + await download_task(names, i); }); } } else { @@ -294,122 +310,7 @@ export async function download_task( for (const i of imagelist) { if (abort.aborted) break; await m.add_new_task(async () => { - const ofiles = db.get_files(i.page_token); - if (ofiles.length) { - const t = ofiles[0]; - if ( - (t.is_original || !download_original_img) && - (await exists(t.path)) - ) { - const p = db.get_pmeta_by_index(task.gid, i.index); - if (!p) { - const op = db.get_pmeta_by_token( - task.gid, - i.page_token, - ); - if (op) { - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } else { - const ops = db.get_pmeta_by_token_only( - i.page_token, - ); - if (ops.length) { - const op = ops[0]; - op.gid = task.gid; - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } - } - } - console.log("Already download page", i.index); - return; - } - } - function load() { - return new Promise((resolve, reject) => { - const errors: unknown[] = []; - function try_load(a: number) { - if (a >= max_retry_count) reject(errors); - i.load().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_load(a + 1); - }); - } - try_load(0); - }); - } - await load(); - assert(i.data); - const pmeta = i.to_pmeta(); - if (pmeta) db.add_pmeta(pmeta); - const download_original = download_original_img && - !i.is_original; - let path = resolve(join(base_path, i.name)); - if (names[i.name] > 1) { - path = add_suffix_to_path(path, i.page_token); - console.log("Changed path to", path); - } - function download_img() { - return new Promise((resolve, reject) => { - async function download() { - const re = await (download_original - ? i.load_original_image() - : i.load_image()); - if (re === undefined) { - throw Error("Failed to fetch image."); - } - if (re.body === null) { - throw Error("Response don't have a body."); - } - const f = await Deno.open(path, { - create: true, - write: true, - truncate: true, - }); - try { - await re.body.pipeTo(f.writable, { - signal: force_abort, - preventClose: true, - }); - } finally { - try { - f.close(); - } catch (_) { - null; - } - } - } - const errors: unknown[] = []; - function try_download(a: number) { - if (a >= max_retry_count) { - reject(errors); - } - download().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_download(a + 1); - }); - } - try_download(0); - }); - } - await download_img(); - const f = download_original - ? i.get_original_file(path) - : i.get_file(path); - if (f === undefined) throw Error("Failed to get file."); - db.add_file(f); - return; + await download_task(names, i); }); } }