diff --git a/client.ts b/client.ts index 1bd9c95..cc353c2 100644 --- a/client.ts +++ b/client.ts @@ -94,10 +94,17 @@ export class Client { * @param gid Gallery ID * @param page_token Page token * @param index Page index + * @param nl Reload token * @returns */ - async fetchSignlePage(gid: number, page_token: string, index: number) { - const url = `https://${this.host}/s/${page_token}/${gid}-${index}`; + async fetchSignlePage( + gid: number, + page_token: string, + index: number, + nl?: string, + ) { + const p = nl ? `?nl=${nl}` : ""; + const url = `https://${this.host}/s/${page_token}/${gid}-${index}${p}`; const re = await this.get(url); if (re.status != 200) { throw new Error( diff --git a/page/GalleryPage.ts b/page/GalleryPage.ts index 67a96ae..2e7eba5 100644 --- a/page/GalleryPage.ts +++ b/page/GalleryPage.ts @@ -1,5 +1,6 @@ import { DOMParser, Element } from "deno_dom/deno-dom-wasm-noinit.ts"; import { Client } from "../client.ts"; +import { EhFile, PMeta } from "../db.ts"; import { initDOMParser, map, parse_bool } from "../utils.ts"; import { parseUrl, UrlType } from "../url.ts"; import { SinglePage } from "./SinglePage.ts"; @@ -15,13 +16,131 @@ class Image { base; #gp; data: SinglePage | undefined; + redirected_url: string | undefined; constructor(base: Page, gp: GalleryPage) { this.base = base; this.#gp = gp; } + get_file(path: string): EhFile | undefined { + const width = this.xres; + if (width === undefined) return undefined; + const height = this.yres; + if (height === undefined) return undefined; + const is_original = this.is_original; + if (is_original === undefined) return undefined; + return { + id: 0, + token: this.page_token, + path, + width, + height, + is_original, + }; + } + get_original_file(path: string): EhFile | undefined { + const width = this.origin_xres; + if (width === undefined) return undefined; + const height = this.origin_yres; + if (height === undefined) return undefined; + return { + id: 0, + token: this.page_token, + path, + width, + height, + is_original: true, + }; + } + get index() { + return this.base.index; + } get is_original() { return this.data?.is_original; } + get name() { + return this.base.name; + } + get original_imgurl() { + return this.data?.original_url; + } + get origin_xres() { + return this.data?.origin_xres; + } + get origin_yres() { + return this.data?.origin_yres; + } + get page_number() { + return this.base.index; + } + get page_token() { + return this.base.token; + } + get src() { + return this.data?.img_url; + } + get xres() { + return this.data?.xres; + } + get yres() { + return this.data?.yres; + } + async load() { + if (this.data === undefined) { + this.data = await this.#gp.client.fetchSignlePage( + this.#gp.gid, + this.page_token, + this.base.index, + ); + } else { + this.data = await this.#gp.client.fetchSignlePage( + this.#gp.gid, + this.page_token, + this.base.index, + this.data.nl, + ); + } + } + async #load_image(u: string) { + const re = await this.#gp.client.get(u); + if (re.status !== 200) { + re.body?.cancel(); + return undefined; + } + return re; + } + async load_image(reload = true) { + const src = this.src; + if (src) { + const re = await this.#load_image(src); + if (re) return re; + } + if (!reload) return; + await this.load(); + const src2 = this.src; + if (src2) return await this.#load_image(src2); + } + async load_original_image() { + if (this.redirected_url) { + const re = await this.#load_image(this.redirected_url); + if (re) return re; + } + const url = this.original_imgurl; + if (!url) return undefined; + this.redirected_url = await this.#gp.client.redirect(url); + return await this.#load_image(this.redirected_url || url); + } + to_pmeta(): PMeta | undefined { + if (!this.data) return undefined; + const gid = this.#gp.gid; + const index = this.base.index; + const token = this.page_token; + const name = this.name; + const width = this.origin_xres; + if (width === undefined) return undefined; + const height = this.origin_yres; + if (height === undefined) return undefined; + return { gid, index, token, name, width, height }; + } } class GalleryPage { diff --git a/page/SinglePage.ts b/page/SinglePage.ts index 1f8a951..5f99b7c 100644 --- a/page/SinglePage.ts +++ b/page/SinglePage.ts @@ -80,7 +80,7 @@ export class SinglePage { return url; } get is_original() { - return this.original_url === null; + return this.original_url === undefined; } get meta() { if (this._meta === undefined) { @@ -108,6 +108,15 @@ export class SinglePage { if (a === null) return null; return a.getAttribute("href"); } + get nl() { + const f = this.doc.getElementById("loadfail"); + if (!f) throw Error("Failed to find loadfail link."); + const n = f.getAttribute("onclick"); + if (!n) throw Error("Failed to get onclick attribute."); + const nl = n.match(/nl\('(.*?)'\)/)?.at(1); + if (!nl) throw Error("Failed to extract nl."); + return nl; + } get origin_xres() { if (this.is_original) return this.xres; if (this.#oxres === undefined) { @@ -130,8 +139,8 @@ export class SinglePage { } get original_url() { const a = this.doc.querySelector("#i7 a"); - if (a == null) return null; - return a.getAttribute("href"); + if (a == null) return undefined; + return a.getAttribute("href") || undefined; } get pageCount() { const e = this.doc.querySelector("#i2>div span:last-child"); diff --git a/page/SinglePage_test.ts b/page/SinglePage_test.ts index 7c1585e..d03b963 100644 --- a/page/SinglePage_test.ts +++ b/page/SinglePage_test.ts @@ -37,4 +37,5 @@ Deno.test({ assertEquals(re2.is_original, false); assertEquals(re2.origin_xres, 4893); assertEquals(re2.origin_yres, 3446); + console.log(np.nl, re.nl, re2.nl); }); diff --git a/tasks/download.ts b/tasks/download.ts index 104fa4e..9417c5c 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -148,7 +148,8 @@ export async function download_task( const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined ? dcfg.remove_previous_gallery : cfg.remove_previous_gallery; - if (mpv_enabled) { + const g = await client.fetchGalleryPage(task.gid, task.token); + if (mpv_enabled || g.mpv_enabled) { const mpv = await client.fetchMPVPage(task.gid, task.token); m.set_total_page(mpv.pagecount); const names = mpv.imagelist.reduce( @@ -280,6 +281,137 @@ export async function download_task( return; }); } + } else { + m.set_total_page(g.length); + const imagelist = await g.imagelist; + const names = imagelist.reduce( + (acc: Record, cur) => { + const curr = cur.name; + return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc; + }, + {}, + ); + for (const i of imagelist) { + if (abort.aborted) break; + await m.add_new_task(async () => { + const ofiles = db.get_files(i.page_token); + if (ofiles.length) { + const t = ofiles[0]; + if ( + (t.is_original || !download_original_img) && + (await exists(t.path)) + ) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, + i.page_token, + ); + if (op) { + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } else { + const ops = db.get_pmeta_by_token_only( + i.page_token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } + } + } + console.log("Already download page", i.index); + return; + } + } + function load() { + return new Promise((resolve, reject) => { + const errors: unknown[] = []; + function try_load(a: number) { + if (a >= max_retry_count) reject(errors); + i.load().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_load(a + 1); + }); + } + try_load(0); + }); + } + await load(); + assert(i.data); + const pmeta = i.to_pmeta(); + if (pmeta) db.add_pmeta(pmeta); + const download_original = download_original_img && + !i.is_original; + let path = resolve(join(base_path, i.name)); + if (names[i.name] > 1) { + path = add_suffix_to_path(path, i.page_token); + console.log("Changed path to", path); + } + function download_img() { + return new Promise((resolve, reject) => { + async function download() { + const re = await (download_original + ? i.load_original_image() + : i.load_image()); + if (re === undefined) { + throw Error("Failed to fetch image."); + } + if (re.body === null) { + throw Error("Response don't have a body."); + } + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, + }); + try { + await re.body.pipeTo(f.writable, { + signal: force_abort, + preventClose: true, + }); + } finally { + try { + f.close(); + } catch (_) { + null; + } + } + } + const errors: unknown[] = []; + function try_download(a: number) { + if (a >= max_retry_count) { + reject(errors); + } + download().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_download(a + 1); + }); + } + try_download(0); + }); + } + await download_img(); + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + db.add_file(f); + return; + }); + } } await m.join(); if (m.has_failed_task) throw Error("Some tasks failed.");