lifegpc/eh-downloader
1
0
Fork 0
mirror of https://github.com/lifegpc/eh-downloader.git synced 2026-07-08 01:31:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed download timeout

Browse Source
This commit is contained in:
lifegpc
2023-12-01 14:39:13 +08:00
parent d3cf414146
commit 7a6d9c9751
7 changed files with 256 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
client.ts
View File

@@ -8,15 +8,23 @@ import { RecoverableError } from "./task_manager.ts";
export type GID = [number, string];
export class Client {
cookies;
host;
ua;
cfg;
get cookies() {
return this.cfg.cookies;
}
get host() {
return this.cfg.ex ? "exhentai.org" : "e-hentai.org";
}
get ua() {
return this.cfg.ua ||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36";
}
get timeout() {
return this.cfg.fetch_timeout;
}
signal;
constructor(cfg: Config, signal?: AbortSignal) {
this.cookies = cfg.cookies;
this.ua = cfg.ua ||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36";
this.host = cfg.ex ? "exhentai.org" : "e-hentai.org";
this.cfg = cfg;
this.signal = signal;
}
get(
@@ -94,6 +102,15 @@ export class Client {
if (!d.signal && this.signal) {
d.signal = this.signal;
}
const osignal = d.signal;
const abortController = new AbortController();
const timeout = setTimeout(() => {
abortController.abort();
}, this.timeout);
osignal?.addEventListener("abort", () => {
abortController.abort(osignal?.reason);
});
d.signal = abortController.signal;
try {
return fetch(url, d);
} catch (e) {
@@ -101,6 +118,8 @@ export class Client {
throw new RecoverableError(e.message, { cause: e.cause });
}
throw e;
} finally {
clearTimeout(timeout);
}
}
}

10
config.ts
View File

@@ -26,6 +26,8 @@ export type ConfigType = {
meili_hosts?: Record<string, string>;
cors_credentials_hosts: Array<string>;
flutter_frontend?: string;
fetch_timeout: number;
download_timeout: number;
};
export enum ThumbnailMethod {
@@ -164,6 +166,12 @@ export class Config {
get flutter_frontend() {
return this._return_string("flutter_frontend");
}
get fetch_timeout() {
return this._return_number("fetch_timeout") || 10000;
}
get download_timeout() {
return this._return_number("download_timeout") || 10000;
}
to_json(): ConfigType {
return {
cookies: typeof this.cookies === "string",
@@ -190,6 +198,8 @@ export class Config {
meili_hosts: this.meili_hosts,
cors_credentials_hosts: this.cors_credentials_hosts,
flutter_frontend: this.flutter_frontend,
fetch_timeout: this.fetch_timeout,
download_timeout: this.download_timeout,
};
}
}

4
server/utils.ts
View File

@@ -22,7 +22,7 @@ function gen_response<T>(
status = (d.status >= 400 && d.status < 600) ? d.status : 400;
}
const h = new Headers(headers);
h.set("Content-Type", "application/json");
h.set("Content-Type", "application/json; charset=UTF-8");
return new Response(JSON.stringify(d), { status, headers: h });
}
@@ -57,7 +57,7 @@ export function gen_error<T = unknown>(
export function return_json<T = unknown>(data: T, status = 200) {
return new Response(JSON.stringify(data), {
status,
headers: { "Content-Type": "application/json" },
headers: { "Content-Type": "application/json; chatset=UTF-8" },
});
}

254
tasks/download.ts
View File

@@ -16,6 +16,7 @@ import {
PromiseStatus,
sleep,
sure_dir,
TimeoutError,
} from "../utils.ts";
import { join, resolve } from "std/path/mod.ts";
import { exists } from "std/fs/exists.ts";
@@ -109,7 +110,14 @@ class DownloadManager {
return re;
}
add_new_details(d: TaskDownloadSingleProgress) {
this.#progress.details.push(d);
const index = this.#progress.details.findIndex((v) =>
v.index === d.index
);
if (index !== -1) {
this.#progress.details[index] = d;
} else {
this.#progress.details.push(d);
}
this.#sendEvent();
}
async add_new_task<T>(f: () => Promise<T>) {
@@ -165,6 +173,7 @@ interface Image {
is_original: boolean | undefined;
name: string;
page_token: string;
redirected_url: string | undefined;
sampled_name: string;
get_file(path: string): EhFile | undefined;
get_original_file(path: string): EhFile | undefined;
@@ -260,126 +269,189 @@ export async function download_task(
return;
}
}
let load_times = 0;
function load() {
return new Promise<void>((resolve, reject) => {
const errors: unknown[] = [];
function try_load(a: number) {
if (a >= max_retry_count) reject(errors);
function try_load() {
if (load_times >= max_retry_count) reject(errors);
i.load().then(resolve).catch((e) => {
if (force_abort.aborted) {
throw Error("aborted.");
}
errors.push(e);
try_load(a + 1);
load_times += 1;
try_load();
});
}
try_load(0);
try_load();
});
}
await load();
assert(i.data);
const pmeta = i.to_pmeta();
if (pmeta) db.add_pmeta(pmeta);
const download_original = download_original_img &&
!i.is_original;
const is_sampled = !download_original_img && !i.is_original;
let path = resolve(
join(base_path, is_sampled ? i.sampled_name : i.name),
);
if (names[i.name] > 1) {
path = add_suffix_to_path(path, i.page_token);
console.log("Changed path to", path);
}
const f = download_original
? i.get_original_file(path)
: i.get_file(path);
if (f === undefined) throw Error("Failed to get file.");
m.add_new_details({
downloaded: 0,
height: f.height,
index: i.index,
is_original: f.is_original,
name: i.name,
token: i.page_token,
total: 0,
width: f.width,
started: 0,
});
function download_img() {
return new Promise<void>((resolve, reject) => {
async function download() {
const re = await (download_original
? i.load_original_image()
: i.load_image());
if (re === undefined) {
throw Error("Failed to fetch image.");
}
m.set_details_started(i.index);
const len = re.headers.get("Content-Length");
if (len) {
const tmp = parseInt(len);
if (!isNaN(tmp)) {
m.set_details_total(i.index, tmp);
async function deal_with_img() {
assert(i.data);
const pmeta = i.to_pmeta();
if (pmeta) db.add_pmeta(pmeta);
const download_original = download_original_img &&
!i.is_original;
const is_sampled = !download_original_img && !i.is_original;
let path = resolve(
join(base_path, is_sampled ? i.sampled_name : i.name),
);
if (names[i.name] > 1) {
path = add_suffix_to_path(path, i.page_token);
console.log("Changed path to", path);
}
const f = download_original
? i.get_original_file(path)
: i.get_file(path);
if (f === undefined) throw Error("Failed to get file.");
m.add_new_details({
downloaded: 0,
height: f.height,
index: i.index,
is_original: f.is_original,
name: i.name,
token: i.page_token,
total: 0,
width: f.width,
started: 0,
});
function download_img() {
return new Promise<void>((resolve, reject) => {
async function download() {
const re = await (download_original
? i.load_original_image()
: i.load_image());
if (re === undefined) {
throw Error("Failed to fetch image.");
}
}
if (re.body === null) {
throw Error("Response don't have a body.");
}
const pr = new ProgressReadable(re.body);
pr.addEventListener("progress", (e) => {
m.set_details_downloaded(i.index, e.detail);
});
pr.addEventListener("finished", () => {
m.remove_details(i.index);
});
try {
const f = await Deno.open(path, {
create: true,
write: true,
truncate: true,
m.set_details_started(i.index);
const len = re.headers.get("Content-Length");
if (len) {
const tmp = parseInt(len);
if (!isNaN(tmp)) {
m.set_details_total(i.index, tmp);
}
}
if (re.body === null) {
throw Error("Response don't have a body.");
}
const pr = new ProgressReadable(
re.body,
cfg.download_timeout,
force_abort,
);
pr.addEventListener("progress", (e) => {
m.set_details_downloaded(i.index, e.detail);
});
pr.addEventListener("finished", () => {
m.remove_details(i.index);
});
try {
await pr.readable.pipeTo(f.writable, {
signal: force_abort,
preventClose: true,
const f = await Deno.open(path, {
create: true,
write: true,
truncate: true,
});
if (pr.error) {
throw (pr.error);
try {
await pr.readable.pipeTo(f.writable, {
signal: pr.signal,
preventClose: true,
});
if (pr.error) {
throw (pr.error);
}
} finally {
try {
f.close();
} catch (_) {
null;
}
}
} catch (e) {
if (pr.is_timeout) {
throw new TimeoutError();
}
throw e;
} finally {
try {
f.close();
pr.readable.cancel();
} catch (_) {
null;
}
}
} finally {
try {
pr.readable.cancel();
} catch (_) {
null;
}
}
const errors: unknown[] = [];
function try_download(a: number) {
if (a >= max_retry_count) {
m.remove_details(i.index);
reject(errors);
}
download().then(resolve).catch((e) => {
if (force_abort.aborted) {
throw Error("aborted.");
}
if (e instanceof DOMException) {
if (e.name == "AbortError") {
reject(new TimeoutError());
return;
}
}
if (e instanceof TimeoutError) {
reject(e);
return;
}
errors.push(e);
try_download(a + 1);
});
}
try_download(0);
});
}
await download_img();
db.add_file(f);
}
let retry = 0;
function try_deal_with_img() {
return new Promise((resolve, reject) => {
function try_() {
load().then(() => {
deal_with_img().then(resolve).catch((e) => {
console.log("Failed to download, retry: ", e);
retry += 1;
if (retry >= max_retry_count) {
reject(e);
return;
}
const download_original = download_original_img &&
!i.is_original;
if (download_original) {
i.redirected_url = undefined;
try2_();
} else try_();
});
}).catch(reject);
}
const errors: unknown[] = [];
function try_download(a: number) {
if (a >= max_retry_count) {
m.remove_details(i.index);
reject(errors);
}
download().then(resolve).catch((e) => {
if (force_abort.aborted) {
throw Error("aborted.");
function try2_() {
deal_with_img().then(resolve).catch((e) => {
console.log("Failed to download, retry: ", e);
retry += 1;
if (retry >= max_retry_count) {
reject(e);
return;
}
errors.push(e);
try_download(a + 1);
const download_original = download_original_img &&
!i.is_original;
if (download_original) {
i.redirected_url = undefined;
try2_();
} else try_();
});
}
try_download(0);
try_();
});
}
await download_img();
db.add_file(f);
await try_deal_with_img();
return;
}
if (mpv_enabled || g.mpv_enabled) {

1
thumbnail/ffmpeg_binary.ts
View File

@@ -18,6 +18,7 @@ export async function fb_generate_thumbnail(
cfg: ThumbnailConfig,
) {
const args = [
"-n",
"-i",
i,
"-vf",

6
utils.ts
View File

@@ -264,3 +264,9 @@ export function map<T, S, V>(
}
return re;
}
export class TimeoutError extends Error {
constructor() {
super("Timeout");
}
}

51
utils/progress_readable.ts
View File

@@ -7,10 +7,31 @@ export class ProgressReadable extends EventTarget {
readable: ReadableStream<Uint8Array>;
readed: number;
error?: unknown;
constructor(readable: ReadableStream<Uint8Array>) {
timeout: number;
get signal() {
return this.#controller.signal;
}
get is_timeout() {
return this.#is_timeout;
}
#controller: AbortController;
#is_timeout: boolean;
#timeout?: number;
constructor(
readable: ReadableStream<Uint8Array>,
timeout: number,
originalSignal?: AbortSignal,
) {
super();
this.readed = 0;
this.timeout = timeout;
this.#is_timeout = false;
const reader = readable.getReader();
this.#controller = new AbortController();
originalSignal?.addEventListener("abort", () => {
this.#controller.abort();
this.#clearTimeout();
});
this.readable = new ReadableStream({
pull: (c) => {
if (c.byobRequest) {
@@ -20,27 +41,51 @@ export class ProgressReadable extends EventTarget {
if (v.done) {
this.dispatchEvent("finished", this.readed);
c.close();
this.#clearTimeout();
return;
} else {
this.readed += v.value.byteLength;
this.dispatchEvent("progress", this.readed);
c.enqueue(v.value);
if (v.value.byteLength != 0) {
this.#clearTimeout();
this.#setTimeout();
}
}
}).catch((e) => {
c.close();
try {
c.close();
} catch (_) {
null;
}
this.error = e;
this.#clearTimeout();
});
}
},
cancel: (reason) => {
try {
readable.cancel(reason);
if (!readable.locked) readable.cancel(reason);
this.#clearTimeout();
} catch (_) {
null;
}
},
type: "bytes",
});
this.#setTimeout();
}
#clearTimeout() {
if (this.#timeout) {
clearTimeout(this.#timeout);
}
}
#setTimeout() {
this.#timeout = setTimeout(() => {
this.#is_timeout = true;
this.#controller.abort();
console.log("aborted");
}, this.timeout);
}
// @ts-ignore Checked type
addEventListener<T extends keyof EventMap>(