lifegpc/eh-downloader
1
0
Fork 0
mirror of https://github.com/lifegpc/eh-downloader.git synced 2026-07-08 01:31:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add stack trace utility and enhance logging across various modules

Browse Source
This commit is contained in:
lifegpc
2024-12-31 15:14:30 +08:00
parent 5131515770
commit 730b673954
15 changed files with 311 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
tasks/download.ts
View File

@@ -24,6 +24,7 @@ import {
import { basename, extname, join, resolve } from "@std/path";
import { exists } from "@std/fs/exists";
import { ProgressReadable } from "../utils/progress_readable.ts";
import { base_logger } from "../utils/logger.ts";
export type DownloadConfig = {
max_download_img_count?: number;
@@ -38,6 +39,8 @@ export const DEFAULT_DOWNLOAD_CONFIG: DownloadConfig = {};
const PROGRESS_UPDATE_INTERVAL = 200;
const logger = base_logger.get_logger("download-task");
class DownloadManager {
#abort: AbortSignal;
#force_abort: AbortSignal;
@@ -78,7 +81,7 @@ class DownloadManager {
async (t) => {
const s = await promiseState(t);
if (s.status === PromiseStatus.Rejected) {
if (!this.#force_abort.aborted) console.log(s.reason);
if (!this.#force_abort.aborted) logger.log(s.reason);
this.#progress.failed_page += 1;
this.#sendEvent();
} else if (s.status === PromiseStatus.Fulfilled) {
@@ -219,7 +222,7 @@ export async function download_task(
manager: TaskManager,
dcfg: DownloadConfig,
) {
console.log("Started to download gallery", task.gid);
logger.log("Started to download gallery", task.gid);
const gdatas = await client.fetchGalleryMetadataByAPI([
task.gid,
task.token,
@@ -288,7 +291,7 @@ export async function download_task(
}
}
}
console.log("Already download page", i.index);
logger.log("Already download page", i.index);
return;
}
}
@@ -301,7 +304,7 @@ export async function download_task(
if (load_times >= max_retry_count) reject(errors);
i.load().then(resolve).catch((e) => {
if (force_abort.aborted) {
console.log("Aborted when loading image:", i);
logger.log("Aborted when loading image:", i);
errors.push(e);
reject(errors);
return;
@@ -326,7 +329,7 @@ export async function download_task(
);
if (names[i.name] > 1) {
path = add_suffix_to_path(path, i.page_token);
console.log("Changed path to", path);
logger.debug("Changed path to", path);
}
const f = download_original
? i.get_original_file(path)
@@ -428,12 +431,12 @@ export async function download_task(
try {
hash = getHashFromUrl(url);
} catch (e) {
console.warn(e);
logger.warn(e);
}
if (hash) {
const fhash = await calFileSha1(path);
if (hash != fhash) {
console.warn(
logger.warn(
`Hash not matched: file hash ${fhash}, original hash ${hash}, url ${url}`,
);
throw new HashError();
@@ -450,7 +453,7 @@ export async function download_task(
}
download().then(resolve).catch((e) => {
if (force_abort.aborted) {
console.log(
logger.log(
"Aborted when downloading image:",
i,
);
@@ -495,7 +498,7 @@ export async function download_task(
reject(e);
return;
}
console.log("Failed to download, retry: ", e);
logger.warn("Failed to download, retry: ", e);
retry += 1;
if (retry >= max_retry_count) {
reject(e);
@@ -512,7 +515,7 @@ export async function download_task(
}
function try2_() {
deal_with_img().then(resolve).catch((e) => {
console.log("Failed to download, retry: ", e);
logger.warn("Failed to download, retry: ", e);
retry += 1;
if (retry >= max_retry_count) {
reject(e);
@@ -584,7 +587,7 @@ export async function download_task(
replaced_gallery.forEach((g) => {
const gmeta = db.get_gmeta_by_gid(g.gid);
if (!gmeta) return;
console.log("Remove gallery ", g.gid);
logger.debug("Remove gallery ", g.gid);
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_remove", { detail: gmeta.gid }),

31
tasks/import.ts
View File

@@ -19,6 +19,7 @@ import { ImportMethod, ImportSize } from "../config.ts";
import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts";
import { EhFile, PMeta } from "../db.ts";
import type { ReadonlyZip } from "../utils/readonly_zip.ts";
import { base_logger } from "../utils/logger.ts";
export type ImportConfig = {
max_import_img_count?: number;
@@ -41,6 +42,8 @@ const VALID_EXTS = [".jpg", ".png", ".gif"];
const PROGRESS_UPDATE_INTERVAL = 200;
const logger = base_logger.get_logger("import-task");
class ImportManager {
#abort: AbortSignal;
#force_abort: AbortSignal;
@@ -78,7 +81,7 @@ class ImportManager {
async (t) => {
const s = await promiseState(t);
if (s.status === PromiseStatus.Rejected) {
if (!this.#force_abort.aborted) console.log(s.reason);
if (!this.#force_abort.aborted) logger.log(s.reason);
this.#progress.failed_page += 1;
this.#sendEvent();
} else if (s.status === PromiseStatus.Fulfilled) {
@@ -262,7 +265,7 @@ class FileLoader {
export async function import_task(task: Task, manager: TaskManager) {
if (!task.details) throw Error("Task details are needed.");
console.log("Started to import gallery", task.gid);
logger.log("Started to import gallery", task.gid);
const icfg: ImportConfig = JSON.parse(task.details);
const cfg = manager.cfg;
const client = manager.client;
@@ -331,12 +334,12 @@ export async function import_task(task: Task, manager: TaskManager) {
async function import_img(i: Page) {
const opath = f.get_file(i.name, i.index);
if (!opath) {
console.log("File not found");
logger.log("File not found");
return;
}
const size = await fb_get_size(opath);
if (!size) {
console.log("Failed to get file size for", opath);
logger.log("Failed to get file size for", opath);
throw Error("Failed to get file size.");
}
const ofiles = db.get_files(i.token);
@@ -373,7 +376,7 @@ export async function import_task(task: Task, manager: TaskManager) {
}
}
}
console.log("Already has page", i.index);
logger.log("Already has page", i.index);
return;
}
}
@@ -394,7 +397,7 @@ export async function import_task(task: Task, manager: TaskManager) {
let path = join(base_path, is_original ? i.name : i.sampled_name);
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
path = add_suffix_to_path(path, i.token);
console.log("Changed path to", path);
logger.debug("Changed path to", path);
}
if (import_method == ImportMethod.Move) {
await Deno.rename(opath, path);
@@ -406,7 +409,7 @@ export async function import_task(task: Task, manager: TaskManager) {
if (cfg.check_file_hash && is_original) {
const sha = await calFileSha1(path);
if (sha.slice(0, i.token.length) != i.token) {
console.warn(
logger.warn(
`Hash not matched: file hash ${sha}, token ${i.token}`,
);
return;
@@ -429,7 +432,7 @@ export async function import_task(task: Task, manager: TaskManager) {
async function import_zip_img(i: Page) {
const opath = f.get_zip(i.name, i.index);
if (!opath) {
console.log("File not found");
logger.log("File not found");
return;
}
const ofiles = db.get_files(i.token);
@@ -466,7 +469,7 @@ export async function import_task(task: Task, manager: TaskManager) {
}
}
}
console.log("Already has page", i.index);
logger.log("Already has page", i.index);
return;
}
}
@@ -478,11 +481,11 @@ export async function import_task(task: Task, manager: TaskManager) {
);
if (names[i.name] > 1) {
path = add_suffix_to_path(path, i.token);
console.log("Changed path to", path);
logger.log("Changed path to", path);
}
const zf = f.open_zip_file(opath);
if (!zf) {
console.log("File not found");
logger.log("File not found");
return;
}
try {
@@ -505,7 +508,7 @@ export async function import_task(task: Task, manager: TaskManager) {
}
const size = await fb_get_size(path);
if (!size) {
console.log("Failed to get file size for", path);
logger.error("Failed to get file size for", path);
throw Error("Failed to get file size.");
}
const is_original = icfg.size == ImportSize.Original ||
@@ -514,7 +517,7 @@ export async function import_task(task: Task, manager: TaskManager) {
if (cfg.check_file_hash && is_original) {
const sha = await calFileSha1(path);
if (sha.slice(0, i.token.length) != i.token) {
console.warn(
logger.warn(
`Hash not matched: file hash ${sha}, token ${i.token}`,
);
return;
@@ -574,7 +577,7 @@ export async function import_task(task: Task, manager: TaskManager) {
replaced_gallery.forEach((g) => {
const gmeta = db.get_gmeta_by_gid(g.gid);
if (!gmeta) return;
console.log("Remove gallery ", g.gid);
logger.debug("Remove gallery ", g.gid);
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_remove", {