diff --git a/db.ts b/db.ts index 8e290f0..f3e2f67 100644 --- a/db.ts +++ b/db.ts @@ -506,6 +506,33 @@ export class EhDb { return r.length ? r[0] : undefined; }); } + check_fix_gallery_page_task() { + return this.transaction(() => { + const r = this.db.queryEntries( + "SELECT * FROM task WHERE type = ?;", + [TaskType.FixGalleryPage], + ); + return r.length ? r[0] : undefined; + }); + } + check_onetime_task() { + return this.transaction(() => { + const r = this.db.queryEntries( + "SELECT * FROM task WHERE type = ? OR type = ?;", + [TaskType.UpdateMeiliSearchData, TaskType.FixGalleryPage], + ); + return r; + }); + } + check_update_meili_search_data_task() { + return this.transaction(() => { + const r = this.db.queryEntries( + "SELECT * FROM task WHERE type = ?;", + [TaskType.UpdateMeiliSearchData], + ); + return r.length ? r[0] : undefined; + }); + } close() { this.db.close(); if (this.#file) { @@ -634,6 +661,19 @@ export class EhDb { ); return s.length ? s[0] : undefined; } + get_pmeta_by_token_only(token: string) { + const s = this.db.queryEntries( + "SELECT * FROM pmeta WHERE token = ?;", + [token], + ); + return s; + } + get_pmeta_count(gid: number) { + return this.db.query<[number]>( + "SELECT COUNT(*) FROM pmeta WHERE gid = ?;", + [gid], + )[0][0]; + } get_random_file() { const s = this.convert_file( this.db.queryEntries( diff --git a/main.ts b/main.ts index feed493..6b31588 100644 --- a/main.ts +++ b/main.ts @@ -19,6 +19,7 @@ function show_help() { console.log( " umsd/update_meili_search_data Sync all gallery metadata to meiliserach server.", ); + console.log(" fgp/fix_gallery_page Fix incorrect gallery page."); console.log("Options:"); console.log(" -h, --help Show this help"); console.log(" -c, --config Specify config file path."); @@ -34,6 +35,7 @@ enum CMD { UpdateTagTranslation, ExportZip, UpdateMeiliSearchData, + FixGalleryPage, } const args = parse(Deno.args, { @@ -65,6 +67,7 @@ if (rcmd == "ez" || rcmd == "export_zip") cmd = CMD.ExportZip; if (rcmd == "umsd" || rcmd == "update_meili_search_data") { cmd = CMD.UpdateMeiliSearchData; } +if (rcmd == "fgp" || rcmd == "fix_gallery_page") cmd = CMD.FixGalleryPage; if (cmd == CMD.Unknown) { throw Error(`Unknown command: ${rcmd}`); } @@ -146,6 +149,15 @@ async function update_meili_search_data() { if (!manager.aborted) manager.close(); } } +async function fix_gallery_page() { + const manager = new TaskManager(settings); + try { + await manager.add_fix_gallery_page_task(); + await manager.run(); + } finally { + if (!manager.aborted) manager.close(); + } +} async function main() { await sure_dir(settings.base); if (cmd == CMD.Download) { @@ -160,6 +172,8 @@ async function main() { await export_zip(); } else if (cmd == CMD.UpdateMeiliSearchData) { await update_meili_search_data(); + } else if (cmd == CMD.FixGalleryPage) { + await fix_gallery_page(); } } diff --git a/task.ts b/task.ts index 09427a7..a362ca8 100644 --- a/task.ts +++ b/task.ts @@ -2,6 +2,7 @@ export enum TaskType { Download, ExportZip, UpdateMeiliSearchData, + FixGalleryPage, } export type Task = { @@ -29,10 +30,16 @@ export type TaskUpdateMeiliSearchDataProgress = { updated_gallery: number; }; +export type TaskFixGalleryPageProgress = { + total_gallery: number; + checked_gallery: number; +}; + export type TaskProgressBasicType = { [TaskType.Download]: TaskDownloadProgess; [TaskType.ExportZip]: TaskExportZipProgress; [TaskType.UpdateMeiliSearchData]: TaskUpdateMeiliSearchDataProgress; + [TaskType.FixGalleryPage]: TaskFixGalleryPageProgress; }; export type TaskProgress = { diff --git a/task_manager.ts b/task_manager.ts index 3bf09f1..39795d2 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -11,6 +11,7 @@ import { export_zip, ExportZipConfig, } from "./tasks/export_zip.ts"; +import { fix_gallery_page } from "./tasks/fix_gallery_page.ts"; import { update_meili_search_data } from "./tasks/update_meili_search_data.ts"; import { DiscriminatedUnion, @@ -123,7 +124,30 @@ export class TaskManager extends EventTarget { }; return await this.#add_task(task); } + async add_fix_gallery_page_task() { + this.#check_closed(); + const otask = await this.db.check_fix_gallery_page_task(); + if (otask !== undefined) { + console.log("The task is already in list."); + return otask; + } + const task: Task = { + gid: 0, + token: "", + id: 0, + pid: Deno.pid, + type: TaskType.FixGalleryPage, + details: null, + }; + return await this.#add_task(task); + } async add_update_meili_search_data_task(gid?: number) { + this.#check_closed(); + const otask = await this.db.check_update_meili_search_data_task(); + if (otask !== undefined) { + console.log("The task is already in list."); + return otask; + } const task: Task = { gid: gid ? gid : 0, token: "", @@ -137,6 +161,8 @@ export class TaskManager extends EventTarget { async check_task(task: Task) { this.#check_closed(); if (await this.check_task_is_running(task)) return; + const ut = (await this.db.check_onetime_task()).map((t) => t.id); + if (ut.length && !ut.includes(task.id)) return; let t = task; if (task.pid !== Deno.pid) { const p = await this.db.set_task_pid(t); @@ -237,14 +263,14 @@ export class TaskManager extends EventTarget { for (const task of my_tasks) { if (this.running_tasks.size == this.max_task_count) break; const checked = await this.check_task(task); - if (checked) this.run_task(checked); + if (checked) await this.run_task(checked); } if (this.running_tasks.size == this.max_task_count) continue; const otasks = await this.db.get_other_pid_tasks(); for (const task of otasks) { if (this.running_tasks.size == this.max_task_count) break; const checked = await this.check_task(task); - if (checked) this.run_task(checked); + if (checked) await this.run_task(checked); } if (this.running_tasks.size == 0) { if (!forever) break; @@ -252,7 +278,7 @@ export class TaskManager extends EventTarget { } } } - run_task(task: Task) { + async run_task(task: Task) { this.#check_closed(); this.dispatchEvent("task_started", task); if (task.type == TaskType.Download) { @@ -290,10 +316,17 @@ export class TaskManager extends EventTarget { }, ); } else if (task.type === TaskType.UpdateMeiliSearchData) { + await this.waiting_unfinished_task(); this.running_tasks.set(task.id, { task: update_meili_search_data(task, this), base: task, }); + } else if (task.type === TaskType.FixGalleryPage) { + await this.waiting_unfinished_task(); + this.running_tasks.set(task.id, { + task: fix_gallery_page(task, this), + base: task, + }); } } async waiting_unfinished_task() { diff --git a/tasks/download.ts b/tasks/download.ts index e3100e1..72292cb 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -148,6 +148,18 @@ export async function download_task( op.name = i.name; db.add_pmeta(op); return; + } else { + const ops = db.get_pmeta_by_token_only( + i.page_token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } } } console.log("Already download page", i.index); diff --git a/tasks/fix_gallery_page.ts b/tasks/fix_gallery_page.ts new file mode 100644 index 0000000..6e035be --- /dev/null +++ b/tasks/fix_gallery_page.ts @@ -0,0 +1,21 @@ +import { Task } from "../task.ts"; +import { TaskManager } from "../task_manager.ts"; +import { asyncForEach } from "../utils.ts"; + +export async function fix_gallery_page(t: Task, m: TaskManager) { + let offset = 0; + let gids = m.db.get_gids(offset); + while (gids.length) { + await asyncForEach(gids, async (gid) => { + const gmeta = m.db.get_gmeta_by_gid(gid); + if (!gmeta) return; + const count = m.db.get_pmeta_count(gid); + if (gmeta.filecount != count) { + await m.add_download_task(gid, gmeta.token); + } + }); + offset += gids.length; + gids = m.db.get_gids(offset); + } + return t; +}