From fe7af51988b4d3659c3399c4f240a3ed9bd4199f Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 15:15:47 +0800 Subject: [PATCH] Add support to remove old galleries --- config.ts | 5 +++ db.ts | 40 ++++++++++++++++++ islands/Settings.tsx | 7 ++++ islands/TaskManager.tsx | 5 +++ meilisearch.ts | 16 +++++++ routes/api/task.ts | 9 ++-- server/task.ts | 8 ++-- task_manager.ts | 24 +++++++++-- tasks/download.ts | 72 ++++++++++++++++++++++++++++---- translation/en/settings.jsonc | 3 +- translation/zh-cn/settings.jsonc | 3 +- 11 files changed, 169 insertions(+), 23 deletions(-) diff --git a/config.ts b/config.ts index 122c62b..4be0d7b 100644 --- a/config.ts +++ b/config.ts @@ -21,6 +21,7 @@ export type ConfigType = { ffmpeg_path: string; thumbnail_method: ThumbnailMethod; thumbnail_dir: string; + remove_previous_gallery: boolean; }; export enum ThumbnailMethod { @@ -128,6 +129,9 @@ export class Config { get thumbnail_dir() { return this._return_string("thumbnail_dir") || "./thumbnails"; } + get remove_previous_gallery() { + return this._return_bool("remove_previous_gallery") || false; + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -149,6 +153,7 @@ export class Config { ffmpeg_path: this.ffmpeg_path, thumbnail_method: this.thumbnail_method, thumbnail_dir: this.thumbnail_dir, + remove_previous_gallery: this.remove_previous_gallery, }; } } diff --git a/db.ts b/db.ts index 300800d..c9cb519 100644 --- a/db.ts +++ b/db.ts @@ -662,6 +662,32 @@ export class EhDb { delete_file(f: EhFile) { this.db.query("DELETE FROM file WHERE id = ?;", [f.id]); } + delete_files(token: string) { + const files = this.get_files(token); + this.db.query("DELETE FROM file WHERE token = ?;", [token]); + this.db.query("DELETE FROM filemeta WHERE token = ?;", [token]); + files.forEach((f) => { + try_remove_sync(f.path); + console.log("Deleted ", f.path); + }); + } + delete_gallery(gid: number) { + this.db.query("DELETE FROM gmeta WHERE gid = ?;", [gid]); + this.db.query("DELETE FROM gtag WHERE gid = ?;", [gid]); + const tokens = new Set( + this.db.query<[string]>("SELECT token FROM pmeta WHERE gid = ?;", [ + gid, + ]).map((v) => v[0]), + ); + this.db.query("DELETE FROM pmeta WHERE gid = ?;", [gid]); + for (const token of tokens) { + const count = this.db.query<[number]>( + "SELECT COUNT(*) FROM pmeta WHERE token = ?;", + [token], + )[0][0]; + if (count === 0) this.delete_files(token); + } + } delete_task(task: Task) { return this.transaction(() => { this.db.query("DELETE FROM task WHERE id = ?;", [task.id]); @@ -805,6 +831,12 @@ export class EhDb { ); return s.length ? s[0] : undefined; } + async get_task(id: number) { + const s = await this.transaction(() => + this.db.queryEntries("SELECT * FROM task WHERE id = ?;", [id]) + ); + return s.length ? s[0] : undefined; + } get_tasks() { return this.transaction(() => this.db.queryEntries("SELECT * FROM task;") @@ -885,4 +917,12 @@ export class EhDb { ]); } } + update_task(task: Task) { + return this.transaction(() => { + this.db.query("UPDATE task SET details = ? WHERE id = ?;", [ + task.details, + task.id, + ]); + }); + } } diff --git a/islands/Settings.tsx b/islands/Settings.tsx index 4e6c9d5..fe4cdd4 100644 --- a/islands/Settings.tsx +++ b/islands/Settings.tsx @@ -121,6 +121,13 @@ export default class Settings extends Component { checked={settings.export_zip_jpn_title} description={t("settings.export_zip_jpn_title")} /> +
{ task.progress = t.detail.detail; sendTaskChangedEvent(t.detail.task_id); } + } else if (t.type == "task_updated") { + const task = tasks.value.get(t.detail.id); + if (task) { + task.base = t.detail; + } } }; self.addEventListener("beforeunload", () => { diff --git a/meilisearch.ts b/meilisearch.ts index 620a41a..43b4d75 100644 --- a/meilisearch.ts +++ b/meilisearch.ts @@ -37,6 +37,7 @@ export class MeiliSearchServer { client; db; handle; + handle2; #gmeta?: Index; #inited = false; target; @@ -50,8 +51,18 @@ export class MeiliSearchServer { this.handle = (e: Event) => { this.#gallery_update(e); }; + this.handle2 = (e: Event) => { + this.#gallery_remove(e); + }; this.target = new EventTarget(); this.target.addEventListener("gallery_update", this.handle); + this.target.addEventListener("gallery_remove", this.handle2); + } + #gallery_remove(e: Event) { + const ev = e as CustomEvent; + this.removeGallery(ev.detail).catch((e) => { + console.log(e); + }); } #gallery_update(e: Event) { const ev = e as CustomEvent; @@ -78,6 +89,7 @@ export class MeiliSearchServer { } close() { this.target.removeEventListener("gallery_update", this.handle); + this.target.removeEventListener("gallery_remove", this.handle2); } async getIndex(uid: string, primaryKey?: string) { try { @@ -112,6 +124,10 @@ export class MeiliSearchServer { this.#updateGMetaSettings(); this.#inited = true; } + async removeGallery(gid: number) { + const gmeta = await this.gmeta; + await this.waitTask(gmeta.deleteDocument(gid)); + } async updateGallery(...gids: number[]) { const gmeta = await this.gmeta; const datas = gids.map((gid) => { diff --git a/routes/api/task.ts b/routes/api/task.ts index 98dc839..d8f1c63 100644 --- a/routes/api/task.ts +++ b/routes/api/task.ts @@ -26,6 +26,7 @@ export const handler: Handlers = { t.removeEventListener("task_finished", handle); t.removeEventListener("task_progress", handle); t.removeEventListener("task_error", handle); + t.removeEventListener("task_updated", handle); ExitTarget.removeEventListener("close", close_handle); }; function sendMessage(mes: TaskServerSocketData) { @@ -45,12 +46,9 @@ export const handler: Handlers = { sendMessage({ type: "close" }); socket.close(); } else if (d.type == "new_download_task") { - t.add_download_task(d.gid, d.token); + t.add_download_task(d.gid, d.token, d.cfg); } else if (d.type == "new_export_zip_task") { - t.add_export_zip_task(d.gid, { - output: d.output, - jpn_title: d.jpn_title, - }); + t.add_export_zip_task(d.gid, d.cfg); } else if (d.type == "task_list") { t.get_task_list().then((tasks) => { sendMessage({ @@ -70,6 +68,7 @@ export const handler: Handlers = { t.addEventListener("task_finished", handle); t.addEventListener("task_progress", handle); t.addEventListener("task_error", handle); + t.addEventListener("task_updated", handle); ExitTarget.addEventListener("close", close_handle); }; return response; diff --git a/server/task.ts b/server/task.ts index 7c0b473..45895bc 100644 --- a/server/task.ts +++ b/server/task.ts @@ -1,5 +1,6 @@ import { Task } from "../task.ts"; import { TaskEventData } from "../task_manager.ts"; +import { DownloadConfig } from "../tasks/download.ts"; import { ExportZipConfig } from "../tasks/export_zip.ts"; import { DiscriminatedUnion } from "../utils.ts"; @@ -9,12 +10,9 @@ export type TaskServerSocketData = TaskEventData | { type: "close" } | { running: number[]; }; -type Gid> = ({ gid: number } & T) extends - infer U ? { [Q in keyof U]: U[Q] } : never; - type EventMap = { - new_download_task: { gid: number; token: string }; - new_export_zip_task: Gid; + new_download_task: { gid: number; token: string; cfg?: DownloadConfig }; + new_export_zip_task: { gid: number; cfg?: ExportZipConfig }; }; export type TaskClientSocketData = DiscriminatedUnion<"type", EventMap> | { diff --git a/task_manager.ts b/task_manager.ts index a93875e..be2a16a 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -5,7 +5,11 @@ import { MeiliSearchServer } from "./meilisearch.ts"; import { check_running } from "./pid_check.ts"; import { add_exit_handler } from "./signal_handler.ts"; import { Task, TaskProgress, TaskProgressBasicType, TaskType } from "./task.ts"; -import { download_task } from "./tasks/download.ts"; +import { + DEFAULT_DOWNLOAD_CONFIG, + download_task, + DownloadConfig, +} from "./tasks/download.ts"; import { DEFAULT_EXPORT_ZIP_CONFIG, export_zip, @@ -29,6 +33,7 @@ type EventMap = { task_finished: Task; task_progress: TaskProgress; task_error: { task: Task; error: string }; + task_updated: Task; }; type Detail> = { @@ -96,7 +101,7 @@ export class TaskManager extends EventTarget { get aborts() { return this.#abort.signal; } - async add_download_task(gid: number, token: string) { + async add_download_task(gid: number, token: string, cfg?: DownloadConfig) { this.#check_closed(); const otask = await this.db.check_download_task(gid, token); if (otask !== undefined) { @@ -109,7 +114,7 @@ export class TaskManager extends EventTarget { id: 0, pid: Deno.pid, type: TaskType.Download, - details: null, + details: cfg ? JSON.stringify(cfg) : null, }; return await this.#add_task(task); } @@ -282,6 +287,9 @@ export class TaskManager extends EventTarget { this.#check_closed(); this.dispatchEvent("task_started", task); if (task.type == TaskType.Download) { + const cfg: DownloadConfig = task.details + ? JSON.parse(task.details) + : DEFAULT_DOWNLOAD_CONFIG; this.running_tasks.set( task.id, { @@ -293,6 +301,7 @@ export class TaskManager extends EventTarget { this.#abort.signal, this.#force_abort.signal, this, + cfg, ), base: task, }, @@ -329,6 +338,15 @@ export class TaskManager extends EventTarget { }); } } + async update_task(t: Task) { + const r = this.running_tasks.get(t.id); + if (r) { + r.base.details = t.details; + } + await this.db.update_task(t); + const a = await this.db.get_task(t.id); + if (a) this.dispatchEvent("task_updated", a); + } async waiting_unfinished_task() { while (1) { await this.check_running_tasks(); diff --git a/tasks/download.ts b/tasks/download.ts index 72292cb..104fa4e 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -15,6 +15,17 @@ import { import { join, resolve } from "std/path/mod.ts"; import { exists } from "std/fs/exists.ts"; +export type DownloadConfig = { + max_download_img_count?: number; + mpv?: boolean; + download_original_img?: boolean; + max_retry_count?: number; + remove_previous_gallery?: boolean; + replaced_gallery?: { gid: number; token: string }[]; +}; + +export const DEFAULT_DOWNLOAD_CONFIG: DownloadConfig = {}; + class DownloadManager { #abort: AbortSignal; #force_abort: AbortSignal; @@ -24,13 +35,13 @@ class DownloadManager { #task: Task; #manager: TaskManager; constructor( - cfg: Config, + max_download_img_count: number, abort: AbortSignal, force_abort: AbortSignal, task: Task, manager: TaskManager, ) { - this.#max_download_count = cfg.max_download_img_count; + this.#max_download_count = max_download_img_count; this.#running_tasks = []; this.#abort = abort; this.#force_abort = force_abort; @@ -97,6 +108,7 @@ export async function download_task( abort: AbortSignal, force_abort: AbortSignal, manager: TaskManager, + dcfg: DownloadConfig, ) { console.log("Started to download gallery", task.gid); const gdatas = await client.fetchGalleryMetadataByAPI([ @@ -116,8 +128,27 @@ export async function download_task( } const base_path = join(cfg.base, task.gid.toString()); await sure_dir(base_path); - const m = new DownloadManager(cfg, abort, force_abort, task, manager); - if (cfg.mpv) { + const max_download_img_count = dcfg.max_download_img_count !== undefined + ? dcfg.max_download_img_count + : cfg.max_download_img_count; + const m = new DownloadManager( + max_download_img_count, + abort, + force_abort, + task, + manager, + ); + const mpv_enabled = dcfg.mpv !== undefined ? dcfg.mpv : cfg.mpv; + const download_original_img = dcfg.download_original_img !== undefined + ? dcfg.download_original_img + : cfg.download_original_img; + const max_retry_count = dcfg.max_retry_count !== undefined + ? dcfg.max_retry_count + : cfg.max_retry_count; + const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined + ? dcfg.remove_previous_gallery + : cfg.remove_previous_gallery; + if (mpv_enabled) { const mpv = await client.fetchMPVPage(task.gid, task.token); m.set_total_page(mpv.pagecount); const names = mpv.imagelist.reduce( @@ -134,7 +165,7 @@ export async function download_task( if (ofiles.length) { const t = ofiles[0]; if ( - (t.is_original || !cfg.download_original_img) && + (t.is_original || !download_original_img) && (await exists(t.path)) ) { const p = db.get_pmeta_by_index(task.gid, i.index); @@ -170,7 +201,7 @@ export async function download_task( return new Promise((resolve, reject) => { const errors: unknown[] = []; function try_load(a: number) { - if (a >= cfg.max_retry_count) reject(errors); + if (a >= max_retry_count) reject(errors); i.load().then(resolve).catch((e) => { if (force_abort.aborted) { throw Error("aborted."); @@ -186,7 +217,7 @@ export async function download_task( assert(i.data); const pmeta = i.to_pmeta(); if (pmeta) db.add_pmeta(pmeta); - const download_original = cfg.download_original_img && + const download_original = download_original_img && !i.is_original; if (download_original) console.log(i.index, i.data.o); let path = resolve(join(base_path, i.name)); @@ -226,7 +257,7 @@ export async function download_task( } const errors: unknown[] = []; function try_download(a: number) { - if (a >= cfg.max_retry_count) { + if (a >= max_retry_count) { reject(errors); } download().then(resolve).catch((e) => { @@ -253,5 +284,30 @@ export async function download_task( await m.join(); if (m.has_failed_task) throw Error("Some tasks failed."); if (abort.aborted || force_abort.aborted) throw Error("aborted"); + if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) { + let replaced_gallery = dcfg.replaced_gallery; + if (replaced_gallery === undefined) { + const fg = await client.fetchGalleryPage( + gmeta.first_gid, + gmeta.first_key, + ); + replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid); + replaced_gallery.push({ + gid: gmeta.first_gid, + token: gmeta.first_key, + }); + } + replaced_gallery.forEach((g) => { + const gmeta = db.get_gmeta_by_gid(g.gid); + if (!gmeta) return; + console.log("Remove gallery ", g.gid); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_remove", { detail: gmeta.gid }), + ); + } + db.delete_gallery(g.gid); + }); + } return task; } diff --git a/translation/en/settings.jsonc b/translation/en/settings.jsonc index 10bc257..3fdf316 100644 --- a/translation/en/settings.jsonc +++ b/translation/en/settings.jsonc @@ -28,5 +28,6 @@ "thumbnail_method": "The method used to generate thumbnail: ", "thumbnail_method0": "ffmpeg binary", "thumbnail_method1": "ffmpeg API", - "thumbnail_dir": "The folder used to store thumbnails: " + "thumbnail_dir": "The folder used to store thumbnails: ", + "remove_previous_gallery": "Remove old galleries which replaced by new ones." } diff --git a/translation/zh-cn/settings.jsonc b/translation/zh-cn/settings.jsonc index 4a5ffa9..aa28c8c 100644 --- a/translation/zh-cn/settings.jsonc +++ b/translation/zh-cn/settings.jsonc @@ -28,5 +28,6 @@ "thumbnail_method": "生成缩略图的方式：", "thumbnail_method0": "FFMPEG二进制", "thumbnail_method1": "FFMPEG API", - "thumbnail_dir": "存放缩略图的文件夹：" + "thumbnail_dir": "存放缩略图的文件夹：", + "remove_previous_gallery": "移除被新画廊替代的旧画廊。" }