diff --git a/config.ts b/config.ts index 9053226..8c8e143 100644 --- a/config.ts +++ b/config.ts @@ -37,6 +37,8 @@ export type ConfigType = { random_file_secret?: string; use_path_based_img_url: boolean; check_file_hash: boolean; + import_method: ImportMethod; + max_import_img_count: number; }; export enum ThumbnailMethod { @@ -44,6 +46,23 @@ export enum ThumbnailMethod { FFMPEG_API, } +export enum ImportMethod { + Copy, + CopyThenDelete, + Move, + Keep, +} + +export enum ImportSize { + Original, + X780 = 780, + X980 = 980, + X1280 = 1280, + Resampled = X1280, + X1600 = 1600, + X2400 = 2400, +} + export class Config { _data; constructor(data: JsonValue) { @@ -212,6 +231,14 @@ export class Config { get check_file_hash() { return this._return_bool("check_file_hash") ?? true; } + get import_method() { + const n = this._return_number("input_method") ?? 1; + if (n < 0 || n > 3) return ImportMethod.CopyThenDelete; + return n as ImportMethod; + } + get max_import_img_count() { + return this._return_number("max_import_img_count") || 3; + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -248,6 +275,8 @@ export class Config { random_file_secret: this.random_file_secret, use_path_based_img_url: this.use_path_based_img_url, check_file_hash: this.check_file_hash, + import_method: this.import_method, + max_import_img_count: this.max_import_img_count, }; } } diff --git a/db.ts b/db.ts index 8e9eaf9..5f8fbcd 100644 --- a/db.ts +++ b/db.ts @@ -814,8 +814,8 @@ export class EhDb { check_download_task(gid: number | bigint, token: string) { return this.transaction(() => { const r = this.db.queryEntries( - "SELECT * FROM task WHERE type = ? AND gid = ? AND token = ?;", - [TaskType.Download, gid, token], + "SELECT * FROM task WHERE (type = ? OR type = ?) AND gid = ? AND token = ?;", + [TaskType.Download, TaskType.Import, gid, token], ); return r.length ? r[0] : undefined; }); diff --git a/page/GalleryPage.ts b/page/GalleryPage.ts index 6404f53..e71981b 100644 --- a/page/GalleryPage.ts +++ b/page/GalleryPage.ts @@ -88,6 +88,9 @@ class Image { get src() { return this.data?.img_url; } + get token() { + return this.base.token; + } get xres() { return this.data?.xres; } diff --git a/page/MPVPage.ts b/page/MPVPage.ts index ee73316..25c0297 100644 --- a/page/MPVPage.ts +++ b/page/MPVPage.ts @@ -136,6 +136,9 @@ class MPVImage { get thumbnail() { return this.base.t; } + get token() { + return this.base.k; + } get xres() { const xres = this.data?.xres; if (!xres) return undefined; diff --git a/routes/api/task.ts b/routes/api/task.ts index 828b966..ec9110c 100644 --- a/routes/api/task.ts +++ b/routes/api/task.ts @@ -12,6 +12,7 @@ import type { DownloadConfig } from "../../tasks/download.ts"; import type { ExportZipConfig } from "../../tasks/export_zip.ts"; import { User, UserPermission } from "../../db.ts"; import { toJSON } from "../../utils.ts"; +import { ImportConfig } from "../../tasks/import.ts"; export const handler: Handlers = { GET(req, ctx) { @@ -165,6 +166,37 @@ export const handler: Handlers = { } catch (e) { return return_error(500, e.message); } + } else if (typ == "import") { + const gid = await parse_big_int(form.get("gid"), null); + const token = await get_string(form.get("token")); + if (gid === null) { + return return_error(2, "gid is required"); + } + if (!token) { + return return_error(3, "token is required"); + } + const cfg = await get_string(form.get("cfg")); + if (!cfg) { + return return_error(4, "cfg is required"); + } + let icfg: ImportConfig | undefined = undefined; + try { + icfg = JSON.parse(cfg); + } catch (_) { + return return_error(4, "cfg is invalid"); + } + if (!icfg) { + return return_error(4, "cfg is required"); + } + try { + const task = await t.add_import_task(gid, token, icfg, true); + if (task === null) { + return return_error(6, "task is already in the list"); + } + return return_data(task, 201); + } catch (e) { + return return_error(500, e.message); + } } else { return return_error(5, "unknown type"); } diff --git a/task.ts b/task.ts index 739efc3..f0e4677 100644 --- a/task.ts +++ b/task.ts @@ -3,6 +3,7 @@ export enum TaskType { ExportZip, UpdateMeiliSearchData, FixGalleryPage, + Import, } export type Task = { @@ -52,11 +53,18 @@ export type TaskFixGalleryPageProgress = { checked_gallery: number; }; +export type TaskImportProgress = { + imported_page: number; + failed_page: number; + total_page: number; +}; + export type TaskProgressBasicType = { [TaskType.Download]: TaskDownloadProgess; [TaskType.ExportZip]: TaskExportZipProgress; [TaskType.UpdateMeiliSearchData]: TaskUpdateMeiliSearchDataProgress; [TaskType.FixGalleryPage]: TaskFixGalleryPageProgress; + [TaskType.Import]: TaskImportProgress; }; export type TaskProgress = { diff --git a/task_manager.ts b/task_manager.ts index b74c4f8..fad2ff1 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -16,6 +16,7 @@ import { ExportZipConfig, } from "./tasks/export_zip.ts"; import { fix_gallery_page } from "./tasks/fix_gallery_page.ts"; +import { ImportConfig } from "./tasks/import.ts"; import { update_meili_search_data } from "./tasks/update_meili_search_data.ts"; import { DiscriminatedUnion, @@ -157,6 +158,28 @@ export class TaskManager extends EventTarget { }; return await this.#add_task(task); } + async add_import_task( + gid: number | bigint, + token: string, + cfg: ImportConfig, + mark_already = false, + ) { + this.#check_closed(); + const otask = await this.db.check_download_task(gid, token); + if (otask !== undefined) { + console.log("The task is already in list."); + return mark_already ? null : otask; + } + const task: Task = { + gid, + token, + id: 0, + pid: Deno.pid, + type: TaskType.Import, + details: toJSON(cfg), + }; + return await this.#add_task(task); + } async add_update_meili_search_data_task(gid?: number | bigint) { this.#check_closed(); const otask = await this.db.check_update_meili_search_data_task(gid); @@ -177,7 +200,9 @@ export class TaskManager extends EventTarget { async check_task(task: Task) { this.#check_closed(); if (await this.check_task_is_running(task)) return; - const ut = (await this.db.check_onetime_task()).map((t) => BigInt(t.id)); + const ut = (await this.db.check_onetime_task()).map((t) => + BigInt(t.id) + ); if (ut.length && !ut.includes(BigInt(task.id))) return; let t = task; if (task.pid != Deno.pid) { diff --git a/tasks/download.ts b/tasks/download.ts index b169a99..7e0ea7a 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -231,9 +231,8 @@ export async function download_task( } const base_path = join(cfg.base, task.gid.toString()); await sure_dir(base_path); - const max_download_img_count = dcfg.max_download_img_count !== undefined - ? dcfg.max_download_img_count - : cfg.max_download_img_count; + const max_download_img_count = dcfg.max_download_img_count ?? + cfg.max_download_img_count; const m = new DownloadManager( max_download_img_count, abort, @@ -241,16 +240,12 @@ export async function download_task( task, manager, ); - const mpv_enabled = dcfg.mpv !== undefined ? dcfg.mpv : cfg.mpv; - const download_original_img = dcfg.download_original_img !== undefined - ? dcfg.download_original_img - : cfg.download_original_img; - const max_retry_count = dcfg.max_retry_count !== undefined - ? dcfg.max_retry_count - : cfg.max_retry_count; - const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined - ? dcfg.remove_previous_gallery - : cfg.remove_previous_gallery; + const mpv_enabled = dcfg.mpv ?? cfg.mpv; + const download_original_img = dcfg.download_original_img ?? + cfg.download_original_img; + const max_retry_count = dcfg.max_retry_count ?? cfg.max_retry_count; + const remove_previous_gallery = dcfg.remove_previous_gallery ?? + cfg.remove_previous_gallery; const g = await client.fetchGalleryPage(task.gid, task.token); async function download_task(names: Record, i: Image) { const ofiles = db.get_files(i.page_token); diff --git a/tasks/export_zip.ts b/tasks/export_zip.ts index 924fd69..f0dd758 100644 --- a/tasks/export_zip.ts +++ b/tasks/export_zip.ts @@ -33,9 +33,7 @@ export async function export_zip( const gid = task.gid; const g = db.get_gmeta_by_gid(gid); if (!g) throw Error("Gallery not found in database."); - const jpn_title = ecfg.jpn_title !== undefined - ? ecfg.jpn_title - : cfg.export_zip_jpn_title; + const jpn_title = ecfg.jpn_title ?? cfg.export_zip_jpn_title; const progress: TaskExportZipProgress = { total_page: Number(g.filecount), added_page: 0, diff --git a/tasks/import.ts b/tasks/import.ts new file mode 100644 index 0000000..5540be6 --- /dev/null +++ b/tasks/import.ts @@ -0,0 +1,421 @@ +import { Task, TaskImportProgress, TaskType } from "../task.ts"; +import { TaskManager } from "../task_manager.ts"; +import { + add_suffix_to_path, + asyncEvery, + asyncFilter, + configureZipJs, + promiseState, + PromiseStatus, + RecoverableError, + sleep, + sure_dir, +} from "../utils.ts"; +import { FS, ZipFileEntry } from "zipjs/index.js"; +import { exists } from "@std/fs/exists"; +import { walk } from "@std/fs/walk"; +import { extname, join, relative } from "@std/path"; +import { ImportMethod, ImportSize } from "../config.ts"; +import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts"; +import { EhFile, PMeta } from "../db.ts"; + +export type ImportConfig = { + max_import_img_count?: number; + mpv?: boolean; + method?: ImportMethod; + remove_previous_gallery?: boolean; + replaced_gallery?: { gid: number | bigint; token: string }[]; + import_path: string; + size: ImportSize; +}; + +interface Page { + index: number; + token: string; + name: string; + sampled_name: string; +} + +const PROGRESS_UPDATE_INTERVAL = 200; + +class ImportManager { + #abort: AbortSignal; + #force_abort: AbortSignal; + #max_import_count; + #running_tasks: Promise[]; + #progress: TaskImportProgress; + #task: Task; + #manager: TaskManager; + #progress_changed: boolean; + #last_send_progress: number; + constructor( + max_import_img_count: number, + abort: AbortSignal, + force_abort: AbortSignal, + task: Task, + manager: TaskManager, + ) { + this.#max_import_count = max_import_img_count; + this.#running_tasks = []; + this.#abort = abort; + this.#force_abort = force_abort; + this.#progress = { + imported_page: 0, + failed_page: 0, + total_page: 0, + }; + this.#task = task; + this.#manager = manager; + this.#progress_changed = false; + this.#last_send_progress = -1; + } + async #check_tasks() { + this.#running_tasks = await asyncFilter( + this.#running_tasks, + async (t) => { + const s = await promiseState(t); + if (s.status === PromiseStatus.Rejected) { + if (!this.#force_abort.aborted) console.log(s.reason); + this.#progress.failed_page += 1; + this.#sendEvent(); + } else if (s.status === PromiseStatus.Fulfilled) { + this.#progress.imported_page += 1; + this.#sendEvent(); + } + return s.status === PromiseStatus.Pending; + }, + ); + if (this.#progress_changed) { + const now = (new Date()).getTime(); + if (now >= this.#last_send_progress + PROGRESS_UPDATE_INTERVAL) { + this.#manager.dispatchTaskProgressEvent( + TaskType.Import, + this.#task.id, + this.#progress, + ); + this.#progress_changed = false; + this.#last_send_progress = now; + } + } + } + #sendEvent() { + this.#progress_changed = true; + const now = (new Date()).getTime(); + if (now < this.#last_send_progress + PROGRESS_UPDATE_INTERVAL) return; + const re = this.#manager.dispatchTaskProgressEvent( + TaskType.Import, + this.#task.id, + this.#progress, + ); + this.#last_send_progress = now; + this.#progress_changed = false; + return re; + } + async add_new_task(f: () => Promise) { + while (1) { + if (this.#abort.aborted) break; + await this.#check_tasks(); + if (this.#running_tasks.length < this.#max_import_count) { + this.#running_tasks.push(f()); + break; + } + await sleep(10); + } + } + get has_failed_task() { + return this.#progress.failed_page > 0; + } + async join() { + while (1) { + await this.#check_tasks(); + if (!this.#running_tasks.length) break; + await sleep(10); + } + } + set_total_page(page: number) { + this.#progress.total_page = page; + this.#sendEvent(); + } +} + +class FileLoader { + #zip?: FS; + #path; + #files: string[] = []; + #inited = false; + #filecount; + #has_prefix = false; + constructor(path: string, filecount: number) { + this.#path = path; + this.#filecount = filecount; + } + #check() { + if (!this.#inited) throw Error("FileLoader not initiailzed."); + } + #get_file(name: string) { + if (this.#files.includes(name)) { + return join(this.#path, name); + } + } + #get_zip(name: string) { + const ext = this.#zip!.getChildByName(name); + if (ext && ext instanceof ZipFileEntry) { + return ext; + } + } + get_file(name: string) { + let t = this.#get_file(name); + if (t) return t; + const ext = extname(name); + if (ext != ".jpg") { + const n = name.slice(0, name.length - 4) + ".jpg"; + t = this.#get_file(n); + if (t) return t; + } + } + get_zip(name: string) { + let t = this.#get_zip(name); + if (t) return t; + const ext = extname(name); + if (ext != ".jpg") { + const n = name.slice(0, name.length - 4) + ".jpg"; + t = this.#get_zip(n); + if (t) return t; + } + } + async init() { + if (await exists(this.#path, { isDirectory: true })) { + for await (const i of walk(this.#path)) { + if (i.path != this.#path) { + this.#files.push(relative(this.#path, i.path)); + } + } + } else { + configureZipJs(); + const r = await Deno.open(this.#path, { read: true }); + try { + this.#zip = new FS(); + const ent = await this.#zip.importReadable(r.readable); + for (const e of ent) { + this.#files.push(e.getFullname()); + } + } finally { + try { + r.close(); + } catch (_) { + null; + } + } + } + let has_prefix = true; + const re = new RegExp(`^\\d{${this.#filecount.toString().length}}_`); + for (const f of this.#files) { + if (!f.match(re)) { + has_prefix = false; + break; + } + } + this.#has_prefix = has_prefix; + this.#inited = true; + return this; + } + get is_zip() { + this.#check(); + return this.#zip !== undefined; + } +} + +export async function import_task(task: Task, manager: TaskManager) { + if (!task.details) throw Error("Task details are needed."); + console.log("Started to import gallery", task.gid); + const icfg: ImportConfig = JSON.parse(task.details); + const cfg = manager.cfg; + const client = manager.client; + const db = manager.db; + const gdatas = await client.fetchGalleryMetadataByAPI([ + task.gid, + task.token, + ]); + const gdata = gdatas.map.get(BigInt(task.gid)); + if (gdata === undefined) throw Error("Gallery metadata not included."); + if (typeof gdata === "string") throw Error(gdata); + const f = + await (new FileLoader(icfg.import_path, parseInt(gdata.filecount))) + .init(); + const gmeta = gdatas.convert(gdata); + db.add_gmeta(gmeta); + await db.add_gtag(task.gid, new Set(gdata.tags)); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_update", { detail: gmeta.gid }), + ); + } + const base_path = join(cfg.base, task.gid.toString()); + await sure_dir(base_path); + let mpv_enabled = icfg.mpv ?? cfg.mpv; + let import_method = icfg.method ?? cfg.import_method; + if ( + f.is_zip && import_method != ImportMethod.Copy && + import_method != ImportMethod.CopyThenDelete + ) { + import_method = ImportMethod.CopyThenDelete; + } + const max_import_img_count = icfg.max_import_img_count ?? + cfg.max_import_img_count; + let imgs: Page[] = []; + const m = new ImportManager( + max_import_img_count, + manager.aborts, + manager.force_aborts, + task, + manager, + ); + if (!mpv_enabled) { + const g = await client.fetchGalleryPage(task.gid, task.token); + if (g.mpv_enabled) { + mpv_enabled = true; + } else { + imgs = await g.imagelist; + m.set_total_page(g.length); + } + } + if (mpv_enabled) { + const g = await client.fetchMPVPage(task.gid, task.token); + imgs = g.imagelist; + m.set_total_page(g.pagecount); + } + const names = imgs.reduce( + (acc: Record, cur) => { + const curr = cur.name; + return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc; + }, + {}, + ); + + async function import_img(i: Page) { + const opath = f.get_file(i.name); + if (!opath) { + console.log("File not found"); + return; + } + const size = await fb_get_size(opath); + if (!size) { + console.log("Failed to get file size for", opath); + throw Error("Failed to get file size."); + } + const ofiles = db.get_files(i.token); + if (ofiles.length) { + const need = await asyncEvery( + ofiles, + async (t) => + (!t.is_original && t.height != size.height && + t.width != size.width) || (!await exists(t.path)), + ); + if (!need) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, + i.token, + ); + if (op) { + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } else { + const ops = db.get_pmeta_by_token_only( + i.token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } + } + } + console.log("Already has page", i.index); + return; + } + } + const pmeta: PMeta = { + gid: task.gid, + height: size.height, + width: size.width, + index: i.index, + name: i.name, + token: i.token, + }; + db.add_pmeta(pmeta); + const oriext = extname(i.name); + const nowext = extname(opath); + const is_original = icfg.size == ImportSize.Original || + (oriext != ".jpg" && oriext == nowext) || + (oriext == nowext && size.width < icfg.size); + let path = join(base_path, is_original ? i.name : i.sampled_name); + if (import_method != ImportMethod.Keep && names[i.name] > 1) { + path = add_suffix_to_path(path, i.token); + console.log("Changed path to", path); + } + if (import_method == ImportMethod.Move) { + await Deno.rename(opath, path); + } else if (import_method != ImportMethod.Keep) { + await Deno.copyFile(opath, path); + if (import_method == ImportMethod.CopyThenDelete) { + await Deno.remove(opath); + } + } else { + path = opath; + } + const file: EhFile = { + height: size.height, + width: size.width, + is_original, + id: 0, + path, + token: i.token, + }; + db.add_file(file); + } + + for (const i of imgs) { + if (manager.aborted) break; + await m.add_new_task(async () => { + await import_img(i); + }); + } + await m.join(); + const remove_previous_gallery = icfg.remove_previous_gallery ?? + cfg.remove_previous_gallery; + if (m.has_failed_task) throw new RecoverableError("Some tasks failed."); + if (manager.aborted || manager.force_aborted) throw Error("aborted"); + if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) { + let replaced_gallery = icfg.replaced_gallery; + if (replaced_gallery === undefined) { + const fg = await client.fetchGalleryPage( + gmeta.first_gid, + gmeta.first_key, + ); + replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid); + replaced_gallery.push({ + gid: gmeta.first_gid, + token: gmeta.first_key, + }); + } + replaced_gallery.forEach((g) => { + const gmeta = db.get_gmeta_by_gid(g.gid); + if (!gmeta) return; + console.log("Remove gallery ", g.gid); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_remove", { detail: gmeta.gid }), + ); + } + db.delete_gallery(g.gid); + }); + } + return task; +}