Add support to import gallery

config.ts

@@ -37,6 +37,8 @@ export type ConfigType = {
     random_file_secret?: string;
     use_path_based_img_url: boolean;
     check_file_hash: boolean;
+    import_method: ImportMethod;
+    max_import_img_count: number;
 };
 
 export enum ThumbnailMethod {
@@ -44,6 +46,23 @@ export enum ThumbnailMethod {
     FFMPEG_API,
 }
 
+export enum ImportMethod {
+    Copy,
+    CopyThenDelete,
+    Move,
+    Keep,
+}
+
+export enum ImportSize {
+    Original,
+    X780 = 780,
+    X980 = 980,
+    X1280 = 1280,
+    Resampled = X1280,
+    X1600 = 1600,
+    X2400 = 2400,
+}
+
 export class Config {
     _data;
     constructor(data: JsonValue) {
@@ -212,6 +231,14 @@ export class Config {
     get check_file_hash() {
         return this._return_bool("check_file_hash") ?? true;
     }
+    get import_method() {
+        const n = this._return_number("input_method") ?? 1;
+        if (n < 0 || n > 3) return ImportMethod.CopyThenDelete;
+        return n as ImportMethod;
+    }
+    get max_import_img_count() {
+        return this._return_number("max_import_img_count") || 3;
+    }
     to_json(): ConfigType {
         return {
             cookies: typeof this.cookies === "string",
@@ -248,6 +275,8 @@ export class Config {
             random_file_secret: this.random_file_secret,
             use_path_based_img_url: this.use_path_based_img_url,
             check_file_hash: this.check_file_hash,
+            import_method: this.import_method,
+            max_import_img_count: this.max_import_img_count,
         };
     }
 }

db.ts

@@ -814,8 +814,8 @@ export class EhDb {
     check_download_task(gid: number | bigint, token: string) {
         return this.transaction(() => {
             const r = this.db.queryEntries<Task>(
-                "SELECT * FROM task WHERE type = ? AND gid = ? AND token = ?;",
-                [TaskType.Download, gid, token],
+                "SELECT * FROM task WHERE (type = ? OR type = ?) AND gid = ? AND token = ?;",
+                [TaskType.Download, TaskType.Import, gid, token],
             );
             return r.length ? r[0] : undefined;
         });

page/GalleryPage.ts

@@ -88,6 +88,9 @@ class Image {
     get src() {
         return this.data?.img_url;
     }
+    get token() {
+        return this.base.token;
+    }
     get xres() {
         return this.data?.xres;
     }

page/MPVPage.ts

@@ -136,6 +136,9 @@ class MPVImage {
     get thumbnail() {
         return this.base.t;
     }
+    get token() {
+        return this.base.k;
+    }
     get xres() {
         const xres = this.data?.xres;
         if (!xres) return undefined;

routes/api/task.ts

@@ -12,6 +12,7 @@ import type { DownloadConfig } from "../../tasks/download.ts";
 import type { ExportZipConfig } from "../../tasks/export_zip.ts";
 import { User, UserPermission } from "../../db.ts";
 import { toJSON } from "../../utils.ts";
+import { ImportConfig } from "../../tasks/import.ts";
 
 export const handler: Handlers = {
     GET(req, ctx) {
@@ -165,6 +166,37 @@ export const handler: Handlers = {
             } catch (e) {
                 return return_error(500, e.message);
             }
+        } else if (typ == "import") {
+            const gid = await parse_big_int(form.get("gid"), null);
+            const token = await get_string(form.get("token"));
+            if (gid === null) {
+                return return_error(2, "gid is required");
+            }
+            if (!token) {
+                return return_error(3, "token is required");
+            }
+            const cfg = await get_string(form.get("cfg"));
+            if (!cfg) {
+                return return_error(4, "cfg is required");
+            }
+            let icfg: ImportConfig | undefined = undefined;
+            try {
+                icfg = JSON.parse(cfg);
+            } catch (_) {
+                return return_error(4, "cfg is invalid");
+            }
+            if (!icfg) {
+                return return_error(4, "cfg is required");
+            }
+            try {
+                const task = await t.add_import_task(gid, token, icfg, true);
+                if (task === null) {
+                    return return_error(6, "task is already in the list");
+                }
+                return return_data(task, 201);
+            } catch (e) {
+                return return_error(500, e.message);
+            }
         } else {
             return return_error(5, "unknown type");
         }

task.ts

@@ -3,6 +3,7 @@ export enum TaskType {
     ExportZip,
     UpdateMeiliSearchData,
     FixGalleryPage,
+    Import,
 }
 
 export type Task<T extends TaskType = TaskType> = {
@@ -52,11 +53,18 @@ export type TaskFixGalleryPageProgress = {
     checked_gallery: number;
 };
 
+export type TaskImportProgress = {
+    imported_page: number;
+    failed_page: number;
+    total_page: number;
+};
+
 export type TaskProgressBasicType = {
     [TaskType.Download]: TaskDownloadProgess;
     [TaskType.ExportZip]: TaskExportZipProgress;
     [TaskType.UpdateMeiliSearchData]: TaskUpdateMeiliSearchDataProgress;
     [TaskType.FixGalleryPage]: TaskFixGalleryPageProgress;
+    [TaskType.Import]: TaskImportProgress;
 };
 
 export type TaskProgress<T extends TaskType = TaskType> = {

task_manager.ts

@@ -16,6 +16,7 @@ import {
     ExportZipConfig,
 } from "./tasks/export_zip.ts";
 import { fix_gallery_page } from "./tasks/fix_gallery_page.ts";
+import { ImportConfig } from "./tasks/import.ts";
 import { update_meili_search_data } from "./tasks/update_meili_search_data.ts";
 import {
     DiscriminatedUnion,
@@ -157,6 +158,28 @@ export class TaskManager extends EventTarget {
         };
         return await this.#add_task(task);
     }
+    async add_import_task(
+        gid: number | bigint,
+        token: string,
+        cfg: ImportConfig,
+        mark_already = false,
+    ) {
+        this.#check_closed();
+        const otask = await this.db.check_download_task(gid, token);
+        if (otask !== undefined) {
+            console.log("The task is already in list.");
+            return mark_already ? null : otask;
+        }
+        const task: Task = {
+            gid,
+            token,
+            id: 0,
+            pid: Deno.pid,
+            type: TaskType.Import,
+            details: toJSON(cfg),
+        };
+        return await this.#add_task(task);
+    }
     async add_update_meili_search_data_task(gid?: number | bigint) {
         this.#check_closed();
         const otask = await this.db.check_update_meili_search_data_task(gid);
@@ -177,7 +200,9 @@ export class TaskManager extends EventTarget {
     async check_task(task: Task) {
         this.#check_closed();
         if (await this.check_task_is_running(task)) return;
-        const ut = (await this.db.check_onetime_task()).map((t) => BigInt(t.id));
+        const ut = (await this.db.check_onetime_task()).map((t) =>
+            BigInt(t.id)
+        );
         if (ut.length && !ut.includes(BigInt(task.id))) return;
         let t = task;
         if (task.pid != Deno.pid) {

tasks/download.ts

@@ -231,9 +231,8 @@ export async function download_task(
     }
     const base_path = join(cfg.base, task.gid.toString());
     await sure_dir(base_path);
-    const max_download_img_count = dcfg.max_download_img_count !== undefined
-        ? dcfg.max_download_img_count
-        : cfg.max_download_img_count;
+    const max_download_img_count = dcfg.max_download_img_count ??
+        cfg.max_download_img_count;
     const m = new DownloadManager(
         max_download_img_count,
         abort,
@@ -241,16 +240,12 @@ export async function download_task(
         task,
         manager,
     );
-    const mpv_enabled = dcfg.mpv !== undefined ? dcfg.mpv : cfg.mpv;
-    const download_original_img = dcfg.download_original_img !== undefined
-        ? dcfg.download_original_img
-        : cfg.download_original_img;
-    const max_retry_count = dcfg.max_retry_count !== undefined
-        ? dcfg.max_retry_count
-        : cfg.max_retry_count;
-    const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined
-        ? dcfg.remove_previous_gallery
-        : cfg.remove_previous_gallery;
+    const mpv_enabled = dcfg.mpv ?? cfg.mpv;
+    const download_original_img = dcfg.download_original_img ??
+        cfg.download_original_img;
+    const max_retry_count = dcfg.max_retry_count ?? cfg.max_retry_count;
+    const remove_previous_gallery = dcfg.remove_previous_gallery ??
+        cfg.remove_previous_gallery;
     const g = await client.fetchGalleryPage(task.gid, task.token);
     async function download_task(names: Record<string, number>, i: Image) {
         const ofiles = db.get_files(i.page_token);

tasks/export_zip.ts

@@ -33,9 +33,7 @@ export async function export_zip(
     const gid = task.gid;
     const g = db.get_gmeta_by_gid(gid);
     if (!g) throw Error("Gallery not found in database.");
-    const jpn_title = ecfg.jpn_title !== undefined
-        ? ecfg.jpn_title
-        : cfg.export_zip_jpn_title;
+    const jpn_title = ecfg.jpn_title ?? cfg.export_zip_jpn_title;
     const progress: TaskExportZipProgress = {
         total_page: Number(g.filecount),
         added_page: 0,

tasks/import.ts

@@ -0,0 +1,421 @@
+import { Task, TaskImportProgress, TaskType } from "../task.ts";
+import { TaskManager } from "../task_manager.ts";
+import {
+    add_suffix_to_path,
+    asyncEvery,
+    asyncFilter,
+    configureZipJs,
+    promiseState,
+    PromiseStatus,
+    RecoverableError,
+    sleep,
+    sure_dir,
+} from "../utils.ts";
+import { FS, ZipFileEntry } from "zipjs/index.js";
+import { exists } from "@std/fs/exists";
+import { walk } from "@std/fs/walk";
+import { extname, join, relative } from "@std/path";
+import { ImportMethod, ImportSize } from "../config.ts";
+import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts";
+import { EhFile, PMeta } from "../db.ts";
+
+export type ImportConfig = {
+    max_import_img_count?: number;
+    mpv?: boolean;
+    method?: ImportMethod;
+    remove_previous_gallery?: boolean;
+    replaced_gallery?: { gid: number | bigint; token: string }[];
+    import_path: string;
+    size: ImportSize;
+};
+
+interface Page {
+    index: number;
+    token: string;
+    name: string;
+    sampled_name: string;
+}
+
+const PROGRESS_UPDATE_INTERVAL = 200;
+
+class ImportManager {
+    #abort: AbortSignal;
+    #force_abort: AbortSignal;
+    #max_import_count;
+    #running_tasks: Promise<unknown>[];
+    #progress: TaskImportProgress;
+    #task: Task;
+    #manager: TaskManager;
+    #progress_changed: boolean;
+    #last_send_progress: number;
+    constructor(
+        max_import_img_count: number,
+        abort: AbortSignal,
+        force_abort: AbortSignal,
+        task: Task,
+        manager: TaskManager,
+    ) {
+        this.#max_import_count = max_import_img_count;
+        this.#running_tasks = [];
+        this.#abort = abort;
+        this.#force_abort = force_abort;
+        this.#progress = {
+            imported_page: 0,
+            failed_page: 0,
+            total_page: 0,
+        };
+        this.#task = task;
+        this.#manager = manager;
+        this.#progress_changed = false;
+        this.#last_send_progress = -1;
+    }
+    async #check_tasks() {
+        this.#running_tasks = await asyncFilter(
+            this.#running_tasks,
+            async (t) => {
+                const s = await promiseState(t);
+                if (s.status === PromiseStatus.Rejected) {
+                    if (!this.#force_abort.aborted) console.log(s.reason);
+                    this.#progress.failed_page += 1;
+                    this.#sendEvent();
+                } else if (s.status === PromiseStatus.Fulfilled) {
+                    this.#progress.imported_page += 1;
+                    this.#sendEvent();
+                }
+                return s.status === PromiseStatus.Pending;
+            },
+        );
+        if (this.#progress_changed) {
+            const now = (new Date()).getTime();
+            if (now >= this.#last_send_progress + PROGRESS_UPDATE_INTERVAL) {
+                this.#manager.dispatchTaskProgressEvent(
+                    TaskType.Import,
+                    this.#task.id,
+                    this.#progress,
+                );
+                this.#progress_changed = false;
+                this.#last_send_progress = now;
+            }
+        }
+    }
+    #sendEvent() {
+        this.#progress_changed = true;
+        const now = (new Date()).getTime();
+        if (now < this.#last_send_progress + PROGRESS_UPDATE_INTERVAL) return;
+        const re = this.#manager.dispatchTaskProgressEvent(
+            TaskType.Import,
+            this.#task.id,
+            this.#progress,
+        );
+        this.#last_send_progress = now;
+        this.#progress_changed = false;
+        return re;
+    }
+    async add_new_task<T>(f: () => Promise<T>) {
+        while (1) {
+            if (this.#abort.aborted) break;
+            await this.#check_tasks();
+            if (this.#running_tasks.length < this.#max_import_count) {
+                this.#running_tasks.push(f());
+                break;
+            }
+            await sleep(10);
+        }
+    }
+    get has_failed_task() {
+        return this.#progress.failed_page > 0;
+    }
+    async join() {
+        while (1) {
+            await this.#check_tasks();
+            if (!this.#running_tasks.length) break;
+            await sleep(10);
+        }
+    }
+    set_total_page(page: number) {
+        this.#progress.total_page = page;
+        this.#sendEvent();
+    }
+}
+
+class FileLoader {
+    #zip?: FS;
+    #path;
+    #files: string[] = [];
+    #inited = false;
+    #filecount;
+    #has_prefix = false;
+    constructor(path: string, filecount: number) {
+        this.#path = path;
+        this.#filecount = filecount;
+    }
+    #check() {
+        if (!this.#inited) throw Error("FileLoader not initiailzed.");
+    }
+    #get_file(name: string) {
+        if (this.#files.includes(name)) {
+            return join(this.#path, name);
+        }
+    }
+    #get_zip(name: string) {
+        const ext = this.#zip!.getChildByName(name);
+        if (ext && ext instanceof ZipFileEntry) {
+            return ext;
+        }
+    }
+    get_file(name: string) {
+        let t = this.#get_file(name);
+        if (t) return t;
+        const ext = extname(name);
+        if (ext != ".jpg") {
+            const n = name.slice(0, name.length - 4) + ".jpg";
+            t = this.#get_file(n);
+            if (t) return t;
+        }
+    }
+    get_zip(name: string) {
+        let t = this.#get_zip(name);
+        if (t) return t;
+        const ext = extname(name);
+        if (ext != ".jpg") {
+            const n = name.slice(0, name.length - 4) + ".jpg";
+            t = this.#get_zip(n);
+            if (t) return t;
+        }
+    }
+    async init() {
+        if (await exists(this.#path, { isDirectory: true })) {
+            for await (const i of walk(this.#path)) {
+                if (i.path != this.#path) {
+                    this.#files.push(relative(this.#path, i.path));
+                }
+            }
+        } else {
+            configureZipJs();
+            const r = await Deno.open(this.#path, { read: true });
+            try {
+                this.#zip = new FS();
+                const ent = await this.#zip.importReadable(r.readable);
+                for (const e of ent) {
+                    this.#files.push(e.getFullname());
+                }
+            } finally {
+                try {
+                    r.close();
+                } catch (_) {
+                    null;
+                }
+            }
+        }
+        let has_prefix = true;
+        const re = new RegExp(`^\\d{${this.#filecount.toString().length}}_`);
+        for (const f of this.#files) {
+            if (!f.match(re)) {
+                has_prefix = false;
+                break;
+            }
+        }
+        this.#has_prefix = has_prefix;
+        this.#inited = true;
+        return this;
+    }
+    get is_zip() {
+        this.#check();
+        return this.#zip !== undefined;
+    }
+}
+
+export async function import_task(task: Task, manager: TaskManager) {
+    if (!task.details) throw Error("Task details are needed.");
+    console.log("Started to import gallery", task.gid);
+    const icfg: ImportConfig = JSON.parse(task.details);
+    const cfg = manager.cfg;
+    const client = manager.client;
+    const db = manager.db;
+    const gdatas = await client.fetchGalleryMetadataByAPI([
+        task.gid,
+        task.token,
+    ]);
+    const gdata = gdatas.map.get(BigInt(task.gid));
+    if (gdata === undefined) throw Error("Gallery metadata not included.");
+    if (typeof gdata === "string") throw Error(gdata);
+    const f =
+        await (new FileLoader(icfg.import_path, parseInt(gdata.filecount)))
+            .init();
+    const gmeta = gdatas.convert(gdata);
+    db.add_gmeta(gmeta);
+    await db.add_gtag(task.gid, new Set(gdata.tags));
+    if (manager.meilisearch) {
+        manager.meilisearch.target.dispatchEvent(
+            new CustomEvent("gallery_update", { detail: gmeta.gid }),
+        );
+    }
+    const base_path = join(cfg.base, task.gid.toString());
+    await sure_dir(base_path);
+    let mpv_enabled = icfg.mpv ?? cfg.mpv;
+    let import_method = icfg.method ?? cfg.import_method;
+    if (
+        f.is_zip && import_method != ImportMethod.Copy &&
+        import_method != ImportMethod.CopyThenDelete
+    ) {
+        import_method = ImportMethod.CopyThenDelete;
+    }
+    const max_import_img_count = icfg.max_import_img_count ??
+        cfg.max_import_img_count;
+    let imgs: Page[] = [];
+    const m = new ImportManager(
+        max_import_img_count,
+        manager.aborts,
+        manager.force_aborts,
+        task,
+        manager,
+    );
+    if (!mpv_enabled) {
+        const g = await client.fetchGalleryPage(task.gid, task.token);
+        if (g.mpv_enabled) {
+            mpv_enabled = true;
+        } else {
+            imgs = await g.imagelist;
+            m.set_total_page(g.length);
+        }
+    }
+    if (mpv_enabled) {
+        const g = await client.fetchMPVPage(task.gid, task.token);
+        imgs = g.imagelist;
+        m.set_total_page(g.pagecount);
+    }
+    const names = imgs.reduce(
+        (acc: Record<string, number>, cur) => {
+            const curr = cur.name;
+            return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
+        },
+        {},
+    );
+
+    async function import_img(i: Page) {
+        const opath = f.get_file(i.name);
+        if (!opath) {
+            console.log("File not found");
+            return;
+        }
+        const size = await fb_get_size(opath);
+        if (!size) {
+            console.log("Failed to get file size for", opath);
+            throw Error("Failed to get file size.");
+        }
+        const ofiles = db.get_files(i.token);
+        if (ofiles.length) {
+            const need = await asyncEvery(
+                ofiles,
+                async (t) =>
+                    (!t.is_original && t.height != size.height &&
+                        t.width != size.width) || (!await exists(t.path)),
+            );
+            if (!need) {
+                const p = db.get_pmeta_by_index(task.gid, i.index);
+                if (!p) {
+                    const op = db.get_pmeta_by_token(
+                        task.gid,
+                        i.token,
+                    );
+                    if (op) {
+                        op.index = i.index;
+                        op.name = i.name;
+                        db.add_pmeta(op);
+                        return;
+                    } else {
+                        const ops = db.get_pmeta_by_token_only(
+                            i.token,
+                        );
+                        if (ops.length) {
+                            const op = ops[0];
+                            op.gid = task.gid;
+                            op.index = i.index;
+                            op.name = i.name;
+                            db.add_pmeta(op);
+                            return;
+                        }
+                    }
+                }
+                console.log("Already has page", i.index);
+                return;
+            }
+        }
+        const pmeta: PMeta = {
+            gid: task.gid,
+            height: size.height,
+            width: size.width,
+            index: i.index,
+            name: i.name,
+            token: i.token,
+        };
+        db.add_pmeta(pmeta);
+        const oriext = extname(i.name);
+        const nowext = extname(opath);
+        const is_original = icfg.size == ImportSize.Original ||
+            (oriext != ".jpg" && oriext == nowext) ||
+            (oriext == nowext && size.width < icfg.size);
+        let path = join(base_path, is_original ? i.name : i.sampled_name);
+        if (import_method != ImportMethod.Keep && names[i.name] > 1) {
+            path = add_suffix_to_path(path, i.token);
+            console.log("Changed path to", path);
+        }
+        if (import_method == ImportMethod.Move) {
+            await Deno.rename(opath, path);
+        } else if (import_method != ImportMethod.Keep) {
+            await Deno.copyFile(opath, path);
+            if (import_method == ImportMethod.CopyThenDelete) {
+                await Deno.remove(opath);
+            }
+        } else {
+            path = opath;
+        }
+        const file: EhFile = {
+            height: size.height,
+            width: size.width,
+            is_original,
+            id: 0,
+            path,
+            token: i.token,
+        };
+        db.add_file(file);
+    }
+
+    for (const i of imgs) {
+        if (manager.aborted) break;
+        await m.add_new_task(async () => {
+            await import_img(i);
+        });
+    }
+    await m.join();
+    const remove_previous_gallery = icfg.remove_previous_gallery ??
+        cfg.remove_previous_gallery;
+    if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
+    if (manager.aborted || manager.force_aborted) throw Error("aborted");
+    if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
+        let replaced_gallery = icfg.replaced_gallery;
+        if (replaced_gallery === undefined) {
+            const fg = await client.fetchGalleryPage(
+                gmeta.first_gid,
+                gmeta.first_key,
+            );
+            replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid);
+            replaced_gallery.push({
+                gid: gmeta.first_gid,
+                token: gmeta.first_key,
+            });
+        }
+        replaced_gallery.forEach((g) => {
+            const gmeta = db.get_gmeta_by_gid(g.gid);
+            if (!gmeta) return;
+            console.log("Remove gallery ", g.gid);
+            if (manager.meilisearch) {
+                manager.meilisearch.target.dispatchEvent(
+                    new CustomEvent("gallery_remove", { detail: gmeta.gid }),
+                );
+            }
+            db.delete_gallery(g.gid);
+        });
+    }
+    return task;
+}