diff --git a/Dockerfile b/Dockerfile index 6aabc96..4cd22d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,6 +67,15 @@ RUN cd ~ && \ make -j$(grep -c ^processor /proc/cpuinfo) && make install && \ cd ~ && rm -rf sqlite-snapshot-202401231504 sqlite-snapshot-202401231504.tar.gz +RUN cd ~ && \ + curl -L "https://libzip.org/download/libzip-1.10.1.tar.gz" -o libzip-1.10.1.tar.gz && \ + tar -xzvf libzip-1.10.1.tar.gz && \ + cd libzip-1.10.1 && \ + cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_REGRESS=OFF -DBUILD_OSSFUZZ=OFF -DBUILD_EXAMPLES=OFF -DBUILD_DOC=OFF \ + -DCMAKE_INSTALL_PREFIX=/clib -DCMAKE_PREFIX_PATH=/clib ../ && \ + make -j$(grep -c ^processor /proc/cpuinfo) && make install && \ + cd ~ && rm -rf libzip-1.10.1 libzip-1.10.1.tar.gz + COPY ./extensions /root/extensions RUN cd /root/extensions && \ diff --git a/import_map.json b/import_map.json index 3822d39..340a61c 100644 --- a/import_map.json +++ b/import_map.json @@ -35,6 +35,7 @@ "@twind/core": "https://esm.sh/@twind/core@1.1.3", "@twind/preset-tailwind": "https://esm.sh/@twind/preset-tailwind@1.1.4/", "@twind/preset-autoprefix": "https://esm.sh/@twind/preset-autoprefix@1.0.7/", - "@lifegpc/sha1": "jsr:/@lifegpc/sha1@1.0.0" + "@lifegpc/sha1": "jsr:/@lifegpc/sha1@1.0.0", + "@lifegpc/libzip/": "jsr:/@lifegpc/libzip@0.0.7/" } } diff --git a/routes/api/status.ts b/routes/api/status.ts index 700da77..ac87cf3 100644 --- a/routes/api/status.ts +++ b/routes/api/status.ts @@ -5,8 +5,10 @@ import { get_host, return_data } from "../../server/utils.ts"; import { check_ffmpeg_binary } from "../../thumbnail/ffmpeg_binary.ts"; import type * as FFMPEG_API from "../../thumbnail/ffmpeg_api.ts"; import { isDocker } from "../../utils.ts"; +import type * as LIBZIP from "@lifegpc/libzip/raw"; let ffmpeg_api: typeof FFMPEG_API | undefined; +let libzip: typeof LIBZIP | undefined; async function check_ffmpeg_api() { if (ffmpeg_api) return true; @@ -18,6 +20,16 @@ async function check_ffmpeg_api() { } } +async function check_libzip() { + if (libzip) return true; + try { + libzip = await import("@lifegpc/libzip/raw"); + return true; + } catch (_) { + return false; + } +} + export const handler: Handlers = { async GET(req, ctx) { const m = get_task_manager(); @@ -51,6 +63,7 @@ export const handler: Handlers = { key: m.cfg.meili_search_api_key || m.cfg.meili_update_api_key, }; } + const libzip_enabled = await check_libzip(); const no_user = c === 0 || c === 0n; const is_docker = isDocker(); return return_data({ @@ -61,6 +74,7 @@ export const handler: Handlers = { meilisearch, no_user, is_docker, + libzip_enabled, }); }, }; diff --git a/server/status.ts b/server/status.ts index 6ff03ba..256b806 100644 --- a/server/status.ts +++ b/server/status.ts @@ -9,4 +9,5 @@ export type StatusData = { }; no_user: boolean; is_docker: boolean; + libzip_enabled: boolean; }; diff --git a/tasks/import.ts b/tasks/import.ts index 2c23c4f..69c0547 100644 --- a/tasks/import.ts +++ b/tasks/import.ts @@ -1,3 +1,4 @@ +// deno-lint-ignore-file no-inner-declarations import { Task, TaskImportProgress, TaskType } from "../task.ts"; import { TaskManager } from "../task_manager.ts"; import { @@ -17,6 +18,7 @@ import { extname, join, relative } from "@std/path"; import { ImportMethod, ImportSize } from "../config.ts"; import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts"; import { EhFile, PMeta } from "../db.ts"; +import type { ReadonlyZip } from "../utils/readonly_zip.ts"; export type ImportConfig = { max_import_img_count?: number; @@ -141,28 +143,44 @@ class ImportManager { class FileLoader { #path; + #zip?: ReadonlyZip; #files: string[] = []; #inited = false; #filecount; #has_prefix = false; + #closed = false; constructor(path: string, filecount: number) { this.#path = path; this.#filecount = filecount; } #check() { if (!this.#inited) throw Error("FileLoader not initiailzed."); + if (this.#closed) throw Error("Already closed."); } #get_file(name: string) { if (this.#files.includes(name)) { return join(this.#path, name); } } - #get_zip(_name: string) { - return null; + #get_zip(name: string) { + if (this.#files.includes(name)) { + return name; + } + } + close() { + if (this.#closed) return; + this.#closed = true; + this.#zip?.close(); } get_file(name: string, index: number) { + this.#check(); if (this.#has_prefix) { - name = `${index.toString().padStart(3, "0")}_${name}`; + name = `${ + index.toString().padStart( + this.#filecount.toString().length, + "0", + ) + }_${name}`; } let t = this.#get_file(name); if (t) return t; @@ -173,7 +191,16 @@ class FileLoader { if (t) return t; } } - get_zip(name: string) { + get_zip(name: string, index: number) { + this.#check(); + if (this.#has_prefix) { + name = `${ + index.toString().padStart( + this.#filecount.toString().length, + "0", + ) + }_${name}`; + } let t = this.#get_zip(name); if (t) return t; const ext = extname(name).toLowerCase(); @@ -190,6 +217,16 @@ class FileLoader { this.#files.push(relative(this.#path, i.path)); } } + } else { + const z = await import("../utils/readonly_zip.ts"); + this.#zip = new z.ReadonlyZip(this.#path); + const count = this.#zip.count; + for (let i = 0n; i < count; i++) { + const f = this.#zip.get_name(i); + if (f) { + this.#files.push(f); + } + } } let has_prefix = true; const re = new RegExp(`^\\d{${this.#filecount.toString().length}}_`); @@ -205,11 +242,21 @@ class FileLoader { } this.#has_prefix = has_prefix; this.#inited = true; + this.#closed = false; return this; } get is_zip() { this.#check(); - return false; + return this.#zip !== undefined; + } + open_zip_file(name: string) { + if (this.#files.includes(name)) { + const s = this.#zip!.open(name); + if (typeof s === "string") { + throw Error(`Failed to open file: ${s}`); + } + return s; + } } } @@ -230,189 +277,316 @@ export async function import_task(task: Task, manager: TaskManager) { const f = await (new FileLoader(icfg.import_path, parseInt(gdata.filecount))) .init(); - const gmeta = gdatas.convert(gdata); - db.add_gmeta(gmeta); - await db.add_gtag(task.gid, new Set(gdata.tags)); - if (manager.meilisearch) { - manager.meilisearch.target.dispatchEvent( - new CustomEvent("gallery_update", { detail: gmeta.gid }), - ); - } - const base_path = join(cfg.base, task.gid.toString()); - await sure_dir(base_path); - let mpv_enabled = icfg.mpv ?? cfg.mpv; - let import_method = icfg.method ?? cfg.import_method; - if ( - f.is_zip && import_method != ImportMethod.Copy && - import_method != ImportMethod.CopyThenDelete - ) { - import_method = ImportMethod.CopyThenDelete; - } - const max_import_img_count = icfg.max_import_img_count ?? - cfg.max_import_img_count; - let imgs: Page[] = []; - const m = new ImportManager( - max_import_img_count, - manager.aborts, - manager.force_aborts, - task, - manager, - ); - if (!mpv_enabled) { - const g = await client.fetchGalleryPage(task.gid, task.token); - if (g.mpv_enabled) { - mpv_enabled = true; - } else { - imgs = await g.imagelist; - m.set_total_page(g.length); - } - } - if (mpv_enabled) { - const g = await client.fetchMPVPage(task.gid, task.token); - imgs = g.imagelist; - m.set_total_page(g.pagecount); - } - const names = imgs.reduce( - (acc: Record, cur) => { - const curr = cur.name; - return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc; - }, - {}, - ); - - async function import_img(i: Page) { - const opath = f.get_file(i.name, i.index); - if (!opath) { - console.log("File not found"); - return; - } - const size = await fb_get_size(opath); - if (!size) { - console.log("Failed to get file size for", opath); - throw Error("Failed to get file size."); - } - const ofiles = db.get_files(i.token); - if (ofiles.length) { - const need = await asyncEvery( - ofiles, - async (t) => - (!t.is_original && t.height != size.height && - t.width != size.width) || (!await exists(t.path)), + try { + const gmeta = gdatas.convert(gdata); + db.add_gmeta(gmeta); + await db.add_gtag(task.gid, new Set(gdata.tags)); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_update", { detail: gmeta.gid }), ); - if (!need) { - const p = db.get_pmeta_by_index(task.gid, i.index); - if (!p) { - const op = db.get_pmeta_by_token( - task.gid, - i.token, - ); - if (op) { - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } else { - const ops = db.get_pmeta_by_token_only( + } + const base_path = join(cfg.base, task.gid.toString()); + await sure_dir(base_path); + let mpv_enabled = icfg.mpv ?? cfg.mpv; + let import_method = icfg.method ?? cfg.import_method; + if ( + f.is_zip && import_method != ImportMethod.Copy && + import_method != ImportMethod.CopyThenDelete + ) { + import_method = ImportMethod.CopyThenDelete; + } + const max_import_img_count = icfg.max_import_img_count ?? + cfg.max_import_img_count; + let imgs: Page[] = []; + const m = new ImportManager( + max_import_img_count, + manager.aborts, + manager.force_aborts, + task, + manager, + ); + if (!mpv_enabled) { + const g = await client.fetchGalleryPage(task.gid, task.token); + if (g.mpv_enabled) { + mpv_enabled = true; + } else { + imgs = await g.imagelist; + m.set_total_page(g.length); + } + } + if (mpv_enabled) { + const g = await client.fetchMPVPage(task.gid, task.token); + imgs = g.imagelist; + m.set_total_page(g.pagecount); + } + const names = imgs.reduce( + (acc: Record, cur) => { + const curr = cur.name; + return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc; + }, + {}, + ); + + async function import_img(i: Page) { + const opath = f.get_file(i.name, i.index); + if (!opath) { + console.log("File not found"); + return; + } + const size = await fb_get_size(opath); + if (!size) { + console.log("Failed to get file size for", opath); + throw Error("Failed to get file size."); + } + const ofiles = db.get_files(i.token); + if (ofiles.length) { + const need = await asyncEvery( + ofiles, + async (t) => + (!t.is_original && t.height != size.height && + t.width != size.width) || (!await exists(t.path)), + ); + if (!need) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, i.token, ); - if (ops.length) { - const op = ops[0]; - op.gid = task.gid; + if (op) { op.index = i.index; op.name = i.name; db.add_pmeta(op); return; + } else { + const ops = db.get_pmeta_by_token_only( + i.token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } } } + console.log("Already has page", i.index); + return; } - console.log("Already has page", i.index); - return; } - } - const pmeta: PMeta = { - gid: task.gid, - height: size.height, - width: size.width, - index: i.index, - name: i.name, - token: i.token, - }; - db.add_pmeta(pmeta); - const oriext = extname(i.name).toLowerCase(); - const nowext = extname(opath).toLowerCase(); - const is_original = icfg.size == ImportSize.Original || - (oriext != ".jpg" && oriext == nowext) || - (oriext == nowext && size.width < icfg.size); - let path = join(base_path, is_original ? i.name : i.sampled_name); - if (import_method != ImportMethod.Keep && names[i.name] > 1) { - path = add_suffix_to_path(path, i.token); - console.log("Changed path to", path); - } - if (import_method == ImportMethod.Move) { - await Deno.rename(opath, path); - } else if (import_method != ImportMethod.Keep) { - await Deno.copyFile(opath, path); - } else { - path = opath; - } - if (cfg.check_file_hash && is_original) { - const sha = await calFileSha1(path); - if (sha.slice(0, i.token.length) != i.token) { - console.warn( - `Hash not matched: file hash ${sha}, token ${i.token}`, - ); - return; + const pmeta: PMeta = { + gid: task.gid, + height: size.height, + width: size.width, + index: i.index, + name: i.name, + token: i.token, + }; + db.add_pmeta(pmeta); + const oriext = extname(i.name).toLowerCase(); + const nowext = extname(opath).toLowerCase(); + const is_original = icfg.size == ImportSize.Original || + (oriext != ".jpg" && oriext == nowext) || + (oriext == nowext && size.width < icfg.size); + let path = join(base_path, is_original ? i.name : i.sampled_name); + if (import_method != ImportMethod.Keep && names[i.name] > 1) { + path = add_suffix_to_path(path, i.token); + console.log("Changed path to", path); } + if (import_method == ImportMethod.Move) { + await Deno.rename(opath, path); + } else if (import_method != ImportMethod.Keep) { + await Deno.copyFile(opath, path); + } else { + path = opath; + } + if (cfg.check_file_hash && is_original) { + const sha = await calFileSha1(path); + if (sha.slice(0, i.token.length) != i.token) { + console.warn( + `Hash not matched: file hash ${sha}, token ${i.token}`, + ); + return; + } + } + if (import_method == ImportMethod.CopyThenDelete) { + await Deno.remove(opath); + } + const file: EhFile = { + height: size.height, + width: size.width, + is_original, + id: 0, + path, + token: i.token, + }; + db.add_file(file); } - if (import_method == ImportMethod.CopyThenDelete) { - await Deno.remove(opath); - } - const file: EhFile = { - height: size.height, - width: size.width, - is_original, - id: 0, - path, - token: i.token, - }; - db.add_file(file); - } - for (const i of imgs) { - if (manager.aborted) break; - await m.add_new_task(async () => { - await import_img(i); - }); - } - await m.join(); - const remove_previous_gallery = icfg.remove_previous_gallery ?? - cfg.remove_previous_gallery; - if (m.has_failed_task) throw new RecoverableError("Some tasks failed."); - if (manager.aborted || manager.force_aborted) throw Error("aborted"); - if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) { - let replaced_gallery = icfg.replaced_gallery; - if (replaced_gallery === undefined) { - const fg = await client.fetchGalleryPage( - gmeta.first_gid, - gmeta.first_key, + async function import_zip_img(i: Page) { + const opath = f.get_zip(i.name, i.index); + if (!opath) { + console.log("File not found"); + return; + } + const ofiles = db.get_files(i.token); + if (ofiles.length) { + const need = await asyncEvery( + ofiles, + async (t) => + !t.is_original && icfg.size === ImportSize.Original || + (!await exists(t.path)), + ); + if (!need) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, + i.token, + ); + if (op) { + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } else { + const ops = db.get_pmeta_by_token_only( + i.token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } + } + } + console.log("Already has page", i.index); + return; + } + } + const oriext = extname(i.name).toLowerCase(); + const nowext = extname(opath).toLowerCase(); + let path = join( + base_path, + i.name.replace(oriext, nowext), ); - replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid); - replaced_gallery.push({ - gid: gmeta.first_gid, - token: gmeta.first_key, + if (names[i.name] > 1) { + path = add_suffix_to_path(path, i.token); + console.log("Changed path to", path); + } + const zf = f.open_zip_file(opath); + if (!zf) { + console.log("File not found"); + return; + } + try { + const out = await Deno.open(path, { + create: true, + write: true, + }); + try { + const buf = new Uint8Array(65536); + let len = zf.read(buf); + while (len > 0) { + await out.write(buf.slice(0, Number(len))); + len = zf.read(buf); + } + } finally { + out.close(); + } + } finally { + zf.close(); + } + const size = await fb_get_size(path); + if (!size) { + console.log("Failed to get file size for", path); + throw Error("Failed to get file size."); + } + const is_original = icfg.size == ImportSize.Original || + (oriext != ".jpg" && oriext == nowext) || + (oriext == nowext && size.width < icfg.size); + if (cfg.check_file_hash && is_original) { + const sha = await calFileSha1(path); + if (sha.slice(0, i.token.length) != i.token) { + console.warn( + `Hash not matched: file hash ${sha}, token ${i.token}`, + ); + return; + } + } + const pmeta: PMeta = { + gid: task.gid, + height: size.height, + width: size.width, + index: i.index, + name: i.name, + token: i.token, + }; + db.add_pmeta(pmeta); + const file: EhFile = { + height: size.height, + width: size.width, + is_original, + id: 0, + path, + token: i.token, + }; + db.add_file(file); + } + + for (const i of imgs) { + if (manager.aborted) break; + await m.add_new_task(async () => { + if (f.is_zip) await import_zip_img(i); + else await import_img(i); }); } - replaced_gallery.forEach((g) => { - const gmeta = db.get_gmeta_by_gid(g.gid); - if (!gmeta) return; - console.log("Remove gallery ", g.gid); - if (manager.meilisearch) { - manager.meilisearch.target.dispatchEvent( - new CustomEvent("gallery_remove", { detail: gmeta.gid }), + await m.join(); + const remove_previous_gallery = icfg.remove_previous_gallery ?? + cfg.remove_previous_gallery; + if (m.has_failed_task) throw new RecoverableError("Some tasks failed."); + if (manager.aborted || manager.force_aborted) throw Error("aborted"); + if (f.is_zip && import_method == ImportMethod.CopyThenDelete) { + f.close(); + await Deno.remove(icfg.import_path); + } + if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) { + let replaced_gallery = icfg.replaced_gallery; + if (replaced_gallery === undefined) { + const fg = await client.fetchGalleryPage( + gmeta.first_gid, + gmeta.first_key, ); + replaced_gallery = fg.new_version.filter((d) => + d.gid < task.gid + ); + replaced_gallery.push({ + gid: gmeta.first_gid, + token: gmeta.first_key, + }); } - db.delete_gallery(g.gid); - }); + replaced_gallery.forEach((g) => { + const gmeta = db.get_gmeta_by_gid(g.gid); + if (!gmeta) return; + console.log("Remove gallery ", g.gid); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_remove", { + detail: gmeta.gid, + }), + ); + } + db.delete_gallery(g.gid); + }); + } + return task; + } finally { + f.close(); } - return task; } diff --git a/utils/readonly_zip.ts b/utils/readonly_zip.ts new file mode 100644 index 0000000..1c9812d --- /dev/null +++ b/utils/readonly_zip.ts @@ -0,0 +1,92 @@ +import * as lz from "@lifegpc/libzip/raw"; + +function get_err(err: number) { + const e = lz.ZipErrorT.new(); + lz.zip_error_init_with_code(e, err); + const s = lz.zip_error_strerror(e); + lz.zip_error_fini(e); + return s; +} + +class ZipFile { + #file: lz.ZipFileT; + #closed = false; + constructor(f: lz.ZipFileT) { + this.#file = f; + } + + #check_close() { + if (this.#closed) throw Error("Already closed."); + } + + close() { + if (this.#closed) return; + this.#closed = true; + const er = lz.zip_fclose(this.#file); + if (er) { + throw Error(`Failed to close file: ${get_err(er)}`); + } + } + + read(buf: Uint8Array) { + this.#check_close(); + const num = lz.zip_fread(this.#file, buf); + if (num == -1) { + throw Error( + `Failed to read file: ${lz.zip_file_strerror(this.#file)}`, + ); + } + return num; + } +} + +export class ReadonlyZip { + #zip: lz.ZipT; + #discarded = false; + #opened_file_list: ZipFile[] = []; + constructor(path: string) { + const ep = lz.IntPointer.new(); + const z = lz.zip_open(path, lz.ZipOpenFlag.RDONLY, ep); + if (!z) { + throw Error(`Failed to open archive: ${get_err(ep.int)}.`); + } + this.#zip = z; + } + close() { + if (this.#discarded) return; + for (const f of this.#opened_file_list) { + f.close(); + } + this.#discarded = true; + lz.zip_discard(this.#zip); + } + get count() { + return lz.zip_get_num_entries(this.#zip, 0); + } + get_index(name: string) { + return lz.zip_name_locate(this.#zip, name, lz.ZipFlags.ENC_UTF_8); + } + get_name(index: number | bigint) { + return lz.zip_get_name(this.#zip, index, 0); + } + open(name: string) { + const z = lz.zip_fopen(this.#zip, name, lz.ZipFlags.ENC_UTF_8); + if (!z) { + const errmsg = lz.zip_strerror(this.#zip); + return errmsg; + } + const f = new ZipFile(z); + this.#opened_file_list.push(f); + return f; + } + open_index(index: number | bigint) { + const z = lz.zip_fopen_index(this.#zip, index, 0); + if (!z) { + const errmsg = lz.zip_strerror(this.#zip); + return errmsg; + } + const f = new ZipFile(z); + this.#opened_file_list.push(f); + return f; + } +}