Add support to import zip file

This commit is contained in:
2024-06-10 18:10:33 +08:00
parent 1d412fd306
commit d3c379632f
6 changed files with 462 additions and 171 deletions

View File

@@ -67,6 +67,15 @@ RUN cd ~ && \
make -j$(grep -c ^processor /proc/cpuinfo) && make install && \
cd ~ && rm -rf sqlite-snapshot-202401231504 sqlite-snapshot-202401231504.tar.gz
RUN cd ~ && \
curl -L "https://libzip.org/download/libzip-1.10.1.tar.gz" -o libzip-1.10.1.tar.gz && \
tar -xzvf libzip-1.10.1.tar.gz && \
cd libzip-1.10.1 && \
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_REGRESS=OFF -DBUILD_OSSFUZZ=OFF -DBUILD_EXAMPLES=OFF -DBUILD_DOC=OFF \
-DCMAKE_INSTALL_PREFIX=/clib -DCMAKE_PREFIX_PATH=/clib ../ && \
make -j$(grep -c ^processor /proc/cpuinfo) && make install && \
cd ~ && rm -rf libzip-1.10.1 libzip-1.10.1.tar.gz
COPY ./extensions /root/extensions
RUN cd /root/extensions && \

View File

@@ -35,6 +35,7 @@
"@twind/core": "https://esm.sh/@twind/[email protected]",
"@twind/preset-tailwind": "https://esm.sh/@twind/[email protected]/",
"@twind/preset-autoprefix": "https://esm.sh/@twind/[email protected]/",
"@lifegpc/sha1": "jsr:/@lifegpc/[email protected]"
"@lifegpc/sha1": "jsr:/@lifegpc/[email protected]",
"@lifegpc/libzip/": "jsr:/@lifegpc/[email protected]/"
}
}

View File

@@ -5,8 +5,10 @@ import { get_host, return_data } from "../../server/utils.ts";
import { check_ffmpeg_binary } from "../../thumbnail/ffmpeg_binary.ts";
import type * as FFMPEG_API from "../../thumbnail/ffmpeg_api.ts";
import { isDocker } from "../../utils.ts";
import type * as LIBZIP from "@lifegpc/libzip/raw";
let ffmpeg_api: typeof FFMPEG_API | undefined;
let libzip: typeof LIBZIP | undefined;
async function check_ffmpeg_api() {
if (ffmpeg_api) return true;
@@ -18,6 +20,16 @@ async function check_ffmpeg_api() {
}
}
async function check_libzip() {
if (libzip) return true;
try {
libzip = await import("@lifegpc/libzip/raw");
return true;
} catch (_) {
return false;
}
}
export const handler: Handlers = {
async GET(req, ctx) {
const m = get_task_manager();
@@ -51,6 +63,7 @@ export const handler: Handlers = {
key: m.cfg.meili_search_api_key || m.cfg.meili_update_api_key,
};
}
const libzip_enabled = await check_libzip();
const no_user = c === 0 || c === 0n;
const is_docker = isDocker();
return return_data<StatusData>({
@@ -61,6 +74,7 @@ export const handler: Handlers = {
meilisearch,
no_user,
is_docker,
libzip_enabled,
});
},
};

View File

@@ -9,4 +9,5 @@ export type StatusData = {
};
no_user: boolean;
is_docker: boolean;
libzip_enabled: boolean;
};

View File

@@ -1,3 +1,4 @@
// deno-lint-ignore-file no-inner-declarations
import { Task, TaskImportProgress, TaskType } from "../task.ts";
import { TaskManager } from "../task_manager.ts";
import {
@@ -17,6 +18,7 @@ import { extname, join, relative } from "@std/path";
import { ImportMethod, ImportSize } from "../config.ts";
import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts";
import { EhFile, PMeta } from "../db.ts";
import type { ReadonlyZip } from "../utils/readonly_zip.ts";
export type ImportConfig = {
max_import_img_count?: number;
@@ -141,28 +143,44 @@ class ImportManager {
class FileLoader {
#path;
#zip?: ReadonlyZip;
#files: string[] = [];
#inited = false;
#filecount;
#has_prefix = false;
#closed = false;
constructor(path: string, filecount: number) {
this.#path = path;
this.#filecount = filecount;
}
#check() {
if (!this.#inited) throw Error("FileLoader not initiailzed.");
if (this.#closed) throw Error("Already closed.");
}
#get_file(name: string) {
if (this.#files.includes(name)) {
return join(this.#path, name);
}
}
#get_zip(_name: string) {
return null;
#get_zip(name: string) {
if (this.#files.includes(name)) {
return name;
}
}
close() {
if (this.#closed) return;
this.#closed = true;
this.#zip?.close();
}
get_file(name: string, index: number) {
this.#check();
if (this.#has_prefix) {
name = `${index.toString().padStart(3, "0")}_${name}`;
name = `${
index.toString().padStart(
this.#filecount.toString().length,
"0",
)
}_${name}`;
}
let t = this.#get_file(name);
if (t) return t;
@@ -173,7 +191,16 @@ class FileLoader {
if (t) return t;
}
}
get_zip(name: string) {
get_zip(name: string, index: number) {
this.#check();
if (this.#has_prefix) {
name = `${
index.toString().padStart(
this.#filecount.toString().length,
"0",
)
}_${name}`;
}
let t = this.#get_zip(name);
if (t) return t;
const ext = extname(name).toLowerCase();
@@ -190,6 +217,16 @@ class FileLoader {
this.#files.push(relative(this.#path, i.path));
}
}
} else {
const z = await import("../utils/readonly_zip.ts");
this.#zip = new z.ReadonlyZip(this.#path);
const count = this.#zip.count;
for (let i = 0n; i < count; i++) {
const f = this.#zip.get_name(i);
if (f) {
this.#files.push(f);
}
}
}
let has_prefix = true;
const re = new RegExp(`^\\d{${this.#filecount.toString().length}}_`);
@@ -205,11 +242,21 @@ class FileLoader {
}
this.#has_prefix = has_prefix;
this.#inited = true;
this.#closed = false;
return this;
}
get is_zip() {
this.#check();
return false;
return this.#zip !== undefined;
}
open_zip_file(name: string) {
if (this.#files.includes(name)) {
const s = this.#zip!.open(name);
if (typeof s === "string") {
throw Error(`Failed to open file: ${s}`);
}
return s;
}
}
}
@@ -230,189 +277,316 @@ export async function import_task(task: Task, manager: TaskManager) {
const f =
await (new FileLoader(icfg.import_path, parseInt(gdata.filecount)))
.init();
const gmeta = gdatas.convert(gdata);
db.add_gmeta(gmeta);
await db.add_gtag(task.gid, new Set(gdata.tags));
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_update", { detail: gmeta.gid }),
);
}
const base_path = join(cfg.base, task.gid.toString());
await sure_dir(base_path);
let mpv_enabled = icfg.mpv ?? cfg.mpv;
let import_method = icfg.method ?? cfg.import_method;
if (
f.is_zip && import_method != ImportMethod.Copy &&
import_method != ImportMethod.CopyThenDelete
) {
import_method = ImportMethod.CopyThenDelete;
}
const max_import_img_count = icfg.max_import_img_count ??
cfg.max_import_img_count;
let imgs: Page[] = [];
const m = new ImportManager(
max_import_img_count,
manager.aborts,
manager.force_aborts,
task,
manager,
);
if (!mpv_enabled) {
const g = await client.fetchGalleryPage(task.gid, task.token);
if (g.mpv_enabled) {
mpv_enabled = true;
} else {
imgs = await g.imagelist;
m.set_total_page(g.length);
}
}
if (mpv_enabled) {
const g = await client.fetchMPVPage(task.gid, task.token);
imgs = g.imagelist;
m.set_total_page(g.pagecount);
}
const names = imgs.reduce(
(acc: Record<string, number>, cur) => {
const curr = cur.name;
return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
},
{},
);
async function import_img(i: Page) {
const opath = f.get_file(i.name, i.index);
if (!opath) {
console.log("File not found");
return;
}
const size = await fb_get_size(opath);
if (!size) {
console.log("Failed to get file size for", opath);
throw Error("Failed to get file size.");
}
const ofiles = db.get_files(i.token);
if (ofiles.length) {
const need = await asyncEvery(
ofiles,
async (t) =>
(!t.is_original && t.height != size.height &&
t.width != size.width) || (!await exists(t.path)),
try {
const gmeta = gdatas.convert(gdata);
db.add_gmeta(gmeta);
await db.add_gtag(task.gid, new Set(gdata.tags));
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_update", { detail: gmeta.gid }),
);
if (!need) {
const p = db.get_pmeta_by_index(task.gid, i.index);
if (!p) {
const op = db.get_pmeta_by_token(
task.gid,
i.token,
);
if (op) {
op.index = i.index;
op.name = i.name;
db.add_pmeta(op);
return;
} else {
const ops = db.get_pmeta_by_token_only(
}
const base_path = join(cfg.base, task.gid.toString());
await sure_dir(base_path);
let mpv_enabled = icfg.mpv ?? cfg.mpv;
let import_method = icfg.method ?? cfg.import_method;
if (
f.is_zip && import_method != ImportMethod.Copy &&
import_method != ImportMethod.CopyThenDelete
) {
import_method = ImportMethod.CopyThenDelete;
}
const max_import_img_count = icfg.max_import_img_count ??
cfg.max_import_img_count;
let imgs: Page[] = [];
const m = new ImportManager(
max_import_img_count,
manager.aborts,
manager.force_aborts,
task,
manager,
);
if (!mpv_enabled) {
const g = await client.fetchGalleryPage(task.gid, task.token);
if (g.mpv_enabled) {
mpv_enabled = true;
} else {
imgs = await g.imagelist;
m.set_total_page(g.length);
}
}
if (mpv_enabled) {
const g = await client.fetchMPVPage(task.gid, task.token);
imgs = g.imagelist;
m.set_total_page(g.pagecount);
}
const names = imgs.reduce(
(acc: Record<string, number>, cur) => {
const curr = cur.name;
return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
},
{},
);
async function import_img(i: Page) {
const opath = f.get_file(i.name, i.index);
if (!opath) {
console.log("File not found");
return;
}
const size = await fb_get_size(opath);
if (!size) {
console.log("Failed to get file size for", opath);
throw Error("Failed to get file size.");
}
const ofiles = db.get_files(i.token);
if (ofiles.length) {
const need = await asyncEvery(
ofiles,
async (t) =>
(!t.is_original && t.height != size.height &&
t.width != size.width) || (!await exists(t.path)),
);
if (!need) {
const p = db.get_pmeta_by_index(task.gid, i.index);
if (!p) {
const op = db.get_pmeta_by_token(
task.gid,
i.token,
);
if (ops.length) {
const op = ops[0];
op.gid = task.gid;
if (op) {
op.index = i.index;
op.name = i.name;
db.add_pmeta(op);
return;
} else {
const ops = db.get_pmeta_by_token_only(
i.token,
);
if (ops.length) {
const op = ops[0];
op.gid = task.gid;
op.index = i.index;
op.name = i.name;
db.add_pmeta(op);
return;
}
}
}
console.log("Already has page", i.index);
return;
}
console.log("Already has page", i.index);
return;
}
}
const pmeta: PMeta = {
gid: task.gid,
height: size.height,
width: size.width,
index: i.index,
name: i.name,
token: i.token,
};
db.add_pmeta(pmeta);
const oriext = extname(i.name).toLowerCase();
const nowext = extname(opath).toLowerCase();
const is_original = icfg.size == ImportSize.Original ||
(oriext != ".jpg" && oriext == nowext) ||
(oriext == nowext && size.width < icfg.size);
let path = join(base_path, is_original ? i.name : i.sampled_name);
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
path = add_suffix_to_path(path, i.token);
console.log("Changed path to", path);
}
if (import_method == ImportMethod.Move) {
await Deno.rename(opath, path);
} else if (import_method != ImportMethod.Keep) {
await Deno.copyFile(opath, path);
} else {
path = opath;
}
if (cfg.check_file_hash && is_original) {
const sha = await calFileSha1(path);
if (sha.slice(0, i.token.length) != i.token) {
console.warn(
`Hash not matched: file hash ${sha}, token ${i.token}`,
);
return;
const pmeta: PMeta = {
gid: task.gid,
height: size.height,
width: size.width,
index: i.index,
name: i.name,
token: i.token,
};
db.add_pmeta(pmeta);
const oriext = extname(i.name).toLowerCase();
const nowext = extname(opath).toLowerCase();
const is_original = icfg.size == ImportSize.Original ||
(oriext != ".jpg" && oriext == nowext) ||
(oriext == nowext && size.width < icfg.size);
let path = join(base_path, is_original ? i.name : i.sampled_name);
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
path = add_suffix_to_path(path, i.token);
console.log("Changed path to", path);
}
if (import_method == ImportMethod.Move) {
await Deno.rename(opath, path);
} else if (import_method != ImportMethod.Keep) {
await Deno.copyFile(opath, path);
} else {
path = opath;
}
if (cfg.check_file_hash && is_original) {
const sha = await calFileSha1(path);
if (sha.slice(0, i.token.length) != i.token) {
console.warn(
`Hash not matched: file hash ${sha}, token ${i.token}`,
);
return;
}
}
if (import_method == ImportMethod.CopyThenDelete) {
await Deno.remove(opath);
}
const file: EhFile = {
height: size.height,
width: size.width,
is_original,
id: 0,
path,
token: i.token,
};
db.add_file(file);
}
if (import_method == ImportMethod.CopyThenDelete) {
await Deno.remove(opath);
}
const file: EhFile = {
height: size.height,
width: size.width,
is_original,
id: 0,
path,
token: i.token,
};
db.add_file(file);
}
for (const i of imgs) {
if (manager.aborted) break;
await m.add_new_task(async () => {
await import_img(i);
});
}
await m.join();
const remove_previous_gallery = icfg.remove_previous_gallery ??
cfg.remove_previous_gallery;
if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
if (manager.aborted || manager.force_aborted) throw Error("aborted");
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
let replaced_gallery = icfg.replaced_gallery;
if (replaced_gallery === undefined) {
const fg = await client.fetchGalleryPage(
gmeta.first_gid,
gmeta.first_key,
async function import_zip_img(i: Page) {
const opath = f.get_zip(i.name, i.index);
if (!opath) {
console.log("File not found");
return;
}
const ofiles = db.get_files(i.token);
if (ofiles.length) {
const need = await asyncEvery(
ofiles,
async (t) =>
!t.is_original && icfg.size === ImportSize.Original ||
(!await exists(t.path)),
);
if (!need) {
const p = db.get_pmeta_by_index(task.gid, i.index);
if (!p) {
const op = db.get_pmeta_by_token(
task.gid,
i.token,
);
if (op) {
op.index = i.index;
op.name = i.name;
db.add_pmeta(op);
return;
} else {
const ops = db.get_pmeta_by_token_only(
i.token,
);
if (ops.length) {
const op = ops[0];
op.gid = task.gid;
op.index = i.index;
op.name = i.name;
db.add_pmeta(op);
return;
}
}
}
console.log("Already has page", i.index);
return;
}
}
const oriext = extname(i.name).toLowerCase();
const nowext = extname(opath).toLowerCase();
let path = join(
base_path,
i.name.replace(oriext, nowext),
);
replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid);
replaced_gallery.push({
gid: gmeta.first_gid,
token: gmeta.first_key,
if (names[i.name] > 1) {
path = add_suffix_to_path(path, i.token);
console.log("Changed path to", path);
}
const zf = f.open_zip_file(opath);
if (!zf) {
console.log("File not found");
return;
}
try {
const out = await Deno.open(path, {
create: true,
write: true,
});
try {
const buf = new Uint8Array(65536);
let len = zf.read(buf);
while (len > 0) {
await out.write(buf.slice(0, Number(len)));
len = zf.read(buf);
}
} finally {
out.close();
}
} finally {
zf.close();
}
const size = await fb_get_size(path);
if (!size) {
console.log("Failed to get file size for", path);
throw Error("Failed to get file size.");
}
const is_original = icfg.size == ImportSize.Original ||
(oriext != ".jpg" && oriext == nowext) ||
(oriext == nowext && size.width < icfg.size);
if (cfg.check_file_hash && is_original) {
const sha = await calFileSha1(path);
if (sha.slice(0, i.token.length) != i.token) {
console.warn(
`Hash not matched: file hash ${sha}, token ${i.token}`,
);
return;
}
}
const pmeta: PMeta = {
gid: task.gid,
height: size.height,
width: size.width,
index: i.index,
name: i.name,
token: i.token,
};
db.add_pmeta(pmeta);
const file: EhFile = {
height: size.height,
width: size.width,
is_original,
id: 0,
path,
token: i.token,
};
db.add_file(file);
}
for (const i of imgs) {
if (manager.aborted) break;
await m.add_new_task(async () => {
if (f.is_zip) await import_zip_img(i);
else await import_img(i);
});
}
replaced_gallery.forEach((g) => {
const gmeta = db.get_gmeta_by_gid(g.gid);
if (!gmeta) return;
console.log("Remove gallery ", g.gid);
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_remove", { detail: gmeta.gid }),
await m.join();
const remove_previous_gallery = icfg.remove_previous_gallery ??
cfg.remove_previous_gallery;
if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
if (manager.aborted || manager.force_aborted) throw Error("aborted");
if (f.is_zip && import_method == ImportMethod.CopyThenDelete) {
f.close();
await Deno.remove(icfg.import_path);
}
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
let replaced_gallery = icfg.replaced_gallery;
if (replaced_gallery === undefined) {
const fg = await client.fetchGalleryPage(
gmeta.first_gid,
gmeta.first_key,
);
replaced_gallery = fg.new_version.filter((d) =>
d.gid < task.gid
);
replaced_gallery.push({
gid: gmeta.first_gid,
token: gmeta.first_key,
});
}
db.delete_gallery(g.gid);
});
replaced_gallery.forEach((g) => {
const gmeta = db.get_gmeta_by_gid(g.gid);
if (!gmeta) return;
console.log("Remove gallery ", g.gid);
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_remove", {
detail: gmeta.gid,
}),
);
}
db.delete_gallery(g.gid);
});
}
return task;
} finally {
f.close();
}
return task;
}

92
utils/readonly_zip.ts Normal file
View File

@@ -0,0 +1,92 @@
import * as lz from "@lifegpc/libzip/raw";
function get_err(err: number) {
const e = lz.ZipErrorT.new();
lz.zip_error_init_with_code(e, err);
const s = lz.zip_error_strerror(e);
lz.zip_error_fini(e);
return s;
}
class ZipFile {
#file: lz.ZipFileT;
#closed = false;
constructor(f: lz.ZipFileT) {
this.#file = f;
}
#check_close() {
if (this.#closed) throw Error("Already closed.");
}
close() {
if (this.#closed) return;
this.#closed = true;
const er = lz.zip_fclose(this.#file);
if (er) {
throw Error(`Failed to close file: ${get_err(er)}`);
}
}
read(buf: Uint8Array) {
this.#check_close();
const num = lz.zip_fread(this.#file, buf);
if (num == -1) {
throw Error(
`Failed to read file: ${lz.zip_file_strerror(this.#file)}`,
);
}
return num;
}
}
export class ReadonlyZip {
#zip: lz.ZipT;
#discarded = false;
#opened_file_list: ZipFile[] = [];
constructor(path: string) {
const ep = lz.IntPointer.new();
const z = lz.zip_open(path, lz.ZipOpenFlag.RDONLY, ep);
if (!z) {
throw Error(`Failed to open archive: ${get_err(ep.int)}.`);
}
this.#zip = z;
}
close() {
if (this.#discarded) return;
for (const f of this.#opened_file_list) {
f.close();
}
this.#discarded = true;
lz.zip_discard(this.#zip);
}
get count() {
return lz.zip_get_num_entries(this.#zip, 0);
}
get_index(name: string) {
return lz.zip_name_locate(this.#zip, name, lz.ZipFlags.ENC_UTF_8);
}
get_name(index: number | bigint) {
return lz.zip_get_name(this.#zip, index, 0);
}
open(name: string) {
const z = lz.zip_fopen(this.#zip, name, lz.ZipFlags.ENC_UTF_8);
if (!z) {
const errmsg = lz.zip_strerror(this.#zip);
return errmsg;
}
const f = new ZipFile(z);
this.#opened_file_list.push(f);
return f;
}
open_index(index: number | bigint) {
const z = lz.zip_fopen_index(this.#zip, index, 0);
if (!z) {
const errmsg = lz.zip_strerror(this.#zip);
return errmsg;
}
const f = new ZipFile(z);
this.#opened_file_list.push(f);
return f;
}
}