mirror of
https://github.com/lifegpc/eh-downloader.git
synced 2026-06-22 03:54:15 +08:00
Add support to import zip file
This commit is contained in:
@@ -67,6 +67,15 @@ RUN cd ~ && \
|
||||
make -j$(grep -c ^processor /proc/cpuinfo) && make install && \
|
||||
cd ~ && rm -rf sqlite-snapshot-202401231504 sqlite-snapshot-202401231504.tar.gz
|
||||
|
||||
RUN cd ~ && \
|
||||
curl -L "https://libzip.org/download/libzip-1.10.1.tar.gz" -o libzip-1.10.1.tar.gz && \
|
||||
tar -xzvf libzip-1.10.1.tar.gz && \
|
||||
cd libzip-1.10.1 && \
|
||||
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_REGRESS=OFF -DBUILD_OSSFUZZ=OFF -DBUILD_EXAMPLES=OFF -DBUILD_DOC=OFF \
|
||||
-DCMAKE_INSTALL_PREFIX=/clib -DCMAKE_PREFIX_PATH=/clib ../ && \
|
||||
make -j$(grep -c ^processor /proc/cpuinfo) && make install && \
|
||||
cd ~ && rm -rf libzip-1.10.1 libzip-1.10.1.tar.gz
|
||||
|
||||
COPY ./extensions /root/extensions
|
||||
|
||||
RUN cd /root/extensions && \
|
||||
|
||||
@@ -35,6 +35,7 @@
|
||||
"@twind/core": "https://esm.sh/@twind/[email protected]",
|
||||
"@twind/preset-tailwind": "https://esm.sh/@twind/[email protected]/",
|
||||
"@twind/preset-autoprefix": "https://esm.sh/@twind/[email protected]/",
|
||||
"@lifegpc/sha1": "jsr:/@lifegpc/[email protected]"
|
||||
"@lifegpc/sha1": "jsr:/@lifegpc/[email protected]",
|
||||
"@lifegpc/libzip/": "jsr:/@lifegpc/[email protected]/"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,10 @@ import { get_host, return_data } from "../../server/utils.ts";
|
||||
import { check_ffmpeg_binary } from "../../thumbnail/ffmpeg_binary.ts";
|
||||
import type * as FFMPEG_API from "../../thumbnail/ffmpeg_api.ts";
|
||||
import { isDocker } from "../../utils.ts";
|
||||
import type * as LIBZIP from "@lifegpc/libzip/raw";
|
||||
|
||||
let ffmpeg_api: typeof FFMPEG_API | undefined;
|
||||
let libzip: typeof LIBZIP | undefined;
|
||||
|
||||
async function check_ffmpeg_api() {
|
||||
if (ffmpeg_api) return true;
|
||||
@@ -18,6 +20,16 @@ async function check_ffmpeg_api() {
|
||||
}
|
||||
}
|
||||
|
||||
async function check_libzip() {
|
||||
if (libzip) return true;
|
||||
try {
|
||||
libzip = await import("@lifegpc/libzip/raw");
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export const handler: Handlers = {
|
||||
async GET(req, ctx) {
|
||||
const m = get_task_manager();
|
||||
@@ -51,6 +63,7 @@ export const handler: Handlers = {
|
||||
key: m.cfg.meili_search_api_key || m.cfg.meili_update_api_key,
|
||||
};
|
||||
}
|
||||
const libzip_enabled = await check_libzip();
|
||||
const no_user = c === 0 || c === 0n;
|
||||
const is_docker = isDocker();
|
||||
return return_data<StatusData>({
|
||||
@@ -61,6 +74,7 @@ export const handler: Handlers = {
|
||||
meilisearch,
|
||||
no_user,
|
||||
is_docker,
|
||||
libzip_enabled,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
@@ -9,4 +9,5 @@ export type StatusData = {
|
||||
};
|
||||
no_user: boolean;
|
||||
is_docker: boolean;
|
||||
libzip_enabled: boolean;
|
||||
};
|
||||
|
||||
514
tasks/import.ts
514
tasks/import.ts
@@ -1,3 +1,4 @@
|
||||
// deno-lint-ignore-file no-inner-declarations
|
||||
import { Task, TaskImportProgress, TaskType } from "../task.ts";
|
||||
import { TaskManager } from "../task_manager.ts";
|
||||
import {
|
||||
@@ -17,6 +18,7 @@ import { extname, join, relative } from "@std/path";
|
||||
import { ImportMethod, ImportSize } from "../config.ts";
|
||||
import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts";
|
||||
import { EhFile, PMeta } from "../db.ts";
|
||||
import type { ReadonlyZip } from "../utils/readonly_zip.ts";
|
||||
|
||||
export type ImportConfig = {
|
||||
max_import_img_count?: number;
|
||||
@@ -141,28 +143,44 @@ class ImportManager {
|
||||
|
||||
class FileLoader {
|
||||
#path;
|
||||
#zip?: ReadonlyZip;
|
||||
#files: string[] = [];
|
||||
#inited = false;
|
||||
#filecount;
|
||||
#has_prefix = false;
|
||||
#closed = false;
|
||||
constructor(path: string, filecount: number) {
|
||||
this.#path = path;
|
||||
this.#filecount = filecount;
|
||||
}
|
||||
#check() {
|
||||
if (!this.#inited) throw Error("FileLoader not initiailzed.");
|
||||
if (this.#closed) throw Error("Already closed.");
|
||||
}
|
||||
#get_file(name: string) {
|
||||
if (this.#files.includes(name)) {
|
||||
return join(this.#path, name);
|
||||
}
|
||||
}
|
||||
#get_zip(_name: string) {
|
||||
return null;
|
||||
#get_zip(name: string) {
|
||||
if (this.#files.includes(name)) {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
close() {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
this.#zip?.close();
|
||||
}
|
||||
get_file(name: string, index: number) {
|
||||
this.#check();
|
||||
if (this.#has_prefix) {
|
||||
name = `${index.toString().padStart(3, "0")}_${name}`;
|
||||
name = `${
|
||||
index.toString().padStart(
|
||||
this.#filecount.toString().length,
|
||||
"0",
|
||||
)
|
||||
}_${name}`;
|
||||
}
|
||||
let t = this.#get_file(name);
|
||||
if (t) return t;
|
||||
@@ -173,7 +191,16 @@ class FileLoader {
|
||||
if (t) return t;
|
||||
}
|
||||
}
|
||||
get_zip(name: string) {
|
||||
get_zip(name: string, index: number) {
|
||||
this.#check();
|
||||
if (this.#has_prefix) {
|
||||
name = `${
|
||||
index.toString().padStart(
|
||||
this.#filecount.toString().length,
|
||||
"0",
|
||||
)
|
||||
}_${name}`;
|
||||
}
|
||||
let t = this.#get_zip(name);
|
||||
if (t) return t;
|
||||
const ext = extname(name).toLowerCase();
|
||||
@@ -190,6 +217,16 @@ class FileLoader {
|
||||
this.#files.push(relative(this.#path, i.path));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const z = await import("../utils/readonly_zip.ts");
|
||||
this.#zip = new z.ReadonlyZip(this.#path);
|
||||
const count = this.#zip.count;
|
||||
for (let i = 0n; i < count; i++) {
|
||||
const f = this.#zip.get_name(i);
|
||||
if (f) {
|
||||
this.#files.push(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
let has_prefix = true;
|
||||
const re = new RegExp(`^\\d{${this.#filecount.toString().length}}_`);
|
||||
@@ -205,11 +242,21 @@ class FileLoader {
|
||||
}
|
||||
this.#has_prefix = has_prefix;
|
||||
this.#inited = true;
|
||||
this.#closed = false;
|
||||
return this;
|
||||
}
|
||||
get is_zip() {
|
||||
this.#check();
|
||||
return false;
|
||||
return this.#zip !== undefined;
|
||||
}
|
||||
open_zip_file(name: string) {
|
||||
if (this.#files.includes(name)) {
|
||||
const s = this.#zip!.open(name);
|
||||
if (typeof s === "string") {
|
||||
throw Error(`Failed to open file: ${s}`);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,189 +277,316 @@ export async function import_task(task: Task, manager: TaskManager) {
|
||||
const f =
|
||||
await (new FileLoader(icfg.import_path, parseInt(gdata.filecount)))
|
||||
.init();
|
||||
const gmeta = gdatas.convert(gdata);
|
||||
db.add_gmeta(gmeta);
|
||||
await db.add_gtag(task.gid, new Set(gdata.tags));
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_update", { detail: gmeta.gid }),
|
||||
);
|
||||
}
|
||||
const base_path = join(cfg.base, task.gid.toString());
|
||||
await sure_dir(base_path);
|
||||
let mpv_enabled = icfg.mpv ?? cfg.mpv;
|
||||
let import_method = icfg.method ?? cfg.import_method;
|
||||
if (
|
||||
f.is_zip && import_method != ImportMethod.Copy &&
|
||||
import_method != ImportMethod.CopyThenDelete
|
||||
) {
|
||||
import_method = ImportMethod.CopyThenDelete;
|
||||
}
|
||||
const max_import_img_count = icfg.max_import_img_count ??
|
||||
cfg.max_import_img_count;
|
||||
let imgs: Page[] = [];
|
||||
const m = new ImportManager(
|
||||
max_import_img_count,
|
||||
manager.aborts,
|
||||
manager.force_aborts,
|
||||
task,
|
||||
manager,
|
||||
);
|
||||
if (!mpv_enabled) {
|
||||
const g = await client.fetchGalleryPage(task.gid, task.token);
|
||||
if (g.mpv_enabled) {
|
||||
mpv_enabled = true;
|
||||
} else {
|
||||
imgs = await g.imagelist;
|
||||
m.set_total_page(g.length);
|
||||
}
|
||||
}
|
||||
if (mpv_enabled) {
|
||||
const g = await client.fetchMPVPage(task.gid, task.token);
|
||||
imgs = g.imagelist;
|
||||
m.set_total_page(g.pagecount);
|
||||
}
|
||||
const names = imgs.reduce(
|
||||
(acc: Record<string, number>, cur) => {
|
||||
const curr = cur.name;
|
||||
return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
async function import_img(i: Page) {
|
||||
const opath = f.get_file(i.name, i.index);
|
||||
if (!opath) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
const size = await fb_get_size(opath);
|
||||
if (!size) {
|
||||
console.log("Failed to get file size for", opath);
|
||||
throw Error("Failed to get file size.");
|
||||
}
|
||||
const ofiles = db.get_files(i.token);
|
||||
if (ofiles.length) {
|
||||
const need = await asyncEvery(
|
||||
ofiles,
|
||||
async (t) =>
|
||||
(!t.is_original && t.height != size.height &&
|
||||
t.width != size.width) || (!await exists(t.path)),
|
||||
try {
|
||||
const gmeta = gdatas.convert(gdata);
|
||||
db.add_gmeta(gmeta);
|
||||
await db.add_gtag(task.gid, new Set(gdata.tags));
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_update", { detail: gmeta.gid }),
|
||||
);
|
||||
if (!need) {
|
||||
const p = db.get_pmeta_by_index(task.gid, i.index);
|
||||
if (!p) {
|
||||
const op = db.get_pmeta_by_token(
|
||||
task.gid,
|
||||
i.token,
|
||||
);
|
||||
if (op) {
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
} else {
|
||||
const ops = db.get_pmeta_by_token_only(
|
||||
}
|
||||
const base_path = join(cfg.base, task.gid.toString());
|
||||
await sure_dir(base_path);
|
||||
let mpv_enabled = icfg.mpv ?? cfg.mpv;
|
||||
let import_method = icfg.method ?? cfg.import_method;
|
||||
if (
|
||||
f.is_zip && import_method != ImportMethod.Copy &&
|
||||
import_method != ImportMethod.CopyThenDelete
|
||||
) {
|
||||
import_method = ImportMethod.CopyThenDelete;
|
||||
}
|
||||
const max_import_img_count = icfg.max_import_img_count ??
|
||||
cfg.max_import_img_count;
|
||||
let imgs: Page[] = [];
|
||||
const m = new ImportManager(
|
||||
max_import_img_count,
|
||||
manager.aborts,
|
||||
manager.force_aborts,
|
||||
task,
|
||||
manager,
|
||||
);
|
||||
if (!mpv_enabled) {
|
||||
const g = await client.fetchGalleryPage(task.gid, task.token);
|
||||
if (g.mpv_enabled) {
|
||||
mpv_enabled = true;
|
||||
} else {
|
||||
imgs = await g.imagelist;
|
||||
m.set_total_page(g.length);
|
||||
}
|
||||
}
|
||||
if (mpv_enabled) {
|
||||
const g = await client.fetchMPVPage(task.gid, task.token);
|
||||
imgs = g.imagelist;
|
||||
m.set_total_page(g.pagecount);
|
||||
}
|
||||
const names = imgs.reduce(
|
||||
(acc: Record<string, number>, cur) => {
|
||||
const curr = cur.name;
|
||||
return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
async function import_img(i: Page) {
|
||||
const opath = f.get_file(i.name, i.index);
|
||||
if (!opath) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
const size = await fb_get_size(opath);
|
||||
if (!size) {
|
||||
console.log("Failed to get file size for", opath);
|
||||
throw Error("Failed to get file size.");
|
||||
}
|
||||
const ofiles = db.get_files(i.token);
|
||||
if (ofiles.length) {
|
||||
const need = await asyncEvery(
|
||||
ofiles,
|
||||
async (t) =>
|
||||
(!t.is_original && t.height != size.height &&
|
||||
t.width != size.width) || (!await exists(t.path)),
|
||||
);
|
||||
if (!need) {
|
||||
const p = db.get_pmeta_by_index(task.gid, i.index);
|
||||
if (!p) {
|
||||
const op = db.get_pmeta_by_token(
|
||||
task.gid,
|
||||
i.token,
|
||||
);
|
||||
if (ops.length) {
|
||||
const op = ops[0];
|
||||
op.gid = task.gid;
|
||||
if (op) {
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
} else {
|
||||
const ops = db.get_pmeta_by_token_only(
|
||||
i.token,
|
||||
);
|
||||
if (ops.length) {
|
||||
const op = ops[0];
|
||||
op.gid = task.gid;
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log("Already has page", i.index);
|
||||
return;
|
||||
}
|
||||
console.log("Already has page", i.index);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const pmeta: PMeta = {
|
||||
gid: task.gid,
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
index: i.index,
|
||||
name: i.name,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_pmeta(pmeta);
|
||||
const oriext = extname(i.name).toLowerCase();
|
||||
const nowext = extname(opath).toLowerCase();
|
||||
const is_original = icfg.size == ImportSize.Original ||
|
||||
(oriext != ".jpg" && oriext == nowext) ||
|
||||
(oriext == nowext && size.width < icfg.size);
|
||||
let path = join(base_path, is_original ? i.name : i.sampled_name);
|
||||
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
|
||||
path = add_suffix_to_path(path, i.token);
|
||||
console.log("Changed path to", path);
|
||||
}
|
||||
if (import_method == ImportMethod.Move) {
|
||||
await Deno.rename(opath, path);
|
||||
} else if (import_method != ImportMethod.Keep) {
|
||||
await Deno.copyFile(opath, path);
|
||||
} else {
|
||||
path = opath;
|
||||
}
|
||||
if (cfg.check_file_hash && is_original) {
|
||||
const sha = await calFileSha1(path);
|
||||
if (sha.slice(0, i.token.length) != i.token) {
|
||||
console.warn(
|
||||
`Hash not matched: file hash ${sha}, token ${i.token}`,
|
||||
);
|
||||
return;
|
||||
const pmeta: PMeta = {
|
||||
gid: task.gid,
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
index: i.index,
|
||||
name: i.name,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_pmeta(pmeta);
|
||||
const oriext = extname(i.name).toLowerCase();
|
||||
const nowext = extname(opath).toLowerCase();
|
||||
const is_original = icfg.size == ImportSize.Original ||
|
||||
(oriext != ".jpg" && oriext == nowext) ||
|
||||
(oriext == nowext && size.width < icfg.size);
|
||||
let path = join(base_path, is_original ? i.name : i.sampled_name);
|
||||
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
|
||||
path = add_suffix_to_path(path, i.token);
|
||||
console.log("Changed path to", path);
|
||||
}
|
||||
if (import_method == ImportMethod.Move) {
|
||||
await Deno.rename(opath, path);
|
||||
} else if (import_method != ImportMethod.Keep) {
|
||||
await Deno.copyFile(opath, path);
|
||||
} else {
|
||||
path = opath;
|
||||
}
|
||||
if (cfg.check_file_hash && is_original) {
|
||||
const sha = await calFileSha1(path);
|
||||
if (sha.slice(0, i.token.length) != i.token) {
|
||||
console.warn(
|
||||
`Hash not matched: file hash ${sha}, token ${i.token}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (import_method == ImportMethod.CopyThenDelete) {
|
||||
await Deno.remove(opath);
|
||||
}
|
||||
const file: EhFile = {
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
is_original,
|
||||
id: 0,
|
||||
path,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_file(file);
|
||||
}
|
||||
if (import_method == ImportMethod.CopyThenDelete) {
|
||||
await Deno.remove(opath);
|
||||
}
|
||||
const file: EhFile = {
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
is_original,
|
||||
id: 0,
|
||||
path,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_file(file);
|
||||
}
|
||||
|
||||
for (const i of imgs) {
|
||||
if (manager.aborted) break;
|
||||
await m.add_new_task(async () => {
|
||||
await import_img(i);
|
||||
});
|
||||
}
|
||||
await m.join();
|
||||
const remove_previous_gallery = icfg.remove_previous_gallery ??
|
||||
cfg.remove_previous_gallery;
|
||||
if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
|
||||
if (manager.aborted || manager.force_aborted) throw Error("aborted");
|
||||
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
|
||||
let replaced_gallery = icfg.replaced_gallery;
|
||||
if (replaced_gallery === undefined) {
|
||||
const fg = await client.fetchGalleryPage(
|
||||
gmeta.first_gid,
|
||||
gmeta.first_key,
|
||||
async function import_zip_img(i: Page) {
|
||||
const opath = f.get_zip(i.name, i.index);
|
||||
if (!opath) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
const ofiles = db.get_files(i.token);
|
||||
if (ofiles.length) {
|
||||
const need = await asyncEvery(
|
||||
ofiles,
|
||||
async (t) =>
|
||||
!t.is_original && icfg.size === ImportSize.Original ||
|
||||
(!await exists(t.path)),
|
||||
);
|
||||
if (!need) {
|
||||
const p = db.get_pmeta_by_index(task.gid, i.index);
|
||||
if (!p) {
|
||||
const op = db.get_pmeta_by_token(
|
||||
task.gid,
|
||||
i.token,
|
||||
);
|
||||
if (op) {
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
} else {
|
||||
const ops = db.get_pmeta_by_token_only(
|
||||
i.token,
|
||||
);
|
||||
if (ops.length) {
|
||||
const op = ops[0];
|
||||
op.gid = task.gid;
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log("Already has page", i.index);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const oriext = extname(i.name).toLowerCase();
|
||||
const nowext = extname(opath).toLowerCase();
|
||||
let path = join(
|
||||
base_path,
|
||||
i.name.replace(oriext, nowext),
|
||||
);
|
||||
replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid);
|
||||
replaced_gallery.push({
|
||||
gid: gmeta.first_gid,
|
||||
token: gmeta.first_key,
|
||||
if (names[i.name] > 1) {
|
||||
path = add_suffix_to_path(path, i.token);
|
||||
console.log("Changed path to", path);
|
||||
}
|
||||
const zf = f.open_zip_file(opath);
|
||||
if (!zf) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const out = await Deno.open(path, {
|
||||
create: true,
|
||||
write: true,
|
||||
});
|
||||
try {
|
||||
const buf = new Uint8Array(65536);
|
||||
let len = zf.read(buf);
|
||||
while (len > 0) {
|
||||
await out.write(buf.slice(0, Number(len)));
|
||||
len = zf.read(buf);
|
||||
}
|
||||
} finally {
|
||||
out.close();
|
||||
}
|
||||
} finally {
|
||||
zf.close();
|
||||
}
|
||||
const size = await fb_get_size(path);
|
||||
if (!size) {
|
||||
console.log("Failed to get file size for", path);
|
||||
throw Error("Failed to get file size.");
|
||||
}
|
||||
const is_original = icfg.size == ImportSize.Original ||
|
||||
(oriext != ".jpg" && oriext == nowext) ||
|
||||
(oriext == nowext && size.width < icfg.size);
|
||||
if (cfg.check_file_hash && is_original) {
|
||||
const sha = await calFileSha1(path);
|
||||
if (sha.slice(0, i.token.length) != i.token) {
|
||||
console.warn(
|
||||
`Hash not matched: file hash ${sha}, token ${i.token}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const pmeta: PMeta = {
|
||||
gid: task.gid,
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
index: i.index,
|
||||
name: i.name,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_pmeta(pmeta);
|
||||
const file: EhFile = {
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
is_original,
|
||||
id: 0,
|
||||
path,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_file(file);
|
||||
}
|
||||
|
||||
for (const i of imgs) {
|
||||
if (manager.aborted) break;
|
||||
await m.add_new_task(async () => {
|
||||
if (f.is_zip) await import_zip_img(i);
|
||||
else await import_img(i);
|
||||
});
|
||||
}
|
||||
replaced_gallery.forEach((g) => {
|
||||
const gmeta = db.get_gmeta_by_gid(g.gid);
|
||||
if (!gmeta) return;
|
||||
console.log("Remove gallery ", g.gid);
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_remove", { detail: gmeta.gid }),
|
||||
await m.join();
|
||||
const remove_previous_gallery = icfg.remove_previous_gallery ??
|
||||
cfg.remove_previous_gallery;
|
||||
if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
|
||||
if (manager.aborted || manager.force_aborted) throw Error("aborted");
|
||||
if (f.is_zip && import_method == ImportMethod.CopyThenDelete) {
|
||||
f.close();
|
||||
await Deno.remove(icfg.import_path);
|
||||
}
|
||||
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
|
||||
let replaced_gallery = icfg.replaced_gallery;
|
||||
if (replaced_gallery === undefined) {
|
||||
const fg = await client.fetchGalleryPage(
|
||||
gmeta.first_gid,
|
||||
gmeta.first_key,
|
||||
);
|
||||
replaced_gallery = fg.new_version.filter((d) =>
|
||||
d.gid < task.gid
|
||||
);
|
||||
replaced_gallery.push({
|
||||
gid: gmeta.first_gid,
|
||||
token: gmeta.first_key,
|
||||
});
|
||||
}
|
||||
db.delete_gallery(g.gid);
|
||||
});
|
||||
replaced_gallery.forEach((g) => {
|
||||
const gmeta = db.get_gmeta_by_gid(g.gid);
|
||||
if (!gmeta) return;
|
||||
console.log("Remove gallery ", g.gid);
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_remove", {
|
||||
detail: gmeta.gid,
|
||||
}),
|
||||
);
|
||||
}
|
||||
db.delete_gallery(g.gid);
|
||||
});
|
||||
}
|
||||
return task;
|
||||
} finally {
|
||||
f.close();
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
92
utils/readonly_zip.ts
Normal file
92
utils/readonly_zip.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
import * as lz from "@lifegpc/libzip/raw";
|
||||
|
||||
function get_err(err: number) {
|
||||
const e = lz.ZipErrorT.new();
|
||||
lz.zip_error_init_with_code(e, err);
|
||||
const s = lz.zip_error_strerror(e);
|
||||
lz.zip_error_fini(e);
|
||||
return s;
|
||||
}
|
||||
|
||||
class ZipFile {
|
||||
#file: lz.ZipFileT;
|
||||
#closed = false;
|
||||
constructor(f: lz.ZipFileT) {
|
||||
this.#file = f;
|
||||
}
|
||||
|
||||
#check_close() {
|
||||
if (this.#closed) throw Error("Already closed.");
|
||||
}
|
||||
|
||||
close() {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
const er = lz.zip_fclose(this.#file);
|
||||
if (er) {
|
||||
throw Error(`Failed to close file: ${get_err(er)}`);
|
||||
}
|
||||
}
|
||||
|
||||
read(buf: Uint8Array) {
|
||||
this.#check_close();
|
||||
const num = lz.zip_fread(this.#file, buf);
|
||||
if (num == -1) {
|
||||
throw Error(
|
||||
`Failed to read file: ${lz.zip_file_strerror(this.#file)}`,
|
||||
);
|
||||
}
|
||||
return num;
|
||||
}
|
||||
}
|
||||
|
||||
export class ReadonlyZip {
|
||||
#zip: lz.ZipT;
|
||||
#discarded = false;
|
||||
#opened_file_list: ZipFile[] = [];
|
||||
constructor(path: string) {
|
||||
const ep = lz.IntPointer.new();
|
||||
const z = lz.zip_open(path, lz.ZipOpenFlag.RDONLY, ep);
|
||||
if (!z) {
|
||||
throw Error(`Failed to open archive: ${get_err(ep.int)}.`);
|
||||
}
|
||||
this.#zip = z;
|
||||
}
|
||||
close() {
|
||||
if (this.#discarded) return;
|
||||
for (const f of this.#opened_file_list) {
|
||||
f.close();
|
||||
}
|
||||
this.#discarded = true;
|
||||
lz.zip_discard(this.#zip);
|
||||
}
|
||||
get count() {
|
||||
return lz.zip_get_num_entries(this.#zip, 0);
|
||||
}
|
||||
get_index(name: string) {
|
||||
return lz.zip_name_locate(this.#zip, name, lz.ZipFlags.ENC_UTF_8);
|
||||
}
|
||||
get_name(index: number | bigint) {
|
||||
return lz.zip_get_name(this.#zip, index, 0);
|
||||
}
|
||||
open(name: string) {
|
||||
const z = lz.zip_fopen(this.#zip, name, lz.ZipFlags.ENC_UTF_8);
|
||||
if (!z) {
|
||||
const errmsg = lz.zip_strerror(this.#zip);
|
||||
return errmsg;
|
||||
}
|
||||
const f = new ZipFile(z);
|
||||
this.#opened_file_list.push(f);
|
||||
return f;
|
||||
}
|
||||
open_index(index: number | bigint) {
|
||||
const z = lz.zip_fopen_index(this.#zip, index, 0);
|
||||
if (!z) {
|
||||
const errmsg = lz.zip_strerror(this.#zip);
|
||||
return errmsg;
|
||||
}
|
||||
const f = new ZipFile(z);
|
||||
this.#opened_file_list.push(f);
|
||||
return f;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user