mirror of
https://github.com/lifegpc/eh-downloader.git
synced 2026-07-08 01:31:00 +08:00
Add support to import zip file
This commit is contained in:
514
tasks/import.ts
514
tasks/import.ts
@@ -1,3 +1,4 @@
|
||||
// deno-lint-ignore-file no-inner-declarations
|
||||
import { Task, TaskImportProgress, TaskType } from "../task.ts";
|
||||
import { TaskManager } from "../task_manager.ts";
|
||||
import {
|
||||
@@ -17,6 +18,7 @@ import { extname, join, relative } from "@std/path";
|
||||
import { ImportMethod, ImportSize } from "../config.ts";
|
||||
import { fb_get_size } from "../thumbnail/ffmpeg_binary.ts";
|
||||
import { EhFile, PMeta } from "../db.ts";
|
||||
import type { ReadonlyZip } from "../utils/readonly_zip.ts";
|
||||
|
||||
export type ImportConfig = {
|
||||
max_import_img_count?: number;
|
||||
@@ -141,28 +143,44 @@ class ImportManager {
|
||||
|
||||
class FileLoader {
|
||||
#path;
|
||||
#zip?: ReadonlyZip;
|
||||
#files: string[] = [];
|
||||
#inited = false;
|
||||
#filecount;
|
||||
#has_prefix = false;
|
||||
#closed = false;
|
||||
constructor(path: string, filecount: number) {
|
||||
this.#path = path;
|
||||
this.#filecount = filecount;
|
||||
}
|
||||
#check() {
|
||||
if (!this.#inited) throw Error("FileLoader not initiailzed.");
|
||||
if (this.#closed) throw Error("Already closed.");
|
||||
}
|
||||
#get_file(name: string) {
|
||||
if (this.#files.includes(name)) {
|
||||
return join(this.#path, name);
|
||||
}
|
||||
}
|
||||
#get_zip(_name: string) {
|
||||
return null;
|
||||
#get_zip(name: string) {
|
||||
if (this.#files.includes(name)) {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
close() {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
this.#zip?.close();
|
||||
}
|
||||
get_file(name: string, index: number) {
|
||||
this.#check();
|
||||
if (this.#has_prefix) {
|
||||
name = `${index.toString().padStart(3, "0")}_${name}`;
|
||||
name = `${
|
||||
index.toString().padStart(
|
||||
this.#filecount.toString().length,
|
||||
"0",
|
||||
)
|
||||
}_${name}`;
|
||||
}
|
||||
let t = this.#get_file(name);
|
||||
if (t) return t;
|
||||
@@ -173,7 +191,16 @@ class FileLoader {
|
||||
if (t) return t;
|
||||
}
|
||||
}
|
||||
get_zip(name: string) {
|
||||
get_zip(name: string, index: number) {
|
||||
this.#check();
|
||||
if (this.#has_prefix) {
|
||||
name = `${
|
||||
index.toString().padStart(
|
||||
this.#filecount.toString().length,
|
||||
"0",
|
||||
)
|
||||
}_${name}`;
|
||||
}
|
||||
let t = this.#get_zip(name);
|
||||
if (t) return t;
|
||||
const ext = extname(name).toLowerCase();
|
||||
@@ -190,6 +217,16 @@ class FileLoader {
|
||||
this.#files.push(relative(this.#path, i.path));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const z = await import("../utils/readonly_zip.ts");
|
||||
this.#zip = new z.ReadonlyZip(this.#path);
|
||||
const count = this.#zip.count;
|
||||
for (let i = 0n; i < count; i++) {
|
||||
const f = this.#zip.get_name(i);
|
||||
if (f) {
|
||||
this.#files.push(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
let has_prefix = true;
|
||||
const re = new RegExp(`^\\d{${this.#filecount.toString().length}}_`);
|
||||
@@ -205,11 +242,21 @@ class FileLoader {
|
||||
}
|
||||
this.#has_prefix = has_prefix;
|
||||
this.#inited = true;
|
||||
this.#closed = false;
|
||||
return this;
|
||||
}
|
||||
get is_zip() {
|
||||
this.#check();
|
||||
return false;
|
||||
return this.#zip !== undefined;
|
||||
}
|
||||
open_zip_file(name: string) {
|
||||
if (this.#files.includes(name)) {
|
||||
const s = this.#zip!.open(name);
|
||||
if (typeof s === "string") {
|
||||
throw Error(`Failed to open file: ${s}`);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,189 +277,316 @@ export async function import_task(task: Task, manager: TaskManager) {
|
||||
const f =
|
||||
await (new FileLoader(icfg.import_path, parseInt(gdata.filecount)))
|
||||
.init();
|
||||
const gmeta = gdatas.convert(gdata);
|
||||
db.add_gmeta(gmeta);
|
||||
await db.add_gtag(task.gid, new Set(gdata.tags));
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_update", { detail: gmeta.gid }),
|
||||
);
|
||||
}
|
||||
const base_path = join(cfg.base, task.gid.toString());
|
||||
await sure_dir(base_path);
|
||||
let mpv_enabled = icfg.mpv ?? cfg.mpv;
|
||||
let import_method = icfg.method ?? cfg.import_method;
|
||||
if (
|
||||
f.is_zip && import_method != ImportMethod.Copy &&
|
||||
import_method != ImportMethod.CopyThenDelete
|
||||
) {
|
||||
import_method = ImportMethod.CopyThenDelete;
|
||||
}
|
||||
const max_import_img_count = icfg.max_import_img_count ??
|
||||
cfg.max_import_img_count;
|
||||
let imgs: Page[] = [];
|
||||
const m = new ImportManager(
|
||||
max_import_img_count,
|
||||
manager.aborts,
|
||||
manager.force_aborts,
|
||||
task,
|
||||
manager,
|
||||
);
|
||||
if (!mpv_enabled) {
|
||||
const g = await client.fetchGalleryPage(task.gid, task.token);
|
||||
if (g.mpv_enabled) {
|
||||
mpv_enabled = true;
|
||||
} else {
|
||||
imgs = await g.imagelist;
|
||||
m.set_total_page(g.length);
|
||||
}
|
||||
}
|
||||
if (mpv_enabled) {
|
||||
const g = await client.fetchMPVPage(task.gid, task.token);
|
||||
imgs = g.imagelist;
|
||||
m.set_total_page(g.pagecount);
|
||||
}
|
||||
const names = imgs.reduce(
|
||||
(acc: Record<string, number>, cur) => {
|
||||
const curr = cur.name;
|
||||
return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
async function import_img(i: Page) {
|
||||
const opath = f.get_file(i.name, i.index);
|
||||
if (!opath) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
const size = await fb_get_size(opath);
|
||||
if (!size) {
|
||||
console.log("Failed to get file size for", opath);
|
||||
throw Error("Failed to get file size.");
|
||||
}
|
||||
const ofiles = db.get_files(i.token);
|
||||
if (ofiles.length) {
|
||||
const need = await asyncEvery(
|
||||
ofiles,
|
||||
async (t) =>
|
||||
(!t.is_original && t.height != size.height &&
|
||||
t.width != size.width) || (!await exists(t.path)),
|
||||
try {
|
||||
const gmeta = gdatas.convert(gdata);
|
||||
db.add_gmeta(gmeta);
|
||||
await db.add_gtag(task.gid, new Set(gdata.tags));
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_update", { detail: gmeta.gid }),
|
||||
);
|
||||
if (!need) {
|
||||
const p = db.get_pmeta_by_index(task.gid, i.index);
|
||||
if (!p) {
|
||||
const op = db.get_pmeta_by_token(
|
||||
task.gid,
|
||||
i.token,
|
||||
);
|
||||
if (op) {
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
} else {
|
||||
const ops = db.get_pmeta_by_token_only(
|
||||
}
|
||||
const base_path = join(cfg.base, task.gid.toString());
|
||||
await sure_dir(base_path);
|
||||
let mpv_enabled = icfg.mpv ?? cfg.mpv;
|
||||
let import_method = icfg.method ?? cfg.import_method;
|
||||
if (
|
||||
f.is_zip && import_method != ImportMethod.Copy &&
|
||||
import_method != ImportMethod.CopyThenDelete
|
||||
) {
|
||||
import_method = ImportMethod.CopyThenDelete;
|
||||
}
|
||||
const max_import_img_count = icfg.max_import_img_count ??
|
||||
cfg.max_import_img_count;
|
||||
let imgs: Page[] = [];
|
||||
const m = new ImportManager(
|
||||
max_import_img_count,
|
||||
manager.aborts,
|
||||
manager.force_aborts,
|
||||
task,
|
||||
manager,
|
||||
);
|
||||
if (!mpv_enabled) {
|
||||
const g = await client.fetchGalleryPage(task.gid, task.token);
|
||||
if (g.mpv_enabled) {
|
||||
mpv_enabled = true;
|
||||
} else {
|
||||
imgs = await g.imagelist;
|
||||
m.set_total_page(g.length);
|
||||
}
|
||||
}
|
||||
if (mpv_enabled) {
|
||||
const g = await client.fetchMPVPage(task.gid, task.token);
|
||||
imgs = g.imagelist;
|
||||
m.set_total_page(g.pagecount);
|
||||
}
|
||||
const names = imgs.reduce(
|
||||
(acc: Record<string, number>, cur) => {
|
||||
const curr = cur.name;
|
||||
return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
async function import_img(i: Page) {
|
||||
const opath = f.get_file(i.name, i.index);
|
||||
if (!opath) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
const size = await fb_get_size(opath);
|
||||
if (!size) {
|
||||
console.log("Failed to get file size for", opath);
|
||||
throw Error("Failed to get file size.");
|
||||
}
|
||||
const ofiles = db.get_files(i.token);
|
||||
if (ofiles.length) {
|
||||
const need = await asyncEvery(
|
||||
ofiles,
|
||||
async (t) =>
|
||||
(!t.is_original && t.height != size.height &&
|
||||
t.width != size.width) || (!await exists(t.path)),
|
||||
);
|
||||
if (!need) {
|
||||
const p = db.get_pmeta_by_index(task.gid, i.index);
|
||||
if (!p) {
|
||||
const op = db.get_pmeta_by_token(
|
||||
task.gid,
|
||||
i.token,
|
||||
);
|
||||
if (ops.length) {
|
||||
const op = ops[0];
|
||||
op.gid = task.gid;
|
||||
if (op) {
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
} else {
|
||||
const ops = db.get_pmeta_by_token_only(
|
||||
i.token,
|
||||
);
|
||||
if (ops.length) {
|
||||
const op = ops[0];
|
||||
op.gid = task.gid;
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log("Already has page", i.index);
|
||||
return;
|
||||
}
|
||||
console.log("Already has page", i.index);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const pmeta: PMeta = {
|
||||
gid: task.gid,
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
index: i.index,
|
||||
name: i.name,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_pmeta(pmeta);
|
||||
const oriext = extname(i.name).toLowerCase();
|
||||
const nowext = extname(opath).toLowerCase();
|
||||
const is_original = icfg.size == ImportSize.Original ||
|
||||
(oriext != ".jpg" && oriext == nowext) ||
|
||||
(oriext == nowext && size.width < icfg.size);
|
||||
let path = join(base_path, is_original ? i.name : i.sampled_name);
|
||||
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
|
||||
path = add_suffix_to_path(path, i.token);
|
||||
console.log("Changed path to", path);
|
||||
}
|
||||
if (import_method == ImportMethod.Move) {
|
||||
await Deno.rename(opath, path);
|
||||
} else if (import_method != ImportMethod.Keep) {
|
||||
await Deno.copyFile(opath, path);
|
||||
} else {
|
||||
path = opath;
|
||||
}
|
||||
if (cfg.check_file_hash && is_original) {
|
||||
const sha = await calFileSha1(path);
|
||||
if (sha.slice(0, i.token.length) != i.token) {
|
||||
console.warn(
|
||||
`Hash not matched: file hash ${sha}, token ${i.token}`,
|
||||
);
|
||||
return;
|
||||
const pmeta: PMeta = {
|
||||
gid: task.gid,
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
index: i.index,
|
||||
name: i.name,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_pmeta(pmeta);
|
||||
const oriext = extname(i.name).toLowerCase();
|
||||
const nowext = extname(opath).toLowerCase();
|
||||
const is_original = icfg.size == ImportSize.Original ||
|
||||
(oriext != ".jpg" && oriext == nowext) ||
|
||||
(oriext == nowext && size.width < icfg.size);
|
||||
let path = join(base_path, is_original ? i.name : i.sampled_name);
|
||||
if (import_method != ImportMethod.Keep && names[i.name] > 1) {
|
||||
path = add_suffix_to_path(path, i.token);
|
||||
console.log("Changed path to", path);
|
||||
}
|
||||
if (import_method == ImportMethod.Move) {
|
||||
await Deno.rename(opath, path);
|
||||
} else if (import_method != ImportMethod.Keep) {
|
||||
await Deno.copyFile(opath, path);
|
||||
} else {
|
||||
path = opath;
|
||||
}
|
||||
if (cfg.check_file_hash && is_original) {
|
||||
const sha = await calFileSha1(path);
|
||||
if (sha.slice(0, i.token.length) != i.token) {
|
||||
console.warn(
|
||||
`Hash not matched: file hash ${sha}, token ${i.token}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (import_method == ImportMethod.CopyThenDelete) {
|
||||
await Deno.remove(opath);
|
||||
}
|
||||
const file: EhFile = {
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
is_original,
|
||||
id: 0,
|
||||
path,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_file(file);
|
||||
}
|
||||
if (import_method == ImportMethod.CopyThenDelete) {
|
||||
await Deno.remove(opath);
|
||||
}
|
||||
const file: EhFile = {
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
is_original,
|
||||
id: 0,
|
||||
path,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_file(file);
|
||||
}
|
||||
|
||||
for (const i of imgs) {
|
||||
if (manager.aborted) break;
|
||||
await m.add_new_task(async () => {
|
||||
await import_img(i);
|
||||
});
|
||||
}
|
||||
await m.join();
|
||||
const remove_previous_gallery = icfg.remove_previous_gallery ??
|
||||
cfg.remove_previous_gallery;
|
||||
if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
|
||||
if (manager.aborted || manager.force_aborted) throw Error("aborted");
|
||||
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
|
||||
let replaced_gallery = icfg.replaced_gallery;
|
||||
if (replaced_gallery === undefined) {
|
||||
const fg = await client.fetchGalleryPage(
|
||||
gmeta.first_gid,
|
||||
gmeta.first_key,
|
||||
async function import_zip_img(i: Page) {
|
||||
const opath = f.get_zip(i.name, i.index);
|
||||
if (!opath) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
const ofiles = db.get_files(i.token);
|
||||
if (ofiles.length) {
|
||||
const need = await asyncEvery(
|
||||
ofiles,
|
||||
async (t) =>
|
||||
!t.is_original && icfg.size === ImportSize.Original ||
|
||||
(!await exists(t.path)),
|
||||
);
|
||||
if (!need) {
|
||||
const p = db.get_pmeta_by_index(task.gid, i.index);
|
||||
if (!p) {
|
||||
const op = db.get_pmeta_by_token(
|
||||
task.gid,
|
||||
i.token,
|
||||
);
|
||||
if (op) {
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
} else {
|
||||
const ops = db.get_pmeta_by_token_only(
|
||||
i.token,
|
||||
);
|
||||
if (ops.length) {
|
||||
const op = ops[0];
|
||||
op.gid = task.gid;
|
||||
op.index = i.index;
|
||||
op.name = i.name;
|
||||
db.add_pmeta(op);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log("Already has page", i.index);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const oriext = extname(i.name).toLowerCase();
|
||||
const nowext = extname(opath).toLowerCase();
|
||||
let path = join(
|
||||
base_path,
|
||||
i.name.replace(oriext, nowext),
|
||||
);
|
||||
replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid);
|
||||
replaced_gallery.push({
|
||||
gid: gmeta.first_gid,
|
||||
token: gmeta.first_key,
|
||||
if (names[i.name] > 1) {
|
||||
path = add_suffix_to_path(path, i.token);
|
||||
console.log("Changed path to", path);
|
||||
}
|
||||
const zf = f.open_zip_file(opath);
|
||||
if (!zf) {
|
||||
console.log("File not found");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const out = await Deno.open(path, {
|
||||
create: true,
|
||||
write: true,
|
||||
});
|
||||
try {
|
||||
const buf = new Uint8Array(65536);
|
||||
let len = zf.read(buf);
|
||||
while (len > 0) {
|
||||
await out.write(buf.slice(0, Number(len)));
|
||||
len = zf.read(buf);
|
||||
}
|
||||
} finally {
|
||||
out.close();
|
||||
}
|
||||
} finally {
|
||||
zf.close();
|
||||
}
|
||||
const size = await fb_get_size(path);
|
||||
if (!size) {
|
||||
console.log("Failed to get file size for", path);
|
||||
throw Error("Failed to get file size.");
|
||||
}
|
||||
const is_original = icfg.size == ImportSize.Original ||
|
||||
(oriext != ".jpg" && oriext == nowext) ||
|
||||
(oriext == nowext && size.width < icfg.size);
|
||||
if (cfg.check_file_hash && is_original) {
|
||||
const sha = await calFileSha1(path);
|
||||
if (sha.slice(0, i.token.length) != i.token) {
|
||||
console.warn(
|
||||
`Hash not matched: file hash ${sha}, token ${i.token}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const pmeta: PMeta = {
|
||||
gid: task.gid,
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
index: i.index,
|
||||
name: i.name,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_pmeta(pmeta);
|
||||
const file: EhFile = {
|
||||
height: size.height,
|
||||
width: size.width,
|
||||
is_original,
|
||||
id: 0,
|
||||
path,
|
||||
token: i.token,
|
||||
};
|
||||
db.add_file(file);
|
||||
}
|
||||
|
||||
for (const i of imgs) {
|
||||
if (manager.aborted) break;
|
||||
await m.add_new_task(async () => {
|
||||
if (f.is_zip) await import_zip_img(i);
|
||||
else await import_img(i);
|
||||
});
|
||||
}
|
||||
replaced_gallery.forEach((g) => {
|
||||
const gmeta = db.get_gmeta_by_gid(g.gid);
|
||||
if (!gmeta) return;
|
||||
console.log("Remove gallery ", g.gid);
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_remove", { detail: gmeta.gid }),
|
||||
await m.join();
|
||||
const remove_previous_gallery = icfg.remove_previous_gallery ??
|
||||
cfg.remove_previous_gallery;
|
||||
if (m.has_failed_task) throw new RecoverableError("Some tasks failed.");
|
||||
if (manager.aborted || manager.force_aborted) throw Error("aborted");
|
||||
if (f.is_zip && import_method == ImportMethod.CopyThenDelete) {
|
||||
f.close();
|
||||
await Deno.remove(icfg.import_path);
|
||||
}
|
||||
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
|
||||
let replaced_gallery = icfg.replaced_gallery;
|
||||
if (replaced_gallery === undefined) {
|
||||
const fg = await client.fetchGalleryPage(
|
||||
gmeta.first_gid,
|
||||
gmeta.first_key,
|
||||
);
|
||||
replaced_gallery = fg.new_version.filter((d) =>
|
||||
d.gid < task.gid
|
||||
);
|
||||
replaced_gallery.push({
|
||||
gid: gmeta.first_gid,
|
||||
token: gmeta.first_key,
|
||||
});
|
||||
}
|
||||
db.delete_gallery(g.gid);
|
||||
});
|
||||
replaced_gallery.forEach((g) => {
|
||||
const gmeta = db.get_gmeta_by_gid(g.gid);
|
||||
if (!gmeta) return;
|
||||
console.log("Remove gallery ", g.gid);
|
||||
if (manager.meilisearch) {
|
||||
manager.meilisearch.target.dispatchEvent(
|
||||
new CustomEvent("gallery_remove", {
|
||||
detail: gmeta.gid,
|
||||
}),
|
||||
);
|
||||
}
|
||||
db.delete_gallery(g.gid);
|
||||
});
|
||||
}
|
||||
return task;
|
||||
} finally {
|
||||
f.close();
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user