lifegpc/eh-downloader
1
0
Fork 0
mirror of https://github.com/lifegpc/eh-downloader.git synced 2026-06-06 05:38:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add export_zip support

Browse Source
This commit is contained in:
lifegpc
2023-05-25 10:15:01 +08:00
parent ab4be7f8b4
commit 38e970dc7d
9 changed files with 183 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
db.ts
View File

@@ -96,7 +96,8 @@ const TASK_TABLE = `CREATE TABLE task (
gid INT,
token TEXT,
pn INT,
pid INT
pid INT,
details TEXT
);`;
const GMETA_TABLE = `CREATE TABLE gmeta (
gid INT,
@@ -155,7 +156,7 @@ export class EhDb {
#lock_file: string | undefined;
#dblock_file: string | undefined;
#_tags: Map<string, number> | undefined;
readonly version = new SemVer("1.0.0-2");
readonly version = new SemVer("1.0.0-3");
constructor(base_path: string) {
const db_path = join(base_path, "data.db");
sure_dir_sync(base_path);
@@ -209,6 +210,9 @@ export class EhDb {
this.add_file(f, false);
});
}
if (v.compare("1.0.0-3") === -1) {
this.db.execute("ALTER TABLE task ADD details TEXT;");
}
this.#write_version();
}
if (
@@ -346,12 +350,26 @@ export class EhDb {
add_task(task: Task) {
return this.transaction(() => {
this.db.query(
"INSERT INTO task (type, gid, token, pn, pid) VALUES (?, ?, ?, ?, ?);",
[task.type, task.gid, task.token, task.pn, task.pid],
"INSERT INTO task (type, gid, token, pn, pid, details) VALUES (?, ?, ?, ?, ?, ?);",
[
task.type,
task.gid,
task.token,
task.pn,
task.pid,
task.details,
],
);
return this.db.queryEntries<Task>(
"SELECT * FROM task WHERE type = ? AND gid = ? AND token = ? AND pn = ? AND pid = ?;",
[task.type, task.gid, task.token, task.pn, task.pid],
"SELECT * FROM task WHERE type = ? AND gid = ? AND token = ? AND pn = ? AND pid = ? AND details = ?;",
[
task.type,
task.gid,
task.token,
task.pn,
task.pid,
task.details,
],
)[0];
});
}
@@ -464,6 +482,12 @@ export class EhDb {
).map((v) => v[0]),
);
}
get_pmeta(gid: number) {
return this.db.queryEntries<PMeta>(
"SELECT * FROM pmeta WHERE gid = ?;",
[gid],
);
}
get_pmeta_by_index(gid: number, index: number) {
const s = this.db.queryEntries<PMeta>(
'SELECT * FROM pmeta WHERE gid = ? AND "index" = ?;',

1
db_test.ts
View File

@@ -16,6 +16,7 @@ Deno.test("DbTest", async () => {
pn: 1,
type: TaskType.Download,
id: 0,
details: null,
}),
);
const gmeta: GMeta = {

3
deno.jsonc
View File

@@ -2,7 +2,8 @@
"imports": {
"std/": "https://deno.land/[email protected]/",
"deno_dom/": "https://deno.land/x/[email protected]/",
"sqlite/": "https://deno.land/x/[email protected]/"
"sqlite/": "https://deno.land/x/[email protected]/",
"zipjs/": "https://deno.land/x/[email protected]/"
},
"tasks": {
"dev": "deno run --watch main.ts",

32
deno.lock generated
View File

@@ -54,6 +54,36 @@
"https://deno.land/x/[email protected]/src/error.ts": "f7a15cb00d7c3797da1aefee3cf86d23e0ae92e73f0ba3165496c3816ab9503a",
"https://deno.land/x/[email protected]/src/function.ts": "e4c83b8ec64bf88bafad2407376b0c6a3b54e777593c70336fb40d43a79865f2",
"https://deno.land/x/[email protected]/src/query.ts": "d58abda928f6582d77bad685ecf551b1be8a15e8e38403e293ec38522e030cad",
"https://deno.land/x/[email protected]/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487"
"https://deno.land/x/[email protected]/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487",
"https://deno.land/x/[email protected]/index.d.ts": "f1c412d1721597eafd2b665ec002ddfb982a8818e32fb6a4d3cab1a86134db40",
"https://deno.land/x/[email protected]/index.js": "7c71926e0c9618e48a22d9dce701131704fd3148a1d2eefd5dba1d786c846a5f",
"https://deno.land/x/[email protected]/lib/core/codec-pool.js": "e5ab8ee3ec800ed751ef1c63a1bd8e50f162aa256a5f625d173d7a32e76e828c",
"https://deno.land/x/[email protected]/lib/core/codec-worker.js": "396e281865a87b708a08d212661f239b8f2dd4ae0d936ff4a681bbbf79a1cc32",
"https://deno.land/x/[email protected]/lib/core/configuration.js": "baa316a63df2f8239f9d52cd4863eaedaddd34ad887b7513588da75d19e84932",
"https://deno.land/x/[email protected]/lib/core/constants.js": "c1aff78c6b9378b26ab4330e85032160ebf1f2d09a99e5e6907626f816e88908",
"https://deno.land/x/[email protected]/lib/core/io.js": "552aa60c7bdae34081a94e6e27cb0515bde7398187fd02eb7d71408698f22f25",
"https://deno.land/x/[email protected]/lib/core/streams/aes-crypto-stream.js": "63988c9f3ce1e043c80e6eb140ebb07bf2ab543ee9a85349651ab74b96aab2cf",
"https://deno.land/x/[email protected]/lib/core/streams/codec-stream.js": "685f1120b94b6295dcd61b195d6202cd24a5344e4588dc52f42e8ac0f9dfe294",
"https://deno.land/x/[email protected]/lib/core/streams/codecs/crc32.js": "dfdde666f72b4a5ffc8cf5b1451e0db578ce4bd90de20df2cff5bfd47758cb23",
"https://deno.land/x/[email protected]/lib/core/streams/codecs/deflate.js": "08c1b24d1845528f6db296570d690ecbe23c6c01c6cb26b561e601e770281c3a",
"https://deno.land/x/[email protected]/lib/core/streams/codecs/inflate.js": "55d00eed332cf2c4f61e2ee23133e3257768d0608572ee3f9641a2921c3a6f67",
"https://deno.land/x/[email protected]/lib/core/streams/codecs/sjcl.js": "462289c5312f01bba8a757a7a0f3d8f349f471183cb4c49fb73d58bba18a5428",
"https://deno.land/x/[email protected]/lib/core/streams/common-crypto.js": "4d462619848d94427fcd486fd94e5c0741af60e476df6720da8224b086eba47e",
"https://deno.land/x/[email protected]/lib/core/streams/crc32-stream.js": "10e26bd18df0e1e89d61a62827a1a1c19f4e541636dd0eccbd85af3afabce289",
"https://deno.land/x/[email protected]/lib/core/streams/stream-adapter.js": "9e7f3fe1601cc447943cd37b5adb6d74c6e9c404d002e707e8eace7bc048929c",
"https://deno.land/x/[email protected]/lib/core/streams/zip-crypto-stream.js": "19305af1e8296e7fa6763f3391d0b8149a1e09c659e1d1ff32a484448b18243c",
"https://deno.land/x/[email protected]/lib/core/streams/zip-entry-stream.js": "fff940b1aefa0cb95962634ab6a52df835361541b454e77747e878bcef17cd9e",
"https://deno.land/x/[email protected]/lib/core/util/cp437-decode.js": "d665ded184037ffe5d255be8f379f90416053e3d0d84fac95b28f4aeaab3d336",
"https://deno.land/x/[email protected]/lib/core/util/decode-text.js": "c04a098fa7c16470c48b6abd4eb4ac48af53547de65e7c8f39b78ae62330ad57",
"https://deno.land/x/[email protected]/lib/core/util/default-mime-type.js": "177ae00e1956d3d00cdefc40eb158cb591d3d24ede452c056d30f98d73d9cd73",
"https://deno.land/x/[email protected]/lib/core/util/encode-text.js": "c51a8947c15b7fe31b0036b69fd68817f54b30ce29502b5c9609d8b15e3b20d9",
"https://deno.land/x/[email protected]/lib/core/util/mime-type.js": "7e4f0d516923f7dcf5cd14a0d06e2b7af9dacd160c707afac82f3ce6c69ae287",
"https://deno.land/x/[email protected]/lib/core/util/stream-codec-shim.js": "845c0659c6ab9ed9b03cae452255e4e0a699001e8b45ccae23cb0ece5f91064c",
"https://deno.land/x/[email protected]/lib/core/zip-entry.js": "d30a535cd1e75ef98094cd04120f178c103cdc4055d23ff747ffc6a154da8d2d",
"https://deno.land/x/[email protected]/lib/core/zip-fs-core.js": "fa57704140bdd73b991917ed63c660b95e8d228e00b519d6bfb592f4e419cb7f",
"https://deno.land/x/[email protected]/lib/core/zip-reader.js": "b3b17325500ea5f06f3828f82a844f514afeb424969f013318cc939f69d94d3b",
"https://deno.land/x/[email protected]/lib/core/zip-writer.js": "34809b421f5deb497ce8cabec730af858c12dde54a6205c78b5591460785dc1e",
"https://deno.land/x/[email protected]/lib/z-worker-inline.js": "df83d91413a2e79f76924f39f26f59e6efbe8f5487d3a91b7e92b6d64236927c",
"https://deno.land/x/[email protected]/lib/zip-fs.js": "a733360302f5fbec9cc01543cb9fcfe7bae3f35a50d0006626ce42fe8183b63f"
}
}

17
main.ts
View File

@@ -22,6 +22,7 @@ enum CMD {
Run,
Optimize,
UpdateTagTranslation,
ExportZip,
}
const args = parse(Deno.args, {
@@ -44,6 +45,7 @@ if (rcmd == "optimize") cmd = CMD.Optimize;
if (rcmd == "utt" || rcmd == "update_tag_translation") {
cmd = CMD.UpdateTagTranslation;
}
if (rcmd == "ez" || rcmd == "export_zip") cmd = CMD.ExportZip;
if (cmd == CMD.Unknown) {
throw Error(`Unknown command: ${rcmd}`);
}
@@ -102,6 +104,19 @@ async function update_tag_translation() {
db.close();
}
}
async function export_zip() {
const manager = new TaskManager(settings);
try {
for (const gid of args._.slice(1)) {
if (typeof gid === "number") {
await manager.add_export_zip_task(gid);
}
}
await manager.run();
} finally {
if (!manager.aborted) manager.close();
}
}
async function main() {
await sure_dir(settings.base);
if (cmd == CMD.Download) {
@@ -112,6 +127,8 @@ async function main() {
optimize();
} else if (cmd == CMD.UpdateTagTranslation) {
await update_tag_translation();
} else if (cmd == CMD.ExportZip) {
await export_zip();
}
}

2
task.ts
View File

@@ -1,5 +1,6 @@
export enum TaskType {
Download,
ExportZip,
}
export type Task = {
@@ -9,4 +10,5 @@ export type Task = {
token: string;
pn: number;
pid: number;
details: string | null;
};

33
task_manager.ts
View File

@@ -5,6 +5,11 @@ import { check_running } from "./pid_check.ts";
import { add_exit_handler } from "./signal_handler.ts";
import { Task, TaskType } from "./task.ts";
import { download_task } from "./tasks/download.ts";
import {
DEFAULT_EXPORT_ZIP_CONFIG,
export_zip,
ExportZipConfig,
} from "./tasks/export_zip.ts";
import { promiseState, PromiseStatus, sleep } from "./utils.ts";
export class AlreadyClosedError extends Error {
@@ -52,6 +57,20 @@ export class TaskManager {
pid: Deno.pid,
pn: 1,
type: TaskType.Download,
details: null,
};
return await this.db.add_task(task);
}
async add_export_zip_task(gid: number, output?: string) {
const cfg: ExportZipConfig = { output };
const task: Task = {
gid,
token: "",
id: 0,
pid: Deno.pid,
pn: 0,
type: TaskType.ExportZip,
details: JSON.stringify(cfg),
};
return await this.db.add_task(task);
}
@@ -145,6 +164,20 @@ export class TaskManager {
this.#force_abort.signal,
),
);
} else if (task.type == TaskType.ExportZip) {
const cfg: ExportZipConfig = task.details
? JSON.parse(task.details)
: DEFAULT_EXPORT_ZIP_CONFIG;
this.running_tasks.set(
task.id,
export_zip(
task,
this.db,
this.cfg,
this.#abort.signal,
cfg,
),
);
}
}
async waiting_unfinished_task() {

61
tasks/export_zip.ts Normal file
View File

@@ -0,0 +1,61 @@
import { join } from "std/path/mod.ts";
import { Uint8ArrayReader, ZipWriter } from "zipjs/index.js";
import { EhDb } from "../db.ts";
import { addZero, asyncForEach } from "../utils.ts";
import { Config } from "../config.ts";
import { Task } from "../task.ts";
export type ExportZipConfig = {
output?: string;
};
export const DEFAULT_EXPORT_ZIP_CONFIG: ExportZipConfig = {};
export async function export_zip(
task: Task,
db: EhDb,
cfg: Config,
signal: AbortSignal,
ecfg: ExportZipConfig,
) {
const gid = task.gid;
const g = db.get_gmeta_by_gid(gid);
if (!g) throw Error("Gallery not found in database.");
const output = ecfg.output === undefined
? join(cfg.base, g.title + ".zip")
: ecfg.output;
const f = await Deno.open(output, {
create: true,
write: true,
truncate: true,
});
try {
const z = new ZipWriter(f.writable, {
signal,
level: 0,
});
const l = g.filecount.toString().length;
await asyncForEach(
db.get_pmeta(gid).sort((a, b) => a.index - b.index),
async (p) => {
const f = db.get_files(gid, p.token);
if (f.length) {
const r = await Deno.readFile(f[0].path, { signal });
await z.add(
`${addZero(p.index, l)}_${p.name}`,
new Uint8ArrayReader(r),
{ signal },
);
}
},
);
await z.close();
} finally {
try {
f.close();
} catch (_) {
null;
}
}
return task;
}

6
utils.ts
View File

@@ -105,3 +105,9 @@ export async function asyncForEach<T, V>(
await callback.apply(thisArg, [arr[i], i, arr]);
}
}
export function addZero(n: string | number, len: number) {
let s = n.toString();
while (s.length < len) s = "0" + s;
return s;
}