From 38e970dc7d427e1bf8a71fed083f0f0e1216edde Mon Sep 17 00:00:00 2001 From: lifegpc Date: Thu, 25 May 2023 10:15:01 +0800 Subject: [PATCH] Add export_zip support --- db.ts | 36 +++++++++++++++++++++----- db_test.ts | 1 + deno.jsonc | 3 ++- deno.lock | 32 +++++++++++++++++++++++- main.ts | 17 +++++++++++++ task.ts | 2 ++ task_manager.ts | 33 ++++++++++++++++++++++++ tasks/export_zip.ts | 61 +++++++++++++++++++++++++++++++++++++++++++++ utils.ts | 6 +++++ 9 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 tasks/export_zip.ts diff --git a/db.ts b/db.ts index 28d0b61..f0c2dec 100644 --- a/db.ts +++ b/db.ts @@ -96,7 +96,8 @@ const TASK_TABLE = `CREATE TABLE task ( gid INT, token TEXT, pn INT, - pid INT + pid INT, + details TEXT );`; const GMETA_TABLE = `CREATE TABLE gmeta ( gid INT, @@ -155,7 +156,7 @@ export class EhDb { #lock_file: string | undefined; #dblock_file: string | undefined; #_tags: Map | undefined; - readonly version = new SemVer("1.0.0-2"); + readonly version = new SemVer("1.0.0-3"); constructor(base_path: string) { const db_path = join(base_path, "data.db"); sure_dir_sync(base_path); @@ -209,6 +210,9 @@ export class EhDb { this.add_file(f, false); }); } + if (v.compare("1.0.0-3") === -1) { + this.db.execute("ALTER TABLE task ADD details TEXT;"); + } this.#write_version(); } if ( @@ -346,12 +350,26 @@ export class EhDb { add_task(task: Task) { return this.transaction(() => { this.db.query( - "INSERT INTO task (type, gid, token, pn, pid) VALUES (?, ?, ?, ?, ?);", - [task.type, task.gid, task.token, task.pn, task.pid], + "INSERT INTO task (type, gid, token, pn, pid, details) VALUES (?, ?, ?, ?, ?, ?);", + [ + task.type, + task.gid, + task.token, + task.pn, + task.pid, + task.details, + ], ); return this.db.queryEntries( - "SELECT * FROM task WHERE type = ? AND gid = ? AND token = ? AND pn = ? AND pid = ?;", - [task.type, task.gid, task.token, task.pn, task.pid], + "SELECT * FROM task WHERE type = ? AND gid = ? AND token = ? AND pn = ? AND pid = ? AND details = ?;", + [ + task.type, + task.gid, + task.token, + task.pn, + task.pid, + task.details, + ], )[0]; }); } @@ -464,6 +482,12 @@ export class EhDb { ).map((v) => v[0]), ); } + get_pmeta(gid: number) { + return this.db.queryEntries( + "SELECT * FROM pmeta WHERE gid = ?;", + [gid], + ); + } get_pmeta_by_index(gid: number, index: number) { const s = this.db.queryEntries( 'SELECT * FROM pmeta WHERE gid = ? AND "index" = ?;', diff --git a/db_test.ts b/db_test.ts index a1019c3..282e3a6 100644 --- a/db_test.ts +++ b/db_test.ts @@ -16,6 +16,7 @@ Deno.test("DbTest", async () => { pn: 1, type: TaskType.Download, id: 0, + details: null, }), ); const gmeta: GMeta = { diff --git a/deno.jsonc b/deno.jsonc index 7124787..f201569 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -2,7 +2,8 @@ "imports": { "std/": "https://deno.land/std@0.188.0/", "deno_dom/": "https://deno.land/x/deno_dom@v0.1.38/", - "sqlite/": "https://deno.land/x/sqlite@v3.7.2/" + "sqlite/": "https://deno.land/x/sqlite@v3.7.2/", + "zipjs/": "https://deno.land/x/zipjs@v2.7.14/" }, "tasks": { "dev": "deno run --watch main.ts", diff --git a/deno.lock b/deno.lock index fb052e1..7284db3 100644 --- a/deno.lock +++ b/deno.lock @@ -54,6 +54,36 @@ "https://deno.land/x/sqlite@v3.7.2/src/error.ts": "f7a15cb00d7c3797da1aefee3cf86d23e0ae92e73f0ba3165496c3816ab9503a", "https://deno.land/x/sqlite@v3.7.2/src/function.ts": "e4c83b8ec64bf88bafad2407376b0c6a3b54e777593c70336fb40d43a79865f2", "https://deno.land/x/sqlite@v3.7.2/src/query.ts": "d58abda928f6582d77bad685ecf551b1be8a15e8e38403e293ec38522e030cad", - "https://deno.land/x/sqlite@v3.7.2/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487" + "https://deno.land/x/sqlite@v3.7.2/src/wasm.ts": "e79d0baa6e42423257fb3c7cc98091c54399254867e0f34a09b5bdef37bd9487", + "https://deno.land/x/zipjs@v2.7.14/index.d.ts": "f1c412d1721597eafd2b665ec002ddfb982a8818e32fb6a4d3cab1a86134db40", + "https://deno.land/x/zipjs@v2.7.14/index.js": "7c71926e0c9618e48a22d9dce701131704fd3148a1d2eefd5dba1d786c846a5f", + "https://deno.land/x/zipjs@v2.7.14/lib/core/codec-pool.js": "e5ab8ee3ec800ed751ef1c63a1bd8e50f162aa256a5f625d173d7a32e76e828c", + "https://deno.land/x/zipjs@v2.7.14/lib/core/codec-worker.js": "396e281865a87b708a08d212661f239b8f2dd4ae0d936ff4a681bbbf79a1cc32", + "https://deno.land/x/zipjs@v2.7.14/lib/core/configuration.js": "baa316a63df2f8239f9d52cd4863eaedaddd34ad887b7513588da75d19e84932", + "https://deno.land/x/zipjs@v2.7.14/lib/core/constants.js": "c1aff78c6b9378b26ab4330e85032160ebf1f2d09a99e5e6907626f816e88908", + "https://deno.land/x/zipjs@v2.7.14/lib/core/io.js": "552aa60c7bdae34081a94e6e27cb0515bde7398187fd02eb7d71408698f22f25", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/aes-crypto-stream.js": "63988c9f3ce1e043c80e6eb140ebb07bf2ab543ee9a85349651ab74b96aab2cf", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/codec-stream.js": "685f1120b94b6295dcd61b195d6202cd24a5344e4588dc52f42e8ac0f9dfe294", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/codecs/crc32.js": "dfdde666f72b4a5ffc8cf5b1451e0db578ce4bd90de20df2cff5bfd47758cb23", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/codecs/deflate.js": "08c1b24d1845528f6db296570d690ecbe23c6c01c6cb26b561e601e770281c3a", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/codecs/inflate.js": "55d00eed332cf2c4f61e2ee23133e3257768d0608572ee3f9641a2921c3a6f67", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/codecs/sjcl.js": "462289c5312f01bba8a757a7a0f3d8f349f471183cb4c49fb73d58bba18a5428", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/common-crypto.js": "4d462619848d94427fcd486fd94e5c0741af60e476df6720da8224b086eba47e", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/crc32-stream.js": "10e26bd18df0e1e89d61a62827a1a1c19f4e541636dd0eccbd85af3afabce289", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/stream-adapter.js": "9e7f3fe1601cc447943cd37b5adb6d74c6e9c404d002e707e8eace7bc048929c", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/zip-crypto-stream.js": "19305af1e8296e7fa6763f3391d0b8149a1e09c659e1d1ff32a484448b18243c", + "https://deno.land/x/zipjs@v2.7.14/lib/core/streams/zip-entry-stream.js": "fff940b1aefa0cb95962634ab6a52df835361541b454e77747e878bcef17cd9e", + "https://deno.land/x/zipjs@v2.7.14/lib/core/util/cp437-decode.js": "d665ded184037ffe5d255be8f379f90416053e3d0d84fac95b28f4aeaab3d336", + "https://deno.land/x/zipjs@v2.7.14/lib/core/util/decode-text.js": "c04a098fa7c16470c48b6abd4eb4ac48af53547de65e7c8f39b78ae62330ad57", + "https://deno.land/x/zipjs@v2.7.14/lib/core/util/default-mime-type.js": "177ae00e1956d3d00cdefc40eb158cb591d3d24ede452c056d30f98d73d9cd73", + "https://deno.land/x/zipjs@v2.7.14/lib/core/util/encode-text.js": "c51a8947c15b7fe31b0036b69fd68817f54b30ce29502b5c9609d8b15e3b20d9", + "https://deno.land/x/zipjs@v2.7.14/lib/core/util/mime-type.js": "7e4f0d516923f7dcf5cd14a0d06e2b7af9dacd160c707afac82f3ce6c69ae287", + "https://deno.land/x/zipjs@v2.7.14/lib/core/util/stream-codec-shim.js": "845c0659c6ab9ed9b03cae452255e4e0a699001e8b45ccae23cb0ece5f91064c", + "https://deno.land/x/zipjs@v2.7.14/lib/core/zip-entry.js": "d30a535cd1e75ef98094cd04120f178c103cdc4055d23ff747ffc6a154da8d2d", + "https://deno.land/x/zipjs@v2.7.14/lib/core/zip-fs-core.js": "fa57704140bdd73b991917ed63c660b95e8d228e00b519d6bfb592f4e419cb7f", + "https://deno.land/x/zipjs@v2.7.14/lib/core/zip-reader.js": "b3b17325500ea5f06f3828f82a844f514afeb424969f013318cc939f69d94d3b", + "https://deno.land/x/zipjs@v2.7.14/lib/core/zip-writer.js": "34809b421f5deb497ce8cabec730af858c12dde54a6205c78b5591460785dc1e", + "https://deno.land/x/zipjs@v2.7.14/lib/z-worker-inline.js": "df83d91413a2e79f76924f39f26f59e6efbe8f5487d3a91b7e92b6d64236927c", + "https://deno.land/x/zipjs@v2.7.14/lib/zip-fs.js": "a733360302f5fbec9cc01543cb9fcfe7bae3f35a50d0006626ce42fe8183b63f" } } diff --git a/main.ts b/main.ts index b59cf32..f456cc6 100644 --- a/main.ts +++ b/main.ts @@ -22,6 +22,7 @@ enum CMD { Run, Optimize, UpdateTagTranslation, + ExportZip, } const args = parse(Deno.args, { @@ -44,6 +45,7 @@ if (rcmd == "optimize") cmd = CMD.Optimize; if (rcmd == "utt" || rcmd == "update_tag_translation") { cmd = CMD.UpdateTagTranslation; } +if (rcmd == "ez" || rcmd == "export_zip") cmd = CMD.ExportZip; if (cmd == CMD.Unknown) { throw Error(`Unknown command: ${rcmd}`); } @@ -102,6 +104,19 @@ async function update_tag_translation() { db.close(); } } +async function export_zip() { + const manager = new TaskManager(settings); + try { + for (const gid of args._.slice(1)) { + if (typeof gid === "number") { + await manager.add_export_zip_task(gid); + } + } + await manager.run(); + } finally { + if (!manager.aborted) manager.close(); + } +} async function main() { await sure_dir(settings.base); if (cmd == CMD.Download) { @@ -112,6 +127,8 @@ async function main() { optimize(); } else if (cmd == CMD.UpdateTagTranslation) { await update_tag_translation(); + } else if (cmd == CMD.ExportZip) { + await export_zip(); } } diff --git a/task.ts b/task.ts index 781ee68..1e2ab32 100644 --- a/task.ts +++ b/task.ts @@ -1,5 +1,6 @@ export enum TaskType { Download, + ExportZip, } export type Task = { @@ -9,4 +10,5 @@ export type Task = { token: string; pn: number; pid: number; + details: string | null; }; diff --git a/task_manager.ts b/task_manager.ts index 7351183..02c78ad 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -5,6 +5,11 @@ import { check_running } from "./pid_check.ts"; import { add_exit_handler } from "./signal_handler.ts"; import { Task, TaskType } from "./task.ts"; import { download_task } from "./tasks/download.ts"; +import { + DEFAULT_EXPORT_ZIP_CONFIG, + export_zip, + ExportZipConfig, +} from "./tasks/export_zip.ts"; import { promiseState, PromiseStatus, sleep } from "./utils.ts"; export class AlreadyClosedError extends Error { @@ -52,6 +57,20 @@ export class TaskManager { pid: Deno.pid, pn: 1, type: TaskType.Download, + details: null, + }; + return await this.db.add_task(task); + } + async add_export_zip_task(gid: number, output?: string) { + const cfg: ExportZipConfig = { output }; + const task: Task = { + gid, + token: "", + id: 0, + pid: Deno.pid, + pn: 0, + type: TaskType.ExportZip, + details: JSON.stringify(cfg), }; return await this.db.add_task(task); } @@ -145,6 +164,20 @@ export class TaskManager { this.#force_abort.signal, ), ); + } else if (task.type == TaskType.ExportZip) { + const cfg: ExportZipConfig = task.details + ? JSON.parse(task.details) + : DEFAULT_EXPORT_ZIP_CONFIG; + this.running_tasks.set( + task.id, + export_zip( + task, + this.db, + this.cfg, + this.#abort.signal, + cfg, + ), + ); } } async waiting_unfinished_task() { diff --git a/tasks/export_zip.ts b/tasks/export_zip.ts new file mode 100644 index 0000000..fdee5f2 --- /dev/null +++ b/tasks/export_zip.ts @@ -0,0 +1,61 @@ +import { join } from "std/path/mod.ts"; +import { Uint8ArrayReader, ZipWriter } from "zipjs/index.js"; +import { EhDb } from "../db.ts"; +import { addZero, asyncForEach } from "../utils.ts"; +import { Config } from "../config.ts"; +import { Task } from "../task.ts"; + +export type ExportZipConfig = { + output?: string; +}; + +export const DEFAULT_EXPORT_ZIP_CONFIG: ExportZipConfig = {}; + +export async function export_zip( + task: Task, + db: EhDb, + cfg: Config, + signal: AbortSignal, + ecfg: ExportZipConfig, +) { + const gid = task.gid; + const g = db.get_gmeta_by_gid(gid); + if (!g) throw Error("Gallery not found in database."); + const output = ecfg.output === undefined + ? join(cfg.base, g.title + ".zip") + : ecfg.output; + const f = await Deno.open(output, { + create: true, + write: true, + truncate: true, + }); + try { + const z = new ZipWriter(f.writable, { + signal, + level: 0, + }); + const l = g.filecount.toString().length; + await asyncForEach( + db.get_pmeta(gid).sort((a, b) => a.index - b.index), + async (p) => { + const f = db.get_files(gid, p.token); + if (f.length) { + const r = await Deno.readFile(f[0].path, { signal }); + await z.add( + `${addZero(p.index, l)}_${p.name}`, + new Uint8ArrayReader(r), + { signal }, + ); + } + }, + ); + await z.close(); + } finally { + try { + f.close(); + } catch (_) { + null; + } + } + return task; +} diff --git a/utils.ts b/utils.ts index 4353f9a..b88273d 100644 --- a/utils.ts +++ b/utils.ts @@ -105,3 +105,9 @@ export async function asyncForEach( await callback.apply(thisArg, [arr[i], i, arr]); } } + +export function addZero(n: string | number, len: number) { + let s = n.toString(); + while (s.length < len) s = "0" + s; + return s; +}