diff --git a/config.ts b/config.ts index e39faad..89dbfa9 100644 --- a/config.ts +++ b/config.ts @@ -51,6 +51,9 @@ export class Config { get max_task_count() { return this._return_number("max_task_count") || 1; } + get mpv() { + return this._return_bool("mpv") || false; + } } export async function load_settings(path: string) { diff --git a/db.ts b/db.ts index d44c583..e288460 100644 --- a/db.ts +++ b/db.ts @@ -60,7 +60,11 @@ export type PMeta = { width: number; height: number; }; -const ALL_TABLES = ["version", "task", "gmeta", "pmeta"]; +type Tag = { + id: number; + tag: string; +}; +const ALL_TABLES = ["version", "task", "gmeta", "pmeta", "tag", "gtag"]; const VERSION_TABLE = `CREATE TABLE version ( id TEXT, ver TEXT, @@ -101,21 +105,31 @@ const PMETA_TABLE = `CREATE TABLE pmeta ( height INT, PRIMARY KEY (gid, token) );`; +const TAG_TABLE = `CREATE TABLE tag ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tag TEXT +);`; +const GTAG_TABLE = `CREATE TABLE gtag ( + gid INT, + id INT, + PRIMARY KEY (gid, id) +);`; export class EhDb { db; - flock_enabled: boolean = eval('typeof Deno.flock !== "undefined"'); + #flock_enabled: boolean = eval('typeof Deno.flock !== "undefined"'); #file: Deno.FsFile | undefined; #dblock: Deno.FsFile | undefined; - _exist_table: Set = new Set(); + #exist_table: Set = new Set(); #lock_file: string | undefined; #dblock_file: string | undefined; + #_tags: Map | undefined; readonly version = new SemVer("1.0.0-0"); constructor(base_path: string) { this.db = new DB(join(base_path, "data.db")); this.db.execute("PRAGMA main.locking_mode=EXCLUSIVE;"); - if (!this._check_database()) this._create_table(); - if (this.flock_enabled) { + if (!this.#check_database()) this.#create_table(); + if (this.#flock_enabled) { this.#lock_file = join(base_path, "db.lock"); this.#dblock_file = join(base_path, "eh.locked"); this.#file = Deno.openSync(this.#lock_file, { @@ -134,34 +148,51 @@ export class EhDb { ); } } - _check_database() { - this._updateExistsTable(); - const v = this._read_version(); + #add_tag(s: string) { + return this.transaction(() => { + this.db.query("INSERT INTO tag (tag) VALUES (?);", [s]); + const r = this.db.queryEntries( + "SELECT * FROM tag WHERE tag = ?;", + [s], + ); + this.#tags.set(s, r[0].id); + return r[0].id; + }); + } + #check_database() { + this.#updateExistsTable(); + const v = this.#read_version(); if (!v) return false; if ( - ALL_TABLES.length !== this._exist_table.size || - !ALL_TABLES.every((x) => this._exist_table.has(x)) + ALL_TABLES.length !== this.#exist_table.size || + !ALL_TABLES.every((x) => this.#exist_table.has(x)) ) return false; return true; } - _create_table() { - if (!this._exist_table.has("version")) { + #create_table() { + if (!this.#exist_table.has("version")) { this.db.execute(VERSION_TABLE); - this._write_version(); + this.#write_version(); } - if (!this._exist_table.has("task")) { + if (!this.#exist_table.has("task")) { this.db.execute(TASK_TABLE); } - if (!this._exist_table.has("gmeta")) { + if (!this.#exist_table.has("gmeta")) { this.db.execute(GMETA_TABLE); } - if (!this._exist_table.has("pmeta")) { + if (!this.#exist_table.has("pmeta")) { this.db.execute(PMETA_TABLE); } - this._updateExistsTable(); + if (!this.#exist_table.has("tag")) { + this.db.execute(TAG_TABLE); + } + if (!this.#exist_table.has("gtag")) { + this.db.execute(GTAG_TABLE); + } + this.#updateExistsTable(); } - _read_version() { - if (!this._exist_table.has("version")) return null; + #read_version() { + if (!this.#exist_table.has("version")) return null; const cur = this.db.query<[string]>( "SELECT ver FROM version WHERE id = ?;", ["eh"], @@ -171,18 +202,27 @@ export class EhDb { } return null; } - _updateExistsTable() { + get #tags() { + if (this.#_tags === undefined) { + const tags = this.db.queryEntries("SELECT * FROM tag;"); + const re = new Map(); + tags.forEach((v) => re.set(v.tag, v.id)); + this.#_tags = re; + return re; + } else return this.#_tags; + } + #updateExistsTable() { const cur = this.db.queryEntries( "SELECT * FROM main.sqlite_master;", ); - this._exist_table.clear(); + this.#exist_table.clear(); for (const i of cur) { if (i.type == "table") { - this._exist_table.add(i.name); + this.#exist_table.add(i.name); } } } - _write_version() { + #write_version() { this.db.transaction(() => { this.db.query("INSERT OR REPLACE INTO version VALUES (?, ?);", [ "eh", @@ -196,6 +236,30 @@ export class EhDb { gmeta, ); } + async add_gtag(gid: number, tags: Set) { + const otags = this.get_gtags(gid); + const deleted: string[] = []; + const added: string[] = []; + for (const o of otags) { + if (!tags.has(o)) deleted.push(o); + } + for (const o of tags) { + if (!otags.has(o)) added.push(o); + } + for (const d of deleted) { + const id = this.#tags.get(d); + if (id === undefined) throw Error("id not found."); + this.db.query("DELETE FROM gtag WHERE gid = ? AND id = ?;", [ + gid, + id, + ]); + } + for (const a of added) { + let id = this.#tags.get(a); + if (id === undefined) id = await this.#add_tag(a); + this.db.query("INSERT INTO gtag VALUES (?, ?);", [gid, id]); + } + } add_pmeta(pmeta: PMeta) { this.db.queryEntries( "INSERT OR REPLACE INTO pmeta VALUES (:gid, :index, :token, :name, :width, :height)", @@ -298,6 +362,14 @@ export class EhDb { ); return s.length ? s[0] : undefined; } + get_gtags(gid: number) { + return new Set( + this.db.query<[string]>( + "SELECT tag.tag FROM gtag INNER JOIN tag ON tag.id = gtag.id WHERE gid = ?;", + [gid], + ).map((v) => v[0]), + ); + } get_pmeta_by_index(gid: number, index: number) { const s = this.db.queryEntries( 'SELECT * FROM pmeta WHERE gid = ? AND "index" = ?;', @@ -326,6 +398,9 @@ export class EhDb { ]) ); } + optimize() { + this.db.execute("VACUUM;"); + } rollback() { this.db.execute("ROLLBACK TRANSACTION;"); } diff --git a/db_test.ts b/db_test.ts index f987830..f103053 100644 --- a/db_test.ts +++ b/db_test.ts @@ -51,5 +51,8 @@ Deno.test("DbTest", async () => { db.add_pmeta(pmeta); assertEquals(pmeta, db.get_pmeta_by_token(pmeta.gid, pmeta.token)); assertEquals(pmeta, db.get_pmeta_by_index(pmeta.gid, pmeta.index)); + const tags = new Set(["std", "df2", "ef3"]); + await db.add_gtag(1, tags); + assertEquals(tags, db.get_gtags(1)); db.close(); }); diff --git a/deno.jsonc b/deno.jsonc index 6bb0f68..39519f6 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -7,8 +7,8 @@ "tasks": { "dev": "deno run --watch main.ts", "test": "deno test --allow-read=./config.json,./test --allow-net --allow-write=./test --allow-run=tasklist.exe --unstable", - "run": "deno run --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --unstable", - "compile": "deno compile --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --unstable" + "run": "deno run --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --allow-net --unstable", + "compile": "deno compile --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --allow-net --unstable" }, "fmt": { "indentWidth": 4, diff --git a/main.ts b/main.ts index 37c17c8..5289409 100644 --- a/main.ts +++ b/main.ts @@ -4,6 +4,7 @@ import { check_file_permissions } from "./permissons.ts"; import { AlreadyClosedError, TaskManager } from "./task_manager.ts"; import { ParsedUrl, parseUrl, UrlType } from "./url.ts"; import { sure_dir } from "./utils.ts"; +import { EhDb } from "./db.ts"; function show_help() { console.log("Usage: main.ts [options]"); @@ -17,6 +18,7 @@ enum CMD { Unknown, Download, Run, + Optimize, } const args = parse(Deno.args, { @@ -35,6 +37,7 @@ const rcmd = args._[0]; let cmd = CMD.Unknown; if (rcmd == "d" || rcmd == "download") cmd = CMD.Download; if (rcmd == "r" || rcmd == "run") cmd = CMD.Run; +if (rcmd == "optimize") cmd = CMD.Optimize; if (cmd == CMD.Unknown) { throw Error(`Unknown command: ${rcmd}`); } @@ -72,12 +75,19 @@ async function run() { manager.close(); } } +function optimize() { + const db = new EhDb(settings.base); + db.optimize(); + db.close(); +} async function main() { await sure_dir(settings.base); if (cmd == CMD.Download) { await download(); } else if (cmd == CMD.Run) { await run(); + } else if (cmd == CMD.Optimize) { + optimize(); } } diff --git a/task_manager.ts b/task_manager.ts index a1b0c30..1fcff23 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -12,11 +12,13 @@ export class AlreadyClosedError extends Error { export class TaskManager { #closed = false; + cfg; client; db; running_tasks: Map>; max_task_count; constructor(cfg: Config) { + this.cfg = cfg; this.client = new Client(cfg); this.db = new EhDb(cfg.base); this.running_tasks = new Map(); @@ -114,7 +116,7 @@ export class TaskManager { if (task.type == TaskType.Download) { this.running_tasks.set( task.id, - download_task(task, this.client, this.db), + download_task(task, this.client, this.db, this.cfg), ); } } diff --git a/tasks/download.ts b/tasks/download.ts index 5a624a9..b9a9fd5 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -1,8 +1,27 @@ import { Client } from "../client.ts"; +import { Config } from "../config.ts"; import { EhDb } from "../db.ts"; import { Task } from "../task.ts"; -export async function download_task(task: Task, client: Client, db: EhDb) { +export async function download_task( + task: Task, + client: Client, + db: EhDb, + cfg: Config, +) { console.log("Started to download gallery", task.gid); + const gdatas = await client.fetchGalleryMetadataByAPI([ + task.gid, + task.token, + ]); + const gdata = gdatas.map.get(task.gid); + if (gdata === undefined) throw Error("Gallery metadata not included."); + if (typeof gdata === "string") throw Error(gdata); + const gmeta = gdatas.convert(gdata); + db.add_gmeta(gmeta); + await db.add_gtag(task.gid, new Set(gdata.tags)); + if (cfg.mpv) { + const mpv = await client.fetchMPVPage(task.gid, task.token); + } return task; }