lifegpc/eh-downloader
1
0
Fork 0
mirror of https://github.com/lifegpc/eh-downloader.git synced 2026-07-08 01:31:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add tags to database

Browse Source
This commit is contained in:
lifegpc
2023-05-21 10:37:09 +08:00
parent 48b6393f9a
commit f1a483ff43
7 changed files with 139 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
config.ts
View File

@@ -51,6 +51,9 @@ export class Config {
get max_task_count() {
return this._return_number("max_task_count") || 1;
}
get mpv() {
return this._return_bool("mpv") || false;
}
}
export async function load_settings(path: string) {

121
db.ts
View File

@@ -60,7 +60,11 @@ export type PMeta = {
width: number;
height: number;
};
const ALL_TABLES = ["version", "task", "gmeta", "pmeta"];
type Tag = {
id: number;
tag: string;
};
const ALL_TABLES = ["version", "task", "gmeta", "pmeta", "tag", "gtag"];
const VERSION_TABLE = `CREATE TABLE version (
id TEXT,
ver TEXT,
@@ -101,21 +105,31 @@ const PMETA_TABLE = `CREATE TABLE pmeta (
height INT,
PRIMARY KEY (gid, token)
);`;
const TAG_TABLE = `CREATE TABLE tag (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tag TEXT
);`;
const GTAG_TABLE = `CREATE TABLE gtag (
gid INT,
id INT,
PRIMARY KEY (gid, id)
);`;
export class EhDb {
db;
flock_enabled: boolean = eval('typeof Deno.flock !== "undefined"');
#flock_enabled: boolean = eval('typeof Deno.flock !== "undefined"');
#file: Deno.FsFile | undefined;
#dblock: Deno.FsFile | undefined;
_exist_table: Set<string> = new Set();
#exist_table: Set<string> = new Set();
#lock_file: string | undefined;
#dblock_file: string | undefined;
#_tags: Map<string, number> | undefined;
readonly version = new SemVer("1.0.0-0");
constructor(base_path: string) {
this.db = new DB(join(base_path, "data.db"));
this.db.execute("PRAGMA main.locking_mode=EXCLUSIVE;");
if (!this._check_database()) this._create_table();
if (this.flock_enabled) {
if (!this.#check_database()) this.#create_table();
if (this.#flock_enabled) {
this.#lock_file = join(base_path, "db.lock");
this.#dblock_file = join(base_path, "eh.locked");
this.#file = Deno.openSync(this.#lock_file, {
@@ -134,34 +148,51 @@ export class EhDb {
);
}
}
_check_database() {
this._updateExistsTable();
const v = this._read_version();
#add_tag(s: string) {
return this.transaction(() => {
this.db.query("INSERT INTO tag (tag) VALUES (?);", [s]);
const r = this.db.queryEntries<Tag>(
"SELECT * FROM tag WHERE tag = ?;",
[s],
);
this.#tags.set(s, r[0].id);
return r[0].id;
});
}
#check_database() {
this.#updateExistsTable();
const v = this.#read_version();
if (!v) return false;
if (
ALL_TABLES.length !== this._exist_table.size ||
!ALL_TABLES.every((x) => this._exist_table.has(x))
ALL_TABLES.length !== this.#exist_table.size ||
!ALL_TABLES.every((x) => this.#exist_table.has(x))
) return false;
return true;
}
_create_table() {
if (!this._exist_table.has("version")) {
#create_table() {
if (!this.#exist_table.has("version")) {
this.db.execute(VERSION_TABLE);
this._write_version();
this.#write_version();
}
if (!this._exist_table.has("task")) {
if (!this.#exist_table.has("task")) {
this.db.execute(TASK_TABLE);
}
if (!this._exist_table.has("gmeta")) {
if (!this.#exist_table.has("gmeta")) {
this.db.execute(GMETA_TABLE);
}
if (!this._exist_table.has("pmeta")) {
if (!this.#exist_table.has("pmeta")) {
this.db.execute(PMETA_TABLE);
}
this._updateExistsTable();
if (!this.#exist_table.has("tag")) {
this.db.execute(TAG_TABLE);
}
if (!this.#exist_table.has("gtag")) {
this.db.execute(GTAG_TABLE);
}
this.#updateExistsTable();
}
_read_version() {
if (!this._exist_table.has("version")) return null;
#read_version() {
if (!this.#exist_table.has("version")) return null;
const cur = this.db.query<[string]>(
"SELECT ver FROM version WHERE id = ?;",
["eh"],
@@ -171,18 +202,27 @@ export class EhDb {
}
return null;
}
_updateExistsTable() {
get #tags() {
if (this.#_tags === undefined) {
const tags = this.db.queryEntries<Tag>("SELECT * FROM tag;");
const re = new Map<string, number>();
tags.forEach((v) => re.set(v.tag, v.id));
this.#_tags = re;
return re;
} else return this.#_tags;
}
#updateExistsTable() {
const cur = this.db.queryEntries<SqliteMaster>(
"SELECT * FROM main.sqlite_master;",
);
this._exist_table.clear();
this.#exist_table.clear();
for (const i of cur) {
if (i.type == "table") {
this._exist_table.add(i.name);
this.#exist_table.add(i.name);
}
}
}
_write_version() {
#write_version() {
this.db.transaction(() => {
this.db.query("INSERT OR REPLACE INTO version VALUES (?, ?);", [
"eh",
@@ -196,6 +236,30 @@ export class EhDb {
gmeta,
);
}
async add_gtag(gid: number, tags: Set<string>) {
const otags = this.get_gtags(gid);
const deleted: string[] = [];
const added: string[] = [];
for (const o of otags) {
if (!tags.has(o)) deleted.push(o);
}
for (const o of tags) {
if (!otags.has(o)) added.push(o);
}
for (const d of deleted) {
const id = this.#tags.get(d);
if (id === undefined) throw Error("id not found.");
this.db.query("DELETE FROM gtag WHERE gid = ? AND id = ?;", [
gid,
id,
]);
}
for (const a of added) {
let id = this.#tags.get(a);
if (id === undefined) id = await this.#add_tag(a);
this.db.query("INSERT INTO gtag VALUES (?, ?);", [gid, id]);
}
}
add_pmeta(pmeta: PMeta) {
this.db.queryEntries(
"INSERT OR REPLACE INTO pmeta VALUES (:gid, :index, :token, :name, :width, :height)",
@@ -298,6 +362,14 @@ export class EhDb {
);
return s.length ? s[0] : undefined;
}
get_gtags(gid: number) {
return new Set(
this.db.query<[string]>(
"SELECT tag.tag FROM gtag INNER JOIN tag ON tag.id = gtag.id WHERE gid = ?;",
[gid],
).map((v) => v[0]),
);
}
get_pmeta_by_index(gid: number, index: number) {
const s = this.db.queryEntries<PMeta>(
'SELECT * FROM pmeta WHERE gid = ? AND "index" = ?;',
@@ -326,6 +398,9 @@ export class EhDb {
])
);
}
optimize() {
this.db.execute("VACUUM;");
}
rollback() {
this.db.execute("ROLLBACK TRANSACTION;");
}

3
db_test.ts
View File

@@ -51,5 +51,8 @@ Deno.test("DbTest", async () => {
db.add_pmeta(pmeta);
assertEquals(pmeta, db.get_pmeta_by_token(pmeta.gid, pmeta.token));
assertEquals(pmeta, db.get_pmeta_by_index(pmeta.gid, pmeta.index));
const tags = new Set(["std", "df2", "ef3"]);
await db.add_gtag(1, tags);
assertEquals(tags, db.get_gtags(1));
db.close();
});

4
deno.jsonc
View File

@@ -7,8 +7,8 @@
"tasks": {
"dev": "deno run --watch main.ts",
"test": "deno test --allow-read=./config.json,./test --allow-net --allow-write=./test --allow-run=tasklist.exe --unstable",
"run": "deno run --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --unstable",
"compile": "deno compile --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --unstable"
"run": "deno run --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --allow-net --unstable",
"compile": "deno compile --allow-read=./config.json,./downloads --allow-write=./downloads --allow-run=tasklist.exe --allow-net --unstable"
},
"fmt": {
"indentWidth": 4,

10
main.ts
View File

@@ -4,6 +4,7 @@ import { check_file_permissions } from "./permissons.ts";
import { AlreadyClosedError, TaskManager } from "./task_manager.ts";
import { ParsedUrl, parseUrl, UrlType } from "./url.ts";
import { sure_dir } from "./utils.ts";
import { EhDb } from "./db.ts";
function show_help() {
console.log("Usage: main.ts [options]");
@@ -17,6 +18,7 @@ enum CMD {
Unknown,
Download,
Run,
Optimize,
}
const args = parse(Deno.args, {
@@ -35,6 +37,7 @@ const rcmd = args._[0];
let cmd = CMD.Unknown;
if (rcmd == "d" || rcmd == "download") cmd = CMD.Download;
if (rcmd == "r" || rcmd == "run") cmd = CMD.Run;
if (rcmd == "optimize") cmd = CMD.Optimize;
if (cmd == CMD.Unknown) {
throw Error(`Unknown command: ${rcmd}`);
}
@@ -72,12 +75,19 @@ async function run() {
manager.close();
}
}
function optimize() {
const db = new EhDb(settings.base);
db.optimize();
db.close();
}
async function main() {
await sure_dir(settings.base);
if (cmd == CMD.Download) {
await download();
} else if (cmd == CMD.Run) {
await run();
} else if (cmd == CMD.Optimize) {
optimize();
}
}

4
task_manager.ts
View File

@@ -12,11 +12,13 @@ export class AlreadyClosedError extends Error {
export class TaskManager {
#closed = false;
cfg;
client;
db;
running_tasks: Map<number, Promise<Task>>;
max_task_count;
constructor(cfg: Config) {
this.cfg = cfg;
this.client = new Client(cfg);
this.db = new EhDb(cfg.base);
this.running_tasks = new Map();
@@ -114,7 +116,7 @@ export class TaskManager {
if (task.type == TaskType.Download) {
this.running_tasks.set(
task.id,
download_task(task, this.client, this.db),
download_task(task, this.client, this.db, this.cfg),
);
}
}

21
tasks/download.ts
View File

@@ -1,8 +1,27 @@
import { Client } from "../client.ts";
import { Config } from "../config.ts";
import { EhDb } from "../db.ts";
import { Task } from "../task.ts";
export async function download_task(task: Task, client: Client, db: EhDb) {
export async function download_task(
task: Task,
client: Client,
db: EhDb,
cfg: Config,
) {
console.log("Started to download gallery", task.gid);
const gdatas = await client.fetchGalleryMetadataByAPI([
task.gid,
task.token,
]);
const gdata = gdatas.map.get(task.gid);
if (gdata === undefined) throw Error("Gallery metadata not included.");
if (typeof gdata === "string") throw Error(gdata);
const gmeta = gdatas.convert(gdata);
db.add_gmeta(gmeta);
await db.add_gtag(task.gid, new Set(gdata.tags));
if (cfg.mpv) {
const mpv = await client.fetchMPVPage(task.gid, task.token);
}
return task;
}