From f46ad802f8801d3bb468ff4b4ad7259fcfa2ac23 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Wed, 14 Jun 2023 13:38:43 +0800 Subject: [PATCH] File now only use token --- db.ts | 61 ++++++++++++++++++++++++++++++++++---------- db_test.ts | 5 ++-- deno.lock | 1 + page/MPVPage.ts | 2 -- server/export_zip.ts | 2 +- tasks/download.ts | 2 +- tasks/export_zip.ts | 2 +- 7 files changed, 54 insertions(+), 21 deletions(-) diff --git a/db.ts b/db.ts index 279cc7a..0845b02 100644 --- a/db.ts +++ b/db.ts @@ -4,7 +4,7 @@ import { unescape } from "std/html/mod.ts"; import { join, resolve } from "std/path/mod.ts"; import { SqliteError } from "sqlite/mod.ts"; import { Status } from "sqlite/src/constants.ts"; -import { sleep, sure_dir_sync } from "./utils.ts"; +import { sleep, sure_dir_sync, try_remove_sync } from "./utils.ts"; import { Task, TaskType } from "./task.ts"; type SqliteMaster = { @@ -69,14 +69,13 @@ export type Tag = { }; export type EhFile = { id: number; - gid: number; token: string; path: string; width: number; height: number; is_original: boolean; }; -export type EhFileRaw = { +export type EhFileRawV1 = { id: number; gid: number; token: string; @@ -85,6 +84,14 @@ export type EhFileRaw = { height: number; is_original: number; }; +export type EhFileRaw = { + id: number; + token: string; + path: string; + width: number; + height: number; + is_original: number; +}; const ALL_TABLES = ["version", "task", "gmeta", "pmeta", "tag", "gtag", "file"]; const VERSION_TABLE = `CREATE TABLE version ( id TEXT, @@ -139,7 +146,6 @@ const GTAG_TABLE = `CREATE TABLE gtag ( );`; const FILE_TABLE = `CREATE TABLE file ( id INTEGER PRIMARY KEY AUTOINCREMENT, - gid INT, token TEXT, path TEXT, width INT, @@ -156,7 +162,7 @@ export class EhDb { #lock_file: string | undefined; #dblock_file: string | undefined; #_tags: Map | undefined; - readonly version = new SemVer("1.0.0-6"); + readonly version = new SemVer("1.0.0-7"); constructor(base_path: string) { const db_path = join(base_path, "data.db"); sure_dir_sync(base_path); @@ -249,6 +255,35 @@ export class EhDb { ); } } + if (v.compare("1.0.0-7") === -1) { + this.db.execute("ALTER TABLE file RENAME TO file_origin;"); + this.db.execute(FILE_TABLE); + let offset = 0; + let files = this.db.queryEntries( + "SELECT * FROM file_origin LIMIT 20 OFFSET 0;", + ); + const d: string[] = []; + while (files.length) { + files.forEach((f) => { + if (!d.includes(f.token)) { + d.push(f.token); + const g: Record = f; + delete g["gid"]; + this.add_file(g as EhFile, false); + } else { + try_remove_sync(f.path); + console.log("Deleted ", f.path); + } + }); + offset += files.length; + files = this.db.queryEntries( + "SELECT * FROM file_origin LIMIT 20 OFFSET ?;", + [offset], + ); + } + this.db.execute("DROP TABLE file_origin;"); + need_optimize = true; + } this.#write_version(); if (need_optimize) this.optimize(); } @@ -354,7 +389,7 @@ export class EhDb { } add_file(f: EhFile, overwrite = true): EhFile { if (overwrite) { - const ofiles = this.get_files(f.gid, f.token); + const ofiles = this.get_files(f.token); if (ofiles.length) { const o = ofiles[0]; f.id = o.id; @@ -365,16 +400,16 @@ export class EhDb { } if (f.id) { this.db.query( - "INSERT OR REPLACE INTO file VALUES (:id, :gid, :token, :path, :width, :height, :is_original);", + "INSERT OR REPLACE INTO file VALUES (:id, :token, :path, :width, :height, :is_original);", f, ); return structuredClone(f); } else { this.db.query( - "INSERT INTO file (gid, token, path, width, height, is_original) VALUES (?, ?, ?, ?, ?, ?);", - [f.gid, f.token, f.path, f.width, f.height, f.is_original], + "INSERT INTO file (token, path, width, height, is_original) VALUES (?, ?, ?, ?, ?);", + [f.token, f.path, f.width, f.height, f.is_original], ); - const s = this.get_files(f.gid, f.token); + const s = this.get_files(f.token); return s[s.length - 1]; } } @@ -505,10 +540,10 @@ export class EhDb { if (!this.#dblock) return; eval(`Deno.funlockSync(${this.#dblock.rid});`); } - get_files(gid: number, token: string) { + get_files(token: string) { return this.convert_file(this.db.queryEntries( - "SELECT * FROM file WHERE gid = ? AND token = ?;", - [gid, token], + "SELECT * FROM file WHERE token = ?;", + [token], )); } get_gids(offset = 0, limit = 20) { diff --git a/db_test.ts b/db_test.ts index 70cc2af..76cccd6 100644 --- a/db_test.ts +++ b/db_test.ts @@ -66,7 +66,6 @@ Deno.test("DbTest", async () => { assertEquals(tags, db.get_gtags(1)); const f: EhFile = { id: 0, - gid: 1, token: "s", path: "a.png", width: 1280, @@ -87,8 +86,8 @@ Deno.test("DbTest", async () => { f3.id = f4.id; assertEquals(f3, f4); assertEquals(f.id, f2.id); - assertEquals(db.get_files(1, "s").length, 2); + assertEquals(db.get_files("s").length, 2); db.add_file(f3); - assertEquals(db.get_files(1, "s").length, 1); + assertEquals(db.get_files("s").length, 1); db.close(); }); diff --git a/deno.lock b/deno.lock index 01d72a1..8e838fb 100644 --- a/deno.lock +++ b/deno.lock @@ -78,6 +78,7 @@ "https://deno.land/std@0.191.0/fs/exists.ts": "29c26bca8584a22876be7cb8844f1b6c8fc35e9af514576b78f5c6884d7ed02d", "https://deno.land/std@0.191.0/html/entities.ts": "1c9fa4d76e36a9bdbe370a65f1612771f3cc2cf802d217b4e633850e2fa25c16", "https://deno.land/std@0.191.0/html/mod.ts": "3f8c71781e32037ab63bd417759d15d31fb9606c6615c85dcabcc515986494ba", + "https://deno.land/std@0.191.0/json/common.ts": "ecd5e87d45b5f0df33238ed8b1746e1444da7f5c86ae53d0f0b04280f41a25bb", "https://deno.land/std@0.191.0/jsonc/mod.ts": "b88dce28eb3645667caa856538ae2fe87af51410822544a0b45a4177ef3bd7dd", "https://deno.land/std@0.191.0/jsonc/parse.ts": "2910e33bc7c3b243e3b6f3a39ce4d6ca84337b277a8df6f2ad2d9e4adbcddc08", "https://deno.land/std@0.191.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", diff --git a/page/MPVPage.ts b/page/MPVPage.ts index 006025c..895889f 100644 --- a/page/MPVPage.ts +++ b/page/MPVPage.ts @@ -58,7 +58,6 @@ class MPVImage { if (is_original === undefined) return undefined; return { id: 0, - gid: this.#mpv.gid, token: this.page_token, path, width, @@ -73,7 +72,6 @@ class MPVImage { if (height === undefined) return undefined; return { id: 0, - gid: this.#mpv.gid, token: this.page_token, path, width, diff --git a/server/export_zip.ts b/server/export_zip.ts index e8f900a..1d403ff 100644 --- a/server/export_zip.ts +++ b/server/export_zip.ts @@ -48,7 +48,7 @@ export function get_export_zip_response( const signalc = new AbortController(); const signal = signalc.signal; const download_task = async (p: PMeta) => { - const f = db.get_files(gid, p.token); + const f = db.get_files(p.token); if (f.length) { const r = await Deno.readFile(f[0].path, { signal }); await zip_writer.add( diff --git a/tasks/download.ts b/tasks/download.ts index 47030ca..e3100e1 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -130,7 +130,7 @@ export async function download_task( for (const i of mpv.imagelist) { if (abort.aborted) break; await m.add_new_task(async () => { - const ofiles = db.get_files(task.gid, i.page_token); + const ofiles = db.get_files(i.page_token); if (ofiles.length) { const t = ofiles[0]; if ( diff --git a/tasks/export_zip.ts b/tasks/export_zip.ts index 08dbc27..0e95dc7 100644 --- a/tasks/export_zip.ts +++ b/tasks/export_zip.ts @@ -62,7 +62,7 @@ export async function export_zip( await asyncForEach( db.get_pmeta(gid).sort((a, b) => a.index - b.index), async (p) => { - const f = db.get_files(gid, p.token); + const f = db.get_files(p.token); if (f.length) { const r = await Deno.readFile(f[0].path, { signal }); await z.add(