diff --git a/Dockerfile b/Dockerfile index 2fea744..efa781b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -91,6 +91,7 @@ ENV PATH=/app/bin:$PATH RUN deno task fetch && deno task server-build && mkdir -p ./thumbnails && chmod 777 ./thumbnails && mkdir -p ./downloads && chmod 777 ./downloads && mkdir -p ./data && chmod 777 ./data ENV DENO_DEPLOYMENT_ID=${DENO_DEPLOYMENT_ID} ENV DOCKER=true +ENV DB_USE_FFI=true EXPOSE 8000 ENTRYPOINT ["/tini", "--", "deno", "task", "server"] diff --git a/db.ts b/db.ts index cfa71e9..747d003 100644 --- a/db.ts +++ b/db.ts @@ -1,4 +1,4 @@ -import { DB } from "sqlite/mod.ts"; +import { Db } from "./utils/db_interface.ts"; import { compare as compare_ver, format as format_ver, @@ -8,7 +8,7 @@ import { unescape } from "std/html/mod.ts"; import { join, resolve } from "std/path/mod.ts"; import { SqliteError } from "sqlite/mod.ts"; import { Status } from "sqlite/src/constants.ts"; -import { sleep, sure_dir_sync, try_remove_sync } from "./utils.ts"; +import { parse_bool, sleep, sure_dir_sync, try_remove_sync } from "./utils.ts"; import { Task, TaskType } from "./task.ts"; import { generate as randomstring } from "randomstring"; @@ -265,7 +265,8 @@ function escape_fields(fields: string, namespace: string) { } export class EhDb { - db; + // @ts-ignore Ignore + db: Db; #flock_enabled: boolean = eval('typeof Deno.flock !== "undefined"'); #file: Deno.FsFile | undefined; #dblock: Deno.FsFile | undefined; @@ -273,16 +274,29 @@ export class EhDb { #lock_file: string | undefined; #dblock_file: string | undefined; #_tags: Map | undefined; + #base_path: string; + #db_path: string; + #use_ffi = false; readonly version = parse_ver("1.0.0-10"); constructor(base_path: string) { - const db_path = join(base_path, "data.db"); + this.#base_path = base_path; + this.#db_path = join(base_path, "data.db"); sure_dir_sync(base_path); - this.db = new DB(db_path); + } + async init() { + this.#use_ffi = parse_bool(Deno.env.get("DB_USE_FFI") ?? "false"); + if (this.#use_ffi) { + const DB = (await import("./utils/db_ffi.ts")).DbFfi; + this.db = new DB(this.#db_path); + } else { + const DB = (await import("./utils/db_wasm.ts")).DbWasm; + this.db = new DB(this.#db_path); + } this.db.execute("PRAGMA main.locking_mode=EXCLUSIVE;"); if (!this.#check_database()) this.#create_table(); - if (this.#flock_enabled) { - this.#lock_file = join(base_path, "db.lock"); - this.#dblock_file = join(base_path, "eh.locked"); + if (!this.#use_ffi && this.#flock_enabled) { + this.#lock_file = join(this.#base_path, "db.lock"); + this.#dblock_file = join(this.#base_path, "eh.locked"); this.#file = Deno.openSync(this.#lock_file, { create: true, write: true, @@ -292,7 +306,7 @@ export class EhDb { write: true, }); this.dblock(); - } else { + } else if (!this.#use_ffi) { console.log( "%cFile locking is disabled. Use --unstable to enable file locking.", "color: yellow;", diff --git a/db_test.ts b/db_test.ts index a263c3b..ef8c057 100644 --- a/db_test.ts +++ b/db_test.ts @@ -8,6 +8,7 @@ Deno.test("DbTest", async () => { await sure_dir("./test/db"); await remove_if_exists("./test/db/data.db"); const db = new EhDb("./test/db"); + await db.init(); console.log( await db.add_task({ gid: 1, diff --git a/import_map.json b/import_map.json index bafc910..d5ccf52 100644 --- a/import_map.json +++ b/import_map.json @@ -28,6 +28,7 @@ "@lit-labs/react/": "https://esm.sh/@lit-labs/react@1.2.1/", "bootstrap/": "https://esm.sh/bootstrap@5.3.0/", "filesize": "https://esm.sh/filesize@10.0.7", - "pwn/": "https://deno.land/x/pwn@1.0.0/" + "pwn/": "https://deno.land/x/pwn@1.0.0/", + "sqlite3/": "https://deno.land/x/sqlite3@0.10.0/" } } diff --git a/main.ts b/main.ts index 6b31588..f1deb1e 100644 --- a/main.ts +++ b/main.ts @@ -78,6 +78,7 @@ if (!check_file_permissions(settings.base)) { } async function download() { const manager = new TaskManager(settings); + await manager.init(); try { const urls: ParsedUrl[] = []; for (const i of args._.slice(1)) { @@ -99,20 +100,23 @@ async function download() { } async function run() { const manager = new TaskManager(settings); + await manager.init(); try { await manager.run(); } finally { if (!manager.aborted) manager.close(); } } -function optimize() { +async function optimize() { const db = new EhDb(settings.db_path || settings.base); + await db.init(); if (args.better_optimize) db.better_optimize(); db.optimize(); db.close(); } async function update_tag_translation() { const db = new EhDb(settings.db_path || settings.base); + await db.init(); const signal = get_abort_signal(); try { const f = await load_eht_file( @@ -129,6 +133,7 @@ async function update_tag_translation() { } async function export_zip() { const manager = new TaskManager(settings); + await manager.init(); try { for (const gid of args._.slice(1)) { if (typeof gid === "number") { @@ -142,6 +147,7 @@ async function export_zip() { } async function update_meili_search_data() { const manager = new TaskManager(settings); + await manager.init(); try { await manager.add_update_meili_search_data_task(); await manager.run(); @@ -151,6 +157,7 @@ async function update_meili_search_data() { } async function fix_gallery_page() { const manager = new TaskManager(settings); + await manager.init(); try { await manager.add_fix_gallery_page_task(); await manager.run(); @@ -165,7 +172,7 @@ async function main() { } else if (cmd == CMD.Run) { await run(); } else if (cmd == CMD.Optimize) { - optimize(); + await optimize(); } else if (cmd == CMD.UpdateTagTranslation) { await update_tag_translation(); } else if (cmd == CMD.ExportZip) { diff --git a/routes/api/_middleware.ts b/routes/api/_middleware.ts index 47dc399..b28b90e 100644 --- a/routes/api/_middleware.ts +++ b/routes/api/_middleware.ts @@ -19,7 +19,9 @@ function handle_auth(req: Request, ctx: MiddlewareHandlerContext) { const check = () => { if (u.pathname === "/api/token" && req.method === "PUT") return true; if (u.pathname === "/api/status" && req.method === "GET") return true; - if (u.pathname === "/api/health_check" && req.method === "GET") return true; + if (u.pathname === "/api/health_check" && req.method === "GET") { + return true; + } return false; }; if (!token) return check(); diff --git a/server.ts b/server.ts index 9b42ed5..6d07f30 100644 --- a/server.ts +++ b/server.ts @@ -30,6 +30,7 @@ export async function startServer(path: string) { cfg_path = path; const cfg = await load_settings(path); task_manager = new TaskManager(cfg); + await task_manager.init(); task_manager.run(true).catch((e) => { if (!(e instanceof AlreadyClosedError)) throw e; }); diff --git a/task_manager.ts b/task_manager.ts index 441a09f..f69a160 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -79,6 +79,9 @@ export class TaskManager extends EventTarget { ); } } + async init() { + await this.db.init(); + } async #add_task(task: Task) { const r = await this.db.add_task(task); this.dispatchEvent("new_task", r); diff --git a/utils.ts b/utils.ts index b02cbbe..5f7509d 100644 --- a/utils.ts +++ b/utils.ts @@ -274,6 +274,8 @@ export class TimeoutError extends Error { let _isDocker: boolean | undefined = undefined; export function isDocker() { - if (_isDocker === undefined) _isDocker = parse_bool(Deno.env.get("DOCKER") ?? "false"); + if (_isDocker === undefined) { + _isDocker = parse_bool(Deno.env.get("DOCKER") ?? "false"); + } return _isDocker; } diff --git a/utils/db_ffi.ts b/utils/db_ffi.ts new file mode 100644 index 0000000..5f07e48 --- /dev/null +++ b/utils/db_ffi.ts @@ -0,0 +1,36 @@ +import { Database, DatabaseOpenOptions } from "sqlite3/mod.ts"; +import { QueryParameterSet, Row, RowObject } from "./db_interface.ts"; + +export class DbFfi { + db; + constructor(path: string, options?: DatabaseOpenOptions) { + this.db = new Database(path, options); + } + + close(_force?: boolean) { + this.db.close(); + } + + execute(sql: string) { + this.db.exec(sql); + } + + query( + sql: string, + params?: QueryParameterSet, + ): Array { + return this.db.prepare(sql).values(params); + } + + queryEntries( + sql: string, + params?: QueryParameterSet, + ): Array { + return this.db.prepare(sql).all(params); + } + + transaction(fn: () => V): V { + const re = this.db.transaction(fn); + return re(); + } +} diff --git a/utils/db_interface.ts b/utils/db_interface.ts new file mode 100644 index 0000000..d16fb43 --- /dev/null +++ b/utils/db_interface.ts @@ -0,0 +1,28 @@ +export type Row = Array; +export type RowObject = Record; +export type QueryParameter = + | boolean + | number + | bigint + | string + | null + | undefined + | Date + | Uint8Array; +export type QueryParameterSet = + | Record + | Array; + +export interface Db { + close(force?: boolean): void; + execute(sql: string): void; + query( + sql: string, + params?: QueryParameterSet, + ): Array; + queryEntries( + sql: string, + params?: QueryParameterSet, + ): Array; + transaction(fn: () => V): V; +} diff --git a/utils/db_wasm.ts b/utils/db_wasm.ts new file mode 100644 index 0000000..d0f19d4 --- /dev/null +++ b/utils/db_wasm.ts @@ -0,0 +1,7 @@ +import { DB, SqliteOptions } from "sqlite/mod.ts"; + +export class DbWasm extends DB { + constructor(dbPath: string, options?: SqliteOptions) { + super(dbPath, options); + } +}