Add support for both wasm/ffi version of sqlite3

Dockerfile

@@ -91,6 +91,7 @@ ENV PATH=/app/bin:$PATH
 RUN deno task fetch && deno task server-build && mkdir -p ./thumbnails && chmod 777 ./thumbnails && mkdir -p ./downloads && chmod 777 ./downloads && mkdir -p ./data && chmod 777 ./data
 ENV DENO_DEPLOYMENT_ID=${DENO_DEPLOYMENT_ID}
 ENV DOCKER=true
+ENV DB_USE_FFI=true
 
 EXPOSE 8000
 ENTRYPOINT ["/tini", "--", "deno", "task", "server"]

db.ts

@@ -1,4 +1,4 @@
-import { DB } from "sqlite/mod.ts";
+import { Db } from "./utils/db_interface.ts";
 import {
     compare as compare_ver,
     format as format_ver,
@@ -8,7 +8,7 @@ import { unescape } from "std/html/mod.ts";
 import { join, resolve } from "std/path/mod.ts";
 import { SqliteError } from "sqlite/mod.ts";
 import { Status } from "sqlite/src/constants.ts";
-import { sleep, sure_dir_sync, try_remove_sync } from "./utils.ts";
+import { parse_bool, sleep, sure_dir_sync, try_remove_sync } from "./utils.ts";
 import { Task, TaskType } from "./task.ts";
 import { generate as randomstring } from "randomstring";
 
@@ -265,7 +265,8 @@ function escape_fields(fields: string, namespace: string) {
 }
 
 export class EhDb {
-    db;
+    // @ts-ignore Ignore
+    db: Db;
     #flock_enabled: boolean = eval('typeof Deno.flock !== "undefined"');
     #file: Deno.FsFile | undefined;
     #dblock: Deno.FsFile | undefined;
@@ -273,16 +274,29 @@ export class EhDb {
     #lock_file: string | undefined;
     #dblock_file: string | undefined;
     #_tags: Map<string, number> | undefined;
+    #base_path: string;
+    #db_path: string;
+    #use_ffi = false;
     readonly version = parse_ver("1.0.0-10");
     constructor(base_path: string) {
-        const db_path = join(base_path, "data.db");
+        this.#base_path = base_path;
+        this.#db_path = join(base_path, "data.db");
         sure_dir_sync(base_path);
-        this.db = new DB(db_path);
+    }
+    async init() {
+        this.#use_ffi = parse_bool(Deno.env.get("DB_USE_FFI") ?? "false");
+        if (this.#use_ffi) {
+            const DB = (await import("./utils/db_ffi.ts")).DbFfi;
+            this.db = new DB(this.#db_path);
+        } else {
+            const DB = (await import("./utils/db_wasm.ts")).DbWasm;
+            this.db = new DB(this.#db_path);
+        }
         this.db.execute("PRAGMA main.locking_mode=EXCLUSIVE;");
         if (!this.#check_database()) this.#create_table();
-        if (this.#flock_enabled) {
-            this.#lock_file = join(base_path, "db.lock");
-            this.#dblock_file = join(base_path, "eh.locked");
+        if (!this.#use_ffi && this.#flock_enabled) {
+            this.#lock_file = join(this.#base_path, "db.lock");
+            this.#dblock_file = join(this.#base_path, "eh.locked");
             this.#file = Deno.openSync(this.#lock_file, {
                 create: true,
                 write: true,
@@ -292,7 +306,7 @@ export class EhDb {
                 write: true,
             });
             this.dblock();
-        } else {
+        } else if (!this.#use_ffi) {
             console.log(
                 "%cFile locking is disabled. Use --unstable to enable file locking.",
                 "color: yellow;",

db_test.ts

@@ -8,6 +8,7 @@ Deno.test("DbTest", async () => {
     await sure_dir("./test/db");
     await remove_if_exists("./test/db/data.db");
     const db = new EhDb("./test/db");
+    await db.init();
     console.log(
         await db.add_task({
             gid: 1,

import_map.json

@@ -28,6 +28,7 @@
         "@lit-labs/react/": "https://esm.sh/@lit-labs/[email protected]/",
         "bootstrap/": "https://esm.sh/[email protected]/",
         "filesize": "https://esm.sh/[email protected]",
-        "pwn/": "https://deno.land/x/[email protected]/"
+        "pwn/": "https://deno.land/x/[email protected]/",
+        "sqlite3/": "https://deno.land/x/[email protected]/"
     }
 }

main.ts

@@ -78,6 +78,7 @@ if (!check_file_permissions(settings.base)) {
 }
 async function download() {
     const manager = new TaskManager(settings);
+    await manager.init();
     try {
         const urls: ParsedUrl[] = [];
         for (const i of args._.slice(1)) {
@@ -99,20 +100,23 @@ async function download() {
 }
 async function run() {
     const manager = new TaskManager(settings);
+    await manager.init();
     try {
         await manager.run();
     } finally {
         if (!manager.aborted) manager.close();
     }
 }
-function optimize() {
+async function optimize() {
     const db = new EhDb(settings.db_path || settings.base);
+    await db.init();
     if (args.better_optimize) db.better_optimize();
     db.optimize();
     db.close();
 }
 async function update_tag_translation() {
     const db = new EhDb(settings.db_path || settings.base);
+    await db.init();
     const signal = get_abort_signal();
     try {
         const f = await load_eht_file(
@@ -129,6 +133,7 @@ async function update_tag_translation() {
 }
 async function export_zip() {
     const manager = new TaskManager(settings);
+    await manager.init();
     try {
         for (const gid of args._.slice(1)) {
             if (typeof gid === "number") {
@@ -142,6 +147,7 @@ async function export_zip() {
 }
 async function update_meili_search_data() {
     const manager = new TaskManager(settings);
+    await manager.init();
     try {
         await manager.add_update_meili_search_data_task();
         await manager.run();
@@ -151,6 +157,7 @@ async function update_meili_search_data() {
 }
 async function fix_gallery_page() {
     const manager = new TaskManager(settings);
+    await manager.init();
     try {
         await manager.add_fix_gallery_page_task();
         await manager.run();
@@ -165,7 +172,7 @@ async function main() {
     } else if (cmd == CMD.Run) {
         await run();
     } else if (cmd == CMD.Optimize) {
-        optimize();
+        await optimize();
     } else if (cmd == CMD.UpdateTagTranslation) {
         await update_tag_translation();
     } else if (cmd == CMD.ExportZip) {

routes/api/_middleware.ts

@@ -19,7 +19,9 @@ function handle_auth(req: Request, ctx: MiddlewareHandlerContext) {
     const check = () => {
         if (u.pathname === "/api/token" && req.method === "PUT") return true;
         if (u.pathname === "/api/status" && req.method === "GET") return true;
-        if (u.pathname === "/api/health_check" && req.method === "GET") return true;
+        if (u.pathname === "/api/health_check" && req.method === "GET") {
+            return true;
+        }
         return false;
     };
     if (!token) return check();

server.ts

@@ -30,6 +30,7 @@ export async function startServer(path: string) {
     cfg_path = path;
     const cfg = await load_settings(path);
     task_manager = new TaskManager(cfg);
+    await task_manager.init();
     task_manager.run(true).catch((e) => {
         if (!(e instanceof AlreadyClosedError)) throw e;
     });

task_manager.ts

@@ -79,6 +79,9 @@ export class TaskManager extends EventTarget {
             );
         }
     }
+    async init() {
+        await this.db.init();
+    }
     async #add_task(task: Task) {
         const r = await this.db.add_task(task);
         this.dispatchEvent("new_task", r);

utils.ts

@@ -274,6 +274,8 @@ export class TimeoutError extends Error {
 let _isDocker: boolean | undefined = undefined;
 
 export function isDocker() {
-    if (_isDocker === undefined) _isDocker = parse_bool(Deno.env.get("DOCKER") ?? "false");
+    if (_isDocker === undefined) {
+        _isDocker = parse_bool(Deno.env.get("DOCKER") ?? "false");
+    }
     return _isDocker;
 }

utils/db_ffi.ts

@@ -0,0 +1,36 @@
+import { Database, DatabaseOpenOptions } from "sqlite3/mod.ts";
+import { QueryParameterSet, Row, RowObject } from "./db_interface.ts";
+
+export class DbFfi {
+    db;
+    constructor(path: string, options?: DatabaseOpenOptions) {
+        this.db = new Database(path, options);
+    }
+
+    close(_force?: boolean) {
+        this.db.close();
+    }
+
+    execute(sql: string) {
+        this.db.exec(sql);
+    }
+
+    query<R extends Row = Row>(
+        sql: string,
+        params?: QueryParameterSet,
+    ): Array<R> {
+        return this.db.prepare(sql).values(params);
+    }
+
+    queryEntries<O extends RowObject = RowObject>(
+        sql: string,
+        params?: QueryParameterSet,
+    ): Array<O> {
+        return this.db.prepare(sql).all(params);
+    }
+
+    transaction<V>(fn: () => V): V {
+        const re = this.db.transaction(fn);
+        return re();
+    }
+}

utils/db_interface.ts

@@ -0,0 +1,28 @@
+export type Row = Array<unknown>;
+export type RowObject = Record<string, unknown>;
+export type QueryParameter =
+    | boolean
+    | number
+    | bigint
+    | string
+    | null
+    | undefined
+    | Date
+    | Uint8Array;
+export type QueryParameterSet =
+    | Record<string, QueryParameter>
+    | Array<QueryParameter>;
+
+export interface Db {
+    close(force?: boolean): void;
+    execute(sql: string): void;
+    query<R extends Row = Row>(
+        sql: string,
+        params?: QueryParameterSet,
+    ): Array<R>;
+    queryEntries<O extends RowObject = RowObject>(
+        sql: string,
+        params?: QueryParameterSet,
+    ): Array<O>;
+    transaction<V>(fn: () => V): V;
+}

utils/db_wasm.ts

@@ -0,0 +1,7 @@
+import { DB, SqliteOptions } from "sqlite/mod.ts";
+
+export class DbWasm extends DB {
+    constructor(dbPath: string, options?: SqliteOptions) {
+        super(dbPath, options);
+    }
+}