diff --git a/config.ts b/config.ts index f04e872..743e5b8 100644 --- a/config.ts +++ b/config.ts @@ -15,6 +15,8 @@ export type ConfigType = { port: number; export_zip_jpn_title: boolean; hostname: string; + meili_host?: string; + meili_update_api_key?: string; }; export class Config { @@ -97,6 +99,12 @@ export class Config { get hostname() { return this._return_string("hostname") || "localhost"; } + get meili_host() { + return this._return_string("meili_host"); + } + get meili_update_api_key() { + return this._return_string("meili_update_api_key"); + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -112,6 +120,8 @@ export class Config { port: this.port, export_zip_jpn_title: this.export_zip_jpn_title, hostname: this.hostname, + meili_host: this.meili_host, + meili_update_api_key: this.meili_update_api_key, }; } } diff --git a/db.ts b/db.ts index 2631ecb..279cc7a 100644 --- a/db.ts +++ b/db.ts @@ -1,5 +1,6 @@ import { DB } from "sqlite/mod.ts"; import { SemVer } from "std/semver/mod.ts"; +import { unescape } from "std/html/mod.ts"; import { join, resolve } from "std/path/mod.ts"; import { SqliteError } from "sqlite/mod.ts"; import { Status } from "sqlite/src/constants.ts"; @@ -155,7 +156,7 @@ export class EhDb { #lock_file: string | undefined; #dblock_file: string | undefined; #_tags: Map | undefined; - readonly version = new SemVer("1.0.0-5"); + readonly version = new SemVer("1.0.0-6"); constructor(base_path: string) { const db_path = join(base_path, "data.db"); sure_dir_sync(base_path); @@ -225,6 +226,29 @@ export class EhDb { if (v.compare("1.0.0-5") === -1) { this.db.execute("ALTER TABLE task DROP pn;"); } + if (v.compare("1.0.0-6") === -1) { + let offset = 0; + let tasks = this.convert_gmeta( + this.db.queryEntries( + "SELECT * FROM gmeta LIMIT 20 OFFSET 0;", + ), + ); + while (tasks.length) { + tasks.forEach((t) => { + t.title = unescape(t.title); + t.title_jpn = unescape(t.title_jpn); + t.uploader = unescape(t.uploader); + this.add_gmeta(t); + }); + offset += tasks.length; + tasks = this.convert_gmeta( + this.db.queryEntries( + "SELECT * FROM gmeta LIMIT 20 OFFSET ?;", + [offset], + ), + ); + } + } this.#write_version(); if (need_optimize) this.optimize(); } @@ -487,6 +511,12 @@ export class EhDb { [gid, token], )); } + get_gids(offset = 0, limit = 20) { + return this.db.query<[number]>( + "SELECT gid FROM gmeta LIMIT ? OFFSET ?;", + [limit, offset], + ).map((n) => n[0]); + } get_gmeta_by_gid(gid: number) { const s = this.convert_gmeta( this.db.queryEntries( diff --git a/deno.json b/deno.json index 1268509..db3a9ba 100644 --- a/deno.json +++ b/deno.json @@ -8,7 +8,8 @@ "run": "deno run --allow-read=./ --allow-write=./ --allow-run=tasklist.exe --allow-env=DENO_DEPLOYMENT_ID --allow-net --unstable", "compile": "deno compile --allow-read=./ --allow-write=./ --allow-run=tasklist.exe --allow-env=DENO_DEPLOYMENT_ID --allow-net --unstable", "compile_full": "deno compile --allow-read --allow-write --allow-run=tasklist.exe --allow-env=DENO_DEPLOYMENT_ID --allow-net --unstable", - "fetch": "deno run --allow-read=./ --allow-write=./ --allow-net fetch_static_files.ts" + "fetch": "deno run --allow-read=./ --allow-write=./ --allow-net fetch_static_files.ts", + "gen_meili_server_key": "deno run --allow-net scripts/gen_meili_server_key.ts" }, "fmt": { "indentWidth": 4, diff --git a/deno.lock b/deno.lock index 57c15c5..01d72a1 100644 --- a/deno.lock +++ b/deno.lock @@ -16,6 +16,9 @@ "https://deno.land/std@0.140.0/path/posix.ts": "293cdaec3ecccec0a9cc2b534302dfe308adb6f10861fa183275d6695faace44", "https://deno.land/std@0.140.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", "https://deno.land/std@0.140.0/path/win32.ts": "31811536855e19ba37a999cd8d1b62078235548d67902ece4aa6b814596dd757", + "https://deno.land/std@0.150.0/media_types/_util.ts": "ce9b4fc4ba1c447dafab619055e20fd88236ca6bdd7834a21f98bd193c3fbfa1", + "https://deno.land/std@0.150.0/media_types/mod.ts": "2d4b6f32a087029272dc59e0a55ae3cc4d1b27b794ccf528e94b1925795b3118", + "https://deno.land/std@0.150.0/media_types/vendor/mime-db.v1.52.0.ts": "724cee25fa40f1a52d3937d6b4fbbfdd7791ff55e1b7ac08d9319d5632c7f5af", "https://deno.land/std@0.173.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", "https://deno.land/std@0.173.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", "https://deno.land/std@0.173.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", @@ -68,27 +71,28 @@ "https://deno.land/std@0.178.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", "https://deno.land/std@0.178.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", "https://deno.land/std@0.178.0/semver/mod.ts": "409a2691f5a411c34e917c1e6d445a6d1d53f3fadf660e44a99dd0bf9b2ef412", - "https://deno.land/std@0.190.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", - "https://deno.land/std@0.190.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", - "https://deno.land/std@0.190.0/flags/mod.ts": "17f444ddbee43c5487568de0c6a076c7729cfe90d96d2ffcd2b8f8adadafb6e8", - "https://deno.land/std@0.190.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e", - "https://deno.land/std@0.190.0/fs/exists.ts": "29c26bca8584a22876be7cb8844f1b6c8fc35e9af514576b78f5c6884d7ed02d", - "https://deno.land/std@0.190.0/json/common.ts": "ecd5e87d45b5f0df33238ed8b1746e1444da7f5c86ae53d0f0b04280f41a25bb", - "https://deno.land/std@0.190.0/jsonc/mod.ts": "b88dce28eb3645667caa856538ae2fe87af51410822544a0b45a4177ef3bd7dd", - "https://deno.land/std@0.190.0/jsonc/parse.ts": "2910e33bc7c3b243e3b6f3a39ce4d6ca84337b277a8df6f2ad2d9e4adbcddc08", - "https://deno.land/std@0.190.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", - "https://deno.land/std@0.190.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", - "https://deno.land/std@0.190.0/path/_util.ts": "d7abb1e0dea065f427b89156e28cdeb32b045870acdf865833ba808a73b576d0", - "https://deno.land/std@0.190.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", - "https://deno.land/std@0.190.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", - "https://deno.land/std@0.190.0/path/mod.ts": "ee161baec5ded6510ee1d1fb6a75a0f5e4b41f3f3301c92c716ecbdf7dae910d", - "https://deno.land/std@0.190.0/path/posix.ts": "8b7c67ac338714b30c816079303d0285dd24af6b284f7ad63da5b27372a2c94d", - "https://deno.land/std@0.190.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", - "https://deno.land/std@0.190.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", - "https://deno.land/std@0.190.0/semver/mod.ts": "200f50cf6872212667df532fb09f0b1a33d3427a5201f75fad30a0d0c6dbcce3", - "https://deno.land/std@0.190.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", - "https://deno.land/std@0.190.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", - "https://deno.land/std@0.190.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f", + "https://deno.land/std@0.191.0/_util/asserts.ts": "178dfc49a464aee693a7e285567b3d0b555dc805ff490505a8aae34f9cfb1462", + "https://deno.land/std@0.191.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", + "https://deno.land/std@0.191.0/flags/mod.ts": "17f444ddbee43c5487568de0c6a076c7729cfe90d96d2ffcd2b8f8adadafb6e8", + "https://deno.land/std@0.191.0/fmt/colors.ts": "d67e3cd9f472535241a8e410d33423980bec45047e343577554d3356e1f0ef4e", + "https://deno.land/std@0.191.0/fs/exists.ts": "29c26bca8584a22876be7cb8844f1b6c8fc35e9af514576b78f5c6884d7ed02d", + "https://deno.land/std@0.191.0/html/entities.ts": "1c9fa4d76e36a9bdbe370a65f1612771f3cc2cf802d217b4e633850e2fa25c16", + "https://deno.land/std@0.191.0/html/mod.ts": "3f8c71781e32037ab63bd417759d15d31fb9606c6615c85dcabcc515986494ba", + "https://deno.land/std@0.191.0/jsonc/mod.ts": "b88dce28eb3645667caa856538ae2fe87af51410822544a0b45a4177ef3bd7dd", + "https://deno.land/std@0.191.0/jsonc/parse.ts": "2910e33bc7c3b243e3b6f3a39ce4d6ca84337b277a8df6f2ad2d9e4adbcddc08", + "https://deno.land/std@0.191.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", + "https://deno.land/std@0.191.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", + "https://deno.land/std@0.191.0/path/_util.ts": "d7abb1e0dea065f427b89156e28cdeb32b045870acdf865833ba808a73b576d0", + "https://deno.land/std@0.191.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", + "https://deno.land/std@0.191.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", + "https://deno.land/std@0.191.0/path/mod.ts": "f065032a7189404fdac3ad1a1551a9ac84751d2f25c431e101787846c86c79ef", + "https://deno.land/std@0.191.0/path/posix.ts": "8b7c67ac338714b30c816079303d0285dd24af6b284f7ad63da5b27372a2c94d", + "https://deno.land/std@0.191.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", + "https://deno.land/std@0.191.0/path/win32.ts": "d186344e5583bcbf8b18af416d13d82b35a317116e6460a5a3953508c3de5bba", + "https://deno.land/std@0.191.0/semver/mod.ts": "200f50cf6872212667df532fb09f0b1a33d3427a5201f75fad30a0d0c6dbcce3", + "https://deno.land/std@0.191.0/testing/_diff.ts": "1a3c044aedf77647d6cac86b798c6417603361b66b54c53331b312caeb447aea", + "https://deno.land/std@0.191.0/testing/_format.ts": "a69126e8a469009adf4cf2a50af889aca364c349797e63174884a52ff75cf4c7", + "https://deno.land/std@0.191.0/testing/asserts.ts": "e16d98b4d73ffc4ed498d717307a12500ae4f2cbe668f1a215632d19fcffc22f", "https://deno.land/x/code_block_writer@11.0.3/mod.ts": "2c3448060e47c9d08604c8f40dee34343f553f33edcdfebbf648442be33205e5", "https://deno.land/x/code_block_writer@11.0.3/utils/string_utils.ts": "60cb4ec8bd335bf241ef785ccec51e809d576ff8e8d29da43d2273b69ce2a6ff", "https://deno.land/x/deno_dom@v0.1.38/build/deno-wasm/deno-wasm.js": "98b1ad24a1c13284557917659402202e5c5258ab1431b3f3a82434ad36ffa05a", @@ -164,6 +168,7 @@ "https://deno.land/x/ts_morph@17.0.1/mod.ts": "adba9b82f24865d15d2c78ef6074b9a7457011719056c9928c800f130a617c93", "https://deno.land/x/ts_morph@17.0.1/ts_morph.d.ts": "a54b0c51b06d84defedf5fdd59c773d803808ae7c9678f7165f7a1a6dfa7f6a3", "https://deno.land/x/ts_morph@17.0.1/ts_morph.js": "1bb80284b9e31a4c5c2078cd533fe9b12b4b2d710267055cb655225aa88fb2df", + "https://deno.land/x/xhr@0.3.0/mod.ts": "094aacd627fd9635cd942053bf8032b5223b909858fa9dc8ffa583752ff63b20", "https://deno.land/x/zipjs@v2.7.14/index.d.ts": "f1c412d1721597eafd2b665ec002ddfb982a8818e32fb6a4d3cab1a86134db40", "https://deno.land/x/zipjs@v2.7.14/index.js": "7c71926e0c9618e48a22d9dce701131704fd3148a1d2eefd5dba1d786c846a5f", "https://deno.land/x/zipjs@v2.7.14/lib/core/codec-pool.js": "e5ab8ee3ec800ed751ef1c63a1bd8e50f162aa256a5f625d173d7a32e76e828c", @@ -200,6 +205,8 @@ "https://esm.sh/accept-language-parser@1.5.0/": "16d82ee0a75451f75b42d9a20db4da0ccae7ccc8cc09a41c73b4488aba010b94", "https://esm.sh/gh/SortableJS/Sortable@1.15.0/Sortable.min.js": "0a3e3cf471bf4566d7a3f9823b1d0d3b2bd075404e93419cd4074bd23310a9b0", "https://esm.sh/lifegpc-md5@1.0.3": "07fb48e0e08189de5f370ab8a1fa9d4dfd1cdb31a1843fba7e6f30d1c28575cd", + "https://esm.sh/lodash@4.17.21/isEqual": "d94a1bba01b5a2061d42ee5c7d27cce9e4c42781b724c48fb9aeb58240272a1b", + "https://esm.sh/meilisearch@0.33.0": "f6a40767720d9f1a65e2b9731ec129296b2e974cb6bdb33f0776818dbf777ac1", "https://esm.sh/preact-material-components@1.6.1/Button": "bc60923d511c6e2e33a7064339b3e643a9c15e3ef232ab063ef570af2ef83dc8", "https://esm.sh/preact-material-components@1.6.1/Checkbox": "bf34f5cd8c6d015916d854d91aab2caf115463e97be9a461f8dd3370ea11a49c", "https://esm.sh/preact-material-components@1.6.1/Dialog": "b0ff8da9c770456748f7e065fecda2fc90f5364ea66cae75ff5f51d57f6a87eb", @@ -330,9 +337,117 @@ "https://esm.sh/v125/@stablelib/int@1.0.1/denonext/int.mjs": "179398595f1fcf9e85c3251583cf23031bf1b5b2040d7ad731de19cdb1e61adc", "https://esm.sh/v125/@stablelib/wipe@1.0.1/denonext/wipe.mjs": "116b9cbca9c97e0a997d7cc3ebd9e60883d99eaa2345d48cafeb783b44808123", "https://esm.sh/v125/array-buffer-to-hex@1.0.0/denonext/array-buffer-to-hex.mjs": "e9c4132a6dc310f33ffc362e0becd7ab12d209962f7028ad1b9b58439cd9b35e", + "https://esm.sh/v125/cross-fetch@3.1.6/denonext/polyfill.js": "f03a84d17c94b7a32e53810751dccb0b41385459bfab9da8efdcfa6b44a8dd96", "https://esm.sh/v125/gh/SortableJS/Sortable@1.15.0/denonext/Sortable.min.js": "aeba191bb6622c4ad41ae5d8f5d4dd6bf15074f264cb3640ed223fbaf052263b", "https://esm.sh/v125/lifegpc-md5@1.0.3/denonext/lifegpc-md5.mjs": "332f72d899b7fa2a74c4a1a4f00616fa5543bcfd9a182314b9408501c3b0bb10", "https://esm.sh/v125/lifegpc-md5@1.0.3/lib/md5.d.ts": "82fd2b0c148e013ade0b46149953270220d0fd61835f2d12a95832fa2e0dcfc0", + "https://esm.sh/v125/lodash@4.17.21/denonext/_DataView.js": "443fc3e0f255bd72f35d2576288a0bb2f73fca0c858e542d0a2770ecb0f6d7ff", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Hash.js": "18837c9815896997e1799ff48ea0dcfe84eac217b9640fdd5200af33551fd837", + "https://esm.sh/v125/lodash@4.17.21/denonext/_ListCache.js": "c857858300e72860dd83fbcad4d738e3c0e6deb57a045f1c68bcee2e914b2fcc", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Map.js": "d4f3dbf3a71e136c5dd1591fcb66c3a1e019cb99f6b7c604eb768cf125e3c45b", + "https://esm.sh/v125/lodash@4.17.21/denonext/_MapCache.js": "d761fabde9c2eeb98c6c8b1d18640651438fcb7a81cb08929280fd697bba7f8e", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Promise.js": "625bac0d5286da49def7849e367f724e14a98f055142d5e95a9978f3b0b6b732", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Set.js": "45dfac4f5109844477f96efd965b5dbbf461b8db22b13b369d5af4eb3ae27864", + "https://esm.sh/v125/lodash@4.17.21/denonext/_SetCache.js": "b5b0d49a0c5fd9ca4676e10e322be07fab36195c66dda8534a2c986d543e998d", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Stack.js": "18b71d289acf042886917feb63def79bbb501ee35afcf644cdef721f6a2b33e0", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Symbol.js": "498dabbb29c80aea7caf8520d61672b10268731d59ee769c1acecc4445c89b6e", + "https://esm.sh/v125/lodash@4.17.21/denonext/_Uint8Array.js": "546de7ce7b76125ad6b7dfa18c4769468358dcc744d4008de339557365d0ce9d", + "https://esm.sh/v125/lodash@4.17.21/denonext/_WeakMap.js": "d493e24471a46a357011c8c7e938229ed129d79cdaa49a90b02b1e175d12cdb7", + "https://esm.sh/v125/lodash@4.17.21/denonext/_arrayFilter.js": "0c3af51e36929973acd94add452a11276df5006cb6d2fb61973993246097d8f2", + "https://esm.sh/v125/lodash@4.17.21/denonext/_arrayLikeKeys.js": "0bf3fd85d781ddb9f89580462c14772e68ae41d7f78a919952af26c5f1eaed14", + "https://esm.sh/v125/lodash@4.17.21/denonext/_arrayPush.js": "4a67cb267644a696b3c01fd74c437c9632eaf36591fce615e1428c046fe12270", + "https://esm.sh/v125/lodash@4.17.21/denonext/_arraySome.js": "5519940cc8bcd2a3de5d583bc2150973f95e5e99eff12252af3d2a5b242302ce", + "https://esm.sh/v125/lodash@4.17.21/denonext/_assocIndexOf.js": "cbff7d08589f809b38b823f925e561b6cc3d3282e778eaf7dec54cb6de73e4bd", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseGetAllKeys.js": "f59b8f7b5cf2a15503a5f7cb56b647e2b67ead1ff7a2e5291479981dbd315aa4", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseGetTag.js": "fff57b8feac50a788c42c18990ad45513f8061f2b887738c506aecc6311877b4", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseIsArguments.js": "f6550784fd1abe77b89bb7e3c85b00bb1a68a2917e8427488d26dc872eafe885", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseIsEqual.js": "f1272cff8a3124e105770b23bc93b49ef11594fcde8fd9f82e5909741347ba42", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseIsEqualDeep.js": "55466272e89801ff68ffafc4ec5eea4761fec21e0f1b9b35c3f6fef8c7f47066", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseIsNative.js": "869f3255c14dd71f122c045fdb807a8caef01bab6ea34aa1e67e36a4b6ab2078", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseIsTypedArray.js": "f1498bc278d7f56f5683b6254ec600ce9aa46e3de0a31d951b89b630b1c0bfe7", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseKeys.js": "2a9350d7c415c40dc19dbc6358b45ba439cc773ac941535b4736bf3db82947bd", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseTimes.js": "c0dd744df78b34e7bacf91d413a6c7329256ed0d72c8e0ca8c4d01a0073ab297", + "https://esm.sh/v125/lodash@4.17.21/denonext/_baseUnary.js": "4cf5e9987a79cfa3483debcef5a7059755024a829566e860dd4be4765df64259", + "https://esm.sh/v125/lodash@4.17.21/denonext/_cacheHas.js": "a5b38e7b5b85cdb62ec5f707af91a31b9f92b8b13645eadfe299eb825e90769e", + "https://esm.sh/v125/lodash@4.17.21/denonext/_coreJsData.js": "f798f5612c88ea0cc2dc9853313cae48c52bd220cb0796e4264e267de1b1a56c", + "https://esm.sh/v125/lodash@4.17.21/denonext/_equalArrays.js": "f1349f88c4d58d00a7e3b222103ed763b5e57e45c1a9bb58d6d3cd6f83e15fd0", + "https://esm.sh/v125/lodash@4.17.21/denonext/_equalByTag.js": "c195c2cc44ac8df4c7c3fd19a59aebf0aa4420d485791c114c877cfea0697c4e", + "https://esm.sh/v125/lodash@4.17.21/denonext/_equalObjects.js": "9294cfc9d5def7eeddbaecc120e9e332b62e9795f6a1b1262022abeeccb6ff25", + "https://esm.sh/v125/lodash@4.17.21/denonext/_freeGlobal.js": "ab432db5d6b7b01850bbfd430f6bbb5e03902db3ff7489f13d0b52db0e0137cb", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getAllKeys.js": "50eae31552d4a606fd5a9d2ca920265cd512b68fdd6ed042efd580ed9ab57670", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getMapData.js": "095fbf4741f36f20202a080438695c1df269569496f6e07c72020d06f0c4217c", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getNative.js": "968ee6eb41b89268ffecd1fbf425302700a1d2329561b4cb3b733e99f653f99d", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getRawTag.js": "b7916538c1835f1c81a89abff95802f0ad5cf06946ce7690c0fede5e534193f8", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getSymbols.js": "b9db2057658c9ed2963e1db19b30e16ee55e2358e0724f2ec67e0a1472b0337a", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getTag.js": "e7cfc4c7edebbf0df0e0dff1183e37dcf337968929d2e4f6ae8b069c215b33ad", + "https://esm.sh/v125/lodash@4.17.21/denonext/_getValue.js": "abc93c7c53ef8d67946aa38bf3c95cf14858a1111d749e0c62c68abd5c5015c8", + "https://esm.sh/v125/lodash@4.17.21/denonext/_hashClear.js": "748cdb00b435050b10ef5a828d527cf1ac83c98c08f557c6a9d4578fc44374ff", + "https://esm.sh/v125/lodash@4.17.21/denonext/_hashDelete.js": "0f6484a4caa0e75abaf24d9fe5b46fb10c1d3686309866bf16497364aa6025c2", + "https://esm.sh/v125/lodash@4.17.21/denonext/_hashGet.js": "0b54ef9bbfb7cad99dd7ee2a4345fdedb4dcbb9a283562e16f3b98c07422ebe6", + "https://esm.sh/v125/lodash@4.17.21/denonext/_hashHas.js": "799280f2108a4b862918c91c9db6cfe02defd18ed97872c9614eb342138d7326", + "https://esm.sh/v125/lodash@4.17.21/denonext/_hashSet.js": "a96d89ad63e3f3ffcea3464e84e05faea8c41da9c13375bb8e89016151649fa9", + "https://esm.sh/v125/lodash@4.17.21/denonext/_isIndex.js": "315a444f0d231a807b0cf71312749fd379361f08e0d374debf56f934dca199e8", + "https://esm.sh/v125/lodash@4.17.21/denonext/_isKeyable.js": "50db32bca53e8716b72766e9a76e8c06af932071d31afb04c55e0a4fd85381e3", + "https://esm.sh/v125/lodash@4.17.21/denonext/_isMasked.js": "b52b8b42ae4e7d251adc9ab487219212bc7031f5f09b31627ce2accfce96da67", + "https://esm.sh/v125/lodash@4.17.21/denonext/_isPrototype.js": "d0c85a8c5039b5cdb3b206fcdc41e10fd2e7773ed18cd457ba101d47db8e06d6", + "https://esm.sh/v125/lodash@4.17.21/denonext/_listCacheClear.js": "eb6e25c159d0ef2b22906084a69d7c9c5855f56707053a84c9e5fa7642b5ea39", + "https://esm.sh/v125/lodash@4.17.21/denonext/_listCacheDelete.js": "15baba98258b133157d8df9b2c0a6183b64cdf6c0136aa590554cfcd6b02cb0a", + "https://esm.sh/v125/lodash@4.17.21/denonext/_listCacheGet.js": "d1c00da1d80ccfa428681ecee4684608d2bc94c44d920c73603dc8a4178263f3", + "https://esm.sh/v125/lodash@4.17.21/denonext/_listCacheHas.js": "395e91d34f914ae8b33e912e0d9d509e98170f68c1a97de80a3a896012155656", + "https://esm.sh/v125/lodash@4.17.21/denonext/_listCacheSet.js": "55aef0b9d70c6d71acc30cdb8ceb43c33affc6079a347c918baa12bbb8489915", + "https://esm.sh/v125/lodash@4.17.21/denonext/_mapCacheClear.js": "423b7b58681e401351159e5334bb63f27b3e6d17c74ba74ef0e64d2aa9ec25e8", + "https://esm.sh/v125/lodash@4.17.21/denonext/_mapCacheDelete.js": "24fac3addf7956f1564ec767fd1a9ba7247c29778645dd73a2792033ca5b085c", + "https://esm.sh/v125/lodash@4.17.21/denonext/_mapCacheGet.js": "2a46198c6cb288a3ad73ab59bda9a980edda7827a0a796706565189866e5f1bf", + "https://esm.sh/v125/lodash@4.17.21/denonext/_mapCacheHas.js": "e2f657f45c888dc2e9c8ff45ac31c252b877b03fffa5170d59ce8e844a0afadc", + "https://esm.sh/v125/lodash@4.17.21/denonext/_mapCacheSet.js": "a5eb7c043312f5b67cde7495b32d0d6d442aa7fd1872299725da97a9ddf9352e", + "https://esm.sh/v125/lodash@4.17.21/denonext/_mapToArray.js": "3caec14039244b138d8c46ab1bce2aeaeeb55e1fafd82f36a6e8a3fd377deaaf", + "https://esm.sh/v125/lodash@4.17.21/denonext/_nativeCreate.js": "0a249a06e87104bfee1c42d7eb8e80fbc8a6d0ec468cabfc276c13d2e56c43cc", + "https://esm.sh/v125/lodash@4.17.21/denonext/_nativeKeys.js": "a5070fb92829c60895e7a9e9932a670db8b192edf81b44ccb3b79438eafb4739", + "https://esm.sh/v125/lodash@4.17.21/denonext/_nodeUtil.js": "992e1e285c9ac2687d2a44d6dcae5a035cc6e4c017aa6de5f5072fd30435355a", + "https://esm.sh/v125/lodash@4.17.21/denonext/_objectToString.js": "e4b2b4479f60ba74790fcf6f89fde6980e78809d1949846738da1d0cbbe041e4", + "https://esm.sh/v125/lodash@4.17.21/denonext/_overArg.js": "f19e1fd1d36b752dd2a6d4a4b4237fda4dece63ecda3f9cb7cdc82fcccf60396", + "https://esm.sh/v125/lodash@4.17.21/denonext/_root.js": "6e85a58b80d04a156f54382bac8276ba67120312e96c5d04587b9275c607b8bc", + "https://esm.sh/v125/lodash@4.17.21/denonext/_setCacheAdd.js": "f1eda094cdfb7faef45f3b6aef4783875d5747ccc95b70ceb2c9e4605335fd65", + "https://esm.sh/v125/lodash@4.17.21/denonext/_setCacheHas.js": "7bab0c16fe5ee1c69cb9207c80c917671d6bdfacdf658012f2625bc7ee8997d2", + "https://esm.sh/v125/lodash@4.17.21/denonext/_setToArray.js": "e083bd44dddf992cde7565461c4ff50e22187befa67bd1220573c78625dc164b", + "https://esm.sh/v125/lodash@4.17.21/denonext/_stackClear.js": "d797082980f4fa19c73e55d03f2108090d71eb01fbe37e43c799c97a0d9e6769", + "https://esm.sh/v125/lodash@4.17.21/denonext/_stackDelete.js": "0cf63f54aa98b77ea5f37a2aac0fc4815d76efd30641ecab1c486033968bfedf", + "https://esm.sh/v125/lodash@4.17.21/denonext/_stackGet.js": "4713035f551afc3d8cecea2218d5572f7945a3668dfc32af7f7a28ba67c205f1", + "https://esm.sh/v125/lodash@4.17.21/denonext/_stackHas.js": "aacd1521a15730ecd8dbf65f26d10a5a8ebc78be66715745e08ac94524886b60", + "https://esm.sh/v125/lodash@4.17.21/denonext/_stackSet.js": "f4cfb1816920afb11b3ff1c8d87ad60751c777acd9cda003f5840e299a67e2dc", + "https://esm.sh/v125/lodash@4.17.21/denonext/_toSource.js": "f25b3fb19186e19a6ca0929ab89ee1687b3a95c066fa84201916d9e7bb2fda77", + "https://esm.sh/v125/lodash@4.17.21/denonext/eq.js": "fca744d89152e768965c5a2187b8c1214dff8ce99a440f17ed9a30cee9d226c9", + "https://esm.sh/v125/lodash@4.17.21/denonext/isArguments.js": "058dd4e845f66788d2d92997caef78580b07df20a1b2be61d43d984afa3c11b6", + "https://esm.sh/v125/lodash@4.17.21/denonext/isArray.js": "0df45788d8e625401f40a6e6b08118bcacc4f66ac7bf2b96ed6cfdb401072997", + "https://esm.sh/v125/lodash@4.17.21/denonext/isArrayLike.js": "80565bf1be81b0c20861946b9e442b80ca388cc31137086f19189f1026e23f01", + "https://esm.sh/v125/lodash@4.17.21/denonext/isBuffer.js": "ccdde50ed65a107bda2d1bad86e09861f72b127cf06c8b1456b72b84d732ad3f", + "https://esm.sh/v125/lodash@4.17.21/denonext/isEqual.js": "27c2df27ed190361e466601aeb166895f40f9ce0b513a02bbb6dfe391cf08208", + "https://esm.sh/v125/lodash@4.17.21/denonext/isFunction.js": "c14e433f92ae98cd44d484b73bfd43f91e6639acac91364ec05f75f94411fb4d", + "https://esm.sh/v125/lodash@4.17.21/denonext/isLength.js": "8edf38a08b3af202c867ac917c92aac70402e6b002816963fbcb6b056c99b2c3", + "https://esm.sh/v125/lodash@4.17.21/denonext/isObject.js": "1473c80b3c01da603950c980d62c64c7c70d99c3a2888ce023eca0cb257fa0f4", + "https://esm.sh/v125/lodash@4.17.21/denonext/isObjectLike.js": "d6fa200df8c6aad4bb0ed8b07162f36596a0b229d5bd19397b04d443ed6238ad", + "https://esm.sh/v125/lodash@4.17.21/denonext/isTypedArray.js": "059404217e3c3fd8beb73e56e9b862cb4aab77821371f96b583e49f11a3e1003", + "https://esm.sh/v125/lodash@4.17.21/denonext/keys.js": "25570a59cbb3039a99540fc5ea02c341283d5e4dcc22740609c4adefcf2fd0d1", + "https://esm.sh/v125/lodash@4.17.21/denonext/stubArray.js": "c3a6c2a1ef38b048ba49a7f647c3973fd1da6831b46e18dafde2f7a66d4dadfa", + "https://esm.sh/v125/lodash@4.17.21/denonext/stubFalse.js": "8b53fd73214efcd5272718f783fec099e43bee32bcc6cd97e71acc341e41e01a", + "https://esm.sh/v125/meilisearch@0.33.0/denonext/meilisearch.mjs": "57b3c71dbeeb495434e3e3ab737cdeb8092de57b5337234053be6f0f85a53647", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/clients/client.d.ts": "9d9eeb302d42cbb79d0324236dd4789f4dabc1487a0d3b927889b97f3c1bed62", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/clients/node-client.d.ts": "d28199cb911fa21166ecf03a13d31e28926d367ac8e8ff0a4462145e0b760f6f", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/enqueued-task.d.ts": "ce04098b3aa08717fde16da795aea6907a88e084c8d559e9205a39b304ae8dc8", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/http-error-handler.d.ts": "10d240286bc55d3efab63a82eaf46a33b1aced402d8b59b8b76f517ba16f784e", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/index.d.ts": "50821f82193cdece3592a48b49f23e8800a0786793443fcfd2589c6f48ffb492", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/meilisearch-api-error.d.ts": "87975cf7862b1e1398e51dbcfedb39fc64c9c34c872eb28b44a65605e618f4d6", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/meilisearch-communication-error.d.ts": "51f815df16cd567ef3f5db2e91a9b867f8b9bb8708967706e1a5cd43fd768da8", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/meilisearch-error.d.ts": "017195a6a4fdbdd00a49af562f529d4b2bf4d0bb1ff59ffd602b1dd717027165", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/meilisearch-timeout-error.d.ts": "c3d15650ce2d0e090b1cbd0fd589ef1d522825652252f7fb08d52c7a0d17c7fe", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/errors/version-hint-message.d.ts": "8c79dd5cd51445fec8e761b5b7cbf512d3cde83a6b1aacb8702a0e2a3cccf5e9", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/http-requests.d.ts": "a9510e51aede9d14f46c10face5ebdd105a2d13b6933bce2bf235a0d6ebd4359", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/index.d.ts": "b4b1d650ac97e2dfccd0ce7c5dff3e6b0b98adcb0d46f8a7ffe07745c1dad42c", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/indexes.d.ts": "bf24e7823a14b072705278c904def355c1c16057db06fdd40e695a2e9f048c54", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/task.d.ts": "ed00d838a0a2ed3a07d265bea08dd17a0176b7ef624c4f6f84029ad6b453117b", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/token.d.ts": "160ef96d5065bc36cd99ce8584f2612ca9fe28ac72d7bf09a37768919d4def18", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/types/index.d.ts": "a0e396a938a5e86b0a838c1f17311c703ff20e21095983a7523549fdb6a2a48f", + "https://esm.sh/v125/meilisearch@0.33.0/dist/types/types/types.d.ts": "eae72c3f246fb8aa28089798ca8570d1877c0480d3db311f3cc458ddc0786978", "https://raw.githubusercontent.com/lucacasonato/esbuild_deno_loader/8031f71afa1bbcd3237a94b11f53a2e5c5c0e7bf/deps.ts": "b7248e5b750be62613a9417f407e65ed43726d83b11f9631d6dbb58634bbd7d1", "https://raw.githubusercontent.com/lucacasonato/esbuild_deno_loader/8031f71afa1bbcd3237a94b11f53a2e5c5c0e7bf/mod.ts": "3e507379372361162f93325a216b86f6098defb5bb60144555b507bca26d061f", "https://raw.githubusercontent.com/lucacasonato/esbuild_deno_loader/8031f71afa1bbcd3237a94b11f53a2e5c5c0e7bf/src/deno.ts": "71bee6b14e72ca193c0686d8b4f1f47d639a64745b6f5c7576f7a3616f436f57", diff --git a/import_map.json b/import_map.json index ce01a13..9231ac1 100644 --- a/import_map.json +++ b/import_map.json @@ -1,6 +1,6 @@ { "imports": { - "std/": "https://deno.land/std@0.190.0/", + "std/": "https://deno.land/std@0.191.0/", "deno_dom/": "https://deno.land/x/deno_dom@v0.1.38/", "sqlite/": "https://deno.land/x/sqlite@v3.7.2/", "zipjs/": "https://deno.land/x/zipjs@v2.7.14/", @@ -17,6 +17,8 @@ "accept-language-parser/": "https://esm.sh/accept-language-parser@1.5.0/", "sortable": "https://esm.sh/gh/SortableJS/Sortable@1.15.0/Sortable.min.js", "esbuild/": "https://deno.land/x/esbuild@v0.18.0/", - "lifegpc-md5": "https://esm.sh/lifegpc-md5@1.0.3" + "lifegpc-md5": "https://esm.sh/lifegpc-md5@1.0.3", + "meilisearch": "https://esm.sh/meilisearch@0.33.0", + "lodash/": "https://esm.sh/lodash@4.17.21/" } } diff --git a/main.ts b/main.ts index f456cc6..332e092 100644 --- a/main.ts +++ b/main.ts @@ -23,6 +23,7 @@ enum CMD { Optimize, UpdateTagTranslation, ExportZip, + UpdateMeiliSearchData, } const args = parse(Deno.args, { @@ -46,6 +47,9 @@ if (rcmd == "utt" || rcmd == "update_tag_translation") { cmd = CMD.UpdateTagTranslation; } if (rcmd == "ez" || rcmd == "export_zip") cmd = CMD.ExportZip; +if (rcmd == "umsd" || rcmd == "update_meili_search_data") { + cmd = CMD.UpdateMeiliSearchData; +} if (cmd == CMD.Unknown) { throw Error(`Unknown command: ${rcmd}`); } @@ -117,6 +121,15 @@ async function export_zip() { if (!manager.aborted) manager.close(); } } +async function update_meili_search_data() { + const manager = new TaskManager(settings); + try { + await manager.add_update_meili_search_data_task(); + await manager.run(); + } finally { + if (!manager.aborted) manager.close(); + } +} async function main() { await sure_dir(settings.base); if (cmd == CMD.Download) { @@ -129,6 +142,8 @@ async function main() { await update_tag_translation(); } else if (cmd == CMD.ExportZip) { await export_zip(); + } else if (cmd == CMD.UpdateMeiliSearchData) { + await update_meili_search_data(); } } diff --git a/meilisearch.ts b/meilisearch.ts new file mode 100644 index 0000000..3fe0d55 --- /dev/null +++ b/meilisearch.ts @@ -0,0 +1,138 @@ +import { + EnqueuedTask, + Index, + MeiliSearch, + MeiliSearchApiError, +} from "meilisearch"; +import { sleep } from "./utils.ts"; +import { EhDb } from "./db.ts"; +import isEqual from "lodash/isEqual"; + +const GMetaSettings: Record = { + displayedAttributes: ["*"], + searchableAttributes: [ + "title", + "title_jpn", + "uploader", + "tags.tag", + "tags.translated", + ], + filterableAttributes: [ + "category", + "expunged", + "filecount", + "filesize", + "gid", + "posted", + "rating", + "tags.id", + "tags.tag", + "tags.translated", + "uploader", + ], + sortableAttributes: ["filecount", "filesize", "gid", "posted", "rating"], +}; + +export class MeiliSearchServer { + client; + db; + handle; + #gmeta?: Index; + #inited = false; + target; + constructor(host: string, api_key: string, db: EhDb, signal?: AbortSignal) { + this.client = new MeiliSearch({ + host, + apiKey: api_key, + requestConfig: { signal }, + }); + this.db = db; + this.handle = (e: Event) => { + this.#gallery_update(e); + }; + this.target = new EventTarget(); + this.target.addEventListener("gallery_update", this.handle); + } + #gallery_update(e: Event) { + const ev = e as CustomEvent; + this.updateGallery(ev.detail); + } + async #updateGMetaSettings() { + if (this.#gmeta) { + const o: Record = await this.#gmeta.getSettings(); + const u: Record = {}; + let need_update = false; + Object.getOwnPropertyNames(GMetaSettings).forEach((k) => { + if (!isEqual(o[k], GMetaSettings[k])) { + u[k] = GMetaSettings[k]; + need_update = true; + } + }); + if (need_update) { + console.log(u); + await this.waitTask(this.#gmeta.updateSettings(u)); + } + } + } + close() { + this.target.removeEventListener("gallery_update", this.handle); + } + async getIndex(uid: string, primaryKey?: string) { + try { + return await this.client.getIndex(uid); + } catch (e) { + if (e instanceof MeiliSearchApiError) { + if (e.code === "index_not_found") { + await this.waitTask( + this.client.createIndex(uid, { primaryKey }), + ); + return await this.client.getIndex(uid); + } + } + throw e; + } + } + get gmeta(): Promise { + return new Promise((resolve, reject) => { + const check = () => { + if (!this.#gmeta) reject(new Error("gmeta not found.")); + else resolve(this.#gmeta); + }; + if (!this.#inited) { + this.init().then(check).catch(reject); + } else { + check(); + } + }); + } + async init() { + this.#gmeta = await this.getIndex("gmeta", "gid"); + this.#updateGMetaSettings(); + this.#inited = true; + } + async updateGallery(...gids: number[]) { + const gmeta = await this.gmeta; + const datas = gids.map((gid) => { + const d = this.db.get_gmeta_by_gid(gid); + if (!d) throw Error("Gallery not found."); + const e = > d; + e.tags = this.db.get_gtags_full(gid); + return e; + }); + await this.waitTask(gmeta.updateDocuments(datas)); + } + async waitTask(task: EnqueuedTask | Promise) { + if (task instanceof Promise) { + task = await task; + } + let status = await this.client.getTask(task.taskUid); + while (status.status === "enqueued" || status.status === "processing") { + await sleep(100); + status = await this.client.getTask(task.taskUid); + } + if (status.status === "failed") { + throw status.error ? status.error : new Error("Task failed."); + } + return status; + } +} diff --git a/page/GalleryMetadata.ts b/page/GalleryMetadata.ts index c5d6baa..69d6e12 100644 --- a/page/GalleryMetadata.ts +++ b/page/GalleryMetadata.ts @@ -1,3 +1,4 @@ +import { unescape } from "std/html/mod.ts"; import { GMeta } from "../db.ts"; export type GalleryMetadataTorrentInfo = { @@ -12,10 +13,13 @@ export type GalleryMetadataSingle = { gid: number; token: string; archiver_key: string; + /// HTML escaped title: string; + /// HTML escaped title_jpn: string; category: string; thumb: string; + /// HTML escaped uploader: string; posted: string; filecount: string; @@ -45,10 +49,10 @@ class GalleryMetadata { return { gid: g.gid, token: g.token, - title: g.title, - title_jpn: g.title_jpn, + title: unescape(g.title), + title_jpn: unescape(g.title_jpn), category: g.category, - uploader: g.uploader, + uploader: unescape(g.uploader), posted: parseInt(g.posted), filecount: parseInt(g.filecount), filesize: g.filesize, diff --git a/page/GalleryMetadata_test.ts b/page/GalleryMetadata_test.ts index daaa4fe..09f1c4e 100644 --- a/page/GalleryMetadata_test.ts +++ b/page/GalleryMetadata_test.ts @@ -1,3 +1,4 @@ +import { assert, assertEquals } from "std/testing/asserts.ts"; import { Client } from "../client.ts"; import { load_settings } from "../config.ts"; import { API_PERMISSION } from "../test_base.ts"; @@ -8,6 +9,17 @@ Deno.test({ }, async () => { const cfg = await load_settings("./config.json"); const client = new Client(cfg); - const re = await client.fetchGalleryMetadataByAPI([2552611, "3132307627"]); + const re = await client.fetchGalleryMetadataByAPI([1389215, "b5e43bd12d"]); console.log(re.obj); + const gdata = re.map.get(1389215); + assert(gdata !== undefined && typeof gdata !== "string"); + assertEquals( + gdata.title, + "(C95) [Shiratamaco (Shiratama)] Usagi Syndrome 4 (Gochuumon wa Usagi desu ka?) [Chinese] [白玉症候群&绅士仓库联合汉化]", + ); + const gmeta = re.convert(gdata); + assertEquals( + gmeta.title, + "(C95) [Shiratamaco (Shiratama)] Usagi Syndrome 4 (Gochuumon wa Usagi desu ka?) [Chinese] [白玉症候群&绅士仓库联合汉化]", + ); }); diff --git a/scripts/gen_meili_server_key.ts b/scripts/gen_meili_server_key.ts new file mode 100644 index 0000000..cc66dae --- /dev/null +++ b/scripts/gen_meili_server_key.ts @@ -0,0 +1,34 @@ +import { MeiliSearch } from "meilisearch"; + +async function gen_key(host: string, master_key: string) { + const client = new MeiliSearch({ host, apiKey: master_key }); + const key = await client.createKey({ + name: "eh_downloader_update_key", + description: "EH Downloader Update Key", + actions: ["*"], + indexes: ["gmeta"], + expiresAt: null, + }); + console.log("Update Key: ", key.key); + const skey = await client.createKey({ + name: "eh_downloader_search_key", + description: "EH Downloader Search Key", + actions: ["search"], + indexes: ["gmeta"], + expiresAt: null, + }); + console.log("Search Key: ", skey.key); +} + +function show_help() { + console.log("gen_meili_server_key "); +} + +if (Deno.args.length < 2) { + show_help(); + Deno.exit(0); +} + +const host = Deno.args[0]; +const master_key = Deno.args[1]; +gen_key(host, master_key); diff --git a/task.ts b/task.ts index f9d04a5..09427a7 100644 --- a/task.ts +++ b/task.ts @@ -1,6 +1,7 @@ export enum TaskType { Download, ExportZip, + UpdateMeiliSearchData, } export type Task = { @@ -23,9 +24,15 @@ export type TaskExportZipProgress = { total_page: number; }; +export type TaskUpdateMeiliSearchDataProgress = { + total_gallery: number; + updated_gallery: number; +}; + export type TaskProgressBasicType = { [TaskType.Download]: TaskDownloadProgess; [TaskType.ExportZip]: TaskExportZipProgress; + [TaskType.UpdateMeiliSearchData]: TaskUpdateMeiliSearchDataProgress; }; export type TaskProgress = { diff --git a/task_manager.ts b/task_manager.ts index ffbe7f5..3bf09f1 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -1,6 +1,7 @@ import { Client } from "./client.ts"; import { Config } from "./config.ts"; import { EhDb } from "./db.ts"; +import { MeiliSearchServer } from "./meilisearch.ts"; import { check_running } from "./pid_check.ts"; import { add_exit_handler } from "./signal_handler.ts"; import { Task, TaskProgress, TaskProgressBasicType, TaskType } from "./task.ts"; @@ -10,6 +11,7 @@ import { export_zip, ExportZipConfig, } from "./tasks/export_zip.ts"; +import { update_meili_search_data } from "./tasks/update_meili_search_data.ts"; import { DiscriminatedUnion, promiseState, @@ -46,6 +48,7 @@ export class TaskManager extends EventTarget { db; running_tasks: Map; max_task_count; + meilisearch?: MeiliSearchServer; #abort; #force_abort; constructor(cfg: Config) { @@ -58,6 +61,14 @@ export class TaskManager extends EventTarget { this.running_tasks = new Map(); this.max_task_count = cfg.max_task_count; add_exit_handler(this); + if (this.cfg.meili_host && this.cfg.meili_update_api_key) { + this.meilisearch = new MeiliSearchServer( + this.cfg.meili_host, + this.cfg.meili_update_api_key, + this.db, + this.force_aborts, + ); + } } async #add_task(task: Task) { const r = await this.db.add_task(task); @@ -112,6 +123,17 @@ export class TaskManager extends EventTarget { }; return await this.#add_task(task); } + async add_update_meili_search_data_task(gid?: number) { + const task: Task = { + gid: gid ? gid : 0, + token: "", + id: 0, + pid: Deno.pid, + type: TaskType.UpdateMeiliSearchData, + details: null, + }; + return await this.#add_task(task); + } async check_task(task: Task) { this.#check_closed(); if (await this.check_task_is_running(task)) return; @@ -267,6 +289,11 @@ export class TaskManager extends EventTarget { base: task, }, ); + } else if (task.type === TaskType.UpdateMeiliSearchData) { + this.running_tasks.set(task.id, { + task: update_meili_search_data(task, this), + base: task, + }); } } async waiting_unfinished_task() { diff --git a/tasks/download.ts b/tasks/download.ts index d88bbb9..47030ca 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -109,6 +109,11 @@ export async function download_task( const gmeta = gdatas.convert(gdata); db.add_gmeta(gmeta); await db.add_gtag(task.gid, new Set(gdata.tags)); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_update", { detail: gmeta.gid }), + ); + } const base_path = join(cfg.base, task.gid.toString()); await sure_dir(base_path); const m = new DownloadManager(cfg, abort, force_abort, task, manager); diff --git a/tasks/update_meili_search_data.ts b/tasks/update_meili_search_data.ts new file mode 100644 index 0000000..e82fdb4 --- /dev/null +++ b/tasks/update_meili_search_data.ts @@ -0,0 +1,21 @@ +import { Task } from "../task.ts"; +import { TaskManager } from "../task_manager.ts"; + +export async function update_meili_search_data( + task: Task, + manager: TaskManager, +) { + if (!manager.meilisearch) throw Error("MeiliServer not found."); + if (task.gid !== 0) { + await manager.meilisearch.updateGallery(task.gid); + } else { + let i = manager.db.get_gids(); + let offset = 0; + while (i.length) { + await manager.meilisearch.updateGallery(...i); + offset += i.length; + i = manager.db.get_gids(offset); + } + } + return task; +}