diff --git a/api.yml b/api.yml index 5136412..39eab6a 100644 --- a/api.yml +++ b/api.yml @@ -999,6 +999,8 @@ components: const: 3 - title: Import const: 4 + - title: UpdateTagTranslation + const: 5 ThumbnailMethod: description: Thumbnail method type: integer @@ -2703,7 +2705,12 @@ paths: properties: type: type: string - enum: [download, export_zip, update_meili_search_data, import] + enum: + - download + - export_zip + - update_meili_search_data + - import + - update_tag_translation description: Task type gid: type: integer diff --git a/db.ts b/db.ts index 1ce41b8..51b98d3 100644 --- a/db.ts +++ b/db.ts @@ -888,8 +888,12 @@ export class EhDb { check_onetime_task() { return this.transaction(() => { const r = this.db.queryEntries( - "SELECT * FROM task WHERE type = ? OR type = ?;", - [TaskType.UpdateMeiliSearchData, TaskType.FixGalleryPage], + "SELECT * FROM task WHERE type = ? OR type = ? OR type = ?;", + [ + TaskType.UpdateMeiliSearchData, + TaskType.FixGalleryPage, + TaskType.UpdateTagTranslation, + ], ); return r; }); @@ -909,6 +913,15 @@ export class EhDb { return r.length ? r[0] : undefined; }); } + check_update_tag_translation_task() { + return this.transaction(() => { + const r = this.db.queryEntries( + "SELECT * FROM task WHERE type = ?;", + [TaskType.UpdateTagTranslation], + ); + return r.length ? r[0] : undefined; + }); + } close() { this.db.close(); if (this.#file) { diff --git a/routes/api/task.ts b/routes/api/task.ts index f739c08..1acb188 100644 --- a/routes/api/task.ts +++ b/routes/api/task.ts @@ -13,6 +13,7 @@ import type { ExportZipConfig } from "../../tasks/export_zip.ts"; import { User, UserPermission } from "../../db.ts"; import { toJSON } from "../../utils.ts"; import { ImportConfig } from "../../tasks/import.ts"; +import { UpdateTagTranslationConfig } from "../../tasks/update_tag_translation.ts"; export const handler: Handlers = { GET(req, ctx) { @@ -72,6 +73,8 @@ export const handler: Handlers = { t.add_download_task(d.gid, d.token, d.cfg); } else if (d.type == "new_export_zip_task") { t.add_export_zip_task(d.gid, d.cfg); + } else if (d.type == "new_update_tag_translation_task") { + t.add_update_tag_translation_task(d.cfg); } else if (d.type == "task_list") { t.get_task_list().then((tasks) => { sendMessage({ @@ -203,6 +206,28 @@ export const handler: Handlers = { } catch (e) { return return_error(500, e.message); } + } else if (typ == "update_tag_translation") { + const cfg = await get_string(form.get("cfg")); + let dcfg: UpdateTagTranslationConfig | undefined = undefined; + if (cfg) { + try { + dcfg = JSON.parse(cfg); + } catch (_) { + return return_error(4, "cfg is invalid"); + } + } + try { + const task = await t.add_update_tag_translation_task( + dcfg, + true, + ); + if (task === null) { + return return_error(6, "task is already in the list"); + } + return return_data(task, 201); + } catch (e) { + return return_error(500, e.message); + } } else { return return_error(5, "unknown type"); } diff --git a/server/task.ts b/server/task.ts index 8f8a28b..ceeac00 100644 --- a/server/task.ts +++ b/server/task.ts @@ -2,6 +2,7 @@ import type { Task } from "../task.ts"; import type { TaskEventData } from "../task_manager.ts"; import type { DownloadConfig } from "../tasks/download.ts"; import type { ExportZipConfig } from "../tasks/export_zip.ts"; +import type { UpdateTagTranslationConfig } from "../tasks/update_tag_translation.ts"; import type { DiscriminatedUnion } from "../utils.ts"; export type TaskServerSocketData = @@ -22,6 +23,7 @@ type EventMap = { cfg?: DownloadConfig; }; new_export_zip_task: { gid: number | bigint; cfg?: ExportZipConfig }; + new_update_tag_translation_task: { cfg?: UpdateTagTranslationConfig }; }; export type TaskClientSocketData = diff --git a/task.ts b/task.ts index f0e4677..5cdc8e4 100644 --- a/task.ts +++ b/task.ts @@ -4,6 +4,7 @@ export enum TaskType { UpdateMeiliSearchData, FixGalleryPage, Import, + UpdateTagTranslation, } export type Task = { @@ -59,12 +60,18 @@ export type TaskImportProgress = { total_page: number; }; +export type TaskUpdateTagTranslationProgress = { + added_tag: number; + total_tag: number; +}; + export type TaskProgressBasicType = { [TaskType.Download]: TaskDownloadProgess; [TaskType.ExportZip]: TaskExportZipProgress; [TaskType.UpdateMeiliSearchData]: TaskUpdateMeiliSearchDataProgress; [TaskType.FixGalleryPage]: TaskFixGalleryPageProgress; [TaskType.Import]: TaskImportProgress; + [TaskType.UpdateTagTranslation]: TaskUpdateTagTranslationProgress; }; export type TaskProgress = { diff --git a/task_manager.ts b/task_manager.ts index d37e87f..5a5af84 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -18,6 +18,11 @@ import { import { fix_gallery_page } from "./tasks/fix_gallery_page.ts"; import { import_task, ImportConfig } from "./tasks/import.ts"; import { update_meili_search_data } from "./tasks/update_meili_search_data.ts"; +import { + DEFAULT_UTT_CONFIG, + update_tag_translation, + UpdateTagTranslationConfig, +} from "./tasks/update_tag_translation.ts"; import { DiscriminatedUnion, promiseState, @@ -197,6 +202,26 @@ export class TaskManager extends EventTarget { }; return await this.#add_task(task); } + async add_update_tag_translation_task( + cfg?: UpdateTagTranslationConfig, + mark_already = false, + ) { + this.#check_closed(); + const otask = await this.db.check_update_tag_translation_task(); + if (otask !== undefined) { + console.log("The task is already in list."); + return mark_already ? null : otask; + } + const task: Task = { + gid: 0, + token: "", + id: 0, + pid: Deno.pid, + type: TaskType.UpdateTagTranslation, + details: toJSON(cfg || DEFAULT_UTT_CONFIG), + }; + return await this.#add_task(task); + } async check_task(task: Task) { this.#check_closed(); if (await this.check_task_is_running(task)) return; @@ -384,6 +409,15 @@ export class TaskManager extends EventTarget { task: import_task(task, this), base: task, }); + } else if (task.type == TaskType.UpdateTagTranslation) { + await this.waiting_unfinished_task(); + const cfg: UpdateTagTranslationConfig = task.details + ? JSON.parse(task.details) + : DEFAULT_UTT_CONFIG; + this.running_tasks.set(BigInt(task.id), { + task: update_tag_translation(task, this, cfg), + base: task, + }); } } async update_task(t: Task) { diff --git a/tasks/update_tag_translation.ts b/tasks/update_tag_translation.ts new file mode 100644 index 0000000..4b74be7 --- /dev/null +++ b/tasks/update_tag_translation.ts @@ -0,0 +1,52 @@ +import { load_eht_file } from "../eh_translation.ts"; +import { Task, TaskType, TaskUpdateTagTranslationProgress } from "../task.ts"; +import { TaskManager } from "../task_manager.ts"; +import { asyncForEach, isDocker } from "../utils.ts"; + +export type UpdateTagTranslationConfig = { + file?: string; +}; + +export const DEFAULT_UTT_CONFIG: UpdateTagTranslationConfig = {}; + +const PROGRESS_UPDATE_INTERVAL = 200; + +export async function update_tag_translation( + task: Task, + manager: TaskManager, + uttcfg: UpdateTagTranslationConfig, +) { + const progress: TaskUpdateTagTranslationProgress = { + added_tag: 0, + total_tag: 0, + }; + let last_send_progress = 0; + const sendEvent = () => { + const now = Date.now(); + if (now < last_send_progress + PROGRESS_UPDATE_INTERVAL) return; + manager.dispatchTaskProgressEvent( + TaskType.UpdateTagTranslation, + task.id, + progress, + ); + last_send_progress = now; + }; + const f = await load_eht_file(uttcfg.file, manager.aborts); + for (const d of f.data) { + progress.total_tag += d.count; + } + sendEvent(); + const file = isDocker() ? "/tmp/utt.lock" : "./utt.lock"; + await Deno.writeTextFile(file, "", { create: true, signal: manager.aborts }); + for (const d of f.data) { + await asyncForEach(Object.getOwnPropertyNames(d.data), async (name) => { + const tag = `${d.namespace}:${name}`; + const t = d.data[name]; + manager.db.update_tags(tag, t.name, t.intro); + progress.added_tag += 1; + sendEvent(); + await Deno.readTextFile(file, { signal: manager.aborts }); + }); + } + return task; +}