From 8072519545535653f3065d66c0566906d5dfe5a0 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 13:15:32 +0800 Subject: [PATCH 01/14] Update --- db.ts | 66 ++++++++++++++++++++++++------------------------- task_manager.ts | 2 +- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/db.ts b/db.ts index 4211626..300800d 100644 --- a/db.ts +++ b/db.ts @@ -582,11 +582,17 @@ export class EhDb { return r; }); } - check_update_meili_search_data_task() { + check_update_meili_search_data_task(gid?: number) { + const args = [TaskType.UpdateMeiliSearchData]; + let wsql = ""; + if (gid !== undefined) { + wsql = " AND gid = ?"; + args.push(gid); + } return this.transaction(() => { const r = this.db.queryEntries( - "SELECT * FROM task WHERE type = ?;", - [TaskType.UpdateMeiliSearchData], + `SELECT * FROM task WHERE type = ?${wsql};`, + args, ); return r.length ? r[0] : undefined; }); @@ -774,38 +780,30 @@ export class EhDb { is_nsfw: boolean | null = null, is_ad: boolean | null = null, ) { - if (is_nsfw === null && is_ad === null) { - const s = this.convert_file( - this.db.queryEntries( - "SELECT * FROM file ORDER BY RANDOM() LIMIT 1;", - ), - ); - return s.length ? s[0] : undefined; - } else if (is_nsfw === null) { - const s = this.convert_file( - this.db.queryEntries( - "SELECT file.* FROM file LEFT JOIN filemeta ON file.token = filemeta.token WHERE IFNULL(filemeta.is_ad, 0) = ? ORDER BY RANDOM() LIMIT 1;", - [is_ad], - ), - ); - return s.length ? s[0] : undefined; - } else if (is_ad === null) { - const s = this.convert_file( - this.db.queryEntries( - "SELECT file.* FROM file LEFT JOIN filemeta ON file.token = filemeta.token WHERE IFNULL(filemeta.is_nsfw, 0) = ? ORDER BY RANDOM() LIMIT 1;", - [is_nsfw], - ), - ); - return s.length ? s[0] : undefined; - } else { - const s = this.convert_file( - this.db.queryEntries( - "SELECT file.* FROM file LEFT JOIN filemeta ON file.token = filemeta.token WHERE IFNULL(filemeta.is_nsfw, 0) = ? AND IFNULL(filemeta.is_ad, 0) = ? ORDER BY RANDOM() LIMIT 1;", - [is_nsfw, is_ad], - ), - ); - return s.length ? s[0] : undefined; + const args = []; + let join_sql = ""; + const where_sql = []; + if (is_nsfw !== null || is_ad !== null) { + join_sql = " LEFT JOIN filemeta ON file.token = filemeta.token"; + if (is_nsfw !== null) { + where_sql.push("IFNULL(filemeta.is_nsfw, 0) = ?"); + args.push(is_nsfw); + } + if (is_ad !== null) { + where_sql.push("IFNULL(filemeta.is_ad, 0) = ?"); + args.push(is_ad); + } } + const wsql = where_sql.length + ? ` WHERE ${where_sql.join(" AND ")}` + : ""; + const s = this.convert_file( + this.db.queryEntries( + `SELECT file.* FROM file${join_sql}${wsql} ORDER BY RANDOM() LIMIT 1;`, + args, + ), + ); + return s.length ? s[0] : undefined; } get_tasks() { return this.transaction(() => diff --git a/task_manager.ts b/task_manager.ts index 39795d2..a93875e 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -143,7 +143,7 @@ export class TaskManager extends EventTarget { } async add_update_meili_search_data_task(gid?: number) { this.#check_closed(); - const otask = await this.db.check_update_meili_search_data_task(); + const otask = await this.db.check_update_meili_search_data_task(gid); if (otask !== undefined) { console.log("The task is already in list."); return otask; From 4e5dbd8390c87dae87e6f324e3a8e10ca4fd0f66 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 13:20:27 +0800 Subject: [PATCH 02/14] Format code --- islands/Settings.tsx | 28 ++++++++++++++++------------ static/common.css | 18 +++++++----------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/islands/Settings.tsx b/islands/Settings.tsx index 40ed4ac..4e6c9d5 100644 --- a/islands/Settings.tsx +++ b/islands/Settings.tsx @@ -102,7 +102,9 @@ export default class Settings extends Component { { > { button { + +.settings .ua>button { position: absolute; top: 30px; left: -10px; } - - -@media (max-width:767px) { - -} +@media (max-width:767px) {} @media (max-width:1280px) { .settings { margin: 0; } -} \ No newline at end of file +} From fe7af51988b4d3659c3399c4f240a3ed9bd4199f Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 15:15:47 +0800 Subject: [PATCH 03/14] Add support to remove old galleries --- config.ts | 5 +++ db.ts | 40 ++++++++++++++++++ islands/Settings.tsx | 7 ++++ islands/TaskManager.tsx | 5 +++ meilisearch.ts | 16 +++++++ routes/api/task.ts | 9 ++-- server/task.ts | 8 ++-- task_manager.ts | 24 +++++++++-- tasks/download.ts | 72 ++++++++++++++++++++++++++++---- translation/en/settings.jsonc | 3 +- translation/zh-cn/settings.jsonc | 3 +- 11 files changed, 169 insertions(+), 23 deletions(-) diff --git a/config.ts b/config.ts index 122c62b..4be0d7b 100644 --- a/config.ts +++ b/config.ts @@ -21,6 +21,7 @@ export type ConfigType = { ffmpeg_path: string; thumbnail_method: ThumbnailMethod; thumbnail_dir: string; + remove_previous_gallery: boolean; }; export enum ThumbnailMethod { @@ -128,6 +129,9 @@ export class Config { get thumbnail_dir() { return this._return_string("thumbnail_dir") || "./thumbnails"; } + get remove_previous_gallery() { + return this._return_bool("remove_previous_gallery") || false; + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -149,6 +153,7 @@ export class Config { ffmpeg_path: this.ffmpeg_path, thumbnail_method: this.thumbnail_method, thumbnail_dir: this.thumbnail_dir, + remove_previous_gallery: this.remove_previous_gallery, }; } } diff --git a/db.ts b/db.ts index 300800d..c9cb519 100644 --- a/db.ts +++ b/db.ts @@ -662,6 +662,32 @@ export class EhDb { delete_file(f: EhFile) { this.db.query("DELETE FROM file WHERE id = ?;", [f.id]); } + delete_files(token: string) { + const files = this.get_files(token); + this.db.query("DELETE FROM file WHERE token = ?;", [token]); + this.db.query("DELETE FROM filemeta WHERE token = ?;", [token]); + files.forEach((f) => { + try_remove_sync(f.path); + console.log("Deleted ", f.path); + }); + } + delete_gallery(gid: number) { + this.db.query("DELETE FROM gmeta WHERE gid = ?;", [gid]); + this.db.query("DELETE FROM gtag WHERE gid = ?;", [gid]); + const tokens = new Set( + this.db.query<[string]>("SELECT token FROM pmeta WHERE gid = ?;", [ + gid, + ]).map((v) => v[0]), + ); + this.db.query("DELETE FROM pmeta WHERE gid = ?;", [gid]); + for (const token of tokens) { + const count = this.db.query<[number]>( + "SELECT COUNT(*) FROM pmeta WHERE token = ?;", + [token], + )[0][0]; + if (count === 0) this.delete_files(token); + } + } delete_task(task: Task) { return this.transaction(() => { this.db.query("DELETE FROM task WHERE id = ?;", [task.id]); @@ -805,6 +831,12 @@ export class EhDb { ); return s.length ? s[0] : undefined; } + async get_task(id: number) { + const s = await this.transaction(() => + this.db.queryEntries("SELECT * FROM task WHERE id = ?;", [id]) + ); + return s.length ? s[0] : undefined; + } get_tasks() { return this.transaction(() => this.db.queryEntries("SELECT * FROM task;") @@ -885,4 +917,12 @@ export class EhDb { ]); } } + update_task(task: Task) { + return this.transaction(() => { + this.db.query("UPDATE task SET details = ? WHERE id = ?;", [ + task.details, + task.id, + ]); + }); + } } diff --git a/islands/Settings.tsx b/islands/Settings.tsx index 4e6c9d5..fe4cdd4 100644 --- a/islands/Settings.tsx +++ b/islands/Settings.tsx @@ -121,6 +121,13 @@ export default class Settings extends Component { checked={settings.export_zip_jpn_title} description={t("settings.export_zip_jpn_title")} /> +
{ task.progress = t.detail.detail; sendTaskChangedEvent(t.detail.task_id); } + } else if (t.type == "task_updated") { + const task = tasks.value.get(t.detail.id); + if (task) { + task.base = t.detail; + } } }; self.addEventListener("beforeunload", () => { diff --git a/meilisearch.ts b/meilisearch.ts index 620a41a..43b4d75 100644 --- a/meilisearch.ts +++ b/meilisearch.ts @@ -37,6 +37,7 @@ export class MeiliSearchServer { client; db; handle; + handle2; #gmeta?: Index; #inited = false; target; @@ -50,8 +51,18 @@ export class MeiliSearchServer { this.handle = (e: Event) => { this.#gallery_update(e); }; + this.handle2 = (e: Event) => { + this.#gallery_remove(e); + }; this.target = new EventTarget(); this.target.addEventListener("gallery_update", this.handle); + this.target.addEventListener("gallery_remove", this.handle2); + } + #gallery_remove(e: Event) { + const ev = e as CustomEvent; + this.removeGallery(ev.detail).catch((e) => { + console.log(e); + }); } #gallery_update(e: Event) { const ev = e as CustomEvent; @@ -78,6 +89,7 @@ export class MeiliSearchServer { } close() { this.target.removeEventListener("gallery_update", this.handle); + this.target.removeEventListener("gallery_remove", this.handle2); } async getIndex(uid: string, primaryKey?: string) { try { @@ -112,6 +124,10 @@ export class MeiliSearchServer { this.#updateGMetaSettings(); this.#inited = true; } + async removeGallery(gid: number) { + const gmeta = await this.gmeta; + await this.waitTask(gmeta.deleteDocument(gid)); + } async updateGallery(...gids: number[]) { const gmeta = await this.gmeta; const datas = gids.map((gid) => { diff --git a/routes/api/task.ts b/routes/api/task.ts index 98dc839..d8f1c63 100644 --- a/routes/api/task.ts +++ b/routes/api/task.ts @@ -26,6 +26,7 @@ export const handler: Handlers = { t.removeEventListener("task_finished", handle); t.removeEventListener("task_progress", handle); t.removeEventListener("task_error", handle); + t.removeEventListener("task_updated", handle); ExitTarget.removeEventListener("close", close_handle); }; function sendMessage(mes: TaskServerSocketData) { @@ -45,12 +46,9 @@ export const handler: Handlers = { sendMessage({ type: "close" }); socket.close(); } else if (d.type == "new_download_task") { - t.add_download_task(d.gid, d.token); + t.add_download_task(d.gid, d.token, d.cfg); } else if (d.type == "new_export_zip_task") { - t.add_export_zip_task(d.gid, { - output: d.output, - jpn_title: d.jpn_title, - }); + t.add_export_zip_task(d.gid, d.cfg); } else if (d.type == "task_list") { t.get_task_list().then((tasks) => { sendMessage({ @@ -70,6 +68,7 @@ export const handler: Handlers = { t.addEventListener("task_finished", handle); t.addEventListener("task_progress", handle); t.addEventListener("task_error", handle); + t.addEventListener("task_updated", handle); ExitTarget.addEventListener("close", close_handle); }; return response; diff --git a/server/task.ts b/server/task.ts index 7c0b473..45895bc 100644 --- a/server/task.ts +++ b/server/task.ts @@ -1,5 +1,6 @@ import { Task } from "../task.ts"; import { TaskEventData } from "../task_manager.ts"; +import { DownloadConfig } from "../tasks/download.ts"; import { ExportZipConfig } from "../tasks/export_zip.ts"; import { DiscriminatedUnion } from "../utils.ts"; @@ -9,12 +10,9 @@ export type TaskServerSocketData = TaskEventData | { type: "close" } | { running: number[]; }; -type Gid> = ({ gid: number } & T) extends - infer U ? { [Q in keyof U]: U[Q] } : never; - type EventMap = { - new_download_task: { gid: number; token: string }; - new_export_zip_task: Gid; + new_download_task: { gid: number; token: string; cfg?: DownloadConfig }; + new_export_zip_task: { gid: number; cfg?: ExportZipConfig }; }; export type TaskClientSocketData = DiscriminatedUnion<"type", EventMap> | { diff --git a/task_manager.ts b/task_manager.ts index a93875e..be2a16a 100644 --- a/task_manager.ts +++ b/task_manager.ts @@ -5,7 +5,11 @@ import { MeiliSearchServer } from "./meilisearch.ts"; import { check_running } from "./pid_check.ts"; import { add_exit_handler } from "./signal_handler.ts"; import { Task, TaskProgress, TaskProgressBasicType, TaskType } from "./task.ts"; -import { download_task } from "./tasks/download.ts"; +import { + DEFAULT_DOWNLOAD_CONFIG, + download_task, + DownloadConfig, +} from "./tasks/download.ts"; import { DEFAULT_EXPORT_ZIP_CONFIG, export_zip, @@ -29,6 +33,7 @@ type EventMap = { task_finished: Task; task_progress: TaskProgress; task_error: { task: Task; error: string }; + task_updated: Task; }; type Detail> = { @@ -96,7 +101,7 @@ export class TaskManager extends EventTarget { get aborts() { return this.#abort.signal; } - async add_download_task(gid: number, token: string) { + async add_download_task(gid: number, token: string, cfg?: DownloadConfig) { this.#check_closed(); const otask = await this.db.check_download_task(gid, token); if (otask !== undefined) { @@ -109,7 +114,7 @@ export class TaskManager extends EventTarget { id: 0, pid: Deno.pid, type: TaskType.Download, - details: null, + details: cfg ? JSON.stringify(cfg) : null, }; return await this.#add_task(task); } @@ -282,6 +287,9 @@ export class TaskManager extends EventTarget { this.#check_closed(); this.dispatchEvent("task_started", task); if (task.type == TaskType.Download) { + const cfg: DownloadConfig = task.details + ? JSON.parse(task.details) + : DEFAULT_DOWNLOAD_CONFIG; this.running_tasks.set( task.id, { @@ -293,6 +301,7 @@ export class TaskManager extends EventTarget { this.#abort.signal, this.#force_abort.signal, this, + cfg, ), base: task, }, @@ -329,6 +338,15 @@ export class TaskManager extends EventTarget { }); } } + async update_task(t: Task) { + const r = this.running_tasks.get(t.id); + if (r) { + r.base.details = t.details; + } + await this.db.update_task(t); + const a = await this.db.get_task(t.id); + if (a) this.dispatchEvent("task_updated", a); + } async waiting_unfinished_task() { while (1) { await this.check_running_tasks(); diff --git a/tasks/download.ts b/tasks/download.ts index 72292cb..104fa4e 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -15,6 +15,17 @@ import { import { join, resolve } from "std/path/mod.ts"; import { exists } from "std/fs/exists.ts"; +export type DownloadConfig = { + max_download_img_count?: number; + mpv?: boolean; + download_original_img?: boolean; + max_retry_count?: number; + remove_previous_gallery?: boolean; + replaced_gallery?: { gid: number; token: string }[]; +}; + +export const DEFAULT_DOWNLOAD_CONFIG: DownloadConfig = {}; + class DownloadManager { #abort: AbortSignal; #force_abort: AbortSignal; @@ -24,13 +35,13 @@ class DownloadManager { #task: Task; #manager: TaskManager; constructor( - cfg: Config, + max_download_img_count: number, abort: AbortSignal, force_abort: AbortSignal, task: Task, manager: TaskManager, ) { - this.#max_download_count = cfg.max_download_img_count; + this.#max_download_count = max_download_img_count; this.#running_tasks = []; this.#abort = abort; this.#force_abort = force_abort; @@ -97,6 +108,7 @@ export async function download_task( abort: AbortSignal, force_abort: AbortSignal, manager: TaskManager, + dcfg: DownloadConfig, ) { console.log("Started to download gallery", task.gid); const gdatas = await client.fetchGalleryMetadataByAPI([ @@ -116,8 +128,27 @@ export async function download_task( } const base_path = join(cfg.base, task.gid.toString()); await sure_dir(base_path); - const m = new DownloadManager(cfg, abort, force_abort, task, manager); - if (cfg.mpv) { + const max_download_img_count = dcfg.max_download_img_count !== undefined + ? dcfg.max_download_img_count + : cfg.max_download_img_count; + const m = new DownloadManager( + max_download_img_count, + abort, + force_abort, + task, + manager, + ); + const mpv_enabled = dcfg.mpv !== undefined ? dcfg.mpv : cfg.mpv; + const download_original_img = dcfg.download_original_img !== undefined + ? dcfg.download_original_img + : cfg.download_original_img; + const max_retry_count = dcfg.max_retry_count !== undefined + ? dcfg.max_retry_count + : cfg.max_retry_count; + const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined + ? dcfg.remove_previous_gallery + : cfg.remove_previous_gallery; + if (mpv_enabled) { const mpv = await client.fetchMPVPage(task.gid, task.token); m.set_total_page(mpv.pagecount); const names = mpv.imagelist.reduce( @@ -134,7 +165,7 @@ export async function download_task( if (ofiles.length) { const t = ofiles[0]; if ( - (t.is_original || !cfg.download_original_img) && + (t.is_original || !download_original_img) && (await exists(t.path)) ) { const p = db.get_pmeta_by_index(task.gid, i.index); @@ -170,7 +201,7 @@ export async function download_task( return new Promise((resolve, reject) => { const errors: unknown[] = []; function try_load(a: number) { - if (a >= cfg.max_retry_count) reject(errors); + if (a >= max_retry_count) reject(errors); i.load().then(resolve).catch((e) => { if (force_abort.aborted) { throw Error("aborted."); @@ -186,7 +217,7 @@ export async function download_task( assert(i.data); const pmeta = i.to_pmeta(); if (pmeta) db.add_pmeta(pmeta); - const download_original = cfg.download_original_img && + const download_original = download_original_img && !i.is_original; if (download_original) console.log(i.index, i.data.o); let path = resolve(join(base_path, i.name)); @@ -226,7 +257,7 @@ export async function download_task( } const errors: unknown[] = []; function try_download(a: number) { - if (a >= cfg.max_retry_count) { + if (a >= max_retry_count) { reject(errors); } download().then(resolve).catch((e) => { @@ -253,5 +284,30 @@ export async function download_task( await m.join(); if (m.has_failed_task) throw Error("Some tasks failed."); if (abort.aborted || force_abort.aborted) throw Error("aborted"); + if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) { + let replaced_gallery = dcfg.replaced_gallery; + if (replaced_gallery === undefined) { + const fg = await client.fetchGalleryPage( + gmeta.first_gid, + gmeta.first_key, + ); + replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid); + replaced_gallery.push({ + gid: gmeta.first_gid, + token: gmeta.first_key, + }); + } + replaced_gallery.forEach((g) => { + const gmeta = db.get_gmeta_by_gid(g.gid); + if (!gmeta) return; + console.log("Remove gallery ", g.gid); + if (manager.meilisearch) { + manager.meilisearch.target.dispatchEvent( + new CustomEvent("gallery_remove", { detail: gmeta.gid }), + ); + } + db.delete_gallery(g.gid); + }); + } return task; } diff --git a/translation/en/settings.jsonc b/translation/en/settings.jsonc index 10bc257..3fdf316 100644 --- a/translation/en/settings.jsonc +++ b/translation/en/settings.jsonc @@ -28,5 +28,6 @@ "thumbnail_method": "The method used to generate thumbnail: ", "thumbnail_method0": "ffmpeg binary", "thumbnail_method1": "ffmpeg API", - "thumbnail_dir": "The folder used to store thumbnails: " + "thumbnail_dir": "The folder used to store thumbnails: ", + "remove_previous_gallery": "Remove old galleries which replaced by new ones." } diff --git a/translation/zh-cn/settings.jsonc b/translation/zh-cn/settings.jsonc index 4a5ffa9..aa28c8c 100644 --- a/translation/zh-cn/settings.jsonc +++ b/translation/zh-cn/settings.jsonc @@ -28,5 +28,6 @@ "thumbnail_method": "生成缩略图的方式：", "thumbnail_method0": "FFMPEG二进制", "thumbnail_method1": "FFMPEG API", - "thumbnail_dir": "存放缩略图的文件夹：" + "thumbnail_dir": "存放缩略图的文件夹：", + "remove_previous_gallery": "移除被新画廊替代的旧画廊。" } From 02a0d975a4ad005c09037b18c627078ffab3608c Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 15:42:49 +0800 Subject: [PATCH 04/14] Add /api/files/[token] --- fresh.gen.ts | 22 ++++++++++++---------- routes/api/files/[token].ts | 22 ++++++++++++++++++++++ server/files.ts | 8 ++++++++ 3 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 routes/api/files/[token].ts create mode 100644 server/files.ts diff --git a/fresh.gen.ts b/fresh.gen.ts index 8f6e270..44c93ef 100644 --- a/fresh.gen.ts +++ b/fresh.gen.ts @@ -11,11 +11,12 @@ import * as $5 from "./routes/api/file/[id].ts"; import * as $6 from "./routes/api/file/random.ts"; import * as $7 from "./routes/api/filemeta.ts"; import * as $8 from "./routes/api/filemeta/[token].ts"; -import * as $9 from "./routes/api/gallery/[gid].ts"; -import * as $10 from "./routes/api/status.ts"; -import * as $11 from "./routes/api/task.ts"; -import * as $12 from "./routes/api/thumbnail/[id].ts"; -import * as $13 from "./routes/index.tsx"; +import * as $9 from "./routes/api/files/[token].ts"; +import * as $10 from "./routes/api/gallery/[gid].ts"; +import * as $11 from "./routes/api/status.ts"; +import * as $12 from "./routes/api/task.ts"; +import * as $13 from "./routes/api/thumbnail/[id].ts"; +import * as $14 from "./routes/index.tsx"; import * as $$0 from "./islands/Container.tsx"; import * as $$1 from "./islands/Settings.tsx"; import * as $$2 from "./islands/TaskManager.tsx"; @@ -31,11 +32,12 @@ const manifest = { "./routes/api/file/random.ts": $6, "./routes/api/filemeta.ts": $7, "./routes/api/filemeta/[token].ts": $8, - "./routes/api/gallery/[gid].ts": $9, - "./routes/api/status.ts": $10, - "./routes/api/task.ts": $11, - "./routes/api/thumbnail/[id].ts": $12, - "./routes/index.tsx": $13, + "./routes/api/files/[token].ts": $9, + "./routes/api/gallery/[gid].ts": $10, + "./routes/api/status.ts": $11, + "./routes/api/task.ts": $12, + "./routes/api/thumbnail/[id].ts": $13, + "./routes/index.tsx": $14, }, islands: { "./islands/Container.tsx": $$0, diff --git a/routes/api/files/[token].ts b/routes/api/files/[token].ts new file mode 100644 index 0000000..29551cc --- /dev/null +++ b/routes/api/files/[token].ts @@ -0,0 +1,22 @@ +import { Handlers } from "$fresh/server.ts"; +import { get_task_manager } from "../../../server.ts"; +import { EhFiles } from "../../../server/files.ts"; +import { return_data } from "../../../server/utils.ts"; + +export const handler: Handlers = { + GET(_req, ctx) { + const tokens = ctx.params.token.split(","); + const m = get_task_manager(); + const data: EhFiles = {}; + for (const token of tokens) { + data[token] = m.db.get_files(token).map((d) => { + /**@ts-ignore */ + delete d.path; + /**@ts-ignore */ + delete d.token; + return d; + }); + } + return return_data(data); + }, +}; diff --git a/server/files.ts b/server/files.ts new file mode 100644 index 0000000..429d866 --- /dev/null +++ b/server/files.ts @@ -0,0 +1,8 @@ +export type EhFileBasic = { + id: number; + width: number; + height: number; + is_original: boolean; +}; + +export type EhFiles = Record; From d38c98aee471ae868782daee751aa5e6f0d8ebe7 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 16:59:44 +0800 Subject: [PATCH 05/14] Update --- client.ts | 6 ++- page/GalleryPage.ts | 91 ++++++++++++++++++++++++++++++++++++++++ page/GalleryPage_test.ts | 18 ++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/client.ts b/client.ts index f15de52..1bd9c95 100644 --- a/client.ts +++ b/client.ts @@ -129,10 +129,12 @@ export class Client { * Fetch a gallery page (use HTML) * @param gid Gallery ID * @param token Token + * @param page Page number * @returns */ - async fetchGalleryPage(gid: number, token: string) { - const url = `https://${this.host}/g/${gid}/${token}/`; + async fetchGalleryPage(gid: number, token: string, page?: number) { + let url = `https://${this.host}/g/${gid}/${token}/`; + if (page) url += `?p=${page}`; const re = await this.get(url); if (re.status != 200) { throw new Error( diff --git a/page/GalleryPage.ts b/page/GalleryPage.ts index 0a2abdf..625959d 100644 --- a/page/GalleryPage.ts +++ b/page/GalleryPage.ts @@ -3,6 +3,24 @@ import { Client } from "../client.ts"; import { initDOMParser, map, parse_bool } from "../utils.ts"; import { parseUrl, UrlType } from "../url.ts"; +type Page = { + token: string; + thumbnail: string; + name: string; +}; + +class Image { + base; + /**Page number*/ + index; + #gp; + constructor(base: Page, index: number, gp: GalleryPage) { + this.base = base; + this.index = index; + this.#gp = gp; + } +} + class GalleryPage { dom; doc; @@ -12,7 +30,9 @@ class GalleryPage { #meta_script: string | undefined = undefined; #gid: number | undefined = undefined; #token: string | undefined = undefined; + #mpv_enabled: boolean | undefined = undefined; #new_version: Array<{ gid: number; token: string }> | undefined = undefined; + #imagelist: Image[] | undefined = undefined; constructor(html: string, client: Client) { const dom = (new DOMParser()).parseFromString(html, "text/html"); if (!dom) { @@ -26,6 +46,39 @@ class GalleryPage { this.doc = doc; this.client = client; } + async #get_imagelist() { + function load_image(doc: Element): Page[] { + const eles = doc.querySelectorAll("#gdt > div[class^=gdt]"); + return map(eles, (e) => { + const b = e as Element; + const a = b.querySelector("a"); + if (!a) throw Error("Link not found."); + const href = a.getAttribute("href"); + if (!href) throw Error("Link not found."); + const u = parseUrl(href); + if (!u || u.type !== UrlType.Single) { + throw Error("Failed to parse url."); + } + const token = u.token; + const img = b.querySelector("img"); + if (!img) throw Error("Image not found."); + const thumbnail = img.getAttribute("src"); + if (!thumbnail) throw Error("Image source not found"); + const name = img.getAttribute("title")?.match(/page \d+: (.*)/i) + ?.at(1); + if (!name) throw Error("name not found"); + return { name, token, thumbnail }; + }); + } + let b = load_image(this.doc); + // deno-lint-ignore no-this-alias + let now: GalleryPage = this; + while (now.has_next_page) { + now = await now.next_page(); + b = b.concat(load_image(now.doc)); + } + return b.map((v, i) => new Image(v, i + 1, this)); + } get favorited() { const o = this.gdd_data.get("Favorited")?.innerText; if (!o) return undefined; @@ -57,6 +110,19 @@ class GalleryPage { } return this.#gid; } + get has_next_page() { + return this.doc.querySelector(".ptt td:last-child a") !== null; + } + get imagelist(): Promise { + return new Promise((resolve, reject) => { + if (this.#imagelist === undefined) { + this.#get_imagelist().then((imagelist) => { + this.#imagelist = imagelist; + resolve(imagelist); + }).catch(reject); + } else resolve(this.#imagelist); + }); + } get language() { const o = this.gdd_data.get("Language")?.innerText; if (!o) return undefined; @@ -83,6 +149,19 @@ class GalleryPage { throw Error("Failed to locate meta script."); } else return this.#meta_script; } + get mpv_enabled() { + if (this.#mpv_enabled === undefined) { + const e = this.doc.querySelector("#gdt a"); + if (!e) throw Error("Page not found."); + const u = e.getAttribute("href"); + if (!u) throw Error("Url not found."); + const p = parseUrl(u); + if (!p) throw Error("Failed to parse url."); + const mpv_enabled = p.type === UrlType.MPV; + this.#mpv_enabled = mpv_enabled; + return mpv_enabled; + } else return this.#mpv_enabled; + } get name() { const ele = this.doc.getElementById("gn"); if (!ele) throw Error("Failed to find gallery's name."); @@ -106,6 +185,18 @@ class GalleryPage { return d; } else return this.#new_version; } + async next_page() { + const url = this.doc.querySelector(".ptt td:last-child a") + ?.getAttribute("href"); + if (!url) throw Error("Url not found."); + const re = await this.client.get(url); + if (re.status != 200) { + throw new Error( + `Fetch ${url} failed, status ${re.status} ${re.statusText}`, + ); + } + return load_gallery_page(await re.text(), this.client); + } get japanese_name() { return this.doc.getElementById("gj")?.innerText; } diff --git a/page/GalleryPage_test.ts b/page/GalleryPage_test.ts index 9e23ada..1097255 100644 --- a/page/GalleryPage_test.ts +++ b/page/GalleryPage_test.ts @@ -37,6 +37,7 @@ Deno.test({ assertEquals(re.gid, 2552611); assertEquals(re.token, "3132307627"); assertEquals(re.new_version.length, 0); + if (!re.mpv_enabled) assertEquals((await re.imagelist).length, 19); }); Deno.test({ @@ -50,4 +51,21 @@ Deno.test({ assertEquals(re.japanese_name, ""); assertEquals(re.length, 42); assertEquals(re.new_version[0], { gid: 2223198, token: "2a5788135e" }); + if (!re.mpv_enabled) assertEquals((await re.imagelist).length, 42); +}); + +Deno.test({ + name: "GalleryPage_test3", + permissions: API_PERMISSION, +}, async () => { + const cfg = await load_settings("./config.json"); + const client = new Client(cfg); + const re = await client.fetchGalleryPage(2576265, "daa01f773d"); + assertEquals(re.name, "[Fanbox] houk1se1 (2021.09.05 - 2023.06.07)"); + assertEquals( + re.japanese_name, + "[Fanbox] ほうき星 (2021.09.05 - 2023.06.07)", + ); + assertEquals(re.length, 820); + if (!re.mpv_enabled) assertEquals((await re.imagelist).length, 820); }); From df41bcb220fda5bba021973c9f3f89113ce83762 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 17:27:17 +0800 Subject: [PATCH 06/14] Bug fix --- routes/api/task.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/api/task.ts b/routes/api/task.ts index d8f1c63..f9ef1dd 100644 --- a/routes/api/task.ts +++ b/routes/api/task.ts @@ -14,7 +14,9 @@ export const handler: Handlers = { const handle = ( e: CustomEvent, ) => { - socket.send(JSON.stringify({ type: e.type, detail: e.detail })); + if (socket.readyState === socket.OPEN) { + socket.send(JSON.stringify({ type: e.type, detail: e.detail })); + } }; const close_handle = () => { sendMessage({ type: "close" }); From 6e52248aa052b076e26cf1c20ce61ef997648823 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 23 Jun 2023 21:01:45 +0800 Subject: [PATCH 07/14] Update --- page/GalleryPage.ts | 18 ++++++---- page/SinglePage.ts | 74 ++++++++++++++++++++++++++++++++++++++++- page/SinglePage_test.ts | 21 ++++++++++++ 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/page/GalleryPage.ts b/page/GalleryPage.ts index 625959d..67a96ae 100644 --- a/page/GalleryPage.ts +++ b/page/GalleryPage.ts @@ -2,23 +2,26 @@ import { DOMParser, Element } from "deno_dom/deno-dom-wasm-noinit.ts"; import { Client } from "../client.ts"; import { initDOMParser, map, parse_bool } from "../utils.ts"; import { parseUrl, UrlType } from "../url.ts"; +import { SinglePage } from "./SinglePage.ts"; type Page = { token: string; thumbnail: string; name: string; + index: number; }; class Image { base; - /**Page number*/ - index; #gp; - constructor(base: Page, index: number, gp: GalleryPage) { + data: SinglePage | undefined; + constructor(base: Page, gp: GalleryPage) { this.base = base; - this.index = index; this.#gp = gp; } + get is_original() { + return this.data?.is_original; + } } class GalleryPage { @@ -56,10 +59,11 @@ class GalleryPage { const href = a.getAttribute("href"); if (!href) throw Error("Link not found."); const u = parseUrl(href); - if (!u || u.type !== UrlType.Single) { + if (!u || u.type !== UrlType.Single || u.index === undefined) { throw Error("Failed to parse url."); } const token = u.token; + const index = u.index; const img = b.querySelector("img"); if (!img) throw Error("Image not found."); const thumbnail = img.getAttribute("src"); @@ -67,7 +71,7 @@ class GalleryPage { const name = img.getAttribute("title")?.match(/page \d+: (.*)/i) ?.at(1); if (!name) throw Error("name not found"); - return { name, token, thumbnail }; + return { name, token, thumbnail, index }; }); } let b = load_image(this.doc); @@ -77,7 +81,7 @@ class GalleryPage { now = await now.next_page(); b = b.concat(load_image(now.doc)); } - return b.map((v, i) => new Image(v, i + 1, this)); + return b.map((v) => new Image(v, this)); } get favorited() { const o = this.gdd_data.get("Favorited")?.innerText; diff --git a/page/SinglePage.ts b/page/SinglePage.ts index 84c859d..1f8a951 100644 --- a/page/SinglePage.ts +++ b/page/SinglePage.ts @@ -2,12 +2,18 @@ import { DOMParser } from "deno_dom/deno-dom-wasm-noinit.ts"; import { Client } from "../client.ts"; import { initDOMParser } from "../utils.ts"; -class SinglePage { +export class SinglePage { dom; doc; client; _meta: string | undefined; _gid: number | undefined; + #i2_data: string | undefined; + #i7_data: string | undefined; + #oxres: number | undefined; + #oyres: number | undefined; + #xres: number | undefined; + #yres: number | undefined; constructor(html: string, client: Client) { const dom = (new DOMParser()).parseFromString(html, "text/html"); if (!dom) { @@ -44,6 +50,28 @@ class SinglePage { } return this._gid; } + get i2_data() { + if (this.#i2_data === undefined) { + const ele = this.doc.querySelector("#i2 > div:not([class=sn])"); + if (!ele) throw Error("Element not found."); + /**@ts-ignore */ + const e = ele as HTMLElement; + const i2_data = e.innerText; + this.#i2_data = i2_data; + return i2_data; + } else return this.#i2_data; + } + get i7_data() { + if (this.#i7_data === undefined) { + const ele = this.doc.querySelector("#i7 a"); + if (!ele) throw Error("Element not found."); + /**@ts-ignore */ + const e = ele as HTMLElement; + const i7_data = e.innerText; + this.#i7_data = i7_data; + return i7_data; + } else return this.#i7_data; + } get img_url() { const img = this.doc.querySelector("#img"); if (!img) throw Error("Unknown image url."); @@ -51,6 +79,9 @@ class SinglePage { if (!url) throw Error("Unknown image url."); return url; } + get is_original() { + return this.original_url === null; + } get meta() { if (this._meta === undefined) { const scripts = this.doc.getElementsByTagName("script"); @@ -63,6 +94,9 @@ class SinglePage { } return this._meta; } + get name() { + return this.i2_data.match(/(.*?) ::/)?.at(1); + } async nextPage() { const url = this.nextPageUrl; if (!url) return null; @@ -74,6 +108,26 @@ class SinglePage { if (a === null) return null; return a.getAttribute("href"); } + get origin_xres() { + if (this.is_original) return this.xres; + if (this.#oxres === undefined) { + const ox = this.i7_data.match(/(\d+) x \d+/)?.at(1); + if (!ox) throw Error("Failed to parse width."); + const oxres = parseInt(ox); + this.#oxres = oxres; + return oxres; + } else return this.#oxres; + } + get origin_yres() { + if (this.is_original) return this.yres; + if (this.#oyres === undefined) { + const oy = this.i7_data.match(/\d+ x (\d+)/)?.at(1); + if (!oy) throw Error("Failed to parse height."); + const oyres = parseInt(oy); + this.#oyres = oyres; + return oyres; + } else return this.#oyres; + } get original_url() { const a = this.doc.querySelector("#i7 a"); if (a == null) return null; @@ -95,6 +149,24 @@ class SinglePage { if (a === null) return null; return a.getAttribute("href"); } + get xres() { + if (this.#xres === undefined) { + const xr = this.i2_data.match(/.*? :: (\d+)/)?.at(1); + if (!xr) throw Error("Failed to parse width."); + const xres = parseInt(xr); + this.#xres = xres; + return xres; + } else return this.#xres; + } + get yres() { + if (this.#yres === undefined) { + const yr = this.i2_data.match(/.*? :: \d+ x (\d+)/)?.at(1); + if (!yr) throw Error("Failed to parse height."); + const yres = parseInt(yr); + this.#yres = yres; + return yres; + } else return this.#yres; + } } export async function load_single_page(html: string, client: Client) { diff --git a/page/SinglePage_test.ts b/page/SinglePage_test.ts index 6986f02..7c1585e 100644 --- a/page/SinglePage_test.ts +++ b/page/SinglePage_test.ts @@ -16,4 +16,25 @@ Deno.test({ assert(np); assertEquals(np.currentIndex, 2); assertEquals(np.gid, re.gid); + if (re.is_original) { + assertEquals(re.xres, 2067); + assertEquals(re.yres, 3001); + assertEquals(re.name, "logo.png"); + } + assertEquals(re.origin_xres, 2067); + assertEquals(re.origin_yres, 3001); + if (np.is_original) { + assertEquals(np.xres, 2067); + assertEquals(np.yres, 3001); + assertEquals(np.name, "1.png"); + } + assertEquals(np.origin_xres, 2067); + assertEquals(np.origin_yres, 3001); + const re2 = await client.fetchSignlePage(2028320, "1f48eb617e", 19); + assertEquals(re2.currentIndex, 19); + assertEquals(re2.gid, 2028320); + assertEquals(re2.name, "18.jpg"); + assertEquals(re2.is_original, false); + assertEquals(re2.origin_xres, 4893); + assertEquals(re2.origin_yres, 3446); }); From edca5f2b9f5fd2cd3f029cff2ba88b5ab2bb2d47 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 10:03:05 +0800 Subject: [PATCH 08/14] Add Git Hooks --- .gitattributes | 1 + hooks/pre-commit | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 .gitattributes create mode 100644 hooks/pre-commit diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e44ee1f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +hooks/* eol=lf diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100644 index 0000000..7eb8b44 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,2 @@ +#!/bin/sh +deno fmt From 837aa9591301a1cc633191eb3713d6ed28c18612 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 10:45:58 +0800 Subject: [PATCH 09/14] Add non-MPV mode support --- client.ts | 11 +++- page/GalleryPage.ts | 119 +++++++++++++++++++++++++++++++++++ page/SinglePage.ts | 15 ++++- page/SinglePage_test.ts | 1 + tasks/download.ts | 134 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 274 insertions(+), 6 deletions(-) diff --git a/client.ts b/client.ts index 1bd9c95..cc353c2 100644 --- a/client.ts +++ b/client.ts @@ -94,10 +94,17 @@ export class Client { * @param gid Gallery ID * @param page_token Page token * @param index Page index + * @param nl Reload token * @returns */ - async fetchSignlePage(gid: number, page_token: string, index: number) { - const url = `https://${this.host}/s/${page_token}/${gid}-${index}`; + async fetchSignlePage( + gid: number, + page_token: string, + index: number, + nl?: string, + ) { + const p = nl ? `?nl=${nl}` : ""; + const url = `https://${this.host}/s/${page_token}/${gid}-${index}${p}`; const re = await this.get(url); if (re.status != 200) { throw new Error( diff --git a/page/GalleryPage.ts b/page/GalleryPage.ts index 67a96ae..2e7eba5 100644 --- a/page/GalleryPage.ts +++ b/page/GalleryPage.ts @@ -1,5 +1,6 @@ import { DOMParser, Element } from "deno_dom/deno-dom-wasm-noinit.ts"; import { Client } from "../client.ts"; +import { EhFile, PMeta } from "../db.ts"; import { initDOMParser, map, parse_bool } from "../utils.ts"; import { parseUrl, UrlType } from "../url.ts"; import { SinglePage } from "./SinglePage.ts"; @@ -15,13 +16,131 @@ class Image { base; #gp; data: SinglePage | undefined; + redirected_url: string | undefined; constructor(base: Page, gp: GalleryPage) { this.base = base; this.#gp = gp; } + get_file(path: string): EhFile | undefined { + const width = this.xres; + if (width === undefined) return undefined; + const height = this.yres; + if (height === undefined) return undefined; + const is_original = this.is_original; + if (is_original === undefined) return undefined; + return { + id: 0, + token: this.page_token, + path, + width, + height, + is_original, + }; + } + get_original_file(path: string): EhFile | undefined { + const width = this.origin_xres; + if (width === undefined) return undefined; + const height = this.origin_yres; + if (height === undefined) return undefined; + return { + id: 0, + token: this.page_token, + path, + width, + height, + is_original: true, + }; + } + get index() { + return this.base.index; + } get is_original() { return this.data?.is_original; } + get name() { + return this.base.name; + } + get original_imgurl() { + return this.data?.original_url; + } + get origin_xres() { + return this.data?.origin_xres; + } + get origin_yres() { + return this.data?.origin_yres; + } + get page_number() { + return this.base.index; + } + get page_token() { + return this.base.token; + } + get src() { + return this.data?.img_url; + } + get xres() { + return this.data?.xres; + } + get yres() { + return this.data?.yres; + } + async load() { + if (this.data === undefined) { + this.data = await this.#gp.client.fetchSignlePage( + this.#gp.gid, + this.page_token, + this.base.index, + ); + } else { + this.data = await this.#gp.client.fetchSignlePage( + this.#gp.gid, + this.page_token, + this.base.index, + this.data.nl, + ); + } + } + async #load_image(u: string) { + const re = await this.#gp.client.get(u); + if (re.status !== 200) { + re.body?.cancel(); + return undefined; + } + return re; + } + async load_image(reload = true) { + const src = this.src; + if (src) { + const re = await this.#load_image(src); + if (re) return re; + } + if (!reload) return; + await this.load(); + const src2 = this.src; + if (src2) return await this.#load_image(src2); + } + async load_original_image() { + if (this.redirected_url) { + const re = await this.#load_image(this.redirected_url); + if (re) return re; + } + const url = this.original_imgurl; + if (!url) return undefined; + this.redirected_url = await this.#gp.client.redirect(url); + return await this.#load_image(this.redirected_url || url); + } + to_pmeta(): PMeta | undefined { + if (!this.data) return undefined; + const gid = this.#gp.gid; + const index = this.base.index; + const token = this.page_token; + const name = this.name; + const width = this.origin_xres; + if (width === undefined) return undefined; + const height = this.origin_yres; + if (height === undefined) return undefined; + return { gid, index, token, name, width, height }; + } } class GalleryPage { diff --git a/page/SinglePage.ts b/page/SinglePage.ts index 1f8a951..5f99b7c 100644 --- a/page/SinglePage.ts +++ b/page/SinglePage.ts @@ -80,7 +80,7 @@ export class SinglePage { return url; } get is_original() { - return this.original_url === null; + return this.original_url === undefined; } get meta() { if (this._meta === undefined) { @@ -108,6 +108,15 @@ export class SinglePage { if (a === null) return null; return a.getAttribute("href"); } + get nl() { + const f = this.doc.getElementById("loadfail"); + if (!f) throw Error("Failed to find loadfail link."); + const n = f.getAttribute("onclick"); + if (!n) throw Error("Failed to get onclick attribute."); + const nl = n.match(/nl\('(.*?)'\)/)?.at(1); + if (!nl) throw Error("Failed to extract nl."); + return nl; + } get origin_xres() { if (this.is_original) return this.xres; if (this.#oxres === undefined) { @@ -130,8 +139,8 @@ export class SinglePage { } get original_url() { const a = this.doc.querySelector("#i7 a"); - if (a == null) return null; - return a.getAttribute("href"); + if (a == null) return undefined; + return a.getAttribute("href") || undefined; } get pageCount() { const e = this.doc.querySelector("#i2>div span:last-child"); diff --git a/page/SinglePage_test.ts b/page/SinglePage_test.ts index 7c1585e..d03b963 100644 --- a/page/SinglePage_test.ts +++ b/page/SinglePage_test.ts @@ -37,4 +37,5 @@ Deno.test({ assertEquals(re2.is_original, false); assertEquals(re2.origin_xres, 4893); assertEquals(re2.origin_yres, 3446); + console.log(np.nl, re.nl, re2.nl); }); diff --git a/tasks/download.ts b/tasks/download.ts index 104fa4e..9417c5c 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -148,7 +148,8 @@ export async function download_task( const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined ? dcfg.remove_previous_gallery : cfg.remove_previous_gallery; - if (mpv_enabled) { + const g = await client.fetchGalleryPage(task.gid, task.token); + if (mpv_enabled || g.mpv_enabled) { const mpv = await client.fetchMPVPage(task.gid, task.token); m.set_total_page(mpv.pagecount); const names = mpv.imagelist.reduce( @@ -280,6 +281,137 @@ export async function download_task( return; }); } + } else { + m.set_total_page(g.length); + const imagelist = await g.imagelist; + const names = imagelist.reduce( + (acc: Record, cur) => { + const curr = cur.name; + return acc[curr] ? ++acc[curr] : acc[curr] = 1, acc; + }, + {}, + ); + for (const i of imagelist) { + if (abort.aborted) break; + await m.add_new_task(async () => { + const ofiles = db.get_files(i.page_token); + if (ofiles.length) { + const t = ofiles[0]; + if ( + (t.is_original || !download_original_img) && + (await exists(t.path)) + ) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, + i.page_token, + ); + if (op) { + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } else { + const ops = db.get_pmeta_by_token_only( + i.page_token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } + } + } + console.log("Already download page", i.index); + return; + } + } + function load() { + return new Promise((resolve, reject) => { + const errors: unknown[] = []; + function try_load(a: number) { + if (a >= max_retry_count) reject(errors); + i.load().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_load(a + 1); + }); + } + try_load(0); + }); + } + await load(); + assert(i.data); + const pmeta = i.to_pmeta(); + if (pmeta) db.add_pmeta(pmeta); + const download_original = download_original_img && + !i.is_original; + let path = resolve(join(base_path, i.name)); + if (names[i.name] > 1) { + path = add_suffix_to_path(path, i.page_token); + console.log("Changed path to", path); + } + function download_img() { + return new Promise((resolve, reject) => { + async function download() { + const re = await (download_original + ? i.load_original_image() + : i.load_image()); + if (re === undefined) { + throw Error("Failed to fetch image."); + } + if (re.body === null) { + throw Error("Response don't have a body."); + } + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, + }); + try { + await re.body.pipeTo(f.writable, { + signal: force_abort, + preventClose: true, + }); + } finally { + try { + f.close(); + } catch (_) { + null; + } + } + } + const errors: unknown[] = []; + function try_download(a: number) { + if (a >= max_retry_count) { + reject(errors); + } + download().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_download(a + 1); + }); + } + try_download(0); + }); + } + await download_img(); + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + db.add_file(f); + return; + }); + } } await m.join(); if (m.has_failed_task) throw Error("Some tasks failed."); From 2fb8b4b74c139721838b1ffffbc2493fc9b58bae Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 10:58:42 +0800 Subject: [PATCH 10/14] Update --- db.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/db.ts b/db.ts index c9cb519..6b50e0b 100644 --- a/db.ts +++ b/db.ts @@ -448,6 +448,12 @@ export class EhDb { ofiles.slice(1).forEach((o) => { this.delete_file(o); }); + ofiles.forEach((o) => { + if (o.path !== f.path) { + try_remove_sync(o.path); + console.log("Deleted ", o.path); + } + }); } } if (f.id) { From 214c41959391d6a52778411c550dd30c274912e8 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 11:35:07 +0800 Subject: [PATCH 11/14] Use interface --- tasks/download.ts | 371 +++++++++++++++++----------------------------- 1 file changed, 136 insertions(+), 235 deletions(-) diff --git a/tasks/download.ts b/tasks/download.ts index 9417c5c..5e89bde 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -1,7 +1,7 @@ import { assert } from "std/testing/asserts.ts"; import { Client } from "../client.ts"; import { Config } from "../config.ts"; -import { EhDb } from "../db.ts"; +import { EhDb, EhFile, PMeta } from "../db.ts"; import { Task, TaskDownloadProgess, TaskType } from "../task.ts"; import { TaskManager } from "../task_manager.ts"; import { @@ -73,7 +73,7 @@ class DownloadManager { this.#progress, ); } - async add_new_task(f: () => Promise) { + async add_new_task(f: () => Promise) { while (1) { if (this.#abort.aborted) break; await this.#check_tasks(); @@ -100,6 +100,20 @@ class DownloadManager { } } +interface Image { + data: unknown; + index: number; + is_original: boolean | undefined; + name: string; + page_token: string; + get_file(path: string): EhFile | undefined; + get_original_file(path: string): EhFile | undefined; + load(): Promise; + load_image(reload?: boolean): Promise; + load_original_image(): Promise; + to_pmeta(): PMeta | undefined; +} + export async function download_task( task: Task, client: Client, @@ -149,6 +163,124 @@ export async function download_task( ? dcfg.remove_previous_gallery : cfg.remove_previous_gallery; const g = await client.fetchGalleryPage(task.gid, task.token); + async function download_task(names: Record, i: Image) { + const ofiles = db.get_files(i.page_token); + if (ofiles.length) { + const t = ofiles[0]; + if ( + (t.is_original || !download_original_img) && + (await exists(t.path)) + ) { + const p = db.get_pmeta_by_index(task.gid, i.index); + if (!p) { + const op = db.get_pmeta_by_token( + task.gid, + i.page_token, + ); + if (op) { + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } else { + const ops = db.get_pmeta_by_token_only( + i.page_token, + ); + if (ops.length) { + const op = ops[0]; + op.gid = task.gid; + op.index = i.index; + op.name = i.name; + db.add_pmeta(op); + return; + } + } + } + console.log("Already download page", i.index); + return; + } + } + function load() { + return new Promise((resolve, reject) => { + const errors: unknown[] = []; + function try_load(a: number) { + if (a >= max_retry_count) reject(errors); + i.load().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_load(a + 1); + }); + } + try_load(0); + }); + } + await load(); + assert(i.data); + const pmeta = i.to_pmeta(); + if (pmeta) db.add_pmeta(pmeta); + const download_original = download_original_img && + !i.is_original; + let path = resolve(join(base_path, i.name)); + if (names[i.name] > 1) { + path = add_suffix_to_path(path, i.page_token); + console.log("Changed path to", path); + } + function download_img() { + return new Promise((resolve, reject) => { + async function download() { + const re = await (download_original + ? i.load_original_image() + : i.load_image()); + if (re === undefined) { + throw Error("Failed to fetch image."); + } + if (re.body === null) { + throw Error("Response don't have a body."); + } + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, + }); + try { + await re.body.pipeTo(f.writable, { + signal: force_abort, + preventClose: true, + }); + } finally { + try { + f.close(); + } catch (_) { + null; + } + } + } + const errors: unknown[] = []; + function try_download(a: number) { + if (a >= max_retry_count) { + reject(errors); + } + download().then(resolve).catch((e) => { + if (force_abort.aborted) { + throw Error("aborted."); + } + errors.push(e); + try_download(a + 1); + }); + } + try_download(0); + }); + } + await download_img(); + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + db.add_file(f); + return; + } if (mpv_enabled || g.mpv_enabled) { const mpv = await client.fetchMPVPage(task.gid, task.token); m.set_total_page(mpv.pagecount); @@ -162,123 +294,7 @@ export async function download_task( for (const i of mpv.imagelist) { if (abort.aborted) break; await m.add_new_task(async () => { - const ofiles = db.get_files(i.page_token); - if (ofiles.length) { - const t = ofiles[0]; - if ( - (t.is_original || !download_original_img) && - (await exists(t.path)) - ) { - const p = db.get_pmeta_by_index(task.gid, i.index); - if (!p) { - const op = db.get_pmeta_by_token( - task.gid, - i.page_token, - ); - if (op) { - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } else { - const ops = db.get_pmeta_by_token_only( - i.page_token, - ); - if (ops.length) { - const op = ops[0]; - op.gid = task.gid; - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } - } - } - console.log("Already download page", i.index); - return; - } - } - function load() { - return new Promise((resolve, reject) => { - const errors: unknown[] = []; - function try_load(a: number) { - if (a >= max_retry_count) reject(errors); - i.load().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_load(a + 1); - }); - } - try_load(0); - }); - } - await load(); - assert(i.data); - const pmeta = i.to_pmeta(); - if (pmeta) db.add_pmeta(pmeta); - const download_original = download_original_img && - !i.is_original; - if (download_original) console.log(i.index, i.data.o); - let path = resolve(join(base_path, i.name)); - if (names[i.name] > 1) { - path = add_suffix_to_path(path, i.page_token); - console.log("Changed path to", path); - } - function download_img() { - return new Promise((resolve, reject) => { - async function download() { - const re = await (download_original - ? i.load_original_image() - : i.load_image()); - if (re === undefined) { - throw Error("Failed to fetch image."); - } - if (re.body === null) { - throw Error("Response don't have a body."); - } - const f = await Deno.open(path, { - create: true, - write: true, - truncate: true, - }); - try { - await re.body.pipeTo(f.writable, { - signal: force_abort, - preventClose: true, - }); - } finally { - try { - f.close(); - } catch (_) { - null; - } - } - } - const errors: unknown[] = []; - function try_download(a: number) { - if (a >= max_retry_count) { - reject(errors); - } - download().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_download(a + 1); - }); - } - try_download(0); - }); - } - await download_img(); - const f = download_original - ? i.get_original_file(path) - : i.get_file(path); - if (f === undefined) throw Error("Failed to get file."); - db.add_file(f); - return; + await download_task(names, i); }); } } else { @@ -294,122 +310,7 @@ export async function download_task( for (const i of imagelist) { if (abort.aborted) break; await m.add_new_task(async () => { - const ofiles = db.get_files(i.page_token); - if (ofiles.length) { - const t = ofiles[0]; - if ( - (t.is_original || !download_original_img) && - (await exists(t.path)) - ) { - const p = db.get_pmeta_by_index(task.gid, i.index); - if (!p) { - const op = db.get_pmeta_by_token( - task.gid, - i.page_token, - ); - if (op) { - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } else { - const ops = db.get_pmeta_by_token_only( - i.page_token, - ); - if (ops.length) { - const op = ops[0]; - op.gid = task.gid; - op.index = i.index; - op.name = i.name; - db.add_pmeta(op); - return; - } - } - } - console.log("Already download page", i.index); - return; - } - } - function load() { - return new Promise((resolve, reject) => { - const errors: unknown[] = []; - function try_load(a: number) { - if (a >= max_retry_count) reject(errors); - i.load().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_load(a + 1); - }); - } - try_load(0); - }); - } - await load(); - assert(i.data); - const pmeta = i.to_pmeta(); - if (pmeta) db.add_pmeta(pmeta); - const download_original = download_original_img && - !i.is_original; - let path = resolve(join(base_path, i.name)); - if (names[i.name] > 1) { - path = add_suffix_to_path(path, i.page_token); - console.log("Changed path to", path); - } - function download_img() { - return new Promise((resolve, reject) => { - async function download() { - const re = await (download_original - ? i.load_original_image() - : i.load_image()); - if (re === undefined) { - throw Error("Failed to fetch image."); - } - if (re.body === null) { - throw Error("Response don't have a body."); - } - const f = await Deno.open(path, { - create: true, - write: true, - truncate: true, - }); - try { - await re.body.pipeTo(f.writable, { - signal: force_abort, - preventClose: true, - }); - } finally { - try { - f.close(); - } catch (_) { - null; - } - } - } - const errors: unknown[] = []; - function try_download(a: number) { - if (a >= max_retry_count) { - reject(errors); - } - download().then(resolve).catch((e) => { - if (force_abort.aborted) { - throw Error("aborted."); - } - errors.push(e); - try_download(a + 1); - }); - } - try_download(0); - }); - } - await download_img(); - const f = download_original - ? i.get_original_file(path) - : i.get_file(path); - if (f === undefined) throw Error("Failed to get file."); - db.add_file(f); - return; + await download_task(names, i); }); } } From 494a0f66e310ffc49719367a1ca35331e2c7c51a Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 13:42:24 +0800 Subject: [PATCH 12/14] Add sampled_name --- page/GalleryPage.ts | 10 ++++++++++ page/MPVPage.ts | 8 ++++++++ tasks/download.ts | 6 +++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/page/GalleryPage.ts b/page/GalleryPage.ts index 2e7eba5..c32495c 100644 --- a/page/GalleryPage.ts +++ b/page/GalleryPage.ts @@ -1,4 +1,5 @@ import { DOMParser, Element } from "deno_dom/deno-dom-wasm-noinit.ts"; +import { extname } from "std/path/mod.ts"; import { Client } from "../client.ts"; import { EhFile, PMeta } from "../db.ts"; import { initDOMParser, map, parse_bool } from "../utils.ts"; @@ -75,6 +76,15 @@ class Image { get page_token() { return this.base.token; } + get sampled_name() { + const name = this.data?.name; + if (name) return name; + const n = this.base.name; + const e = extname(n); + const b = n.slice(0, n.length - e.length); + if (n === ".gif") return `${b}.gif`; + return `${b}.jpg`; + } get src() { return this.data?.img_url; } diff --git a/page/MPVPage.ts b/page/MPVPage.ts index 895889f..f32ea1a 100644 --- a/page/MPVPage.ts +++ b/page/MPVPage.ts @@ -1,4 +1,5 @@ import { DOMParser } from "deno_dom/deno-dom-wasm-noinit.ts"; +import { extname } from "std/path/mod.ts"; import { Client } from "../client.ts"; import { initDOMParser } from "../utils.ts"; import { EhFile, PMeta } from "../db.ts"; @@ -122,6 +123,13 @@ class MPVImage { get page_token() { return this.base.k; } + get sampled_name() { + const n = this.base.n; + const e = extname(n); + const b = n.slice(0, n.length - e.length); + if (n === ".gif") return `${b}.gif`; + return `${b}.jpg`; + } get src() { return this.data?.i; } diff --git a/tasks/download.ts b/tasks/download.ts index 5e89bde..0ed71a7 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -106,6 +106,7 @@ interface Image { is_original: boolean | undefined; name: string; page_token: string; + sampled_name: string; get_file(path: string): EhFile | undefined; get_original_file(path: string): EhFile | undefined; load(): Promise; @@ -222,7 +223,10 @@ export async function download_task( if (pmeta) db.add_pmeta(pmeta); const download_original = download_original_img && !i.is_original; - let path = resolve(join(base_path, i.name)); + const is_sampled = !download_original_img && !i.is_original; + let path = resolve( + join(base_path, is_sampled ? i.sampled_name : i.name), + ); if (names[i.name] > 1) { path = add_suffix_to_path(path, i.page_token); console.log("Changed path to", path); From 01d364a1ec1c5ca307d521b62c0b2d3283f58e81 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 16:06:32 +0800 Subject: [PATCH 13/14] Update --- components/NewTask.tsx | 30 ++++++++++++++++++++++++++++++ islands/Container.tsx | 30 +++++++++--------------------- islands/TaskManager.tsx | 27 +++++++++++++++++++++++++++ server/state.ts | 30 ++++++++++++++++++++++++++++++ static/common.css | 12 ++++++++++++ 5 files changed, 108 insertions(+), 21 deletions(-) create mode 100644 components/NewTask.tsx create mode 100644 server/state.ts diff --git a/components/NewTask.tsx b/components/NewTask.tsx new file mode 100644 index 0000000..6cc0f6d --- /dev/null +++ b/components/NewTask.tsx @@ -0,0 +1,30 @@ +import { Component, ContextType } from "preact"; +import { GlobalCtx } from "./GlobalContext.tsx"; +import Fab from "preact-material-components/Fab"; +import Icon from "preact-material-components/Icon"; +import { set_state } from "../server/state.ts"; + +export type NewTaskProps = { + show: boolean; +}; + +export default class NewTask extends Component { + static contextType = GlobalCtx; + declare context: ContextType; + render() { + if (!this.props.show) return null; + return ( +
+ + { + set_state((p) => p.slice(0, p.length - 4)); + }} + > + close + + +
+ ); + } +} diff --git a/islands/Container.tsx b/islands/Container.tsx index d267445..aa423f9 100644 --- a/islands/Container.tsx +++ b/islands/Container.tsx @@ -1,6 +1,6 @@ import { Head } from "$fresh/runtime.ts"; import { Component, ContextType } from "preact"; -import { StateUpdater, useEffect, useState } from "preact/hooks"; +import { useEffect, useState } from "preact/hooks"; import Icon from "preact-material-components/Icon"; import List from "preact-material-components/List"; import TopAppBar from "preact-material-components/TopAppBar"; @@ -9,6 +9,8 @@ import { GlobalCtx } from "../components/GlobalContext.tsx"; import Settings from "./Settings.tsx"; import t, { i18n_map, I18NMap } from "../server/i18n.ts"; import TaskManager from "./TaskManager.tsx"; +import { initState, set_state } from "../server/state.ts"; +import NewTask from "../components/NewTask.tsx"; export type ContainerProps = { i18n: I18NMap; @@ -21,26 +23,8 @@ export default class Container extends Component { i18n_map.value = this.props.i18n; const [display, set_display] = useState(false); const [state, set_state1] = useState("#/"); - const set_state: StateUpdater = (updater) => { - const v = typeof updater === "function" ? updater(state) : updater; - set_state1(v); - history.pushState(v, "", v); - }; useEffect(() => { - const hash = document.location.hash; - if (!hash || hash == "#") { - set_state("#/"); - } else { - set_state1(hash); - } - self.addEventListener("popstate", (e) => { - const s = e.state; - if (typeof s === "string") { - set_state1(s); - } else { - set_state1("#/"); - } - }); + initState(set_state1); }, []); return (
@@ -96,7 +80,11 @@ export default class Container extends Component {
- + +
); diff --git a/islands/TaskManager.tsx b/islands/TaskManager.tsx index fec5307..1378ceb 100644 --- a/islands/TaskManager.tsx +++ b/islands/TaskManager.tsx @@ -7,13 +7,25 @@ import { Sortable } from "sortable"; import { TaskClientSocketData, TaskServerSocketData } from "../server/task.ts"; import { get_ws_host } from "../server/utils.ts"; import Task from "../components/Task.tsx"; +import Fab from "preact-material-components/Fab"; +import Icon from "preact-material-components/Icon"; +import { set_state } from "../server/state.ts"; export type TaskManagerProps = { + base: string; show: boolean; }; const tasks = signal(new Map()); const task_list = signal(new Array()); +export const task_ws = signal(undefined); +export function sendTaskMessage(mes: TaskClientSocketData) { + const ws = task_ws.value; + if (ws && ws.readyState === ws.OPEN) { + ws.send(JSON.stringify(mes)); + return true; + } else return false; +} type SortableEvent = CustomEvent & { oldIndex: number | undefined; @@ -46,6 +58,7 @@ export default class TaskManager extends Component { useEffect(() => { const ws = new WebSocket(`${get_ws_host()}/api/task`); console.log(ws); + task_ws.value = ws; function sendMessage(mes: TaskClientSocketData) { ws.send(JSON.stringify(mes)); } @@ -153,6 +166,20 @@ export default class TaskManager extends Component { if (!this.props.show) return null; return (
+
+
+
+ + { + set_state(`${this.props.base}/new`); + }} + > + add + + +
+
| undefined = undefined; + +export function initState(l: StateUpdater) { + const hash = document.location.hash; + listener = l; + if (!hash || hash == "#") { + set_state("#/"); + } else { + set_state(hash); + } + self.addEventListener("popstate", (e) => { + const s = e.state; + if (typeof s === "string") { + l(s); + } else { + l("#/"); + } + }); +} + +export const set_state: StateUpdater = (updater) => { + const v = typeof updater === "function" ? updater(state.value) : updater; + state.value = v; + history.pushState(v, "", v); + if (listener) listener(v); +}; diff --git a/static/common.css b/static/common.css index 5b0e1aa..6cae7de 100644 --- a/static/common.css +++ b/static/common.css @@ -104,3 +104,15 @@ margin: 0; } } + +.task_manager #task_head { + display: flex; + justify-content: space-between; + align-items: center; +} + +.new_task .close { + position: absolute; + right: 10px; + top: 10px; +} From 627b38d85df7d4c0098d2d4571288861e4899c07 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 25 Jun 2023 16:40:12 +0800 Subject: [PATCH 14/14] Add ?current=1 support for /api/config --- routes/api/config.ts | 18 +++++++++++------- routes/api/deploy_id.ts | 5 ++--- server/utils.ts | 7 +++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/routes/api/config.ts b/routes/api/config.ts index 30a3917..a8179ba 100644 --- a/routes/api/config.ts +++ b/routes/api/config.ts @@ -1,6 +1,8 @@ import { Handlers } from "$fresh/server.ts"; import { ConfigType, load_settings, save_settings } from "../../config.ts"; import { get_cfg_path, get_task_manager } from "../../server.ts"; +import { parse_bool } from "../../server/parse_form.ts"; +import { return_json } from "../../server/utils.ts"; const UNSAFE_TYPE: (keyof ConfigType)[] = [ "base", @@ -14,12 +16,16 @@ const UNSAFE_TYPE: (keyof ConfigType)[] = [ const UNSAFE_TYPE2 = UNSAFE_TYPE as string[]; export const handler: Handlers = { - async GET(_req, _ctx) { + async GET(req, _ctx) { + const u = new URL(req.url); + const current = await parse_bool(u.searchParams.get("current"), false); + if (current) { + const t = get_task_manager(); + return return_json(t.cfg.to_json()); + } const path = get_cfg_path(); const cfg = await load_settings(path); - return new Response(JSON.stringify(cfg.to_json()), { - headers: { "Content-Type": "application/json" }, - }); + return return_json(cfg.to_json()); }, async POST(req, _ctx) { const content_type = req.headers.get("Content-Type"); @@ -39,9 +45,7 @@ export const handler: Handlers = { } }); await save_settings(path, cfg, m.force_aborts); - return new Response(JSON.stringify({ is_unsafe }), { - headers: { "Content-Type": "application/json" }, - }); + return return_json({ is_unsafe }); } else { return new Response("Bad Request", { status: 400 }); } diff --git a/routes/api/deploy_id.ts b/routes/api/deploy_id.ts index 2a71664..9020c6d 100644 --- a/routes/api/deploy_id.ts +++ b/routes/api/deploy_id.ts @@ -1,10 +1,9 @@ import { Handlers } from "$fresh/server.ts"; +import { return_json } from "../../server/utils.ts"; export const handler: Handlers = { GET(_req, _ctx) { const data = { id: Deno.env.get("DENO_DEPLOYMENT_ID") }; - return new Response(JSON.stringify(data), { - headers: { "Content-Type": "application/json" }, - }); + return return_json(data); }, }; diff --git a/server/utils.ts b/server/utils.ts index 8f2f517..e028c99 100644 --- a/server/utils.ts +++ b/server/utils.ts @@ -33,3 +33,10 @@ export function return_error( export function return_data(data: T, status = 200) { return gen_response({ ok: true, status: 0, data }, status); } + +export function return_json(data: T, status = 200) { + return new Response(JSON.stringify(data), { + status, + headers: { "Content-Type": "application/json" }, + }); +}