lifegpc/eh-downloader
1
0
Fork 0
mirror of https://github.com/lifegpc/eh-downloader.git synced 2026-07-08 01:31:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add support to remove old galleries

Browse Source
This commit is contained in:
lifegpc
2023-06-23 15:15:47 +08:00
parent 4e5dbd8390
commit fe7af51988
11 changed files with 169 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
config.ts
View File

@@ -21,6 +21,7 @@ export type ConfigType = {
ffmpeg_path: string;
thumbnail_method: ThumbnailMethod;
thumbnail_dir: string;
remove_previous_gallery: boolean;
};
export enum ThumbnailMethod {
@@ -128,6 +129,9 @@ export class Config {
get thumbnail_dir() {
return this._return_string("thumbnail_dir") || "./thumbnails";
}
get remove_previous_gallery() {
return this._return_bool("remove_previous_gallery") || false;
}
to_json(): ConfigType {
return {
cookies: typeof this.cookies === "string",
@@ -149,6 +153,7 @@ export class Config {
ffmpeg_path: this.ffmpeg_path,
thumbnail_method: this.thumbnail_method,
thumbnail_dir: this.thumbnail_dir,
remove_previous_gallery: this.remove_previous_gallery,
};
}
}

40
db.ts
View File

@@ -662,6 +662,32 @@ export class EhDb {
delete_file(f: EhFile) {
this.db.query("DELETE FROM file WHERE id = ?;", [f.id]);
}
delete_files(token: string) {
const files = this.get_files(token);
this.db.query("DELETE FROM file WHERE token = ?;", [token]);
this.db.query("DELETE FROM filemeta WHERE token = ?;", [token]);
files.forEach((f) => {
try_remove_sync(f.path);
console.log("Deleted ", f.path);
});
}
delete_gallery(gid: number) {
this.db.query("DELETE FROM gmeta WHERE gid = ?;", [gid]);
this.db.query("DELETE FROM gtag WHERE gid = ?;", [gid]);
const tokens = new Set(
this.db.query<[string]>("SELECT token FROM pmeta WHERE gid = ?;", [
gid,
]).map((v) => v[0]),
);
this.db.query("DELETE FROM pmeta WHERE gid = ?;", [gid]);
for (const token of tokens) {
const count = this.db.query<[number]>(
"SELECT COUNT(*) FROM pmeta WHERE token = ?;",
[token],
)[0][0];
if (count === 0) this.delete_files(token);
}
}
delete_task(task: Task) {
return this.transaction(() => {
this.db.query("DELETE FROM task WHERE id = ?;", [task.id]);
@@ -805,6 +831,12 @@ export class EhDb {
);
return s.length ? s[0] : undefined;
}
async get_task(id: number) {
const s = await this.transaction(() =>
this.db.queryEntries<Task>("SELECT * FROM task WHERE id = ?;", [id])
);
return s.length ? s[0] : undefined;
}
get_tasks() {
return this.transaction(() =>
this.db.queryEntries<Task>("SELECT * FROM task;")
@@ -885,4 +917,12 @@ export class EhDb {
]);
}
}
update_task(task: Task) {
return this.transaction(() => {
this.db.query("UPDATE task SET details = ? WHERE id = ?;", [
task.details,
task.id,
]);
});
}
}

7
islands/Settings.tsx
View File

@@ -121,6 +121,13 @@ export default class Settings extends Component<SettingsProps> {
checked={settings.export_zip_jpn_title}
description={t("settings.export_zip_jpn_title")}
/>
<SettingsCheckbox
name="remove_previous_gallery"
checked={settings.remove_previous_gallery}
description={t(
"settings.remove_previous_gallery",
)}
/>
</div>
<div class="text-box">
<SettingsSelect

5
islands/TaskManager.tsx
View File

@@ -139,6 +139,11 @@ export default class TaskManager extends Component<TaskManagerProps> {
task.progress = t.detail.detail;
sendTaskChangedEvent(t.detail.task_id);
}
} else if (t.type == "task_updated") {
const task = tasks.value.get(t.detail.id);
if (task) {
task.base = t.detail;
}
}
};
self.addEventListener("beforeunload", () => {

16
meilisearch.ts
View File

@@ -37,6 +37,7 @@ export class MeiliSearchServer {
client;
db;
handle;
handle2;
#gmeta?: Index;
#inited = false;
target;
@@ -50,8 +51,18 @@ export class MeiliSearchServer {
this.handle = (e: Event) => {
this.#gallery_update(e);
};
this.handle2 = (e: Event) => {
this.#gallery_remove(e);
};
this.target = new EventTarget();
this.target.addEventListener("gallery_update", this.handle);
this.target.addEventListener("gallery_remove", this.handle2);
}
#gallery_remove(e: Event) {
const ev = e as CustomEvent<number>;
this.removeGallery(ev.detail).catch((e) => {
console.log(e);
});
}
#gallery_update(e: Event) {
const ev = e as CustomEvent<number>;
@@ -78,6 +89,7 @@ export class MeiliSearchServer {
}
close() {
this.target.removeEventListener("gallery_update", this.handle);
this.target.removeEventListener("gallery_remove", this.handle2);
}
async getIndex(uid: string, primaryKey?: string) {
try {
@@ -112,6 +124,10 @@ export class MeiliSearchServer {
this.#updateGMetaSettings();
this.#inited = true;
}
async removeGallery(gid: number) {
const gmeta = await this.gmeta;
await this.waitTask(gmeta.deleteDocument(gid));
}
async updateGallery(...gids: number[]) {
const gmeta = await this.gmeta;
const datas = gids.map((gid) => {

9
routes/api/task.ts
View File

@@ -26,6 +26,7 @@ export const handler: Handlers<Task[]> = {
t.removeEventListener("task_finished", handle);
t.removeEventListener("task_progress", handle);
t.removeEventListener("task_error", handle);
t.removeEventListener("task_updated", handle);
ExitTarget.removeEventListener("close", close_handle);
};
function sendMessage(mes: TaskServerSocketData) {
@@ -45,12 +46,9 @@ export const handler: Handlers<Task[]> = {
sendMessage({ type: "close" });
socket.close();
} else if (d.type == "new_download_task") {
t.add_download_task(d.gid, d.token);
t.add_download_task(d.gid, d.token, d.cfg);
} else if (d.type == "new_export_zip_task") {
t.add_export_zip_task(d.gid, {
output: d.output,
jpn_title: d.jpn_title,
});
t.add_export_zip_task(d.gid, d.cfg);
} else if (d.type == "task_list") {
t.get_task_list().then((tasks) => {
sendMessage({
@@ -70,6 +68,7 @@ export const handler: Handlers<Task[]> = {
t.addEventListener("task_finished", handle);
t.addEventListener("task_progress", handle);
t.addEventListener("task_error", handle);
t.addEventListener("task_updated", handle);
ExitTarget.addEventListener("close", close_handle);
};
return response;

8
server/task.ts
View File

@@ -1,5 +1,6 @@
import { Task } from "../task.ts";
import { TaskEventData } from "../task_manager.ts";
import { DownloadConfig } from "../tasks/download.ts";
import { ExportZipConfig } from "../tasks/export_zip.ts";
import { DiscriminatedUnion } from "../utils.ts";
@@ -9,12 +10,9 @@ export type TaskServerSocketData = TaskEventData | { type: "close" } | {
running: number[];
};
type Gid<T extends Record<PropertyKey, unknown>> = ({ gid: number } & T) extends
infer U ? { [Q in keyof U]: U[Q] } : never;
type EventMap = {
new_download_task: { gid: number; token: string };
new_export_zip_task: Gid<ExportZipConfig>;
new_download_task: { gid: number; token: string; cfg?: DownloadConfig };
new_export_zip_task: { gid: number; cfg?: ExportZipConfig };
};
export type TaskClientSocketData = DiscriminatedUnion<"type", EventMap> | {

24
task_manager.ts
View File

@@ -5,7 +5,11 @@ import { MeiliSearchServer } from "./meilisearch.ts";
import { check_running } from "./pid_check.ts";
import { add_exit_handler } from "./signal_handler.ts";
import { Task, TaskProgress, TaskProgressBasicType, TaskType } from "./task.ts";
import { download_task } from "./tasks/download.ts";
import {
DEFAULT_DOWNLOAD_CONFIG,
download_task,
DownloadConfig,
} from "./tasks/download.ts";
import {
DEFAULT_EXPORT_ZIP_CONFIG,
export_zip,
@@ -29,6 +33,7 @@ type EventMap = {
task_finished: Task;
task_progress: TaskProgress;
task_error: { task: Task; error: string };
task_updated: Task;
};
type Detail<T extends Record<PropertyKey, unknown>> = {
@@ -96,7 +101,7 @@ export class TaskManager extends EventTarget {
get aborts() {
return this.#abort.signal;
}
async add_download_task(gid: number, token: string) {
async add_download_task(gid: number, token: string, cfg?: DownloadConfig) {
this.#check_closed();
const otask = await this.db.check_download_task(gid, token);
if (otask !== undefined) {
@@ -109,7 +114,7 @@ export class TaskManager extends EventTarget {
id: 0,
pid: Deno.pid,
type: TaskType.Download,
details: null,
details: cfg ? JSON.stringify(cfg) : null,
};
return await this.#add_task(task);
}
@@ -282,6 +287,9 @@ export class TaskManager extends EventTarget {
this.#check_closed();
this.dispatchEvent("task_started", task);
if (task.type == TaskType.Download) {
const cfg: DownloadConfig = task.details
? JSON.parse(task.details)
: DEFAULT_DOWNLOAD_CONFIG;
this.running_tasks.set(
task.id,
{
@@ -293,6 +301,7 @@ export class TaskManager extends EventTarget {
this.#abort.signal,
this.#force_abort.signal,
this,
cfg,
),
base: task,
},
@@ -329,6 +338,15 @@ export class TaskManager extends EventTarget {
});
}
}
async update_task(t: Task) {
const r = this.running_tasks.get(t.id);
if (r) {
r.base.details = t.details;
}
await this.db.update_task(t);
const a = await this.db.get_task(t.id);
if (a) this.dispatchEvent("task_updated", a);
}
async waiting_unfinished_task() {
while (1) {
await this.check_running_tasks();

72
tasks/download.ts
View File

@@ -15,6 +15,17 @@ import {
import { join, resolve } from "std/path/mod.ts";
import { exists } from "std/fs/exists.ts";
export type DownloadConfig = {
max_download_img_count?: number;
mpv?: boolean;
download_original_img?: boolean;
max_retry_count?: number;
remove_previous_gallery?: boolean;
replaced_gallery?: { gid: number; token: string }[];
};
export const DEFAULT_DOWNLOAD_CONFIG: DownloadConfig = {};
class DownloadManager {
#abort: AbortSignal;
#force_abort: AbortSignal;
@@ -24,13 +35,13 @@ class DownloadManager {
#task: Task;
#manager: TaskManager;
constructor(
cfg: Config,
max_download_img_count: number,
abort: AbortSignal,
force_abort: AbortSignal,
task: Task,
manager: TaskManager,
) {
this.#max_download_count = cfg.max_download_img_count;
this.#max_download_count = max_download_img_count;
this.#running_tasks = [];
this.#abort = abort;
this.#force_abort = force_abort;
@@ -97,6 +108,7 @@ export async function download_task(
abort: AbortSignal,
force_abort: AbortSignal,
manager: TaskManager,
dcfg: DownloadConfig,
) {
console.log("Started to download gallery", task.gid);
const gdatas = await client.fetchGalleryMetadataByAPI([
@@ -116,8 +128,27 @@ export async function download_task(
}
const base_path = join(cfg.base, task.gid.toString());
await sure_dir(base_path);
const m = new DownloadManager(cfg, abort, force_abort, task, manager);
if (cfg.mpv) {
const max_download_img_count = dcfg.max_download_img_count !== undefined
? dcfg.max_download_img_count
: cfg.max_download_img_count;
const m = new DownloadManager(
max_download_img_count,
abort,
force_abort,
task,
manager,
);
const mpv_enabled = dcfg.mpv !== undefined ? dcfg.mpv : cfg.mpv;
const download_original_img = dcfg.download_original_img !== undefined
? dcfg.download_original_img
: cfg.download_original_img;
const max_retry_count = dcfg.max_retry_count !== undefined
? dcfg.max_retry_count
: cfg.max_retry_count;
const remove_previous_gallery = dcfg.remove_previous_gallery !== undefined
? dcfg.remove_previous_gallery
: cfg.remove_previous_gallery;
if (mpv_enabled) {
const mpv = await client.fetchMPVPage(task.gid, task.token);
m.set_total_page(mpv.pagecount);
const names = mpv.imagelist.reduce(
@@ -134,7 +165,7 @@ export async function download_task(
if (ofiles.length) {
const t = ofiles[0];
if (
(t.is_original || !cfg.download_original_img) &&
(t.is_original || !download_original_img) &&
(await exists(t.path))
) {
const p = db.get_pmeta_by_index(task.gid, i.index);
@@ -170,7 +201,7 @@ export async function download_task(
return new Promise<void>((resolve, reject) => {
const errors: unknown[] = [];
function try_load(a: number) {
if (a >= cfg.max_retry_count) reject(errors);
if (a >= max_retry_count) reject(errors);
i.load().then(resolve).catch((e) => {
if (force_abort.aborted) {
throw Error("aborted.");
@@ -186,7 +217,7 @@ export async function download_task(
assert(i.data);
const pmeta = i.to_pmeta();
if (pmeta) db.add_pmeta(pmeta);
const download_original = cfg.download_original_img &&
const download_original = download_original_img &&
!i.is_original;
if (download_original) console.log(i.index, i.data.o);
let path = resolve(join(base_path, i.name));
@@ -226,7 +257,7 @@ export async function download_task(
}
const errors: unknown[] = [];
function try_download(a: number) {
if (a >= cfg.max_retry_count) {
if (a >= max_retry_count) {
reject(errors);
}
download().then(resolve).catch((e) => {
@@ -253,5 +284,30 @@ export async function download_task(
await m.join();
if (m.has_failed_task) throw Error("Some tasks failed.");
if (abort.aborted || force_abort.aborted) throw Error("aborted");
if (remove_previous_gallery && gmeta.first_gid && gmeta.first_key) {
let replaced_gallery = dcfg.replaced_gallery;
if (replaced_gallery === undefined) {
const fg = await client.fetchGalleryPage(
gmeta.first_gid,
gmeta.first_key,
);
replaced_gallery = fg.new_version.filter((d) => d.gid < task.gid);
replaced_gallery.push({
gid: gmeta.first_gid,
token: gmeta.first_key,
});
}
replaced_gallery.forEach((g) => {
const gmeta = db.get_gmeta_by_gid(g.gid);
if (!gmeta) return;
console.log("Remove gallery ", g.gid);
if (manager.meilisearch) {
manager.meilisearch.target.dispatchEvent(
new CustomEvent("gallery_remove", { detail: gmeta.gid }),
);
}
db.delete_gallery(g.gid);
});
}
return task;
}

3
translation/en/settings.jsonc
View File

@@ -28,5 +28,6 @@
"thumbnail_method": "The method used to generate thumbnail: ",
"thumbnail_method0": "ffmpeg binary",
"thumbnail_method1": "ffmpeg API",
"thumbnail_dir": "The folder used to store thumbnails: "
"thumbnail_dir": "The folder used to store thumbnails: ",
"remove_previous_gallery": "Remove old galleries which replaced by new ones."
}

3
translation/zh-cn/settings.jsonc
View File

@@ -28,5 +28,6 @@
"thumbnail_method": "生成缩略图的方式：",
"thumbnail_method0": "FFMPEG二进制",
"thumbnail_method1": "FFMPEG API",
"thumbnail_dir": "存放缩略图的文件夹："
"thumbnail_dir": "存放缩略图的文件夹：",
"remove_previous_gallery": "移除被新画廊替代的旧画廊。"
}