diff --git a/components/Task.tsx b/components/Task.tsx index 2bceaf2..d6ab4a7 100644 --- a/components/Task.tsx +++ b/components/Task.tsx @@ -72,7 +72,7 @@ export default class Task extends Component { if (task.base.type === TaskType.Download) { const d = task .progress as TaskProgressBasicType[TaskType.Download]; - progress_div = ( + const b_progress_div = ( { ); + progress_div = ( +
+ {b_progress_div} + {d.details.map((v) => { + return ( +
+
{v.name}
+ + + +
+ ); + })} +
+ ); } } return ( diff --git a/task.ts b/task.ts index 88f387d..9523f22 100644 --- a/task.ts +++ b/task.ts @@ -14,10 +14,23 @@ export type Task = { details: string | null; }; +export type TaskDownloadSingleProgress = { + index: number; + token: string; + name: string; + width: number; + height: number; + is_original: boolean; + total: number; + started: number; + downloaded: number; +}; + export type TaskDownloadProgess = { downloaded_page: number; failed_page: number; total_page: number; + details: TaskDownloadSingleProgress[]; }; export type TaskExportZipProgress = { diff --git a/tasks/download.ts b/tasks/download.ts index 6f41bf0..dc3b93f 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -2,7 +2,12 @@ import { assert } from "std/assert/mod.ts"; import { Client } from "../client.ts"; import type { Config } from "../config.ts"; import type { EhDb, EhFile, PMeta } from "../db.ts"; -import { Task, TaskDownloadProgess, TaskType } from "../task.ts"; +import { + Task, + TaskDownloadProgess, + TaskDownloadSingleProgress, + TaskType, +} from "../task.ts"; import { RecoverableError, TaskManager } from "../task_manager.ts"; import { add_suffix_to_path, @@ -14,6 +19,7 @@ import { } from "../utils.ts"; import { join, resolve } from "std/path/mod.ts"; import { exists } from "std/fs/exists.ts"; +import { ProgressReadable } from "../utils/progress_readable.ts"; export type DownloadConfig = { max_download_img_count?: number; @@ -26,6 +32,8 @@ export type DownloadConfig = { export const DEFAULT_DOWNLOAD_CONFIG: DownloadConfig = {}; +const PROGRESS_UPDATE_INTERVAL = 200; + class DownloadManager { #abort: AbortSignal; #force_abort: AbortSignal; @@ -34,6 +42,8 @@ class DownloadManager { #progress: TaskDownloadProgess; #task: Task; #manager: TaskManager; + #progress_changed: boolean; + #last_send_progress: number; constructor( max_download_img_count: number, abort: AbortSignal, @@ -45,9 +55,16 @@ class DownloadManager { this.#running_tasks = []; this.#abort = abort; this.#force_abort = force_abort; - this.#progress = { downloaded_page: 0, failed_page: 0, total_page: 0 }; + this.#progress = { + downloaded_page: 0, + failed_page: 0, + total_page: 0, + details: [], + }; this.#task = task; this.#manager = manager; + this.#progress_changed = false; + this.#last_send_progress = -1; } async #check_tasks() { this.#running_tasks = await asyncFilter( @@ -65,13 +82,35 @@ class DownloadManager { return s.status === PromiseStatus.Pending; }, ); + if (this.#progress_changed) { + const now = (new Date()).getTime(); + if (now >= this.#last_send_progress + PROGRESS_UPDATE_INTERVAL) { + this.#manager.dispatchTaskProgressEvent( + TaskType.Download, + this.#task.id, + this.#progress, + ); + } + this.#progress_changed = false; + this.#last_send_progress = now; + } } #sendEvent() { - return this.#manager.dispatchTaskProgressEvent( + this.#progress_changed = true; + const now = (new Date()).getTime(); + if (now < this.#last_send_progress + PROGRESS_UPDATE_INTERVAL) return; + const re = this.#manager.dispatchTaskProgressEvent( TaskType.Download, this.#task.id, this.#progress, ); + this.#last_send_progress = now; + this.#progress_changed = false; + return re; + } + add_new_details(d: TaskDownloadSingleProgress) { + this.#progress.details.push(d); + this.#sendEvent(); } async add_new_task(f: () => Promise) { while (1) { @@ -94,6 +133,26 @@ class DownloadManager { await sleep(10); } } + remove_details(index: number) { + this.#progress.details = this.#progress.details.filter((v) => + v.index !== index + ); + } + set_details_downloaded(index: number, downloaded: number) { + const d = this.#progress.details.find((v) => v.index === index); + if (d) d.downloaded = downloaded; + this.#sendEvent(); + } + set_details_started(index: number) { + const d = this.#progress.details.find((v) => v.index === index); + if (d) d.started = (new Date()).getTime(); + this.#sendEvent(); + } + set_details_total(index: number, total: number) { + const d = this.#progress.details.find((v) => v.index === index); + if (d) d.total = total; + this.#sendEvent(); + } set_total_page(page: number) { this.#progress.total_page = page; this.#sendEvent(); @@ -231,6 +290,21 @@ export async function download_task( path = add_suffix_to_path(path, i.page_token); console.log("Changed path to", path); } + const f = download_original + ? i.get_original_file(path) + : i.get_file(path); + if (f === undefined) throw Error("Failed to get file."); + m.add_new_details({ + downloaded: 0, + height: f.height, + index: i.index, + is_original: f.is_original, + name: i.name, + token: i.page_token, + total: 0, + width: f.width, + started: 0, + }); function download_img() { return new Promise((resolve, reject) => { async function download() { @@ -240,22 +314,45 @@ export async function download_task( if (re === undefined) { throw Error("Failed to fetch image."); } + m.set_details_started(i.index); + const len = re.headers.get("Content-Length"); + if (len) { + const tmp = parseInt(len); + if (!isNaN(tmp)) { + m.set_details_total(i.index, tmp); + } + } if (re.body === null) { throw Error("Response don't have a body."); } - const f = await Deno.open(path, { - create: true, - write: true, - truncate: true, + const pr = new ProgressReadable(re.body); + pr.addEventListener("progress", (e) => { + m.set_details_downloaded(i.index, e.detail); + }); + pr.addEventListener("finished", () => { + m.remove_details(i.index); }); try { - await re.body.pipeTo(f.writable, { - signal: force_abort, - preventClose: true, + const f = await Deno.open(path, { + create: true, + write: true, + truncate: true, }); + try { + await pr.readable.pipeTo(f.writable, { + signal: force_abort, + preventClose: true, + }); + } finally { + try { + f.close(); + } catch (_) { + null; + } + } } finally { try { - f.close(); + pr.readable.cancel(); } catch (_) { null; } @@ -278,10 +375,6 @@ export async function download_task( }); } await download_img(); - const f = download_original - ? i.get_original_file(path) - : i.get_file(path); - if (f === undefined) throw Error("Failed to get file."); db.add_file(f); return; } diff --git a/utils/progress_readable.ts b/utils/progress_readable.ts new file mode 100644 index 0000000..498d0dd --- /dev/null +++ b/utils/progress_readable.ts @@ -0,0 +1,60 @@ +type EventMap = { + "finished": number; + "progress": number; +}; + +export class ProgressReadable extends EventTarget { + readable: ReadableStream; + readed: number; + constructor(readable: ReadableStream) { + super(); + this.readed = 0; + const reader = readable.getReader(); + this.readable = new ReadableStream({ + pull: async (c) => { + if (c.byobRequest) { + throw Error("Unimplemented."); + } else { + const v = await reader.read(); + if (v.done) { + this.dispatchEvent("finished", this.readed); + c.close(); + return; + } else { + this.readed += v.value.byteLength; + this.dispatchEvent("progress", this.readed); + c.enqueue(v.value); + } + } + }, + cancel: (reason) => { + readable.cancel(reason); + }, + type: "bytes", + }); + } + // @ts-ignore Checked type + addEventListener( + type: T, + callback: (e: CustomEvent) => void | Promise, + options?: boolean | AddEventListenerOptions, + ): void { + super.addEventListener(type, callback, options); + } + // @ts-ignore Checked type + dispatchEvent(type: T, detail: EventMap[T]) { + return super.dispatchEvent(new CustomEvent(type, { detail })); + } + // @ts-ignore Checked type + removeEventListener( + type: T, + callback: (e: CustomEvent) => void | Promise, + options?: boolean | EventListenerOptions, + ): void { + super.removeEventListener( + type, + callback, + options, + ); + } +}