From 3808a3c067d04d6bf2cb768bd59d02c3ba86ed6e Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 17 Jul 2022 02:41:12 +0000 Subject: [PATCH] make sure all download operations use task manager Make sure DownloaderHelper is impled Send --- src/download.rs | 117 ++++++++++++++++++++++++--------------- src/downloader/helper.rs | 25 +++++---- 2 files changed, 86 insertions(+), 56 deletions(-) diff --git a/src/download.rs b/src/download.rs index aa82b7a..d53d20c 100644 --- a/src/download.rs +++ b/src/download.rs @@ -237,34 +237,33 @@ impl Main { let src = (&ugoira_data["originalSrc"]) .as_str() .try_err(gettext("Can not find source link for ugoira."))?; - let file_name = get_file_name_from_url(src).try_err(format!( - "{} {}", - gettext("Failed to get file name from url:"), - src - ))?; - let file_name = base.join(file_name); - let downloader = Downloader::new( - src, - json::object! { "referer": "https://www.pixiv.net/" }, - Some(&file_name), - helper.overwrite(), - )?; - match downloader { - DownloaderResult::Ok(d) => { - d.handle_options(&helper, None); - d.download(); - d.join().await?; - if d.is_panic() { - return Err(PixivDownloaderError::from( - d.get_panic() - .try_err(gettext("Failed to get error message."))?, - )); - } - } - DownloaderResult::Canceled => {} - } + let dh = DownloaderHelper::builder(src)? + .headers(json::object! { "referer": "https://www.pixiv.net/" }) + .build(); + let tasks = TaskManager::default(); + tasks + .add_task(Self::download_file( + dh, + Some(get_progress_bar()), + Arc::clone(&base), + )) + .await; + tasks.join().await; + let mut tasks = tasks.take_finished_tasks(); + let task = tasks.get_mut(0).try_err(gettext("No finished task."))?; + let task = task.as_any_mut(); + let task = task + .downcast_mut::>>() + .try_err("Failed to downcast task.")?; + task.await??; #[cfg(feature = "ugoira")] { + let file_name = get_file_name_from_url(src).try_err(format!( + "{} {}", + gettext("Failed to get file name from url:"), + src + ))?; + let file_name = base.join(file_name); let metadata = match get_video_metadata(Arc::clone(&datas).as_ref()) { Ok(m) => m, Err(e) => { @@ -352,20 +351,38 @@ impl Main { } else if pages_data.is_some() { let mut np = 0u16; let pages_data = pages_data.as_ref().unwrap(); + let tasks = TaskManager::default(); for page in pages_data.members() { let link = page["urls"]["original"] .as_str() .try_err(gettext("Failed to get original picture's link."))?; - Self::download_artwork_link( - link.to_owned(), - np, - None, - Arc::clone(&datas), - Arc::clone(&base), - ) - .await?; + tasks + .add_task(Self::download_artwork_link( + link.to_owned(), + np, + Some(get_progress_bar()), + Arc::clone(&datas), + Arc::clone(&base), + )) + .await; + tasks.join().await; np += 1; } + let mut re = Ok(()); + let tasks = tasks.take_finished_tasks(); + for mut task in tasks { + let t = task.as_any_mut(); + if let Some(task) = t.downcast_mut::>>() + { + let r = task.await; + let r = match r { + Ok(r) => r, + Err(e) => Err(PixivDownloaderError::from(e)), + }; + concat_pixiv_downloader_error!(re, r); + } + } + return re; } else { let link = if ajax_ver { (&re["urls"]["original"]).as_str() @@ -373,23 +390,33 @@ impl Main { (&re["illust"][format!("{}", id)]["urls"]["original"]).as_str() } .try_err(gettext("Failed to get original picture's link."))?; - Self::download_artwork_link( - link.to_owned(), - 0, - None, - Arc::clone(&datas), - Arc::clone(&base), - ) - .await?; + let tasks = TaskManager::default(); + tasks + .add_task(Self::download_artwork_link( + link.to_owned(), + 0, + Some(get_progress_bar()), + Arc::clone(&datas), + Arc::clone(&base), + )) + .await; + tasks.join().await; + let mut tasks = tasks.take_finished_tasks(); + let task = tasks.get_mut(0).try_err(gettext("No tasks finished."))?; + let task = task.as_any_mut(); + let task = task + .downcast_mut::>>() + .try_err("Failed to downcast the result.")?; + task.await??; } Ok(()) } - /// Download a fanbox file link + /// Download a file link /// * `dh` - Link and other informations /// * `progress_bars` - Multiple progress bars /// * `base` - The directory of the target - pub async fn download_fanbox_file( + pub async fn download_file( dh: DownloaderHelper, progress_bars: Option>, base: Arc, @@ -570,7 +597,7 @@ impl Main { let dh = f .download_url()? .try_err(gettext("Failed to get url of the file."))?; - Self::download_fanbox_file(dh, None, Arc::clone(&base)).await?; + Self::download_file(dh, None, Arc::clone(&base)).await?; } } FanboxPost::Image(img) => { diff --git a/src/downloader/helper.rs b/src/downloader/helper.rs index bc5e4e9..a545e67 100644 --- a/src/downloader/helper.rs +++ b/src/downloader/helper.rs @@ -8,8 +8,10 @@ use crate::gettext; use crate::webclient::ToHeaders; use crate::webclient::WebClient; use reqwest::IntoUrl; +use std::collections::HashMap; use std::convert::TryFrom; use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use url::Url; @@ -20,9 +22,9 @@ pub struct DownloaderHelper { /// Web client pub client: Option>, /// New headers wants to apply - pub headers: Option>, + pub headers: Option>, /// Recommand file name - pub file_name: Option>>, + pub file_name: Option, } pub struct DownloaderHelperBuilder { @@ -52,19 +54,20 @@ impl DownloaderHelper { let file = self .get_local_file_path(base) .try_err(gettext("Failed to get file name from url."))?; - let headers = match &self.headers { - Some(headers) => headers.to_headers(), - None => None, - }; match &self.client { Some(client) => Downloader::::new2( Arc::clone(client), self.url.clone(), - headers, + self.headers.clone(), + Some(&file), + overwrite, + ), + None => Downloader::::new( + self.url.clone(), + self.headers.clone(), Some(&file), overwrite, ), - None => Downloader::::new(self.url.clone(), headers, Some(&file), overwrite), } } @@ -74,7 +77,7 @@ impl DownloaderHelper { ) -> Option { let base = base.as_ref(); match &self.file_name { - Some(file_name) => Some(base.join(file_name.as_ref())), + Some(file_name) => Some(base.join(file_name)), None => match crate::utils::get_file_name_from_url(self.url.clone()) { Some(file_name) => Some(base.join(file_name)), None => None, @@ -87,12 +90,12 @@ impl DownloaderHelper { } pub fn set_file_name + ?Sized>(&mut self, p: &P) { - self.file_name.replace(Box::new(p.as_ref().to_owned())); + self.file_name.replace(p.as_ref().to_owned()); } pub fn set_headers(&mut self, headers: H) { self.headers.replace_with(match headers.to_headers() { - Some(headers) => Some(Box::new(headers)), + Some(headers) => Some(headers), None => None, }); }