From 969c8985865f43ce9f08af0a5ba77c6eeb490f1f Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 17 Jul 2022 03:16:34 +0000 Subject: [PATCH] download_fanbox_post now use task manager --- src/data/fanbox.rs | 2 +- src/download.rs | 70 +++++++++++++++++++++++++++++----------- src/downloader/helper.rs | 25 +++++++------- 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/src/data/fanbox.rs b/src/data/fanbox.rs index 75489de..b0c79a9 100644 --- a/src/data/fanbox.rs +++ b/src/data/fanbox.rs @@ -14,7 +14,7 @@ pub struct FanboxData { /// Raw data pub raw: JsonValue, #[cfg(feature = "exif")] - pub exif_data: Option>, + pub exif_data: Option>, } impl FanboxData { diff --git a/src/download.rs b/src/download.rs index d53d20c..43241e2 100644 --- a/src/download.rs +++ b/src/download.rs @@ -545,6 +545,8 @@ impl Main { data_file .save(&json_file) .try_err(gettext("Failed to save post data to file."))?; + let tasks = TaskManager::default(); + let download_mutiple_images = helper.download_multiple_images(); match post { FanboxPost::Article(article) => { let article = Arc::new(article); @@ -572,14 +574,18 @@ impl Main { let dh = img .download_original_url()? .try_err(gettext("Can not get original url for image"))?; - Self::download_fanbox_image( - dh, - np, - None, - Arc::clone(&datas), - Arc::clone(&base), - ) - .await?; + tasks + .add_task(Self::download_fanbox_image( + dh, + np, + Some(get_progress_bar()), + Arc::clone(&datas), + Arc::clone(&base), + )) + .await; + if !download_mutiple_images { + tasks.join().await; + } np += 1; } _ => {} @@ -597,7 +603,16 @@ impl Main { let dh = f .download_url()? .try_err(gettext("Failed to get url of the file."))?; - Self::download_file(dh, None, Arc::clone(&base)).await?; + tasks + .add_task(Self::download_file( + dh, + Some(get_progress_bar()), + Arc::clone(&base), + )) + .await; + if !download_mutiple_images { + tasks.join().await; + } } } FanboxPost::Image(img) => { @@ -617,14 +632,18 @@ impl Main { let dh = img .download_original_url()? .try_err(gettext("Can not get original url for image"))?; - Self::download_fanbox_image( - dh, - np, - None, - Arc::clone(&datas), - Arc::clone(&base), - ) - .await?; + tasks + .add_task(Self::download_fanbox_image( + dh, + np, + Some(get_progress_bar()), + Arc::clone(&datas), + Arc::clone(&base), + )) + .await; + if !download_mutiple_images { + tasks.join().await; + } np += 1; } } @@ -634,6 +653,21 @@ impl Main { ))); } } - Ok(()) + tasks.join().await; + let mut re = Ok(()); + let tasks = tasks.take_finished_tasks(); + for mut task in tasks { + let task = task.as_any_mut(); + if let Some(task) = task.downcast_mut::>>() + { + let r = task.await; + let r = match r { + Ok(r) => r, + Err(e) => Err(PixivDownloaderError::from(e)), + }; + concat_pixiv_downloader_error!(re, r); + } + } + re } } diff --git a/src/downloader/helper.rs b/src/downloader/helper.rs index a545e67..8f9701a 100644 --- a/src/downloader/helper.rs +++ b/src/downloader/helper.rs @@ -8,10 +8,8 @@ use crate::gettext; use crate::webclient::ToHeaders; use crate::webclient::WebClient; use reqwest::IntoUrl; -use std::collections::HashMap; use std::convert::TryFrom; use std::path::Path; -use std::path::PathBuf; use std::sync::Arc; use url::Url; @@ -22,9 +20,9 @@ pub struct DownloaderHelper { /// Web client pub client: Option>, /// New headers wants to apply - pub headers: Option>, + pub headers: Option>, /// Recommand file name - pub file_name: Option, + pub file_name: Option + Send + Sync>>, } pub struct DownloaderHelperBuilder { @@ -54,20 +52,19 @@ impl DownloaderHelper { let file = self .get_local_file_path(base) .try_err(gettext("Failed to get file name from url."))?; + let headers = match &self.headers { + Some(headers) => headers.to_headers(), + None => None, + }; match &self.client { Some(client) => Downloader::::new2( Arc::clone(client), self.url.clone(), - self.headers.clone(), - Some(&file), - overwrite, - ), - None => Downloader::::new( - self.url.clone(), - self.headers.clone(), + headers, Some(&file), overwrite, ), + None => Downloader::::new(self.url.clone(), headers, Some(&file), overwrite), } } @@ -77,7 +74,7 @@ impl DownloaderHelper { ) -> Option { let base = base.as_ref(); match &self.file_name { - Some(file_name) => Some(base.join(file_name)), + Some(file_name) => Some(base.join(file_name.as_ref())), None => match crate::utils::get_file_name_from_url(self.url.clone()) { Some(file_name) => Some(base.join(file_name)), None => None, @@ -90,12 +87,12 @@ impl DownloaderHelper { } pub fn set_file_name + ?Sized>(&mut self, p: &P) { - self.file_name.replace(p.as_ref().to_owned()); + self.file_name.replace(Box::new(p.as_ref().to_owned())); } pub fn set_headers(&mut self, headers: H) { self.headers.replace_with(match headers.to_headers() { - Some(headers) => Some(headers), + Some(headers) => Some(Box::new(headers)), None => None, }); }