diff --git a/proc_macros/proc_macros.rs b/proc_macros/proc_macros.rs index a12e1bf..3c9685f 100644 --- a/proc_macros/proc_macros.rs +++ b/proc_macros/proc_macros.rs @@ -149,7 +149,7 @@ pub fn fanbox_api_test(item: TokenStream) -> TokenStream { panic!("Failed to initiailze the client."); } if !client.check_login().await { - println!("The client is not logined. Skip test."); + println!("The client is not logged in. Skip test."); return; } #(#stmts)* @@ -197,7 +197,7 @@ pub fn fanbox_api_quick_test(item: TokenStream) -> TokenStream { panic!("Failed to initiailze the client."); } if !client.check_login().await { - println!("The client is not logined. Skip test."); + println!("The client is not logged in. Skip test."); return; } match #expr.await { diff --git a/src/download.rs b/src/download.rs index 5202274..00092b1 100644 --- a/src/download.rs +++ b/src/download.rs @@ -42,6 +42,8 @@ impl Main { pub async fn download(&mut self) -> i32 { let pw = Arc::new(PixivWebClient::new()); let fc = Arc::new(FanboxClient::new()); + let tasks = TaskManager::new_post(); + let download_multiple_posts = get_helper().download_multiple_posts(); for id in self.cmd.as_ref().unwrap().ids.iter() { match id { PixivID::Artwork(id) => { @@ -54,22 +56,14 @@ impl Main { return 1; } if !pw.logined() { - println!("{}", gettext("Warning: Web api client not logined, some future may not work.")); + println!("{}", gettext("Warning: Web api client not logged in, some future may not work.")); } } - let r = self.download_artwork(Arc::clone(&pw), id.clone()).await; - let r = if r.is_ok() { - 0 - } else { - println!( - "{} {}", - gettext("Failed to download artwork:"), - r.unwrap_err() - ); - 1 - }; - if r != 0 { - return r; + tasks + .add_task(Self::download_artwork(Arc::clone(&pw), id.clone())) + .await; + if !download_multiple_posts { + tasks.join().await; } } PixivID::FanboxPost(id) => { @@ -83,24 +77,39 @@ impl Main { return 1; } if !fc.logined() { - println!("{}", gettext("Warning: Fanbox client is not logined.")); + println!("{}", gettext("Warning: Fanbox client is not logged in.")); } - let r = self.download_fanbox_post(Arc::clone(&fc), id.clone()).await; - let r = match r { - Ok(_) => 0, - Err(e) => { - println!("{} {}", gettext("Failed to download post:"), e); - 1 - } - }; - if r != 0 { - return r; + tasks + .add_task(Self::download_fanbox_post(Arc::clone(&fc), id.clone())) + .await; + if !download_multiple_posts { + tasks.join().await; } } } } } - 0 + let mut re = 0; + tasks.join().await; + let tasks = tasks.take_finished_tasks(); + for mut task in tasks { + let task = task.as_any_mut(); + if let Some(task) = task.downcast_mut::>>() + { + let result = match task.await { + Ok(result) => result, + Err(e) => Err(PixivDownloaderError::from(e)), + }; + match result { + Ok(_) => {} + Err(e) => { + println!("{} {}", gettext("Failed to download post:"), e); + re = 1; + } + } + } + } + re } /// Download artwork link @@ -171,7 +180,6 @@ impl Main { } pub async fn download_artwork( - &self, pw: Arc, id: u64, ) -> Result<(), PixivDownloaderError> { @@ -498,7 +506,6 @@ impl Main { } pub async fn download_fanbox_post( - &self, fc: Arc, id: FanboxPostID, ) -> Result<(), PixivDownloaderError> { diff --git a/src/opthelper.rs b/src/opthelper.rs index 045611d..030dd13 100644 --- a/src/opthelper.rs +++ b/src/opthelper.rs @@ -71,6 +71,24 @@ impl OptHelper { self._cors_entries.get_ref().clone() } + /// Whether to download multiple posts/artworks at the same time. + pub fn download_multiple_posts(&self) -> bool { + match self.opt.get_ref().download_multiple_posts { + Some(r) => { + return r; + } + None => {} + } + if self.settings.get_ref().have_bool("download-multiple-posts") { + return self + .settings + .get_ref() + .get_bool("download-multiple-posts") + .unwrap(); + } + false + } + /// Return max retry count of each part when downloading in multiple thread mode. pub fn download_part_retry(&self) -> Option { if self.opt.get_ref().download_part_retry.is_some() { @@ -128,7 +146,24 @@ impl OptHelper { } } - /// Return the maximun number of tasks to download simultaneously. + /// Return the maximum number of tasks to download posts/artworks at the same time. + pub fn max_download_post_tasks(&self) -> usize { + match self.opt.get_ref().max_download_post_tasks { + Some(r) => { + return r; + } + None => {} + } + match self.settings.get_ref().get("max-download-post-tasks") { + Some(re) => { + return re.as_usize().unwrap(); + } + None => {} + } + 3 + } + + /// Return the maximum number of tasks to download files at the same time. pub fn max_download_tasks(&self) -> usize { match self.opt.get_ref().max_download_tasks { Some(r) => { @@ -145,7 +180,7 @@ impl OptHelper { 5 } - /// Return the maximun threads when downloading file. + /// Return the maximum threads when downloading file. pub fn max_threads(&self) -> u64 { match self.opt.get_ref().max_threads { Some(r) => { @@ -331,11 +366,7 @@ impl OptHelper { } None => {} } - if self - .settings - .get_ref() - .have_bool("download-multiple-files") - { + if self.settings.get_ref().have_bool("download-multiple-files") { return self .settings .get_ref() diff --git a/src/opts.rs b/src/opts.rs index 47d9519..b03c12b 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -85,15 +85,19 @@ pub struct CommandOpts { pub multiple_threads_download: Option, /// Max retry count of each part when downloading in multiple thread mode. pub download_part_retry: Option, - /// The maximun threads when downloading file. + /// The maximum threads when downloading file. pub max_threads: Option, /// The size of the each part when downloading file. pub part_size: Option, #[cfg(feature = "server")] /// Server listen address pub server: Option, - /// Maximun number of tasks to download simultaneously + /// maximum number of tasks to download files at the same time pub max_download_tasks: Option, + /// Whether to download multiple posts/artworks at the same time. + pub download_multiple_posts: Option, + /// The maximum number of tasks to download posts/artworks at the same time. + pub max_download_post_tasks: Option, } impl CommandOpts { @@ -123,6 +127,8 @@ impl CommandOpts { #[cfg(feature = "server")] server: None, max_download_tasks: None, + download_multiple_posts: None, + max_download_post_tasks: None, } } @@ -398,7 +404,7 @@ pub fn parse_cmd() -> Option { opts.optopt( "m", "max-threads", - gettext("The maximun threads when downloading file."), + gettext("The maximum threads when downloading file."), "COUNT", ); opts.optopt( @@ -412,7 +418,7 @@ pub fn parse_cmd() -> Option { "max-download-tasks", format!( "{} ({} {})", - gettext("The maximun number of tasks to download simultaneously."), + gettext("The maximum number of tasks to download files at the same time."), gettext("Default:"), "5" ) @@ -421,6 +427,34 @@ pub fn parse_cmd() -> Option { HasArg::Maybe, getopts::Occur::Optional, ); + opts.opt( + "", + "download-multiple-posts", + format!( + "{} ({} {})", + gettext("Download multiple posts/artworks at the same time."), + gettext("Default:"), + "yes" + ) + .as_str(), + "yes/no", + HasArg::Maybe, + getopts::Occur::Optional, + ); + opts.opt( + "", + "max-download-post-tasks", + format!( + "{} ({} {})", + gettext("The maximum number of tasks to download posts/artworks at the same time."), + gettext("Default:"), + 3 + ) + .as_str(), + "yes/no", + HasArg::Maybe, + getopts::Occur::Optional, + ); let result = match opts.parse(&argv[1..]) { Ok(m) => m, Err(err) => { @@ -645,6 +679,32 @@ pub fn parse_cmd() -> Option { return None; } } + match parse_optional_opt(&result, "download-multiple-posts", true, parse_bool) { + Ok(b) => re.as_mut().unwrap().download_multiple_posts = b, + Err(e) => { + println!( + "{} {}", + gettext("Failed to parse :") + .replace("", "download-multiple-posts") + .as_str(), + e + ); + return None; + } + } + match parse_optional_opt(&result, "max-download-post-tasks", 3, parse_nonempty_usize) { + Ok(r) => re.as_mut().unwrap().max_download_post_tasks = r, + Err(e) => { + println!( + "{} {}", + gettext("Failed to parse :") + .replace("", "max-download-post-tasks") + .as_str(), + e + ); + return None; + } + } re } diff --git a/src/pixiv_web.rs b/src/pixiv_web.rs index 9524efd..9b26ed5 100644 --- a/src/pixiv_web.rs +++ b/src/pixiv_web.rs @@ -74,7 +74,7 @@ impl PixivWebClient { self.auto_init(); let r = self .client - .get_with_param("https://www.pixiv.net/", self.params.get_ref(), None) + .get_with_param("https://www.pixiv.net/", self.get_params(), None) .await; if r.is_none() { return false; @@ -186,7 +186,7 @@ impl PixivWebClient { .client .get_with_param( format!("https://www.pixiv.net/ajax/illust/{}", id), - self.params.get_ref(), + self.get_params(), None, ) .await; @@ -211,7 +211,7 @@ impl PixivWebClient { .client .get_with_param( format!("https://www.pixiv.net/artworks/{}", id), - self.params.get_ref(), + self.get_params(), None, ) .await; @@ -256,7 +256,7 @@ impl PixivWebClient { .client .get_with_param( format!("https://www.pixiv.net/ajax/illust/{}/pages", id), - self.params.get_ref(), + self.get_params(), None, ) .await; @@ -275,13 +275,17 @@ impl PixivWebClient { v } + pub fn get_params(&self) -> Option { + self.params.get_ref().clone() + } + pub async fn get_ugoira(&self, id: u64) -> Option { self.auto_init(); let r = self .client .get_with_param( format!("https://www.pixiv.net/ajax/illust/{}/ugoira_meta", id), - self.params.get_ref(), + self.get_params(), None, ) .await; diff --git a/src/settings_list.rs b/src/settings_list.rs index 1af4a56..df74cb3 100644 --- a/src/settings_list.rs +++ b/src/settings_list.rs @@ -33,14 +33,16 @@ pub fn get_settings_list() -> Vec { SettingDes::new("download-retry-interval", gettext("The interval (in seconds) between two retries when downloading files."), JsonValueType::Multiple, Some(check_retry_interval)).unwrap(), SettingDes::new("multiple-threads-download", gettext("Whether to enable multiple threads download."), JsonValueType::Boolean, None).unwrap(), SettingDes::new("download-part-retry", gettext("Max retry count of each part when downloading in multiple thread mode."), JsonValueType::Number, Some(check_i64)).unwrap(), - SettingDes::new("max-threads", gettext("The maximun threads when downloading file."), JsonValueType::Number, Some(check_u64)).unwrap(), + SettingDes::new("max-threads", gettext("The maximum threads when downloading file."), JsonValueType::Number, Some(check_u64)).unwrap(), SettingDes::new("part-size", gettext("The size of the each part when downloading file."), JsonValueType::Number, Some(check_parse_size_u32)).unwrap(), SettingDes::new("proxy", gettext("Proxy settings."), JsonValueType::Array, Some(check_proxy)).unwrap(), #[cfg(feature = "server")] SettingDes::new("server", gettext("Server address."), JsonValueType::Str, Some(check_socket_addr)).unwrap(), #[cfg(feature = "server")] SettingDes::new("cors-entries", gettext("The domains allowed to send CORS requests."), JsonValueType::Array, Some(check_cors_entries)).unwrap(), - SettingDes::new("max-download-tasks", gettext("The maximun number of tasks to download simultaneously."), JsonValueType::Number, Some(check_nozero_usize)).unwrap(), + SettingDes::new("max-download-tasks", gettext("The maximum number of tasks to download files at the same time."), JsonValueType::Number, Some(check_nozero_usize)).unwrap(), + SettingDes::new("download-multiple-posts", gettext("Download multiple posts/artworks at the same time."), JsonValueType::Boolean, None).unwrap(), + SettingDes::new("max-download-post-tasks", gettext("The maximum number of tasks to download posts/artworks at the same time."), JsonValueType::Number, Some(check_nozero_usize)).unwrap(), ] } diff --git a/src/task_manager.rs b/src/task_manager.rs index a9de463..3ab0b26 100644 --- a/src/task_manager.rs +++ b/src/task_manager.rs @@ -16,6 +16,8 @@ lazy_static! { #[doc(hidden)] static ref TOTAL_DOWNLOAD_TASK_COUNT: Arc> = Arc::new(Mutex::new(0)); #[doc(hidden)] + static ref TOTAL_POST_TASK_COUNT: Arc> = Arc::new(Mutex::new(0)); + #[doc(hidden)] static ref PROGRESS_BAR: Arc = Arc::new(MultiProgress::new()); } @@ -63,6 +65,22 @@ impl GetMaxCount for MaxDownloadTasks { } } +pub struct MaxDownloadPostTasks { + _unused: [u8; 0], +} + +impl MaxDownloadPostTasks { + pub fn new() -> Self { + MaxDownloadPostTasks { _unused: [] } + } +} + +impl GetMaxCount for MaxDownloadPostTasks { + fn get_max_count(&self) -> usize { + get_helper().max_download_post_tasks() + } +} + /// Task manager pub struct TaskManager { /// Current running task @@ -71,12 +89,15 @@ pub struct TaskManager { finished_tasks: RwLock>>, /// Total task count task_count: Arc>, - max_count: Box, + max_count: Box, } impl TaskManager { /// Create a new instance - pub fn new(task_count: Arc>, max_count: T) -> Self { + pub fn new( + task_count: Arc>, + max_count: T, + ) -> Self { Self { tasks: RwLock::new(Vec::new()), finished_tasks: RwLock::new(Vec::new()), @@ -85,6 +106,11 @@ impl TaskManager { } } + /// Create a new instance with post max count + pub fn new_post() -> Self { + Self::new(get_total_post_task_count(), MaxDownloadPostTasks::new()) + } + /// Add a new task. pub async fn add_task(&self, future: F) where @@ -161,6 +187,10 @@ pub fn get_total_download_task_count() -> Arc> { Arc::clone(&TOTAL_DOWNLOAD_TASK_COUNT) } +pub fn get_total_post_task_count() -> Arc> { + Arc::clone(&TOTAL_POST_TASK_COUNT) +} + impl Default for TaskManager { fn default() -> Self { Self::new(get_total_download_task_count(), MaxDownloadTasks::new())