lifegpc/pixiv_downloader
1
0
Fork 0
mirror of https://github.com/lifegpc/pixiv_downloader.git synced 2026-06-21 03:14:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add new option download-multiple-posts and max-download-post-tasks

Browse Source
This commit is contained in:
lifegpc
2022-07-17 06:00:12 +00:00
committed by GitHub
parent d6be8107ee
commit b8fac850c9
7 changed files with 184 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
proc_macros/proc_macros.rs
View File

@@ -149,7 +149,7 @@ pub fn fanbox_api_test(item: TokenStream) -> TokenStream {
panic!("Failed to initiailze the client.");
}
if !client.check_login().await {
println!("The client is not logined. Skip test.");
println!("The client is not logged in. Skip test.");
return;
}
#(#stmts)*
@@ -197,7 +197,7 @@ pub fn fanbox_api_quick_test(item: TokenStream) -> TokenStream {
panic!("Failed to initiailze the client.");
}
if !client.check_login().await {
println!("The client is not logined. Skip test.");
println!("The client is not logged in. Skip test.");
return;
}
match #expr.await {

63
src/download.rs
View File

@@ -42,6 +42,8 @@ impl Main {
pub async fn download(&mut self) -> i32 {
let pw = Arc::new(PixivWebClient::new());
let fc = Arc::new(FanboxClient::new());
let tasks = TaskManager::new_post();
let download_multiple_posts = get_helper().download_multiple_posts();
for id in self.cmd.as_ref().unwrap().ids.iter() {
match id {
PixivID::Artwork(id) => {
@@ -54,22 +56,14 @@ impl Main {
return 1;
}
if !pw.logined() {
println!("{}", gettext("Warning: Web api client not logined, some future may not work."));
println!("{}", gettext("Warning: Web api client not logged in, some future may not work."));
}
}
let r = self.download_artwork(Arc::clone(&pw), id.clone()).await;
let r = if r.is_ok() {
0
} else {
println!(
"{} {}",
gettext("Failed to download artwork:"),
r.unwrap_err()
);
1
};
if r != 0 {
return r;
tasks
.add_task(Self::download_artwork(Arc::clone(&pw), id.clone()))
.await;
if !download_multiple_posts {
tasks.join().await;
}
}
PixivID::FanboxPost(id) => {
@@ -83,24 +77,39 @@ impl Main {
return 1;
}
if !fc.logined() {
println!("{}", gettext("Warning: Fanbox client is not logined."));
println!("{}", gettext("Warning: Fanbox client is not logged in."));
}
let r = self.download_fanbox_post(Arc::clone(&fc), id.clone()).await;
let r = match r {
Ok(_) => 0,
Err(e) => {
println!("{} {}", gettext("Failed to download post:"), e);
1
}
};
if r != 0 {
return r;
tasks
.add_task(Self::download_fanbox_post(Arc::clone(&fc), id.clone()))
.await;
if !download_multiple_posts {
tasks.join().await;
}
}
}
}
}
0
let mut re = 0;
tasks.join().await;
let tasks = tasks.take_finished_tasks();
for mut task in tasks {
let task = task.as_any_mut();
if let Some(task) = task.downcast_mut::<JoinHandle<Result<(), PixivDownloaderError>>>()
{
let result = match task.await {
Ok(result) => result,
Err(e) => Err(PixivDownloaderError::from(e)),
};
match result {
Ok(_) => {}
Err(e) => {
println!("{} {}", gettext("Failed to download post:"), e);
re = 1;
}
}
}
}
re
}
/// Download artwork link
@@ -171,7 +180,6 @@ impl Main {
}
pub async fn download_artwork(
&self,
pw: Arc<PixivWebClient>,
id: u64,
) -> Result<(), PixivDownloaderError> {
@@ -498,7 +506,6 @@ impl Main {
}
pub async fn download_fanbox_post(
&self,
fc: Arc<FanboxClient>,
id: FanboxPostID,
) -> Result<(), PixivDownloaderError> {

45
src/opthelper.rs
View File

@@ -71,6 +71,24 @@ impl OptHelper {
self._cors_entries.get_ref().clone()
}
/// Whether to download multiple posts/artworks at the same time.
pub fn download_multiple_posts(&self) -> bool {
match self.opt.get_ref().download_multiple_posts {
Some(r) => {
return r;
}
None => {}
}
if self.settings.get_ref().have_bool("download-multiple-posts") {
return self
.settings
.get_ref()
.get_bool("download-multiple-posts")
.unwrap();
}
false
}
/// Return max retry count of each part when downloading in multiple thread mode.
pub fn download_part_retry(&self) -> Option<i64> {
if self.opt.get_ref().download_part_retry.is_some() {
@@ -128,7 +146,24 @@ impl OptHelper {
}
}
/// Return the maximun number of tasks to download simultaneously.
/// Return the maximum number of tasks to download posts/artworks at the same time.
pub fn max_download_post_tasks(&self) -> usize {
match self.opt.get_ref().max_download_post_tasks {
Some(r) => {
return r;
}
None => {}
}
match self.settings.get_ref().get("max-download-post-tasks") {
Some(re) => {
return re.as_usize().unwrap();
}
None => {}
}
3
}
/// Return the maximum number of tasks to download files at the same time.
pub fn max_download_tasks(&self) -> usize {
match self.opt.get_ref().max_download_tasks {
Some(r) => {
@@ -145,7 +180,7 @@ impl OptHelper {
5
}
/// Return the maximun threads when downloading file.
/// Return the maximum threads when downloading file.
pub fn max_threads(&self) -> u64 {
match self.opt.get_ref().max_threads {
Some(r) => {
@@ -331,11 +366,7 @@ impl OptHelper {
}
None => {}
}
if self
.settings
.get_ref()
.have_bool("download-multiple-files")
{
if self.settings.get_ref().have_bool("download-multiple-files") {
return self
.settings
.get_ref()

68
src/opts.rs
View File

@@ -85,15 +85,19 @@ pub struct CommandOpts {
pub multiple_threads_download: Option<bool>,
/// Max retry count of each part when downloading in multiple thread mode.
pub download_part_retry: Option<i64>,
/// The maximun threads when downloading file.
/// The maximum threads when downloading file.
pub max_threads: Option<u64>,
/// The size of the each part when downloading file.
pub part_size: Option<u32>,
#[cfg(feature = "server")]
/// Server listen address
pub server: Option<SocketAddr>,
/// Maximun number of tasks to download simultaneously
/// maximum number of tasks to download files at the same time
pub max_download_tasks: Option<usize>,
/// Whether to download multiple posts/artworks at the same time.
pub download_multiple_posts: Option<bool>,
/// The maximum number of tasks to download posts/artworks at the same time.
pub max_download_post_tasks: Option<usize>,
}
impl CommandOpts {
@@ -123,6 +127,8 @@ impl CommandOpts {
#[cfg(feature = "server")]
server: None,
max_download_tasks: None,
download_multiple_posts: None,
max_download_post_tasks: None,
}
}
@@ -398,7 +404,7 @@ pub fn parse_cmd() -> Option<CommandOpts> {
opts.optopt(
"m",
"max-threads",
gettext("The maximun threads when downloading file."),
gettext("The maximum threads when downloading file."),
"COUNT",
);
opts.optopt(
@@ -412,7 +418,7 @@ pub fn parse_cmd() -> Option<CommandOpts> {
"max-download-tasks",
format!(
"{} ({} {})",
gettext("The maximun number of tasks to download simultaneously."),
gettext("The maximum number of tasks to download files at the same time."),
gettext("Default:"),
"5"
)
@@ -421,6 +427,34 @@ pub fn parse_cmd() -> Option<CommandOpts> {
HasArg::Maybe,
getopts::Occur::Optional,
);
opts.opt(
"",
"download-multiple-posts",
format!(
"{} ({} {})",
gettext("Download multiple posts/artworks at the same time."),
gettext("Default:"),
"yes"
)
.as_str(),
"yes/no",
HasArg::Maybe,
getopts::Occur::Optional,
);
opts.opt(
"",
"max-download-post-tasks",
format!(
"{} ({} {})",
gettext("The maximum number of tasks to download posts/artworks at the same time."),
gettext("Default:"),
3
)
.as_str(),
"yes/no",
HasArg::Maybe,
getopts::Occur::Optional,
);
let result = match opts.parse(&argv[1..]) {
Ok(m) => m,
Err(err) => {
@@ -645,6 +679,32 @@ pub fn parse_cmd() -> Option<CommandOpts> {
return None;
}
}
match parse_optional_opt(&result, "download-multiple-posts", true, parse_bool) {
Ok(b) => re.as_mut().unwrap().download_multiple_posts = b,
Err(e) => {
println!(
"{} {}",
gettext("Failed to parse <opt>:")
.replace("<opt>", "download-multiple-posts")
.as_str(),
e
);
return None;
}
}
match parse_optional_opt(&result, "max-download-post-tasks", 3, parse_nonempty_usize) {
Ok(r) => re.as_mut().unwrap().max_download_post_tasks = r,
Err(e) => {
println!(
"{} {}",
gettext("Failed to parse <opt>:")
.replace("<opt>", "max-download-post-tasks")
.as_str(),
e
);
return None;
}
}
re
}

14
src/pixiv_web.rs
View File

@@ -74,7 +74,7 @@ impl PixivWebClient {
self.auto_init();
let r = self
.client
.get_with_param("https://www.pixiv.net/", self.params.get_ref(), None)
.get_with_param("https://www.pixiv.net/", self.get_params(), None)
.await;
if r.is_none() {
return false;
@@ -186,7 +186,7 @@ impl PixivWebClient {
.client
.get_with_param(
format!("https://www.pixiv.net/ajax/illust/{}", id),
self.params.get_ref(),
self.get_params(),
None,
)
.await;
@@ -211,7 +211,7 @@ impl PixivWebClient {
.client
.get_with_param(
format!("https://www.pixiv.net/artworks/{}", id),
self.params.get_ref(),
self.get_params(),
None,
)
.await;
@@ -256,7 +256,7 @@ impl PixivWebClient {
.client
.get_with_param(
format!("https://www.pixiv.net/ajax/illust/{}/pages", id),
self.params.get_ref(),
self.get_params(),
None,
)
.await;
@@ -275,13 +275,17 @@ impl PixivWebClient {
v
}
pub fn get_params(&self) -> Option<JsonValue> {
self.params.get_ref().clone()
}
pub async fn get_ugoira(&self, id: u64) -> Option<JsonValue> {
self.auto_init();
let r = self
.client
.get_with_param(
format!("https://www.pixiv.net/ajax/illust/{}/ugoira_meta", id),
self.params.get_ref(),
self.get_params(),
None,
)
.await;

6
src/settings_list.rs
View File

@@ -33,14 +33,16 @@ pub fn get_settings_list() -> Vec<SettingDes> {
SettingDes::new("download-retry-interval", gettext("The interval (in seconds) between two retries when downloading files."), JsonValueType::Multiple, Some(check_retry_interval)).unwrap(),
SettingDes::new("multiple-threads-download", gettext("Whether to enable multiple threads download."), JsonValueType::Boolean, None).unwrap(),
SettingDes::new("download-part-retry", gettext("Max retry count of each part when downloading in multiple thread mode."), JsonValueType::Number, Some(check_i64)).unwrap(),
SettingDes::new("max-threads", gettext("The maximun threads when downloading file."), JsonValueType::Number, Some(check_u64)).unwrap(),
SettingDes::new("max-threads", gettext("The maximum threads when downloading file."), JsonValueType::Number, Some(check_u64)).unwrap(),
SettingDes::new("part-size", gettext("The size of the each part when downloading file."), JsonValueType::Number, Some(check_parse_size_u32)).unwrap(),
SettingDes::new("proxy", gettext("Proxy settings."), JsonValueType::Array, Some(check_proxy)).unwrap(),
#[cfg(feature = "server")]
SettingDes::new("server", gettext("Server address."), JsonValueType::Str, Some(check_socket_addr)).unwrap(),
#[cfg(feature = "server")]
SettingDes::new("cors-entries", gettext("The domains allowed to send CORS requests."), JsonValueType::Array, Some(check_cors_entries)).unwrap(),
SettingDes::new("max-download-tasks", gettext("The maximun number of tasks to download simultaneously."), JsonValueType::Number, Some(check_nozero_usize)).unwrap(),
SettingDes::new("max-download-tasks", gettext("The maximum number of tasks to download files at the same time."), JsonValueType::Number, Some(check_nozero_usize)).unwrap(),
SettingDes::new("download-multiple-posts", gettext("Download multiple posts/artworks at the same time."), JsonValueType::Boolean, None).unwrap(),
SettingDes::new("max-download-post-tasks", gettext("The maximum number of tasks to download posts/artworks at the same time."), JsonValueType::Number, Some(check_nozero_usize)).unwrap(),
]
}

34
src/task_manager.rs
View File

@@ -16,6 +16,8 @@ lazy_static! {
#[doc(hidden)]
static ref TOTAL_DOWNLOAD_TASK_COUNT: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
#[doc(hidden)]
static ref TOTAL_POST_TASK_COUNT: Arc<Mutex<usize>> = Arc::new(Mutex::new(0));
#[doc(hidden)]
static ref PROGRESS_BAR: Arc<MultiProgress> = Arc::new(MultiProgress::new());
}
@@ -63,6 +65,22 @@ impl GetMaxCount for MaxDownloadTasks {
}
}
pub struct MaxDownloadPostTasks {
_unused: [u8; 0],
}
impl MaxDownloadPostTasks {
pub fn new() -> Self {
MaxDownloadPostTasks { _unused: [] }
}
}
impl GetMaxCount for MaxDownloadPostTasks {
fn get_max_count(&self) -> usize {
get_helper().max_download_post_tasks()
}
}
/// Task manager
pub struct TaskManager {
/// Current running task
@@ -71,12 +89,15 @@ pub struct TaskManager {
finished_tasks: RwLock<Vec<Box<dyn IsFinishedAny + Send + Sync>>>,
/// Total task count
task_count: Arc<Mutex<usize>>,
max_count: Box<dyn GetMaxCount>,
max_count: Box<dyn GetMaxCount + Send + Sync>,
}
impl TaskManager {
/// Create a new instance
pub fn new<T: GetMaxCount + 'static>(task_count: Arc<Mutex<usize>>, max_count: T) -> Self {
pub fn new<T: GetMaxCount + Send + Sync + 'static>(
task_count: Arc<Mutex<usize>>,
max_count: T,
) -> Self {
Self {
tasks: RwLock::new(Vec::new()),
finished_tasks: RwLock::new(Vec::new()),
@@ -85,6 +106,11 @@ impl TaskManager {
}
}
/// Create a new instance with post max count
pub fn new_post() -> Self {
Self::new(get_total_post_task_count(), MaxDownloadPostTasks::new())
}
/// Add a new task.
pub async fn add_task<F>(&self, future: F)
where
@@ -161,6 +187,10 @@ pub fn get_total_download_task_count() -> Arc<Mutex<usize>> {
Arc::clone(&TOTAL_DOWNLOAD_TASK_COUNT)
}
pub fn get_total_post_task_count() -> Arc<Mutex<usize>> {
Arc::clone(&TOTAL_POST_TASK_COUNT)
}
impl Default for TaskManager {
fn default() -> Self {
Self::new(get_total_download_task_count(), MaxDownloadTasks::new())