From 54fdca5b15e71d9b215c60a5ac1ac7575c815c6e Mon Sep 17 00:00:00 2001 From: lifegpc Date: Thu, 10 Mar 2022 17:21:28 +0800 Subject: [PATCH] add option use_webpage --- Language/pixiv_downloader.pot | 110 ++++++++++++++------------- Language/pixiv_downloader.zh_CN.po | 116 +++++++++++++++-------------- src/data/data.rs | 16 +++- src/download.rs | 30 ++++++-- src/opthelper.rs | 11 +++ src/opts.rs | 5 ++ src/pixiv_web.rs | 16 +++- src/settings.rs | 14 ++++ src/settings_list.rs | 1 + 9 files changed, 200 insertions(+), 119 deletions(-) diff --git a/Language/pixiv_downloader.pot b/Language/pixiv_downloader.pot index 3c03cd3..5498358 100644 --- a/Language/pixiv_downloader.pot +++ b/Language/pixiv_downloader.pot @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: pixiv_downloader\n" -"POT-Creation-Date: 2022-03-09 22:44+0800\n" +"POT-Creation-Date: 2022-03-10 17:17+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -69,19 +69,19 @@ msgstr "" msgid "Can not parse expired time:" msgstr "" -#: cookies.rs:366 data/json.rs:53 settings.rs:404 webclient.rs:288 +#: cookies.rs:366 data/json.rs:53 settings.rs:418 webclient.rs:288 msgid "Failed to remove file:" msgstr "" -#: cookies.rs:372 data/json.rs:64 settings.rs:411 webclient.rs:294 +#: cookies.rs:372 data/json.rs:64 settings.rs:425 webclient.rs:294 msgid "Failed to create file:" msgstr "" -#: cookies.rs:379 data/json.rs:70 settings.rs:417 webclient.rs:307 +#: cookies.rs:379 data/json.rs:70 settings.rs:431 webclient.rs:307 msgid "Failed to write file:" msgstr "" -#: data/data.rs:62 +#: data/data.rs:70 msgid "Failed to unescape string:" msgstr "" @@ -89,40 +89,40 @@ msgstr "" msgid "Failed to initialize pixiv web api client." msgstr "" -#: download.rs:25 +#: download.rs:27 msgid "Warning: Web api client not logined, some future may not work." msgstr "" -#: download.rs:49 +#: download.rs:63 msgid "Failed to get page count." msgstr "" -#: download.rs:58 +#: download.rs:72 msgid "Failed to get pages' data." msgstr "" -#: download.rs:67 +#: download.rs:85 msgid "Failed to save metadata to JSON file." msgstr "" -#: download.rs:77 download.rs:120 +#: download.rs:95 download.rs:142 msgid "Failed to get original picture's link." msgstr "" -#: download.rs:83 download.rs:126 +#: download.rs:101 download.rs:148 msgid "Failed to get file name from url:" msgstr "" -#: download.rs:90 download.rs:96 download.rs:133 download.rs:139 +#: download.rs:108 download.rs:114 download.rs:155 download.rs:161 #: pixiv_web.rs:153 msgid "Failed to download image:" msgstr "" -#: download.rs:101 download.rs:144 +#: download.rs:119 download.rs:166 msgid "Downloaded image:" msgstr "" -#: download.rs:110 download.rs:153 +#: download.rs:128 download.rs:175 msgid "Failed to add exif data to image:" msgstr "" @@ -130,87 +130,91 @@ msgstr "" msgid "Failed to parse duration from string." msgstr "" -#: opts.rs:85 +#: opts.rs:88 msgid "Warning: The specified config file not found." msgstr "" -#: opts.rs:109 +#: opts.rs:112 msgid "Usage:" msgstr "" -#: opts.rs:111 +#: opts.rs:114 msgid "Download an artwork" msgstr "" -#: opts.rs:113 +#: opts.rs:116 msgid "Fix the config file" msgstr "" -#: opts.rs:115 +#: opts.rs:118 msgid "Print all available settings" msgstr "" -#: opts.rs:123 +#: opts.rs:126 msgid "Print help message." msgstr "" -#: opts.rs:127 +#: opts.rs:130 msgid "The location of config file." msgstr "" -#: opts.rs:133 settings_list.rs:10 +#: opts.rs:136 settings_list.rs:10 msgid "The location of cookies file. Used for web API." msgstr "" -#: opts.rs:139 settings_list.rs:11 +#: opts.rs:142 settings_list.rs:11 msgid "The language of translated tags." msgstr "" -#: opts.rs:142 +#: opts.rs:145 msgid "Verbose logging." msgstr "" -#: opts.rs:143 +#: opts.rs:146 msgid "Overwrite existing file." msgstr "" -#: opts.rs:144 +#: opts.rs:147 msgid "Skip overwrite existing file." msgstr "" -#: opts.rs:148 settings_list.rs:12 +#: opts.rs:151 settings_list.rs:12 msgid "Max retry count if request failed." msgstr "" -#: opts.rs:154 settings_list.rs:13 +#: opts.rs:157 settings_list.rs:13 msgid "The interval (in seconds) between two retries." msgstr "" -#: opts.rs:176 +#: opts.rs:160 settings_list.rs:14 +msgid "Use data from webpage first." +msgstr "" + +#: opts.rs:180 msgid "Unknown command." msgstr "" -#: opts.rs:186 +#: opts.rs:190 msgid "Failed to parse ID:" msgstr "" -#: opts.rs:192 +#: opts.rs:196 msgid "No URL or ID specified." msgstr "" -#: opts.rs:200 +#: opts.rs:204 msgid "No detailed command specified." msgstr "" -#: opts.rs:213 +#: opts.rs:217 msgid "Unknown config subcommand." msgstr "" -#: opts.rs:249 +#: opts.rs:253 msgid "Retry count must be an non-negative integer:" msgstr "" -#: opts.rs:258 +#: opts.rs:262 msgid "Failed to parse retry interval:" msgstr "" @@ -250,23 +254,23 @@ msgstr "" msgid "Failed to detect error." msgstr "" -#: pixiv_web.rs:169 pixiv_web.rs:174 -msgid "Failed to get artwork page:" -msgstr "" - -#: pixiv_web.rs:180 -msgid "Failed to parse artwork page." -msgstr "" - -#: pixiv_web.rs:184 +#: pixiv_web.rs:168 pixiv_web.rs:198 msgid "Artwork's data:" msgstr "" -#: pixiv_web.rs:198 +#: pixiv_web.rs:183 pixiv_web.rs:188 +msgid "Failed to get artwork page:" +msgstr "" + +#: pixiv_web.rs:194 +msgid "Failed to parse artwork page." +msgstr "" + +#: pixiv_web.rs:212 msgid "Artwork's page data:" msgstr "" -#: pixiv_web.rs:222 +#: pixiv_web.rs:236 msgid "Warning: Failed to save cookies file:" msgstr "" @@ -290,31 +294,31 @@ msgstr "" msgid "Can not insert setting to JSON object." msgstr "" -#: settings.rs:339 +#: settings.rs:353 msgid "Settings file is empty." msgstr "" -#: settings.rs:346 +#: settings.rs:360 msgid "Can not read from settings file." msgstr "" -#: settings.rs:355 +#: settings.rs:369 msgid "Can not parse settings file." msgstr "" -#: settings.rs:364 +#: settings.rs:378 msgid "Unknown settings file." msgstr "" -#: settings.rs:375 +#: settings.rs:389 msgid "\"\" is invalid, you can use \"pixiv_downloader config fix\" to remove all invalid value." msgstr "" -#: settings.rs:394 +#: settings.rs:408 msgid "Can not convert settings to JSON object." msgstr "" -#: settings.rs:422 +#: settings.rs:436 msgid "Failed to flush file:" msgstr "" diff --git a/Language/pixiv_downloader.zh_CN.po b/Language/pixiv_downloader.zh_CN.po index b9fead0..fa296d4 100644 --- a/Language/pixiv_downloader.zh_CN.po +++ b/Language/pixiv_downloader.zh_CN.po @@ -1,8 +1,8 @@ msgid "" msgstr "" "Project-Id-Version: pixiv_downloader\n" -"POT-Creation-Date: 2022-03-09 22:44+0800\n" -"PO-Revision-Date: 2022-03-09 22:45+0800\n" +"POT-Creation-Date: 2022-03-10 17:17+0800\n" +"PO-Revision-Date: 2022-03-10 17:20+0800\n" "Last-Translator: lifegpc \n" "Language-Team: \n" "Language: zh_CN\n" @@ -70,19 +70,19 @@ msgstr "无效的Cookie：" msgid "Can not parse expired time:" msgstr "无法解析过期时间：" -#: cookies.rs:366 data/json.rs:53 settings.rs:404 webclient.rs:288 +#: cookies.rs:366 data/json.rs:53 settings.rs:418 webclient.rs:288 msgid "Failed to remove file:" msgstr "无法删除文件：" -#: cookies.rs:372 data/json.rs:64 settings.rs:411 webclient.rs:294 +#: cookies.rs:372 data/json.rs:64 settings.rs:425 webclient.rs:294 msgid "Failed to create file:" msgstr "无法创建文件：" -#: cookies.rs:379 data/json.rs:70 settings.rs:417 webclient.rs:307 +#: cookies.rs:379 data/json.rs:70 settings.rs:431 webclient.rs:307 msgid "Failed to write file:" msgstr "无法写入文件：" -#: data/data.rs:62 +#: data/data.rs:70 msgid "Failed to unescape string:" msgstr "无法反转义字符串：" @@ -90,128 +90,132 @@ msgstr "无法反转义字符串：" msgid "Failed to initialize pixiv web api client." msgstr "无法初始化 Pixiv 网页 API 客户端。" -#: download.rs:25 +#: download.rs:27 msgid "Warning: Web api client not logined, some future may not work." msgstr "警告：Web API 客户端未登录，一些功能可能无法工作。" -#: download.rs:49 +#: download.rs:63 msgid "Failed to get page count." msgstr "无法获取页数。" -#: download.rs:58 +#: download.rs:72 msgid "Failed to get pages' data." msgstr "无法获取每页数据。" -#: download.rs:67 +#: download.rs:85 msgid "Failed to save metadata to JSON file." msgstr "无法将元数据保存到 JSON 文件。" -#: download.rs:77 download.rs:120 +#: download.rs:95 download.rs:142 msgid "Failed to get original picture's link." msgstr "无法获取原图链接。" -#: download.rs:83 download.rs:126 +#: download.rs:101 download.rs:148 msgid "Failed to get file name from url:" msgstr "无法从 URL 获取文件名：" -#: download.rs:90 download.rs:96 download.rs:133 download.rs:139 +#: download.rs:108 download.rs:114 download.rs:155 download.rs:161 #: pixiv_web.rs:153 msgid "Failed to download image:" msgstr "无法下载图片：" -#: download.rs:101 download.rs:144 +#: download.rs:119 download.rs:166 msgid "Downloaded image:" msgstr "已下载图片：" -#: download.rs:110 download.rs:153 +#: download.rs:128 download.rs:175 msgid "Failed to add exif data to image:" msgstr "无法往图片增加 EXIF 数据：" #: dur.rs:73 msgid "Failed to parse duration from string." -msgstr "无法解析从字符串解析时长：" +msgstr "无法解析从字符串解析时长。" -#: opts.rs:85 +#: opts.rs:88 msgid "Warning: The specified config file not found." msgstr "警告：没有找到指定的设置文件。" -#: opts.rs:109 +#: opts.rs:112 msgid "Usage:" msgstr "使用方法：" -#: opts.rs:111 +#: opts.rs:114 msgid "Download an artwork" msgstr "下载一个作品" -#: opts.rs:113 +#: opts.rs:116 msgid "Fix the config file" msgstr "修复设置文件" -#: opts.rs:115 +#: opts.rs:118 msgid "Print all available settings" msgstr "打印所有可用的设置" -#: opts.rs:123 +#: opts.rs:126 msgid "Print help message." msgstr "打印帮助信息。" -#: opts.rs:127 +#: opts.rs:130 msgid "The location of config file." msgstr "设置文件的位置。" -#: opts.rs:133 settings_list.rs:10 +#: opts.rs:136 settings_list.rs:10 msgid "The location of cookies file. Used for web API." msgstr "cookies 文件的位置。用于网页 API。" -#: opts.rs:139 settings_list.rs:11 +#: opts.rs:142 settings_list.rs:11 msgid "The language of translated tags." msgstr "翻译后的标签语言。" -#: opts.rs:142 +#: opts.rs:145 msgid "Verbose logging." msgstr "启用详细内容输出。" -#: opts.rs:143 +#: opts.rs:146 msgid "Overwrite existing file." msgstr "覆盖已有文件。" -#: opts.rs:144 +#: opts.rs:147 msgid "Skip overwrite existing file." msgstr "跳过覆盖已有文件。" -#: opts.rs:148 settings_list.rs:12 +#: opts.rs:151 settings_list.rs:12 msgid "Max retry count if request failed." msgstr "请求失败时最大重试次数。" -#: opts.rs:154 settings_list.rs:13 +#: opts.rs:157 settings_list.rs:13 msgid "The interval (in seconds) between two retries." msgstr "两次尝试的间隔时间（单位：秒）。" -#: opts.rs:176 +#: opts.rs:160 settings_list.rs:14 +msgid "Use data from webpage first." +msgstr "优先使用来自网页的数据。" + +#: opts.rs:180 msgid "Unknown command." msgstr "未知指令。" -#: opts.rs:186 +#: opts.rs:190 msgid "Failed to parse ID:" msgstr "无法解析 ID：" -#: opts.rs:192 +#: opts.rs:196 msgid "No URL or ID specified." msgstr "没有指定网址或 ID。" -#: opts.rs:200 +#: opts.rs:204 msgid "No detailed command specified." msgstr "没有指定更详细的指令。" -#: opts.rs:213 +#: opts.rs:217 msgid "Unknown config subcommand." msgstr "未知的 config 子指令。" -#: opts.rs:249 +#: opts.rs:253 msgid "Retry count must be an non-negative integer:" msgstr "重试次数不能是负数：" -#: opts.rs:258 +#: opts.rs:262 msgid "Failed to parse retry interval:" msgstr "无法解析间隔时间：" @@ -251,23 +255,23 @@ msgstr "网络错误：" msgid "Failed to detect error." msgstr "检测 error 字段失败。" -#: pixiv_web.rs:169 pixiv_web.rs:174 -msgid "Failed to get artwork page:" -msgstr "无法获取作品页：" - -#: pixiv_web.rs:180 -msgid "Failed to parse artwork page." -msgstr "无法解析作品页。" - -#: pixiv_web.rs:184 +#: pixiv_web.rs:168 pixiv_web.rs:198 msgid "Artwork's data:" msgstr "作品数据：" -#: pixiv_web.rs:198 +#: pixiv_web.rs:183 pixiv_web.rs:188 +msgid "Failed to get artwork page:" +msgstr "无法获取作品页：" + +#: pixiv_web.rs:194 +msgid "Failed to parse artwork page." +msgstr "无法解析作品页。" + +#: pixiv_web.rs:212 msgid "Artwork's page data:" msgstr "作品页面数据：" -#: pixiv_web.rs:222 +#: pixiv_web.rs:236 msgid "Warning: Failed to save cookies file:" msgstr "警告：无法保存 cookies 文件：" @@ -277,7 +281,7 @@ msgstr "无法获取 JSON 对象。" #: retry_interval.rs:37 retry_interval.rs:55 msgid "Failed to parse JSON number." -msgstr "无法解析 JSON 数字：" +msgstr "无法解析 JSON 数字。" #: retry_interval.rs:61 retry_interval.rs:65 msgid "Unsupported JSON type." @@ -291,23 +295,23 @@ msgstr "多种类型" msgid "Can not insert setting to JSON object." msgstr "无法将设置插入 JSON 对象。" -#: settings.rs:339 +#: settings.rs:353 msgid "Settings file is empty." msgstr "设置文件是空的。" -#: settings.rs:346 +#: settings.rs:360 msgid "Can not read from settings file." msgstr "无法读取设置文件。" -#: settings.rs:355 +#: settings.rs:369 msgid "Can not parse settings file." msgstr "无法解析设置文件。" -#: settings.rs:364 +#: settings.rs:378 msgid "Unknown settings file." msgstr "未知的设置文件。" -#: settings.rs:375 +#: settings.rs:389 msgid "" "\"\" is invalid, you can use \"pixiv_downloader config fix\" to remove " "all invalid value." @@ -315,11 +319,11 @@ msgstr "" "\"\" 不合法，您可以使用 \"pixiv_downloader config fix\" 来移除所有不合" "法的值。" -#: settings.rs:394 +#: settings.rs:408 msgid "Can not convert settings to JSON object." msgstr "无法将设置转换为 JSON 对象。" -#: settings.rs:422 +#: settings.rs:436 msgid "Failed to flush file:" msgstr "无法刷新文件缓冲区：" diff --git a/src/data/data.rs b/src/data/data.rs index ce56747..a8d0038 100644 --- a/src/data/data.rs +++ b/src/data/data.rs @@ -37,22 +37,30 @@ impl PixivData { pub fn from_web_page_data(&mut self, value: &JsonValue, allow_overwrite: bool) { let id: u64 = (&self.id).try_into().unwrap(); let ids = format!("{}", id); + self.from_web_page_ajax_data(&value["illust"][ids.as_str()], allow_overwrite) + } + + /// Read data from JSON object. + /// The object is from `https://www.pixiv.net/illust/ajax/` + /// * `value` - The JSON object + /// * `allow_overwrite` - Allow overwrite the data existing. + pub fn from_web_page_ajax_data(&mut self, value: &JsonValue, allow_overwrite: bool) { if self.title.is_none() || allow_overwrite { - let title = value["illust"][ids.as_str()]["illustTitle"].as_str(); + let title = value["illustTitle"].as_str(); if title.is_some() { self.title = Some(String::from(title.unwrap())); } } if self.author.is_none() || allow_overwrite { - let author = value["illust"][ids.as_str()]["userName"].as_str(); + let author = value["userName"].as_str(); if author.is_some() { self.author = Some(String::from(author.unwrap())); } } if self.description.is_none() || allow_overwrite { - let mut description = value["illust"][ids.as_str()]["description"].as_str(); + let mut description = value["description"].as_str(); if description.is_none() { - description = value["illust"][ids.as_str()]["illustComment"].as_str(); + description = value["illustComment"].as_str(); } if description.is_some() { let re = unescape(description.unwrap()); diff --git a/src/download.rs b/src/download.rs index 3113fcf..fa7b2c6 100644 --- a/src/download.rs +++ b/src/download.rs @@ -41,12 +41,24 @@ impl Main { } pub fn download_artwork(&self, pw: &mut PixivWebClient, id: u64) -> i32 { - let re = pw.get_artwork(id); + let mut re = None; + let pages; + let mut ajax_ver = true; + if pw.helper.use_webpage() { + re = pw.get_artwork(id); + if re.is_some() { + ajax_ver = false; + } + } if re.is_none() { - return 1; + re = pw.get_artwork_ajax(id); } let re = re.unwrap(); - let pages = (&re["illust"][format!("{}", id).as_str()]["pageCount"]).as_u64(); + if ajax_ver { + pages = (&re["pageCount"]).as_u64(); + } else { + pages = (&re["illust"][format!("{}", id).as_str()]["pageCount"]).as_u64(); + } if pages.is_none() { println!("{}", gettext("Failed to get page count.")); return 1; @@ -63,7 +75,11 @@ impl Main { let base = PathBuf::from("."); let json_file = base.join(format!("{}.json", id)); let mut datas = PixivData::new(id).unwrap(); - datas.from_web_page_data(&re, true); + if ajax_ver { + datas.from_web_page_ajax_data(&re, true); + } else { + datas.from_web_page_data(&re, true); + } let json_data = JSONDataFile::from(&datas); if !json_data.save(&json_file) { println!("{}", gettext("Failed to save metadata to JSON file.")); @@ -117,7 +133,11 @@ impl Main { } } } else { - let link = (&re["illust"][format!("{}", id)]["urls"]["original"]).as_str(); + let link = if ajax_ver { + (&re["urls"]["original"]).as_str() + } else { + (&re["illust"][format!("{}", id)]["urls"]["original"]).as_str() + }; if link.is_none() { println!("{}", gettext("Failed to get original picture's link.")); return 1; diff --git a/src/opthelper.rs b/src/opthelper.rs index 0d3b60e..ad2b992 100644 --- a/src/opthelper.rs +++ b/src/opthelper.rs @@ -78,4 +78,15 @@ impl<'a> OptHelper<'a> { } self.default_retry_interval.clone() } + + /// Return whether to use data from webpage first. + pub fn use_webpage(&self) -> bool { + if self.opt.use_webpage { + return true; + } + if self.settings.have_bool("use-webpage") { + return self.settings.get_bool("use-webpage").unwrap(); + } + false + } } diff --git a/src/opts.rs b/src/opts.rs index df14246..ff1c541 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -57,6 +57,8 @@ pub struct CommandOpts { pub retry: Option, /// Retry interval pub retry_interval: Option>, + /// Use data from webpage first + pub use_webpage: bool, } impl CommandOpts { @@ -72,6 +74,7 @@ impl CommandOpts { overwrite: None, retry: None, retry_interval: None, + use_webpage: false, } } @@ -154,6 +157,7 @@ pub fn parse_cmd() -> Option { gettext("The interval (in seconds) between two retries."), "LIST", ); + opts.optflag("", "use-webpage", gettext("Use data from webpage first.")); let result = match opts.parse(&argv[1..]) { Ok(m) => m, Err(err) => { @@ -260,5 +264,6 @@ pub fn parse_cmd() -> Option { } re.as_mut().unwrap().retry_interval = Some(r.unwrap()); } + re.as_mut().unwrap().use_webpage = result.opt_present("use-webpage"); re } diff --git a/src/pixiv_web.rs b/src/pixiv_web.rs index 377a5ce..2e6bdd6 100644 --- a/src/pixiv_web.rs +++ b/src/pixiv_web.rs @@ -156,6 +156,20 @@ impl<'a> PixivWebClient<'a> { Some(r) } + pub fn get_artwork_ajax(&mut self, id: u64) -> Option { + self.auto_init(); + let r = self.client.get(format!("https://www.pixiv.net/ajax/illust/{}", id), None); + if r.is_none() { + return None; + } + let r = r.unwrap(); + let v = self.deal_json(r); + if self.helper.verbose() && v.is_some() { + println!("{} {}", gettext("Artwork's data:"), v.as_ref().unwrap().pretty(2)); + } + v + } + pub fn get_artwork(&mut self, id: u64) -> Option { self.auto_init(); let r = self.client.get(format!("https://www.pixiv.net/artworks/{}", id), None); @@ -194,7 +208,7 @@ impl<'a> PixivWebClient<'a> { } let r = r.unwrap(); let v = self.deal_json(r); - if self.helper.verbose() { + if self.helper.verbose() && v.is_some() { println!("{} {}", gettext("Artwork's page data:"), v.as_ref().unwrap().pretty(2)); } v diff --git a/src/settings.rs b/src/settings.rs index 027e723..274c440 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -292,6 +292,13 @@ impl SettingStore { self.data.get(key) } + pub fn get_bool(&self, key: &str) -> Option { + match self.data.get(key) { + Some(obj) => { obj.as_bool() } + None => { None } + } + } + pub fn get_str(&self, key: &str) -> Option { let obj = self.data.get(key); if obj.is_none() { @@ -309,6 +316,13 @@ impl SettingStore { self.data.have(key) } + pub fn have_bool(&self, key: &str) -> bool { + match self.data.get(key) { + Some(obj) => { obj.is_boolean() } + None => { false } + } + } + pub fn have_str(&self, key: &str) -> bool { let obj = self.data.get(key); if obj.is_none() { diff --git a/src/settings_list.rs b/src/settings_list.rs index 66ef5c7..f4e8f3d 100644 --- a/src/settings_list.rs +++ b/src/settings_list.rs @@ -11,6 +11,7 @@ pub fn get_settings_list() -> Vec { SettingDes::new("language", gettext("The language of translated tags."), JsonValueType::Str, None).unwrap(), SettingDes::new("retry", gettext("Max retry count if request failed."), JsonValueType::Number, Some(check_u64)).unwrap(), SettingDes::new("retry-interval", gettext("The interval (in seconds) between two retries."), JsonValueType::Multiple, Some(check_retry_interval)).unwrap(), + SettingDes::new("use-webpage", gettext("Use data from webpage first."), JsonValueType::Boolean, None).unwrap(), ] }