diff --git a/Cargo.lock b/Cargo.lock index 333d6a4..da3b506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -394,6 +394,17 @@ version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.21" @@ -413,9 +424,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -890,11 +903,13 @@ version = "0.0.1" dependencies = [ "chrono", "dateparser", + "futures-util", "getopts", "gettext", "html_parser", "json", "lazy_static", + "regex", "reqwest", "spin_on", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 31e9d1a..f8383df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,14 +8,16 @@ edition = "2018" [dependencies] chrono = "0.4" dateparser = "0.1.6" +futures-util = "0.3" getopts = "0.2" gettext = "0.4" html_parser = "0.6.2" lazy_static = "1.4" json = "0.12" +regex = "1" reqwest = { version = "0.11", features = ["brotli", "deflate", "gzip", "rustls-tls", "socks", "stream"] } spin_on = "0.1.1" -tokio = { version = "1.17", features = ["rt", "macros", "rt-multi-thread"] } +tokio = { version = "1.17", features = ["rt", "macros", "rt-multi-thread", "time"] } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["winnls", "stringapiset"] } diff --git a/Language/pixiv_downloader.pot b/Language/pixiv_downloader.pot index 45e3c66..16d270e 100644 --- a/Language/pixiv_downloader.pot +++ b/Language/pixiv_downloader.pot @@ -1,15 +1,15 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: bili\n" -"POT-Creation-Date: 2022-02-26 23:42+0800\n" +"Project-Id-Version: pixiv_downloader\n" +"POT-Creation-Date: 2022-02-27 17:47+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Generator: Poedit 3.0\n" +"X-Generator: Poedit 3.0.1\n" "X-Poedit-Basepath: ../src\n" "X-Poedit-SearchPath-0: .\n" @@ -69,95 +69,171 @@ msgstr "" msgid "Can not parse expired time:" msgstr "" -#: cookies.rs:366 settings.rs:392 +#: cookies.rs:366 data/json.rs:39 settings.rs:392 webclient.rs:257 msgid "Failed to remove file:" msgstr "" -#: cookies.rs:372 settings.rs:399 +#: cookies.rs:372 data/json.rs:50 settings.rs:399 webclient.rs:263 msgid "Failed to create file:" msgstr "" -#: cookies.rs:379 settings.rs:405 +#: cookies.rs:379 data/json.rs:56 settings.rs:405 webclient.rs:276 msgid "Failed to write file:" msgstr "" -#: download.rs:9 pixiv.rs:47 +#: download.rs:15 pixiv_web.rs:53 msgid "Failed to initialize pixiv web api client." msgstr "" -#: opts.rs:69 +#: download.rs:22 +msgid "Warning: Web api client not logined, some future may not work." +msgstr "" + +#: download.rs:45 +msgid "Failed to get page count." +msgstr "" + +#: download.rs:54 +msgid "Failed to get pages' data." +msgstr "" + +#: download.rs:61 +msgid "Failed to save metadata to JSON file." +msgstr "" + +#: download.rs:69 +msgid "Failed to get original picture's link." +msgstr "" + +#: download.rs:75 +msgid "Failed to get file name from url:" +msgstr "" + +#: download.rs:82 download.rs:88 pixiv_web.rs:148 +msgid "Failed to download image:" +msgstr "" + +#: download.rs:93 +msgid "Downloaded image:" +msgstr "" + +#: opts.rs:76 msgid "Warning: The specified config file not found." msgstr "" -#: opts.rs:93 +#: opts.rs:100 msgid "Usage:" msgstr "" -#: opts.rs:95 +#: opts.rs:102 msgid "Download an artwork" msgstr "" -#: opts.rs:97 +#: opts.rs:104 msgid "Fix the config file" msgstr "" -#: opts.rs:99 +#: opts.rs:106 msgid "Print all available settings" msgstr "" -#: opts.rs:107 +#: opts.rs:114 msgid "Print help message." msgstr "" -#: opts.rs:111 +#: opts.rs:118 msgid "The location of config file." msgstr "" -#: opts.rs:117 settings_list.rs:8 +#: opts.rs:124 settings_list.rs:8 msgid "The location of cookies file. Used for web API." msgstr "" -#: opts.rs:123 settings_list.rs:9 +#: opts.rs:130 settings_list.rs:9 msgid "The language of translated tags." msgstr "" -#: opts.rs:145 +#: opts.rs:133 +msgid "Verbose logging." +msgstr "" + +#: opts.rs:134 +msgid "Overwrite existing file." +msgstr "" + +#: opts.rs:135 +msgid "Skip overwrite existing file." +msgstr "" + +#: opts.rs:155 msgid "Unknown command." msgstr "" -#: opts.rs:156 -msgid "No URL specified." +#: opts.rs:165 +msgid "Failed to parse ID:" msgstr "" -#: opts.rs:164 +#: opts.rs:171 +msgid "No URL or ID specified." +msgstr "" + +#: opts.rs:179 msgid "No detailed command specified." msgstr "" -#: opts.rs:177 +#: opts.rs:192 msgid "Unknown config subcommand." msgstr "" -#: parser\mainpage.rs:53 parser/mainpage.rs:53 +#: parser/metadata.rs:54 pixiv_web.rs:107 msgid "Failed to parse JSON:" msgstr "" -#: parser\mainpage.rs:75 parser/mainpage.rs:75 +#: parser/metadata.rs:76 msgid "Failed to parse HTML:" msgstr "" -#: parser\mainpage.rs:80 parser/mainpage.rs:80 +#: parser/metadata.rs:81 msgid "Some errors occured during parsing:" msgstr "" -#: pixiv.rs:63 pixiv.rs:68 +#: pixiv_web.rs:68 pixiv_web.rs:73 msgid "Failed to get main page:" msgstr "" -#: pixiv.rs:74 +#: pixiv_web.rs:79 msgid "Failed to parse main page." msgstr "" -#: pixiv.rs:88 +#: pixiv_web.rs:83 +msgid "Main page's data:" +msgstr "" + +#: pixiv_web.rs:98 +msgid "Network error:" +msgstr "" + +#: pixiv_web.rs:117 +msgid "Failed to detect error." +msgstr "" + +#: pixiv_web.rs:164 pixiv_web.rs:169 +msgid "Failed to get artwork page:" +msgstr "" + +#: pixiv_web.rs:175 +msgid "Failed to parse artwork page." +msgstr "" + +#: pixiv_web.rs:179 +msgid "Artwork's data:" +msgstr "" + +#: pixiv_web.rs:193 +msgid "Artwork's page data:" +msgstr "" + +#: pixiv_web.rs:217 msgid "Warning: Failed to save cookies file:" msgstr "" @@ -201,26 +277,54 @@ msgstr "" msgid "Pixiv's refresh tokens. Used to login." msgstr "" -#: webclient.rs:59 +#: utils.rs:29 +msgid "Do you want to delete file \"\"?" +msgstr "" + +#: utils.rs:49 webclient.rs:143 +msgid "Can not parse URL:" +msgstr "" + +#: utils.rs:56 +msgid "Failed to get file name from path:" +msgstr "" + +#: webclient.rs:103 msgid "Failed to parse Set-Cookie header." msgstr "" -#: webclient.rs:64 +#: webclient.rs:108 msgid "Failed to convert to string:" msgstr "" -#: webclient.rs:94 +#: webclient.rs:150 +msgid "Parameters should be object or array:" +msgstr "" + +#: webclient.rs:170 +msgid "Parameters should be array:" +msgstr "" + +#: webclient.rs:174 +msgid "Parameters need at least a value:" +msgstr "" + +#: webclient.rs:209 msgid "Error when request:" msgstr "" -#: main.rs:46 +#: webclient.rs:270 +msgid "Error when downloading file:" +msgstr "" + +#: main.rs:50 msgid "Failed to save config file:" msgstr "" -#: main.rs:57 +#: main.rs:61 msgid "All available settings:" msgstr "" -#: main.rs:89 +#: main.rs:93 msgid "Can not read config file:" msgstr "" diff --git a/Language/pixiv_downloader.zh_CN.po b/Language/pixiv_downloader.zh_CN.po index 0a123df..70c962a 100644 --- a/Language/pixiv_downloader.zh_CN.po +++ b/Language/pixiv_downloader.zh_CN.po @@ -1,17 +1,17 @@ msgid "" msgstr "" -"Project-Id-Version: bili\n" -"POT-Creation-Date: 2022-02-26 23:42+0800\n" -"PO-Revision-Date: 2022-02-26 23:43+0800\n" +"Project-Id-Version: pixiv_downloader\n" +"POT-Creation-Date: 2022-02-27 17:47+0800\n" +"PO-Revision-Date: 2022-02-27 17:48+0800\n" "Last-Translator: lifegpc \n" "Language-Team: \n" "Language: zh_CN\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Generator: Poedit 3.0\n" -"X-Poedit-Basepath: ../src\n" "Plural-Forms: nplurals=1; plural=0;\n" +"X-Generator: Poedit 3.0.1\n" +"X-Poedit-Basepath: ../src\n" "X-Poedit-SearchPath-0: .\n" #: cookies.rs:75 cookies.rs:217 @@ -70,95 +70,171 @@ msgstr "无效的Cookie：" msgid "Can not parse expired time:" msgstr "无法解析过期时间：" -#: cookies.rs:366 settings.rs:392 +#: cookies.rs:366 data/json.rs:39 settings.rs:392 webclient.rs:257 msgid "Failed to remove file:" msgstr "无法删除文件：" -#: cookies.rs:372 settings.rs:399 +#: cookies.rs:372 data/json.rs:50 settings.rs:399 webclient.rs:263 msgid "Failed to create file:" msgstr "无法创建文件：" -#: cookies.rs:379 settings.rs:405 +#: cookies.rs:379 data/json.rs:56 settings.rs:405 webclient.rs:276 msgid "Failed to write file:" msgstr "无法写入文件：" -#: download.rs:9 pixiv.rs:47 +#: download.rs:15 pixiv_web.rs:53 msgid "Failed to initialize pixiv web api client." msgstr "无法初始化 Pixiv 网页 API 客户端。" -#: opts.rs:69 +#: download.rs:22 +msgid "Warning: Web api client not logined, some future may not work." +msgstr "警告：Web API 客户端未登录，一些功能可能无法工作。" + +#: download.rs:45 +msgid "Failed to get page count." +msgstr "无法获取页数。" + +#: download.rs:54 +msgid "Failed to get pages' data." +msgstr "无法获取每页数据。" + +#: download.rs:61 +msgid "Failed to save metadata to JSON file." +msgstr "无法将元数据保存到 JSON 文件。" + +#: download.rs:69 +msgid "Failed to get original picture's link." +msgstr "无法获取原图链接。" + +#: download.rs:75 +msgid "Failed to get file name from url:" +msgstr "无法从 URL 获取文件名：" + +#: download.rs:82 download.rs:88 pixiv_web.rs:148 +msgid "Failed to download image:" +msgstr "无法下载图片：" + +#: download.rs:93 +msgid "Downloaded image:" +msgstr "已下载图片：" + +#: opts.rs:76 msgid "Warning: The specified config file not found." msgstr "警告：没有找到指定的设置文件。" -#: opts.rs:93 +#: opts.rs:100 msgid "Usage:" msgstr "使用方法：" -#: opts.rs:95 +#: opts.rs:102 msgid "Download an artwork" msgstr "下载一个作品" -#: opts.rs:97 +#: opts.rs:104 msgid "Fix the config file" msgstr "修复设置文件" -#: opts.rs:99 +#: opts.rs:106 msgid "Print all available settings" msgstr "打印所有可用的设置" -#: opts.rs:107 +#: opts.rs:114 msgid "Print help message." msgstr "打印帮助信息。" -#: opts.rs:111 +#: opts.rs:118 msgid "The location of config file." msgstr "设置文件的位置。" -#: opts.rs:117 settings_list.rs:8 +#: opts.rs:124 settings_list.rs:8 msgid "The location of cookies file. Used for web API." msgstr "cookies 文件的位置。用于网页 API。" -#: opts.rs:123 settings_list.rs:9 +#: opts.rs:130 settings_list.rs:9 msgid "The language of translated tags." msgstr "翻译后的标签语言。" -#: opts.rs:145 +#: opts.rs:133 +msgid "Verbose logging." +msgstr "启用详细内容输出。" + +#: opts.rs:134 +msgid "Overwrite existing file." +msgstr "覆盖已有文件。" + +#: opts.rs:135 +msgid "Skip overwrite existing file." +msgstr "跳过覆盖已有文件。" + +#: opts.rs:155 msgid "Unknown command." msgstr "未知指令。" -#: opts.rs:156 -msgid "No URL specified." -msgstr "没有指定网址。" +#: opts.rs:165 +msgid "Failed to parse ID:" +msgstr "无法解析 ID：" -#: opts.rs:164 +#: opts.rs:171 +msgid "No URL or ID specified." +msgstr "没有指定网址或 ID。" + +#: opts.rs:179 msgid "No detailed command specified." msgstr "没有指定更详细的指令。" -#: opts.rs:177 +#: opts.rs:192 msgid "Unknown config subcommand." msgstr "未知的 config 子指令。" -#: parser\mainpage.rs:53 parser/mainpage.rs:53 +#: parser/metadata.rs:54 pixiv_web.rs:107 msgid "Failed to parse JSON:" msgstr "无法解析 JSON：" -#: parser\mainpage.rs:75 parser/mainpage.rs:75 +#: parser/metadata.rs:76 msgid "Failed to parse HTML:" msgstr "无法解析 HTML：" -#: parser\mainpage.rs:80 parser/mainpage.rs:80 +#: parser/metadata.rs:81 msgid "Some errors occured during parsing:" msgstr "在解析中发生了一些错误：" -#: pixiv.rs:63 pixiv.rs:68 +#: pixiv_web.rs:68 pixiv_web.rs:73 msgid "Failed to get main page:" msgstr "无法获取主页：" -#: pixiv.rs:74 +#: pixiv_web.rs:79 msgid "Failed to parse main page." msgstr "无法解析主页。" -#: pixiv.rs:88 +#: pixiv_web.rs:83 +msgid "Main page's data:" +msgstr "主页的数据：" + +#: pixiv_web.rs:98 +msgid "Network error:" +msgstr "网络错误：" + +#: pixiv_web.rs:117 +msgid "Failed to detect error." +msgstr "检测 error 字段失败。" + +#: pixiv_web.rs:164 pixiv_web.rs:169 +msgid "Failed to get artwork page:" +msgstr "无法获取作品页：" + +#: pixiv_web.rs:175 +msgid "Failed to parse artwork page." +msgstr "无法解析作品页。" + +#: pixiv_web.rs:179 +msgid "Artwork's data:" +msgstr "作品数据：" + +#: pixiv_web.rs:193 +msgid "Artwork's page data:" +msgstr "作品页面数据：" + +#: pixiv_web.rs:217 msgid "Warning: Failed to save cookies file:" msgstr "警告：无法保存 cookies 文件：" @@ -206,26 +282,54 @@ msgstr "无法刷新文件缓冲区：" msgid "Pixiv's refresh tokens. Used to login." msgstr "Pixiv 的 refresh tokens。用于登录。" -#: webclient.rs:59 +#: utils.rs:29 +msgid "Do you want to delete file \"\"?" +msgstr "你想要删除文件 吗？" + +#: utils.rs:49 webclient.rs:143 +msgid "Can not parse URL:" +msgstr "无法解析 URL：" + +#: utils.rs:56 +msgid "Failed to get file name from path:" +msgstr "无法从路径获取文件名：" + +#: webclient.rs:103 msgid "Failed to parse Set-Cookie header." msgstr "无法解析 Set-Cookie 头部。" -#: webclient.rs:64 +#: webclient.rs:108 msgid "Failed to convert to string:" msgstr "无法转换为字符串：" -#: webclient.rs:94 +#: webclient.rs:150 +msgid "Parameters should be object or array:" +msgstr "参数应该是对象或数组：" + +#: webclient.rs:170 +msgid "Parameters should be array:" +msgstr "参数应该是数组：" + +#: webclient.rs:174 +msgid "Parameters need at least a value:" +msgstr "参数需要至少含有一个值：" + +#: webclient.rs:209 msgid "Error when request:" msgstr "请求时发生错误：" -#: main.rs:46 +#: webclient.rs:270 +msgid "Error when downloading file:" +msgstr "下载文件时发生错误：" + +#: main.rs:50 msgid "Failed to save config file:" msgstr "无法保存设置文件：" -#: main.rs:57 +#: main.rs:61 msgid "All available settings:" msgstr "所有可用的设置：" -#: main.rs:89 +#: main.rs:93 msgid "Can not read config file:" msgstr "无法读取设置文件：" diff --git a/src/cookies.rs b/src/cookies.rs index 9fb966c..3fbc075 100644 --- a/src/cookies.rs +++ b/src/cookies.rs @@ -234,7 +234,7 @@ impl Cookie { } else { self._domain.clone() }; - if subdomain && !host.starts_with(&domain) && host != &domain[1..] { + if subdomain && !host.ends_with(&domain) && host != &domain[1..] { return false; } if !subdomain && host != domain { diff --git a/src/data/json.rs b/src/data/json.rs new file mode 100644 index 0000000..cf1f5f2 --- /dev/null +++ b/src/data/json.rs @@ -0,0 +1,76 @@ +use crate::gettext; +use crate::pixiv_link::PixivID; +use crate::pixiv_link::ToPixivID; +use json::JsonValue; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::fs::File; +use std::fs::remove_file; +use std::io::Write; +use std::path::Path; + +pub trait ToJson { + fn to_json(&self) -> Option; +} + +/// Store metadata informations in JSON file +pub struct JSONDataFile { + id: PixivID, + maps: HashMap, +} + +impl JSONDataFile { + pub fn new(id: T) -> Option { + let i = id.to_pixiv_id(); + if i.is_none() { + return None; + } + Some(Self { + id: i.unwrap(), + maps: HashMap::new(), + }) + } + + pub fn save + ?Sized>(&self, path: &S) -> bool { + let p = Path::new(path); + if p.exists() { + let r = remove_file(p); + if r.is_err() { + println!("{} {}", gettext("Failed to remove file:"), r.unwrap_err()); + return false; + } + } + let value = self.to_json(); + if value.is_none() { + return false; + } + let value = value.unwrap(); + let f = File::create(p); + if f.is_err() { + println!("{} {}", gettext("Failed to create file:"), f.unwrap_err()); + return false; + } + let mut f = f.unwrap(); + let r = f.write(value.pretty(2).as_bytes()); + if r.is_err() { + println!("{} {}", gettext("Failed to write file:"), r.unwrap_err()); + return false; + } + true + } +} + +impl ToJson for JSONDataFile { + fn to_json(&self) -> Option { + let mut value = json::object! {}; + if value.insert("id", self.id.to_json()).is_err() { + return None; + } + for (k, v) in self.maps.iter() { + if value.insert(k, v.clone()).is_err() { + return None; + } + } + Some(value) + } +} diff --git a/src/data/mod.rs b/src/data/mod.rs new file mode 100644 index 0000000..22fdbb3 --- /dev/null +++ b/src/data/mod.rs @@ -0,0 +1 @@ +pub mod json; diff --git a/src/download.rs b/src/download.rs index dd0a8de..e225ee5 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,6 +1,12 @@ +use crate::data::json::JSONDataFile; use crate::gettext; -use crate::pixiv::PixivWebClient; +use crate::pixiv_link::PixivID; +use crate::pixiv_web::PixivWebClient; +use crate::utils::get_file_name_from_url; +use crate::webclient::WebClient; use crate::Main; +use json::JsonValue; +use std::path::PathBuf; impl Main { pub fn download(&mut self) -> i32 { @@ -10,6 +16,86 @@ impl Main { return 1; } pw.check_login(); + if !pw.logined() { + println!( + "{}", + gettext("Warning: Web api client not logined, some future may not work.") + ); + } + for id in self.cmd.as_ref().unwrap().ids.iter() { + match id { + PixivID::Artwork(id) => { + let r = self.download_artwork(&mut pw, id.clone()); + if r != 0 { + return r; + } + } + } + } + 0 + } + + pub fn download_artwork(&self, pw: &mut PixivWebClient, id: u64) -> i32 { + let re = pw.get_artwork(id); + if re.is_none() { + return 1; + } + let pages = (&(re.as_ref().unwrap())["illust"][format!("{}", id).as_str()]["sl"]).as_u64(); + if pages.is_none() { + println!("{}", gettext("Failed to get page count.")); + return 1; + } + let pages = pages.unwrap(); + let mut pages_data: Option = None; + if pages > 1 { + pages_data = pw.get_illust_pages(id); + } + if pages > 1 && pages_data.is_none() { + println!("{}", gettext("Failed to get pages' data.")); + return 1; + } + let base = PathBuf::from("."); + let json_file = base.join(format!("{}.json", id)); + let json_data = JSONDataFile::new(id).unwrap(); + if !json_data.save(&json_file) { + println!("{}", gettext("Failed to save metadata to JSON file.")); + return 1; + } + if pages_data.is_some() { + let pages_data = pages_data.as_ref().unwrap(); + for page in pages_data.members() { + let link = &page["urls"]["original"]; + if !link.is_string() { + println!("{}", gettext("Failed to get original picture's link.")); + return 1; + } + let link = link.as_str().unwrap(); + let file_name = get_file_name_from_url(link); + if file_name.is_none() { + println!("{} {}", gettext("Failed to get file name from url:"), link); + return 1; + } + let file_name = file_name.unwrap(); + let file_name = base.join(file_name); + let r = pw.download_image(link); + if r.is_none() { + println!("{} {}", gettext("Failed to download image:"), link); + return 1; + } + let r = r.unwrap(); + let re = WebClient::download_stream(&file_name, r, pw.helper.overwrite()); + if re.is_err() { + println!("{} {}", gettext("Failed to download image:"), link); + return 1; + } + println!( + "{} {} -> {}", + gettext("Downloaded image:"), + link, + file_name.to_str().unwrap_or("(null)") + ) + } + } 0 } } diff --git a/src/i18n.rs b/src/i18n.rs index 1ea11a2..9353742 100644 --- a/src/i18n.rs +++ b/src/i18n.rs @@ -150,6 +150,7 @@ impl I18n { } lazy_static! { + #[doc(hidden)] static ref I18NT: I18n = I18n::new(); } diff --git a/src/main.rs b/src/main.rs index 7af7c64..6346773 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,22 @@ extern crate chrono; extern crate dateparser; +extern crate futures_util; extern crate json; #[macro_use] extern crate lazy_static; extern crate tokio; +extern crate regex; extern crate reqwest; mod cookies; +mod data; mod download; mod i18n; mod opthelper; mod opts; mod parser; -mod pixiv; +mod pixiv_link; +mod pixiv_web; mod settings; mod settings_list; mod utils; diff --git a/src/opthelper.rs b/src/opthelper.rs index 3bbfff6..1f950c7 100644 --- a/src/opthelper.rs +++ b/src/opthelper.rs @@ -1,13 +1,17 @@ use crate::opts::CommandOpts; use crate::settings::SettingStore; +/// An sturct to access all available settings/command line switches #[derive(Clone)] pub struct OptHelper<'a> { + /// Command Line Options opt: &'a CommandOpts, + /// Settings settings: &'a SettingStore, } impl<'a> OptHelper<'a> { + /// return cookies location, no any check pub fn cookies(&self) -> Option { if self.opt.cookies.is_some() { self.opt.cookies.clone() @@ -18,6 +22,7 @@ impl<'a> OptHelper<'a> { } } + /// return language pub fn language(&self) -> Option { if self.opt.language.is_some() { self.opt.language.clone() @@ -31,4 +36,12 @@ impl<'a> OptHelper<'a> { pub fn new(opt: &'a CommandOpts, settings: &'a SettingStore) -> Self { Self { opt, settings } } + + pub fn overwrite(&self) -> Option { + self.opt.overwrite + } + + pub fn verbose(&self) -> bool { + self.opt.verbose + } } diff --git a/src/opts.rs b/src/opts.rs index e7e4137..cac37eb 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -1,6 +1,7 @@ extern crate getopts; use crate::gettext; +use crate::pixiv_link::PixivID; use crate::utils::check_file_exists; use crate::utils::get_exe_path_else_current; use getopts::Options; @@ -35,8 +36,8 @@ impl PartialEq for &ConfigCommand { pub struct CommandOpts { /// Command pub cmd: Command, - /// URLs - pub urls: Vec, + /// IDs + pub ids: Vec, /// Config location pub _config: Option, /// Config command @@ -45,17 +46,23 @@ pub struct CommandOpts { pub cookies: Option, /// The language of translated tags pub language: Option, + /// Verbose logging + pub verbose: bool, + /// Whether to overwrite file + pub overwrite: Option, } impl CommandOpts { pub fn new(cmd: Command) -> Self { Self { cmd, - urls: Vec::new(), + ids: Vec::new(), _config: None, config_cmd: None, cookies: None, language: None, + verbose: false, + overwrite: None, } } @@ -87,7 +94,7 @@ impl CommandOpts { pub fn print_usage(prog: &str, opts: &Options) { let brief = format!( "{} -{} download [options] [] {} +{} download [options] [] {} {} config fix [options] {} {} config help [options] {}", gettext("Usage:"), @@ -123,6 +130,9 @@ pub fn parse_cmd() -> Option { gettext("The language of translated tags."), "LANG", ); + opts.optflag("v", "verbose", gettext("Verbose logging.")); + opts.optflag("y", "yes", gettext("Overwrite existing file.")); + opts.optflag("n", "no", gettext("Skip overwrite existing file.")); let result = match opts.parse(&argv[1..]) { Ok(m) => m, Err(err) => { @@ -148,16 +158,21 @@ pub fn parse_cmd() -> Option { } match re.as_ref().unwrap().cmd { Command::Download => { - let mut urls = Vec::new(); + let mut ids = Vec::new(); for url in result.free.iter().skip(1) { - urls.push(url.to_string()); + let id = PixivID::parse(url); + if id.is_none() { + println!("{} {}", gettext("Failed to parse ID:"), url); + return None; + } + ids.push(id.unwrap()); } - if urls.is_empty() { - println!("{}", gettext("No URL specified.")); + if ids.is_empty() { + println!("{}", gettext("No URL or ID specified.")); print_usage(&argv[0], &opts); return None; } - re.as_mut().unwrap().urls = urls; + re.as_mut().unwrap().ids = ids; } Command::Config => { if result.free.len() < 2 { @@ -189,5 +204,21 @@ pub fn parse_cmd() -> Option { if result.opt_present("language") { re.as_mut().unwrap().language = Some(result.opt_str("language").unwrap()); } + re.as_mut().unwrap().verbose = result.opt_present("verbose"); + let yes = result.opt_present("yes"); + let no = result.opt_present("no"); + re.as_mut().unwrap().overwrite = if yes && no { + if result.opt_positions("yes").last().unwrap() > result.opt_positions("no").last().unwrap() { + Some(true) + } else { + Some(false) + } + } else if yes { + Some(true) + } else if no { + Some(false) + } else { + None + }; re } diff --git a/src/parser/mainpage.rs b/src/parser/metadata.rs similarity index 82% rename from src/parser/mainpage.rs rename to src/parser/metadata.rs index 87db79b..dc0bdbc 100644 --- a/src/parser/mainpage.rs +++ b/src/parser/metadata.rs @@ -2,14 +2,19 @@ use crate::gettext; use html_parser::Dom; use html_parser::Node; use json::JsonValue; +use std::default::Default; -pub struct MainPageParser { +pub struct MetaDataParser { + pub key: String, pub value: Option, } -impl MainPageParser { - pub fn new() -> Self { +impl MetaDataParser { + /// Create an new instance + /// * `key` - meta's name + pub fn new(key: &str) -> Self { Self { + key: key.to_string(), value: None, } } @@ -26,13 +31,14 @@ impl MainPageParser { if name.is_none() { return false; } - if name.as_ref().unwrap() != "global-data" { + if name.as_ref().unwrap() != self.key.as_str() { return false; } if e.id.is_none() { return false; } - if e.id.as_ref().unwrap() != "meta-global-data" { + let mkey = format!("meta-{}", self.key.as_str()); + if e.id.as_ref().unwrap() != mkey.as_str() { return false; } let c = e.attributes.get("content"); @@ -85,3 +91,9 @@ impl MainPageParser { false } } + +impl Default for MetaDataParser { + fn default() -> Self { + Self::new("global-data") + } +} diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a713231..8adefde 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,3 +1,3 @@ extern crate html_parser; -pub mod mainpage; +pub mod metadata; diff --git a/src/pixiv.rs b/src/pixiv.rs deleted file mode 100644 index d5beb10..0000000 --- a/src/pixiv.rs +++ /dev/null @@ -1,91 +0,0 @@ -use crate::gettext; -use crate::opthelper::OptHelper; -use crate::parser::mainpage::MainPageParser; -use crate::webclient::WebClient; -use crate::Main; -use spin_on::spin_on; - -/// A client which use Pixiv's web API -pub struct PixivWebClient<'a> { - client: WebClient, - helper: OptHelper<'a>, - /// true if in is initialized - inited: bool, -} - -impl<'a> PixivWebClient<'a> { - pub fn new(m: &'a Main) -> Self { - Self { - client: WebClient::new(), - helper: OptHelper::new(m.cmd.as_ref().unwrap(), m.settings.as_ref().unwrap()), - inited: false, - } - } - - pub fn init(&mut self) -> bool { - let c = self.helper.cookies(); - if c.is_some() { - if !self.client.read_cookies(c.as_ref().unwrap()) { - return false; - } - } - self.client.set_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"); - let l = self.helper.language(); - if l.is_some() { - self.client.set_header("Accept-Language", l.as_ref().unwrap()); - } else { - self.client.set_header("Accept-Language", "ja"); - } - self.inited = true; - true - } - - pub fn auto_init(&mut self) { - if !self.inited { - let r = self.init(); - if !r { - panic!("{}", gettext("Failed to initialize pixiv web api client.")); - } - } - } - - pub fn check_login(&mut self) -> bool { - self.auto_init(); - let r = self.client.get("https://www.pixiv.net/"); - if r.is_none() { - return false; - } - let r = r.unwrap(); - let status = r.status(); - let code = status.as_u16(); - if code >= 400 { - println!("{} {}", gettext("Failed to get main page:"), status); - return false; - } - let data = spin_on(r.text_with_charset("UTF-8")); - if data.is_err() { - println!("{} {}", gettext("Failed to get main page:"), data.unwrap_err()); - return false; - } - let data = data.unwrap(); - let mut p = MainPageParser::new(); - if !p.parse(data.as_str()) { - println!("{}", gettext("Failed to parse main page.")); - return false; - } - true - } -} - -impl <'a> Drop for PixivWebClient<'a> { - fn drop(&mut self) { - if self.inited { - let c = self.helper.cookies(); - if c.is_some() { - if !self.client.save_cookies(c.as_ref().unwrap()) { - println!("{} {}", gettext("Warning: Failed to save cookies file:"), c.as_ref().unwrap()); - } - } - } - } -} diff --git a/src/pixiv_link.rs b/src/pixiv_link.rs new file mode 100644 index 0000000..f110d05 --- /dev/null +++ b/src/pixiv_link.rs @@ -0,0 +1,73 @@ +use crate::data::json::ToJson; +use json::JsonValue; +use regex::Regex; + +lazy_static! { + #[doc(hidden)] + static ref RE: Regex = Regex::new("^(https?://)?(www\\.)?pixiv\\.net/artworks/(?P\\d+)").unwrap(); +} + +/// Repesent an Pixiv ID +#[derive(Clone, Debug)] +pub enum PixivID { + /// Artwork Id, include illust and manga + Artwork(u64), +} + +pub trait ToPixivID { + fn to_pixiv_id(&self) -> Option; +} + +impl PixivID { + pub fn parse(s: &str) -> Option { + let p = s.trim(); + let num = p.parse::(); + if num.is_ok() { + return Some(PixivID::Artwork(num.unwrap())); + } + let re = RE.captures(s); + if re.is_some() { + let r = re.unwrap().name("id"); + if r.is_some() { + let r = r.unwrap().as_str(); + let num = r.parse::(); + return Some(PixivID::Artwork(num.unwrap())); + } + } + None + } +} + +impl ToJson for PixivID { + fn to_json(&self) -> Option { + match *self { + PixivID::Artwork(id) => { + Some(json::value!({"type": "artwork", "id": id})) + } + } + } +} + +impl ToPixivID for &PixivID { + fn to_pixiv_id(&self) -> Option { + Some((*self).clone()) + } +} + +impl ToPixivID for &str { + fn to_pixiv_id(&self) -> Option { + PixivID::parse(*self) + } +} + +impl ToPixivID for String { + fn to_pixiv_id(&self) -> Option { + PixivID::parse(self) + } +} + +impl ToPixivID for u64 { + fn to_pixiv_id(&self) -> Option { + Some(PixivID::Artwork(*self)) + } +} diff --git a/src/pixiv_web.rs b/src/pixiv_web.rs new file mode 100644 index 0000000..71eca90 --- /dev/null +++ b/src/pixiv_web.rs @@ -0,0 +1,222 @@ +use crate::gettext; +use crate::opthelper::OptHelper; +use crate::parser::metadata::MetaDataParser; +use crate::webclient::WebClient; +use crate::Main; +use json::JsonValue; +use reqwest::IntoUrl; +use reqwest::Response; +use spin_on::spin_on; + +/// A client which use Pixiv's web API +pub struct PixivWebClient<'a> { + client: WebClient, + pub helper: OptHelper<'a>, + /// true if in is initialized + inited: bool, + data: Option, +} + +impl<'a> PixivWebClient<'a> { + pub fn new(m: &'a Main) -> Self { + Self { + client: WebClient::new(), + helper: OptHelper::new(m.cmd.as_ref().unwrap(), m.settings.as_ref().unwrap()), + inited: false, + data: None, + } + } + + pub fn init(&mut self) -> bool { + let c = self.helper.cookies(); + if c.is_some() { + if !self.client.read_cookies(c.as_ref().unwrap()) { + return false; + } + } + self.client.set_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36"); + let l = self.helper.language(); + if l.is_some() { + self.client.set_header("Accept-Language", l.as_ref().unwrap()); + } else { + self.client.set_header("Accept-Language", "ja"); + } + self.client.verbose = self.helper.verbose(); + self.inited = true; + true + } + + pub fn auto_init(&mut self) { + if !self.inited { + let r = self.init(); + if !r { + panic!("{}", gettext("Failed to initialize pixiv web api client.")); + } + } + } + + pub fn check_login(&mut self) -> bool { + self.auto_init(); + let r = self.client.get("https://www.pixiv.net/", None); + if r.is_none() { + return false; + } + let r = r.unwrap(); + let status = r.status(); + let code = status.as_u16(); + if code >= 400 { + println!("{} {}", gettext("Failed to get main page:"), status); + return false; + } + let data = spin_on(r.text_with_charset("UTF-8")); + if data.is_err() { + println!("{} {}", gettext("Failed to get main page:"), data.unwrap_err()); + return false; + } + let data = data.unwrap(); + let mut p = MetaDataParser::default(); + if !p.parse(data.as_str()) { + println!("{}", gettext("Failed to parse main page.")); + return false; + } + if self.helper.verbose() { + println!("{}\n{}", gettext("Main page's data:"), p.value.as_ref().unwrap().pretty(2).as_str()); + } + self.data = Some(p.value.unwrap()); + true + } + + pub fn deal_json(&mut self, r: Response) -> Option { + let status = r.status(); + let code = status.as_u16(); + let is_status_err = code >= 400; + let data = spin_on(r.text_with_charset("UTF-8")); + if data.is_err() { + if is_status_err { + println!("HTTP ERROR {}", status); + } + println!("{} {}", gettext("Network error:"), data.unwrap_err()); + return None; + } + let data = data.unwrap(); + let re = json::parse(data.as_str()); + if re.is_err() { + if is_status_err { + println!("HTTP ERROR {}", status); + } else { + println!("{} {}", gettext("Failed to parse JSON:"), re.unwrap_err()); + } + return None; + } + let value = re.unwrap(); + let error = (&value["error"]).as_bool(); + if error.is_none() { + if is_status_err { + println!("HTTP ERROR {}", status); + } + println!("{}", gettext("Failed to detect error.")); + return None; + } + let error = error.unwrap(); + if error { + let message = (&value["message"]).as_str(); + if is_status_err { + println!("HTTP ERROR {}", status); + } + if message.is_some() { + println!("{}", message.unwrap()); + } + return None; + } + let body = &value["body"]; + if body.is_empty() || body.is_null() { + return Some(value); + } + Some(body.clone()) + } + + pub fn download_image(&mut self, url: U) -> Option { + self.auto_init(); + let r = self.client.get(url, json::object!{"referer": "https://www.pixiv.net/"}); + if r.is_none() { + return None; + } + let r = r.unwrap(); + let status = r.status(); + let code = status.as_u16(); + if code >= 400 { + println!("{} {}", gettext("Failed to download image:"), status); + return None; + } + Some(r) + } + + pub fn get_artwork(&mut self, id: u64) -> Option { + self.auto_init(); + let r = self.client.get(format!("https://www.pixiv.net/artworks/{}", id), None); + if r.is_none() { + return None; + } + let r = r.unwrap(); + let status = r.status(); + let code = status.as_u16(); + if code >= 400 { + println!("{} {}", gettext("Failed to get artwork page:"), status); + return None; + } + let data = spin_on(r.text_with_charset("UTF-8")); + if data.is_err() { + println!("{} {}", gettext("Failed to get artwork page:"), data.unwrap_err()); + return None; + } + let data = data.unwrap(); + let mut p = MetaDataParser::new("preload-data"); + if !p.parse(data.as_str()) { + println!("{}", gettext("Failed to parse artwork page.")); + return None; + } + if self.helper.verbose() { + println!("{} {}", gettext("Artwork's data:"), p.value.as_ref().unwrap().pretty(2)); + } + Some(p.value.unwrap()) + } + + pub fn get_illust_pages(&mut self, id: u64) -> Option { + self.auto_init(); + let r = self.client.get(format!("https://www.pixiv.net/ajax/illust/{}/pages", id), None); + if r.is_none() { + return None; + } + let r = r.unwrap(); + let v = self.deal_json(r); + if self.helper.verbose() { + println!("{} {}", gettext("Artwork's page data:"), v.as_ref().unwrap().pretty(2)); + } + v + } + + pub fn logined(&self) -> bool { + if self.data.is_none() { + return false; + } + let value = self.data.as_ref().unwrap(); + let d = &value["userData"]; + if d.is_object() { + return true; + } + false + } +} + +impl <'a> Drop for PixivWebClient<'a> { + fn drop(&mut self) { + if self.inited { + let c = self.helper.cookies(); + if c.is_some() { + if !self.client.save_cookies(c.as_ref().unwrap()) { + println!("{} {}", gettext("Warning: Failed to save cookies file:"), c.as_ref().unwrap()); + } + } + } + } +} diff --git a/src/utils.rs b/src/utils.rs index db40b92..e95e0b1 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,3 +1,5 @@ +use crate::gettext; +use reqwest::IntoUrl; use std::env; use std::path::Path; use std::path::PathBuf; @@ -22,3 +24,41 @@ pub fn check_file_exists(path: &str) -> bool { let p = Path::new(path); p.exists() } + +pub fn ask_need_overwrite(path: &str) -> bool { + let s = gettext("Do you want to delete file \"\"?").replace("", path); + print!("{}(y/n)", s.as_str()); + let mut d = String::from(""); + loop { + let re = std::io::stdin().read_line(&mut d); + if re.is_err() { + continue; + } + let d = d.trim().to_lowercase(); + if d == "y" { + return true; + } else { + return false; + } + } +} + +pub fn get_file_name_from_url(url: U) -> Option { + let u = url.into_url(); + if u.is_err() { + println!("{} {}", gettext("Can not parse URL:"), u.unwrap_err()); + return None; + } + let u = u.unwrap(); + let path = Path::new(u.path()); + let re = path.file_name(); + if re.is_none() { + println!("{} {}", gettext("Failed to get file name from path:"), u.path()); + return None; + } + let r = re.unwrap().to_str(); + if r.is_none() { + return None; + } + Some(String::from(r.unwrap())) +} diff --git a/src/webclient.rs b/src/webclient.rs index daad765..696e75b 100644 --- a/src/webclient.rs +++ b/src/webclient.rs @@ -3,9 +3,51 @@ extern crate spin_on; use crate::cookies::Cookie; use crate::cookies::CookieJar; use crate::gettext; +use crate::utils::ask_need_overwrite; +use futures_util::StreamExt; +use json::JsonValue; use reqwest::{Client, IntoUrl, RequestBuilder, Response}; -use std::collections::HashMap; use spin_on::spin_on; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::fs::File; +use std::fs::remove_file; +use std::io::Write; +use std::path::Path; + +pub trait ToHeaders { + fn to_headers(&self) -> Option>; +} + +impl ToHeaders for Option> { + fn to_headers(&self) -> Option> { + self.clone() + } +} + +impl ToHeaders for HashMap { + fn to_headers(&self) -> Option> { + Some(self.clone()) + } +} + +impl ToHeaders for JsonValue { + fn to_headers(&self) -> Option> { + if !self.is_object() { + return None; + } + let mut h = HashMap::new(); + for (k, v) in self.entries() { + let d = if v.is_string() { + String::from(v.as_str().unwrap()) + } else { + v.dump() + }; + h.insert(String::from(k), d); + } + Some(h) + } +} /// Generate `cookie` header for a url /// * `c` - Cookies @@ -31,6 +73,7 @@ pub struct WebClient { /// HTTP Headers pub headers: HashMap, cookies: CookieJar, + pub verbose: bool, } impl WebClient { @@ -39,6 +82,7 @@ impl WebClient { client: Client::new(), headers: HashMap::new(), cookies: CookieJar::new(), + verbose: false, } } @@ -82,10 +126,81 @@ impl WebClient { pub fn set_header(&mut self, key: &str, value: &str) -> Option { self.headers.insert(String::from(key), String::from(value)) } + /// Send GET requests with parameters + /// * `param` - GET parameters. Should be a JSON object/array. If value in map is not a string, will dump it + /// # Examples + /// ``` + /// let client = WebClient::new(); + /// client.verbose = true; + /// client.get_with_param("https://test.com/a", json::object!{"data": "param1"}); + /// client.get_with_param("https://test.com/a", json::object!{"daa": {"ad": "test"}}); + /// client.get_with_param("https://test.com/a", json::array![["daa", "param1"]]); + /// ``` + /// It will GET `https://test.com/a?data=param1`, `https://test.com/a?daa=%7B%22ad%22%3A%22test%22%7D`, `https://test.com/a?daa=param1` + pub fn get_with_param(&mut self, url: U, param: JsonValue) -> Option { + let u = url.into_url(); + if u.is_err() { + println!("{} \"{}\"", gettext("Can not parse URL:"), u.unwrap_err()); + return None; + } + let mut u = u.unwrap(); + if !param.is_object() && !param.is_array() { + println!( + "{} \"{}\"", + gettext("Parameters should be object or array:"), + param + ); + return None; + } + { + let mut query = u.query_pairs_mut(); + if param.is_object() { + for (k, v) in param.entries() { + let s: String; + if v.is_string() { + s = String::from(v.as_str().unwrap()); + } else { + s = v.dump(); + } + query.append_pair(k, s.as_str()); + } + } else { + for v in param.members() { + if !v.is_object() { + println!("{} \"{}\"", gettext("Parameters should be array:"), v); + return None; + } + if v.len() < 2 { + println!("{} \"{}\"", gettext("Parameters need at least a value:"), v); + return None; + } + let okey = &v[0]; + let key: String; + if okey.is_string() { + key = String::from(okey.as_str().unwrap()); + } else { + key = okey.dump(); + } + let mut mems = v.members(); + mems.next(); + for val in mems { + let s: String; + if val.is_string() { + s = String::from(val.as_str().unwrap()); + } else { + s = val.dump(); + } + query.append_pair(key.as_str(), s.as_str()); + } + } + } + } + self.get(u.as_str(), None) + } /// Send GET requests - pub fn get(&mut self, url: U) -> Option { - let r = self.aget(url); + pub fn get(&mut self, url: U, headers: H) -> Option { + let r = self.aget(url, headers); let r = r.send(); let r = spin_on(r); match r { @@ -97,19 +212,71 @@ impl WebClient { } let r = r.unwrap(); self.handle_set_cookie(&r); + if self.verbose { + println!("{}", r.status()); + } Some(r) } - pub fn aget(&mut self, url: U) -> RequestBuilder { + pub fn aget(&mut self, url: U, headers: H) -> RequestBuilder { let s = url.as_str(); + if self.verbose { + println!("GET {}", s); + } let mut r = self.client.get(s); for (k, v) in self.headers.iter() { r = r.header(k, v); } + let headers = headers.to_headers(); + if headers.is_some() { + let h = headers.unwrap(); + for (k, v) in h.iter() { + r = r.header(k, v); + } + } let c = gen_cookie_header(&mut self.cookies, s); if c.len() > 0 { r = r.header("Cookie", c.as_str()); } r } + + pub fn download_stream + ?Sized>(file_name: &S, r: Response, overwrite: Option) -> Result<(), ()> { + let p = Path::new(file_name); + if p.exists() { + let overwrite = if overwrite.is_none() { + ask_need_overwrite(p.to_str().unwrap()) + } else { + overwrite.unwrap() + }; + if !overwrite { + return Ok(()); + } + let re = remove_file(p); + if re.is_err() { + println!("{} {}", gettext("Failed to remove file:"), re.unwrap_err()); + return Err(()); + } + } + let f = File::create(p); + if f.is_err() { + println!("{} {}", gettext("Failed to create file:"), f.unwrap_err()); + return Err(()); + } + let mut f = f.unwrap(); + let mut stream = r.bytes_stream(); + while let Some(data) = spin_on(stream.next()) { + if data.is_err() { + println!("{} {}", gettext("Error when downloading file:"), data.unwrap_err()); + return Err(()); + } + let data = data.unwrap(); + let r = f.write(&data); + if r.is_err() { + println!("{} {}", gettext("Failed to write file:"), r.unwrap_err()); + return Err(()); + } + } + Ok(()) + } }