From 9493f475dfd50c92e34dfdffbbdb10dd2a8bca80 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Mon, 28 Mar 2022 21:11:03 +0800 Subject: [PATCH] add support for download multiply images --- Cargo.lock | 9 +- Cargo.toml | 2 +- Language/pixiv_downloader.pot | 226 ++++++++++++++-------------- Language/pixiv_downloader.zh_CN.po | 228 +++++++++++++++-------------- src/data/data.rs | 8 +- src/data/json.rs | 11 +- src/download.rs | 146 ++++++++++++++---- src/opthelper.rs | 21 ++- src/opts.rs | 11 +- src/pixiv_web.rs | 28 +++- src/settings_list.rs | 1 + src/webclient.rs | 124 +++++++++++++++- 12 files changed, 543 insertions(+), 272 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 63f7e7c..dc349a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,7 +278,9 @@ dependencies = [ "encode_unicode", "libc", "once_cell", + "regex", "terminal_size", + "unicode-width", "winapi", ] @@ -766,14 +768,13 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.16.2" +version = "0.17.0-rc.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" +checksum = "088977cb4de4e09ea0232bd51c844490475ad94344fb0c60784f1fb380059b6d" dependencies = [ "console", - "lazy_static", "number_prefix", - "regex", + "unicode-width", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 463cc63..a2e226e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ futures-util = "0.3" getopts = "0.2" gettext = "0.4" html_parser = "0.6.2" -indicatif = "0.16" +indicatif = "0.17.0-rc.10" int-enum = { version = "0.4", optional = true } json = "0.12" lazy_static = "1.4" diff --git a/Language/pixiv_downloader.pot b/Language/pixiv_downloader.pot index 2e5d5ed..811e6f5 100644 --- a/Language/pixiv_downloader.pot +++ b/Language/pixiv_downloader.pot @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: pixiv_downloader\n" -"POT-Creation-Date: 2022-03-26 15:53+0800\n" +"POT-Creation-Date: 2022-03-28 21:09+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -115,15 +115,18 @@ msgstr "" msgid "Can not parse expired time:" msgstr "" -#: cookies.rs:366 data/json.rs:50 settings.rs:418 webclient.rs:302 +#: cookies.rs:366 data/json.rs:51 settings.rs:418 webclient.rs:343 +#: webclient.rs:345 webclient.rs:420 msgid "Failed to remove file:" msgstr "" -#: cookies.rs:372 data/json.rs:61 settings.rs:425 webclient.rs:312 +#: cookies.rs:372 data/json.rs:62 settings.rs:425 webclient.rs:354 +#: webclient.rs:356 webclient.rs:430 msgid "Failed to create file:" msgstr "" -#: cookies.rs:379 data/json.rs:67 settings.rs:431 webclient.rs:329 +#: cookies.rs:379 data/json.rs:68 settings.rs:431 webclient.rs:381 +#: webclient.rs:383 webclient.rs:447 msgid "Failed to write file:" msgstr "" @@ -131,84 +134,85 @@ msgstr "" msgid "Failed to unescape string:" msgstr "" -#: download.rs:25 pixiv_web.rs:58 +#: download.rs:30 pixiv_web.rs:58 msgid "Failed to initialize pixiv web api client." msgstr "" -#: download.rs:34 +#: download.rs:39 msgid "Warning: Web api client not logined, some future may not work." msgstr "" -#: download.rs:73 -msgid "Failed to get page count." -msgstr "" - -#: download.rs:82 -msgid "Failed to get pages' data." -msgstr "" - -#: download.rs:95 -msgid "Failed to save metadata to JSON file." -msgstr "" - -#: download.rs:110 -msgid "Failed to get ugoira's data." -msgstr "" - -#: download.rs:116 -msgid "Can not find source link for ugoira." -msgstr "" - -#: download.rs:122 download.rs:200 download.rs:274 -msgid "Failed to get file name from url:" -msgstr "" - -#: download.rs:138 download.rs:144 -msgid "Failed to download ugoira:" -msgstr "" - -#: download.rs:149 -msgid "Downloaded ugoira:" -msgstr "" - -#: download.rs:159 -msgid "Warning: Failed to generate video's metadata:" -msgstr "" - -#: download.rs:169 -msgid "Failed to convert from ugoira to mp4 video file:" -msgstr "" - -#: download.rs:172 -msgid "Converted -> " -msgstr "" - -#: download.rs:175 -msgid "Failed to parse frames:" -msgstr "" - -#: download.rs:182 -msgid "Warning: Unknown illust type:" -msgstr "" - -#: download.rs:185 -msgid "Warning: Failed to get illust's type." -msgstr "" - -#: download.rs:194 download.rs:268 +#: download.rs:58 download.rs:286 download.rs:360 msgid "Failed to get original picture's link." msgstr "" -#: download.rs:215 download.rs:254 download.rs:294 download.rs:324 +#: download.rs:64 download.rs:197 download.rs:292 download.rs:366 +msgid "Failed to get file name from url:" +msgstr "" + +#: download.rs:79 download.rs:115 download.rs:307 download.rs:346 +#: download.rs:386 download.rs:416 msgid "Failed to add exif data to image:" msgstr "" -#: download.rs:234 download.rs:240 download.rs:304 download.rs:310 -#: pixiv_web.rs:153 +#: download.rs:100 download.rs:107 download.rs:326 download.rs:332 +#: download.rs:396 download.rs:402 pixiv_web.rs:153 pixiv_web.rs:169 msgid "Failed to download image:" msgstr "" -#: download.rs:245 download.rs:315 +#: download.rs:147 +msgid "Failed to get page count." +msgstr "" + +#: download.rs:156 +msgid "Failed to get pages' data." +msgstr "" + +#: download.rs:170 +msgid "Failed to save metadata to JSON file." +msgstr "" + +#: download.rs:185 +msgid "Failed to get ugoira's data." +msgstr "" + +#: download.rs:191 +msgid "Can not find source link for ugoira." +msgstr "" + +#: download.rs:213 download.rs:219 +msgid "Failed to download ugoira:" +msgstr "" + +#: download.rs:224 +msgid "Downloaded ugoira:" +msgstr "" + +#: download.rs:234 +msgid "Warning: Failed to generate video's metadata:" +msgstr "" + +#: download.rs:244 +msgid "Failed to convert from ugoira to mp4 video file:" +msgstr "" + +#: download.rs:247 +msgid "Converted -> " +msgstr "" + +#: download.rs:250 +msgid "Failed to parse frames:" +msgstr "" + +#: download.rs:257 +msgid "Warning: Unknown illust type:" +msgstr "" + +#: download.rs:260 +msgid "Warning: Failed to get illust's type." +msgstr "" + +#: download.rs:337 download.rs:407 webclient.rs:390 msgid "Downloaded image:" msgstr "" @@ -220,103 +224,107 @@ msgstr "" msgid "Failed to parse value." msgstr "" -#: opts.rs:98 +#: opts.rs:101 msgid "Warning: The specified config file not found." msgstr "" -#: opts.rs:122 +#: opts.rs:125 msgid "Usage:" msgstr "" -#: opts.rs:124 +#: opts.rs:127 msgid "Download an artwork" msgstr "" -#: opts.rs:126 +#: opts.rs:129 msgid "Fix the config file" msgstr "" -#: opts.rs:128 +#: opts.rs:131 msgid "Print all available settings" msgstr "" -#: opts.rs:136 +#: opts.rs:139 msgid "Print help message." msgstr "" -#: opts.rs:140 +#: opts.rs:143 msgid "The location of config file." msgstr "" -#: opts.rs:146 settings_list.rs:13 +#: opts.rs:149 settings_list.rs:13 msgid "The location of cookies file. Used for web API." msgstr "" -#: opts.rs:152 settings_list.rs:14 +#: opts.rs:155 settings_list.rs:14 msgid "The language of translated tags." msgstr "" -#: opts.rs:155 +#: opts.rs:158 msgid "Verbose logging." msgstr "" -#: opts.rs:156 +#: opts.rs:159 msgid "Overwrite existing file." msgstr "" -#: opts.rs:157 +#: opts.rs:160 msgid "Skip overwrite existing file." msgstr "" -#: opts.rs:161 settings_list.rs:15 +#: opts.rs:164 settings_list.rs:15 msgid "Max retry count if request failed." msgstr "" -#: opts.rs:167 settings_list.rs:16 +#: opts.rs:170 settings_list.rs:16 msgid "The interval (in seconds) between two retries." msgstr "" -#: opts.rs:170 settings_list.rs:17 +#: opts.rs:173 settings_list.rs:17 msgid "Use data from webpage first." msgstr "" -#: opts.rs:175 settings_list.rs:20 +#: opts.rs:178 settings_list.rs:20 msgid "Add/Update exif information to image files even when overwrite are disabled." msgstr "" -#: opts.rs:180 settings_list.rs:22 +#: opts.rs:183 settings_list.rs:22 msgid "Whether to enable progress bar." msgstr "" -#: opts.rs:202 +#: opts.rs:189 settings_list.rs:23 +msgid "Download multiple images at the same time." +msgstr "" + +#: opts.rs:210 msgid "Unknown command." msgstr "" -#: opts.rs:212 +#: opts.rs:220 msgid "Failed to parse ID:" msgstr "" -#: opts.rs:218 +#: opts.rs:226 msgid "No URL or ID specified." msgstr "" -#: opts.rs:226 +#: opts.rs:234 msgid "No detailed command specified." msgstr "" -#: opts.rs:239 +#: opts.rs:247 msgid "Unknown config subcommand." msgstr "" -#: opts.rs:275 +#: opts.rs:283 msgid "Retry count must be an non-negative integer:" msgstr "" -#: opts.rs:284 +#: opts.rs:292 msgid "Failed to parse retry interval:" msgstr "" -#: opts.rs:298 +#: opts.rs:306 msgid "Failed to parse :" msgstr "" @@ -356,27 +364,27 @@ msgstr "" msgid "Failed to detect error." msgstr "" -#: pixiv_web.rs:168 pixiv_web.rs:198 +#: pixiv_web.rs:184 pixiv_web.rs:214 msgid "Artwork's data:" msgstr "" -#: pixiv_web.rs:183 pixiv_web.rs:188 +#: pixiv_web.rs:199 pixiv_web.rs:204 msgid "Failed to get artwork page:" msgstr "" -#: pixiv_web.rs:194 +#: pixiv_web.rs:210 msgid "Failed to parse artwork page." msgstr "" -#: pixiv_web.rs:212 +#: pixiv_web.rs:228 msgid "Artwork's page data:" msgstr "" -#: pixiv_web.rs:226 +#: pixiv_web.rs:242 msgid "Ugoira's data:" msgstr "" -#: pixiv_web.rs:250 +#: pixiv_web.rs:266 msgid "Warning: Failed to save cookies file:" msgstr "" @@ -500,7 +508,7 @@ msgstr "" msgid "Do you want to delete file \"\"?" msgstr "" -#: utils.rs:51 webclient.rs:154 +#: utils.rs:51 webclient.rs:156 msgid "Can not parse URL:" msgstr "" @@ -508,43 +516,43 @@ msgstr "" msgid "Failed to get file name from path:" msgstr "" -#: webclient.rs:114 +#: webclient.rs:116 msgid "Failed to parse Set-Cookie header." msgstr "" -#: webclient.rs:119 +#: webclient.rs:121 msgid "Failed to convert to string:" msgstr "" -#: webclient.rs:161 +#: webclient.rs:163 msgid "Parameters should be object or array:" msgstr "" -#: webclient.rs:181 +#: webclient.rs:183 msgid "Parameters should be array:" msgstr "" -#: webclient.rs:185 +#: webclient.rs:187 msgid "Parameters need at least a value:" msgstr "" -#: webclient.rs:224 +#: webclient.rs:226 webclient.rs:247 msgid "Retry after seconds." msgstr "" -#: webclient.rs:228 +#: webclient.rs:230 webclient.rs:251 msgid "Retry times now." msgstr "" -#: webclient.rs:242 +#: webclient.rs:264 webclient.rs:282 msgid "Error when request:" msgstr "" -#: webclient.rs:308 +#: webclient.rs:334 webclient.rs:426 msgid "Downloading \"\"." msgstr "" -#: webclient.rs:319 +#: webclient.rs:366 webclient.rs:368 webclient.rs:437 msgid "Error when downloading file:" msgstr "" diff --git a/Language/pixiv_downloader.zh_CN.po b/Language/pixiv_downloader.zh_CN.po index 6c5ec6f..ba2faea 100644 --- a/Language/pixiv_downloader.zh_CN.po +++ b/Language/pixiv_downloader.zh_CN.po @@ -1,8 +1,8 @@ msgid "" msgstr "" "Project-Id-Version: pixiv_downloader\n" -"POT-Creation-Date: 2022-03-26 15:53+0800\n" -"PO-Revision-Date: 2022-03-26 15:54+0800\n" +"POT-Creation-Date: 2022-03-28 21:09+0800\n" +"PO-Revision-Date: 2022-03-28 21:10+0800\n" "Last-Translator: lifegpc \n" "Language-Team: \n" "Language: zh_CN\n" @@ -116,15 +116,18 @@ msgstr "无效的Cookie：" msgid "Can not parse expired time:" msgstr "无法解析过期时间：" -#: cookies.rs:366 data/json.rs:50 settings.rs:418 webclient.rs:302 +#: cookies.rs:366 data/json.rs:51 settings.rs:418 webclient.rs:343 +#: webclient.rs:345 webclient.rs:420 msgid "Failed to remove file:" msgstr "无法删除文件：" -#: cookies.rs:372 data/json.rs:61 settings.rs:425 webclient.rs:312 +#: cookies.rs:372 data/json.rs:62 settings.rs:425 webclient.rs:354 +#: webclient.rs:356 webclient.rs:430 msgid "Failed to create file:" msgstr "无法创建文件：" -#: cookies.rs:379 data/json.rs:67 settings.rs:431 webclient.rs:329 +#: cookies.rs:379 data/json.rs:68 settings.rs:431 webclient.rs:381 +#: webclient.rs:383 webclient.rs:447 msgid "Failed to write file:" msgstr "无法写入文件：" @@ -132,84 +135,85 @@ msgstr "无法写入文件：" msgid "Failed to unescape string:" msgstr "无法反转义字符串：" -#: download.rs:25 pixiv_web.rs:58 +#: download.rs:30 pixiv_web.rs:58 msgid "Failed to initialize pixiv web api client." msgstr "无法初始化 Pixiv 网页 API 客户端。" -#: download.rs:34 +#: download.rs:39 msgid "Warning: Web api client not logined, some future may not work." msgstr "警告：Web API 客户端未登录，一些功能可能无法工作。" -#: download.rs:73 -msgid "Failed to get page count." -msgstr "无法获取页数。" - -#: download.rs:82 -msgid "Failed to get pages' data." -msgstr "无法获取每页数据。" - -#: download.rs:95 -msgid "Failed to save metadata to JSON file." -msgstr "无法将元数据保存到 JSON 文件。" - -#: download.rs:110 -msgid "Failed to get ugoira's data." -msgstr "无法获取动图数据。" - -#: download.rs:116 -msgid "Can not find source link for ugoira." -msgstr "无法获取动图的链接。" - -#: download.rs:122 download.rs:200 download.rs:274 -msgid "Failed to get file name from url:" -msgstr "无法从 URL 获取文件名：" - -#: download.rs:138 download.rs:144 -msgid "Failed to download ugoira:" -msgstr "无法下载动图：" - -#: download.rs:149 -msgid "Downloaded ugoira:" -msgstr "已下载动图：" - -#: download.rs:159 -msgid "Warning: Failed to generate video's metadata:" -msgstr "警告：无法生成视频元数据：" - -#: download.rs:169 -msgid "Failed to convert from ugoira to mp4 video file:" -msgstr "无法将动图转换为 mp4 视频文件：" - -#: download.rs:172 -msgid "Converted -> " -msgstr "已转换 -> " - -#: download.rs:175 -msgid "Failed to parse frames:" -msgstr "无法解析帧数据：" - -#: download.rs:182 -msgid "Warning: Unknown illust type:" -msgstr "警告：未知的插图类型：" - -#: download.rs:185 -msgid "Warning: Failed to get illust's type." -msgstr "警告：无法获取插图类型。" - -#: download.rs:194 download.rs:268 +#: download.rs:58 download.rs:286 download.rs:360 msgid "Failed to get original picture's link." msgstr "无法获取原图链接。" -#: download.rs:215 download.rs:254 download.rs:294 download.rs:324 +#: download.rs:64 download.rs:197 download.rs:292 download.rs:366 +msgid "Failed to get file name from url:" +msgstr "无法从 URL 获取文件名：" + +#: download.rs:79 download.rs:115 download.rs:307 download.rs:346 +#: download.rs:386 download.rs:416 msgid "Failed to add exif data to image:" msgstr "无法往图片增加 EXIF 数据：" -#: download.rs:234 download.rs:240 download.rs:304 download.rs:310 -#: pixiv_web.rs:153 +#: download.rs:100 download.rs:107 download.rs:326 download.rs:332 +#: download.rs:396 download.rs:402 pixiv_web.rs:153 pixiv_web.rs:169 msgid "Failed to download image:" msgstr "无法下载图片：" -#: download.rs:245 download.rs:315 +#: download.rs:147 +msgid "Failed to get page count." +msgstr "无法获取页数。" + +#: download.rs:156 +msgid "Failed to get pages' data." +msgstr "无法获取每页数据。" + +#: download.rs:170 +msgid "Failed to save metadata to JSON file." +msgstr "无法将元数据保存到 JSON 文件。" + +#: download.rs:185 +msgid "Failed to get ugoira's data." +msgstr "无法获取动图数据。" + +#: download.rs:191 +msgid "Can not find source link for ugoira." +msgstr "无法获取动图的链接。" + +#: download.rs:213 download.rs:219 +msgid "Failed to download ugoira:" +msgstr "无法下载动图：" + +#: download.rs:224 +msgid "Downloaded ugoira:" +msgstr "已下载动图：" + +#: download.rs:234 +msgid "Warning: Failed to generate video's metadata:" +msgstr "警告：无法生成视频元数据：" + +#: download.rs:244 +msgid "Failed to convert from ugoira to mp4 video file:" +msgstr "无法将动图转换为 mp4 视频文件：" + +#: download.rs:247 +msgid "Converted -> " +msgstr "已转换 -> " + +#: download.rs:250 +msgid "Failed to parse frames:" +msgstr "无法解析帧数据：" + +#: download.rs:257 +msgid "Warning: Unknown illust type:" +msgstr "警告：未知的插图类型：" + +#: download.rs:260 +msgid "Warning: Failed to get illust's type." +msgstr "警告：无法获取插图类型。" + +#: download.rs:337 download.rs:407 webclient.rs:390 msgid "Downloaded image:" msgstr "已下载图片：" @@ -221,104 +225,108 @@ msgstr "无法解析从字符串解析时长。" msgid "Failed to parse value." msgstr "无法解析值。" -#: opts.rs:98 +#: opts.rs:101 msgid "Warning: The specified config file not found." msgstr "警告：没有找到指定的设置文件。" -#: opts.rs:122 +#: opts.rs:125 msgid "Usage:" msgstr "使用方法：" -#: opts.rs:124 +#: opts.rs:127 msgid "Download an artwork" msgstr "下载一个作品" -#: opts.rs:126 +#: opts.rs:129 msgid "Fix the config file" msgstr "修复设置文件" -#: opts.rs:128 +#: opts.rs:131 msgid "Print all available settings" msgstr "打印所有可用的设置" -#: opts.rs:136 +#: opts.rs:139 msgid "Print help message." msgstr "打印帮助信息。" -#: opts.rs:140 +#: opts.rs:143 msgid "The location of config file." msgstr "设置文件的位置。" -#: opts.rs:146 settings_list.rs:13 +#: opts.rs:149 settings_list.rs:13 msgid "The location of cookies file. Used for web API." msgstr "cookies 文件的位置。用于网页 API。" -#: opts.rs:152 settings_list.rs:14 +#: opts.rs:155 settings_list.rs:14 msgid "The language of translated tags." msgstr "翻译后的标签语言。" -#: opts.rs:155 +#: opts.rs:158 msgid "Verbose logging." msgstr "启用详细内容输出。" -#: opts.rs:156 +#: opts.rs:159 msgid "Overwrite existing file." msgstr "覆盖已有文件。" -#: opts.rs:157 +#: opts.rs:160 msgid "Skip overwrite existing file." msgstr "跳过覆盖已有文件。" -#: opts.rs:161 settings_list.rs:15 +#: opts.rs:164 settings_list.rs:15 msgid "Max retry count if request failed." msgstr "请求失败时最大重试次数。" -#: opts.rs:167 settings_list.rs:16 +#: opts.rs:170 settings_list.rs:16 msgid "The interval (in seconds) between two retries." msgstr "两次尝试的间隔时间（单位：秒）。" -#: opts.rs:170 settings_list.rs:17 +#: opts.rs:173 settings_list.rs:17 msgid "Use data from webpage first." msgstr "优先使用来自网页的数据。" -#: opts.rs:175 settings_list.rs:20 +#: opts.rs:178 settings_list.rs:20 msgid "" "Add/Update exif information to image files even when overwrite are disabled." msgstr "添加或更新图片的 EXIF 信息，即使不覆盖图片文件。" -#: opts.rs:180 settings_list.rs:22 +#: opts.rs:183 settings_list.rs:22 msgid "Whether to enable progress bar." msgstr "是否启用进度条。" -#: opts.rs:202 +#: opts.rs:189 settings_list.rs:23 +msgid "Download multiple images at the same time." +msgstr "同时下载多张图片。" + +#: opts.rs:210 msgid "Unknown command." msgstr "未知指令。" -#: opts.rs:212 +#: opts.rs:220 msgid "Failed to parse ID:" msgstr "无法解析 ID：" -#: opts.rs:218 +#: opts.rs:226 msgid "No URL or ID specified." msgstr "没有指定网址或 ID。" -#: opts.rs:226 +#: opts.rs:234 msgid "No detailed command specified." msgstr "没有指定更详细的指令。" -#: opts.rs:239 +#: opts.rs:247 msgid "Unknown config subcommand." msgstr "未知的 config 子指令。" -#: opts.rs:275 +#: opts.rs:283 msgid "Retry count must be an non-negative integer:" msgstr "重试次数不能是负数：" -#: opts.rs:284 +#: opts.rs:292 msgid "Failed to parse retry interval:" msgstr "无法解析间隔时间：" -#: opts.rs:298 +#: opts.rs:306 msgid "Failed to parse :" msgstr "无法解析 ：" @@ -358,27 +366,27 @@ msgstr "网络错误：" msgid "Failed to detect error." msgstr "检测 error 字段失败。" -#: pixiv_web.rs:168 pixiv_web.rs:198 +#: pixiv_web.rs:184 pixiv_web.rs:214 msgid "Artwork's data:" msgstr "作品数据：" -#: pixiv_web.rs:183 pixiv_web.rs:188 +#: pixiv_web.rs:199 pixiv_web.rs:204 msgid "Failed to get artwork page:" msgstr "无法获取作品页：" -#: pixiv_web.rs:194 +#: pixiv_web.rs:210 msgid "Failed to parse artwork page." msgstr "无法解析作品页。" -#: pixiv_web.rs:212 +#: pixiv_web.rs:228 msgid "Artwork's page data:" msgstr "作品页面数据：" -#: pixiv_web.rs:226 +#: pixiv_web.rs:242 msgid "Ugoira's data:" msgstr "动图数据：" -#: pixiv_web.rs:250 +#: pixiv_web.rs:266 msgid "Warning: Failed to save cookies file:" msgstr "警告：无法保存 cookies 文件：" @@ -506,7 +514,7 @@ msgstr "无法转换路径。" msgid "Do you want to delete file \"\"?" msgstr "你想要删除文件 吗？" -#: utils.rs:51 webclient.rs:154 +#: utils.rs:51 webclient.rs:156 msgid "Can not parse URL:" msgstr "无法解析 URL：" @@ -514,43 +522,43 @@ msgstr "无法解析 URL：" msgid "Failed to get file name from path:" msgstr "无法从路径获取文件名：" -#: webclient.rs:114 +#: webclient.rs:116 msgid "Failed to parse Set-Cookie header." msgstr "无法解析 Set-Cookie 头部。" -#: webclient.rs:119 +#: webclient.rs:121 msgid "Failed to convert to string:" msgstr "无法转换为字符串：" -#: webclient.rs:161 +#: webclient.rs:163 msgid "Parameters should be object or array:" msgstr "参数应该是对象或数组：" -#: webclient.rs:181 +#: webclient.rs:183 msgid "Parameters should be array:" msgstr "参数应该是数组：" -#: webclient.rs:185 +#: webclient.rs:187 msgid "Parameters need at least a value:" msgstr "参数需要至少含有一个值：" -#: webclient.rs:224 +#: webclient.rs:226 webclient.rs:247 msgid "Retry after seconds." msgstr " 秒后重试。" -#: webclient.rs:228 +#: webclient.rs:230 webclient.rs:251 msgid "Retry times now." msgstr "正在进行第 次重试。" -#: webclient.rs:242 +#: webclient.rs:264 webclient.rs:282 msgid "Error when request:" msgstr "请求时发生错误：" -#: webclient.rs:308 +#: webclient.rs:334 webclient.rs:426 msgid "Downloading \"\"." msgstr "正在下载 \"\"。" -#: webclient.rs:319 +#: webclient.rs:366 webclient.rs:368 webclient.rs:437 msgid "Error when downloading file:" msgstr "下载文件时发生错误：" diff --git a/src/data/data.rs b/src/data/data.rs index 320bbf1..bb9bc04 100644 --- a/src/data/data.rs +++ b/src/data/data.rs @@ -8,7 +8,7 @@ use std::convert::TryInto; use xml::unescape; /// Pixiv's basic data -pub struct PixivData<'a> { +pub struct PixivData { /// ID pub id: PixivID, /// The title @@ -16,11 +16,11 @@ pub struct PixivData<'a> { /// The author pub author: Option, pub description: Option, - helper: OptHelper<'a>, + helper: OptHelper, } -impl<'a> PixivData<'a> { - pub fn new(id: T, helper: OptHelper<'a>) -> Option { +impl PixivData { + pub fn new(id: T, helper: OptHelper) -> Option { let i = id.to_pixiv_id(); if i.is_none() { return None; diff --git a/src/data/json.rs b/src/data/json.rs index 0dad2f1..455d953 100644 --- a/src/data/json.rs +++ b/src/data/json.rs @@ -12,6 +12,7 @@ use std::fs::File; use std::fs::remove_file; use std::io::Write; use std::path::Path; +use std::sync::Arc; /// Store metadata informations in JSON file pub struct JSONDataFile { @@ -71,13 +72,19 @@ impl JSONDataFile { } } -impl<'a> From> for JSONDataFile { +impl From for JSONDataFile { fn from(p: PixivData) -> Self { JSONDataFile::from(&p) } } -impl<'a> From<&'a PixivData<'a>> for JSONDataFile { +impl From> for JSONDataFile { + fn from(p: Arc) -> Self { + JSONDataFile::from(p.as_ref()) + } +} + +impl From<&PixivData> for JSONDataFile { fn from(p: &PixivData) -> Self { let mut f = Self { id: p.id.clone(), diff --git a/src/download.rs b/src/download.rs index ea43769..b08879d 100644 --- a/src/download.rs +++ b/src/download.rs @@ -7,6 +7,7 @@ use crate::data::json::JSONDataFile; #[cfg(feature = "ugoira")] use crate::data::video::get_video_metadata; use crate::gettext; +use crate::opthelper::OptHelper; use crate::pixiv_link::PixivID; use crate::pixiv_web::PixivWebClient; #[cfg(feature = "ugoira")] @@ -15,20 +16,24 @@ use crate::utils::ask_need_overwrite; use crate::utils::get_file_name_from_url; use crate::webclient::WebClient; use crate::Main; +use indicatif::MultiProgress; use json::JsonValue; +use spin_on::spin_on; use std::path::PathBuf; +use std::sync::Arc; +use futures_util::lock::Mutex; impl Main { pub fn download(&mut self) -> i32 { - let mut pw = PixivWebClient::new(&self); - if !pw.init() { + let pw = Arc::new(Mutex::new(PixivWebClient::new(&self))); + if !pw.try_lock().unwrap().init() { println!("{}", gettext("Failed to initialize pixiv web api client.")); return 1; } - if !pw.check_login() { + if !pw.try_lock().unwrap().check_login() { return 1; } - if !pw.logined() { + if !pw.try_lock().unwrap().logined() { println!( "{}", gettext("Warning: Web api client not logined, some future may not work.") @@ -37,7 +42,7 @@ impl Main { for id in self.cmd.as_ref().unwrap().ids.iter() { match id { PixivID::Artwork(id) => { - let r = self.download_artwork(&mut pw, id.clone()); + let r = self.download_artwork(Arc::clone(&pw), id.clone()); if r != 0 { return r; } @@ -47,18 +52,87 @@ impl Main { 0 } - pub fn download_artwork(&self, pw: &mut PixivWebClient, id: u64) -> i32 { + pub async fn download_artwork_page(pw: Arc>, page: JsonValue, np: u16, progress_bars: Arc, datas: Arc, base: Arc, helper: Arc) -> i32 { + let link = &page["urls"]["original"]; + if !link.is_string() { + println!("{}", gettext("Failed to get original picture's link.")); + return 1; + } + let link = link.as_str().unwrap(); + let file_name = get_file_name_from_url(link); + if file_name.is_none() { + println!("{} {}", gettext("Failed to get file name from url:"), link); + return 1; + } + let file_name = file_name.unwrap(); + let file_name = base.join(file_name); + if file_name.exists() { + match helper.overwrite() { + Some(overwrite) => { + if !overwrite { + #[cfg(feature = "exif")] + { + if helper.update_exif() { + if add_exifdata_to_image(&file_name, &datas, np).is_err() { + println!( + "{} {}", + gettext("Failed to add exif data to image:"), + file_name.to_str().unwrap_or("(null)") + ); + } + } + } + return 0; + } + } + None => { + if !ask_need_overwrite(file_name.to_str().unwrap()) { + return 0; + } + } + } + } + let r; + { + let mut pw = pw.lock().await; + r = pw.adownload_image(link).await; + if r.is_none() { + println!("{} {}", gettext("Failed to download image:"), link); + return 1; + } + } + let r = r.unwrap(); + let re = WebClient::adownload_stream(&file_name, r, &helper, Some(progress_bars)).await; + if re.is_err() { + println!("{} {}", gettext("Failed to download image:"), link); + return 1; + } + #[cfg(feature = "exif")] + { + if add_exifdata_to_image(&file_name, &datas, np).is_err() { + println!( + "{} {}", + gettext("Failed to add exif data to image:"), + file_name.to_str().unwrap_or("(null)") + ); + } + } + 0 + } + + pub fn download_artwork(&self, pw: Arc>, id: u64) -> i32 { let mut re = None; let pages; let mut ajax_ver = true; - if pw.helper.use_webpage() { - re = pw.get_artwork(id); + let helper = Arc::new(pw.try_lock().unwrap().helper.clone()); + if helper.use_webpage() { + re = pw.try_lock().unwrap().get_artwork(id); if re.is_some() { ajax_ver = false; } } if re.is_none() { - re = pw.get_artwork_ajax(id); + re = pw.try_lock().unwrap().get_artwork_ajax(id); } if re.is_none() { return 1; @@ -76,21 +150,22 @@ impl Main { let pages = pages.unwrap(); let mut pages_data: Option = None; if pages > 1 { - pages_data = pw.get_illust_pages(id); + pages_data = pw.try_lock().unwrap().get_illust_pages(id); } if pages > 1 && pages_data.is_none() { println!("{}", gettext("Failed to get pages' data.")); return 1; } - let base = PathBuf::from("."); + let base = Arc::new(PathBuf::from(".")); let json_file = base.join(format!("{}.json", id)); - let mut datas = PixivData::new(id, pw.helper.clone()).unwrap(); + let mut datas = PixivData::new(id, (*helper).clone()).unwrap(); if ajax_ver { datas.from_web_page_ajax_data(&re, true); } else { datas.from_web_page_data(&re, true); } - let json_data = JSONDataFile::from(&datas); + let datas = Arc::new(datas); + let json_data = JSONDataFile::from(Arc::clone(&datas)); if !json_data.save(&json_file) { println!("{}", gettext("Failed to save metadata to JSON file.")); return 1; @@ -105,7 +180,7 @@ impl Main { match illust_type { 0 => { } 2 => { - let ugoira_data = pw.get_ugoira(id); + let ugoira_data = pw.try_lock().unwrap().get_ugoira(id); if ugoira_data.is_none() { println!("{}", gettext("Failed to get ugoira's data.")); return 1; @@ -125,7 +200,7 @@ impl Main { let file_name = file_name.unwrap(); let file_name = base.join(file_name); let dw = if file_name.exists() { - match pw.helper.overwrite() { + match helper.overwrite() { Some(overwrite) => { overwrite } None => { ask_need_overwrite(file_name.to_str().unwrap()) } } @@ -133,13 +208,13 @@ impl Main { true }; if dw { - let r = pw.download_image(src); + let r = pw.try_lock().unwrap().download_image(src); if r.is_none() { println!("{} {}", gettext("Failed to download ugoira:"), src); return 1; } let r = r.unwrap(); - let re = WebClient::download_stream(&file_name, r, &pw.helper); + let re = WebClient::download_stream(&file_name, r, &helper); if re.is_err() { println!("{} {}", gettext("Failed to download ugoira:"), src); return 1; @@ -153,7 +228,7 @@ impl Main { } #[cfg(feature = "ugoira")] { - let metadata = match get_video_metadata(&datas) { + let metadata = match get_video_metadata(Arc::clone(&datas).as_ref()) { Ok(m) => { m } Err(e) => { println!("{} {}", gettext("Warning: Failed to generate video's metadata:"), e); @@ -184,7 +259,24 @@ impl Main { } else { println!("{}", gettext("Warning: Failed to get illust's type.")); } - if pages_data.is_some() { + if pages_data.is_some() && helper.download_multiple_images() { + let mut np = 0u16; + let pages_data = pages_data.as_ref().unwrap(); + let progress_bars = Arc::new(MultiProgress::new()); + let mut tasks = Vec::new(); + for page in pages_data.members() { + let f = tokio::spawn(Self::download_artwork_page(Arc::clone(&pw), page.clone(), np, Arc::clone(&progress_bars), Arc::clone(&datas), Arc::clone(&base), Arc::clone(&helper))); + tasks.push(f); + np += 1; + } + let mut re = 0; + for task in tasks { + let r = spin_on(task); + re |= r.unwrap_or(1); + } + return re; + } + else if pages_data.is_some() { #[cfg(feature = "exif")] let mut np = 0u16; let pages_data = pages_data.as_ref().unwrap(); @@ -203,12 +295,12 @@ impl Main { let file_name = file_name.unwrap(); let file_name = base.join(file_name); if file_name.exists() { - match pw.helper.overwrite() { + match helper.overwrite() { Some(overwrite) => { if !overwrite { #[cfg(feature = "exif")] { - if pw.helper.update_exif() { + if helper.update_exif() { if add_exifdata_to_image(&file_name, &datas, np).is_err() { println!( "{} {}", @@ -229,13 +321,13 @@ impl Main { } } } - let r = pw.download_image(link); + let r = pw.try_lock().unwrap().download_image(link); if r.is_none() { println!("{} {}", gettext("Failed to download image:"), link); return 1; } let r = r.unwrap(); - let re = WebClient::download_stream(&file_name, r, &pw.helper); + let re = WebClient::download_stream(&file_name, r, &helper); if re.is_err() { println!("{} {}", gettext("Failed to download image:"), link); return 1; @@ -277,7 +369,7 @@ impl Main { let file_name = file_name.unwrap(); let file_name = base.join(file_name); if file_name.exists() { - let overwrite = match pw.helper.overwrite() { + let overwrite = match helper.overwrite() { Some(overwrite) => { overwrite } @@ -287,7 +379,7 @@ impl Main { }; if !overwrite { #[cfg(feature = "exif")] - if pw.helper.update_exif() { + if helper.update_exif() { if add_exifdata_to_image(&file_name, &datas, 0).is_err() { println!( "{} {}", @@ -299,13 +391,13 @@ impl Main { return 0; } } - let r = pw.download_image(link); + let r = pw.try_lock().unwrap().download_image(link); if r.is_none() { println!("{} {}", gettext("Failed to download image:"), link); return 1; } let r = r.unwrap(); - let re = WebClient::download_stream(&file_name, r, &pw.helper); + let re = WebClient::download_stream(&file_name, r, &helper); if re.is_err() { println!("{} {}", gettext("Failed to download image:"), link); return 1; diff --git a/src/opthelper.rs b/src/opthelper.rs index 4622d1f..bc537a9 100644 --- a/src/opthelper.rs +++ b/src/opthelper.rs @@ -11,17 +11,17 @@ use std::time::Duration; /// An sturct to access all available settings/command line switches #[derive(Clone, Debug)] -pub struct OptHelper<'a> { +pub struct OptHelper { /// Command Line Options - opt: &'a CommandOpts, + opt: CommandOpts, /// Settings - settings: &'a SettingStore, + settings: SettingStore, default_retry_interval: NonTailList, _author_name_filters: Option>, _use_progress_bar: Option, } -impl<'a> OptHelper<'a> { +impl OptHelper { pub fn author_name_filters(&self) -> Option<&Vec> { if self.settings.have("author-name-filters") { return self._author_name_filters.as_ref(); @@ -51,7 +51,7 @@ impl<'a> OptHelper<'a> { } } - pub fn new(opt: &'a CommandOpts, settings: &'a SettingStore) -> Self { + pub fn new(opt: CommandOpts, settings: SettingStore) -> Self { let mut l = NonTailList::default(); l += Duration::new(3, 0); let _author_name_filters = if settings.have("author-name-filters") { @@ -146,4 +146,15 @@ impl<'a> OptHelper<'a> { } String::from("[{elapsed_precise}] [{wide_bar:.green/yellow}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta}) {msg:40}") } + + /// Return whether to download multiple images at the same time. + pub fn download_multiple_images(&self) -> bool { + if self.opt.download_multiple_images { + return true; + } + if self.settings.have_bool("download-multiple-images") { + return self.settings.get_bool("download-multiple-images").unwrap(); + } + false + } } diff --git a/src/opts.rs b/src/opts.rs index b901b3d..45d3453 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -36,7 +36,7 @@ impl PartialEq for &ConfigCommand { } } -#[derive(Debug)] +#[derive(Clone, Debug)] /// Command Line Options pub struct CommandOpts { /// Command @@ -66,6 +66,8 @@ pub struct CommandOpts { pub update_exif: bool, /// Whether to enable progress bar pub use_progress_bar: Option, + /// Whether to download multiple images at the same time + pub download_multiple_images: bool, } impl CommandOpts { @@ -85,6 +87,7 @@ impl CommandOpts { #[cfg(feature = "exif")] update_exif: false, use_progress_bar: None, + download_multiple_images: false, } } @@ -180,6 +183,11 @@ pub fn parse_cmd() -> Option { gettext("Whether to enable progress bar."), "yes/no/auto", ); + opts.optflag( + "", + "download-multiple-images", + gettext("Download multiple images at the same time."), + ); let result = match opts.parse(&argv[1..]) { Ok(m) => m, Err(err) => { @@ -300,5 +308,6 @@ pub fn parse_cmd() -> Option { } re.as_mut().unwrap().use_progress_bar = Some(r.unwrap()); } + re.as_mut().unwrap().download_multiple_images = result.opt_present("download-multiple-images"); re } diff --git a/src/pixiv_web.rs b/src/pixiv_web.rs index d113181..b71ae60 100644 --- a/src/pixiv_web.rs +++ b/src/pixiv_web.rs @@ -9,19 +9,19 @@ use reqwest::Response; use spin_on::spin_on; /// A client which use Pixiv's web API -pub struct PixivWebClient<'a> { +pub struct PixivWebClient { client: WebClient, - pub helper: OptHelper<'a>, + pub helper: OptHelper, /// true if in is initialized inited: bool, data: Option, } -impl<'a> PixivWebClient<'a> { - pub fn new(m: &'a Main) -> Self { +impl PixivWebClient { + pub fn new(m: &Main) -> Self { Self { client: WebClient::new(), - helper: OptHelper::new(m.cmd.as_ref().unwrap(), m.settings.as_ref().unwrap()), + helper: OptHelper::new(m.cmd.as_ref().unwrap().clone(), m.settings.as_ref().unwrap().clone()), inited: false, data: None, } @@ -156,6 +156,22 @@ impl<'a> PixivWebClient<'a> { Some(r) } + pub async fn adownload_image(&mut self, url: U) -> Option { + self.auto_init(); + let r = self.client.aget(url, json::object!{"referer": "https://www.pixiv.net/"}).await; + if r.is_none() { + return None; + } + let r = r.unwrap(); + let status = r.status(); + let code = status.as_u16(); + if code >= 400 { + println!("{} {}", gettext("Failed to download image:"), status); + return None; + } + Some(r) + } + pub fn get_artwork_ajax(&mut self, id: u64) -> Option { self.auto_init(); let r = self.client.get(format!("https://www.pixiv.net/ajax/illust/{}", id), None); @@ -241,7 +257,7 @@ impl<'a> PixivWebClient<'a> { } } -impl <'a> Drop for PixivWebClient<'a> { +impl Drop for PixivWebClient { fn drop(&mut self) { if self.inited { let c = self.helper.cookies(); diff --git a/src/settings_list.rs b/src/settings_list.rs index 6ad43b8..2c168f4 100644 --- a/src/settings_list.rs +++ b/src/settings_list.rs @@ -20,6 +20,7 @@ pub fn get_settings_list() -> Vec { SettingDes::new("update-exif", gettext("Add/Update exif information to image files even when overwrite are disabled."), JsonValueType::Boolean, None).unwrap(), SettingDes::new("progress-bar-template", gettext("Progress bar's template. See for more informations.").replace("", "https://docs.rs/indicatif/latest/indicatif/#templates").as_str(), JsonValueType::Str, Some(check_nonempty_str)).unwrap(), SettingDes::new("use-progress-bar", gettext("Whether to enable progress bar."), JsonValueType::Multiple, Some(check_user_or_not)).unwrap(), + SettingDes::new("download-multiple-images", gettext("Download multiple images at the same time."), JsonValueType::Boolean, None).unwrap(), ] } diff --git a/src/webclient.rs b/src/webclient.rs index 7fa2520..b5a1db0 100644 --- a/src/webclient.rs +++ b/src/webclient.rs @@ -6,6 +6,7 @@ use crate::gettext; use crate::list::NonTailList; use crate::opthelper::OptHelper; use futures_util::StreamExt; +use indicatif::MultiProgress; use indicatif::ProgressBar; use indicatif::ProgressStyle; use json::JsonValue; @@ -18,6 +19,7 @@ use std::fs::File; use std::fs::remove_file; use std::io::Write; use std::path::Path; +use std::sync::Arc; use std::time::Duration; pub trait ToHeaders { @@ -231,9 +233,29 @@ impl WebClient { None } + pub async fn aget(&mut self, url: U, headers: H) -> Option { + let mut count = 0u64; + while count <= self.retry { + let r = self._aget2(url.clone(), headers.clone()).await; + if r.is_some() { + return r; + } + count += 1; + if count <= self.retry { + let t = self.retry_interval.as_ref().unwrap()[(count - 1).try_into().unwrap()]; + if !t.is_zero() { + println!("{}", gettext("Retry after seconds.").replace("", format!("{}", t.as_secs_f64()).as_str()).as_str()); + tokio::time::sleep(t).await; + } + } + println!("{}", gettext("Retry times now.").replace("", format!("{}", count).as_str()).as_str()); + } + None + } + /// Send GET requests pub fn _get(&mut self, url: U, headers: H) -> Option { - let r = self.aget(url, headers); + let r = self._aget(url, headers); let r = r.send(); let r = spin_on(r); match r { @@ -251,7 +273,25 @@ impl WebClient { Some(r) } - pub fn aget(&mut self, url: U, headers: H) -> RequestBuilder { + pub async fn _aget2(&mut self, url: U, headers: H) -> Option { + let r = self._aget(url, headers); + let r = r.send().await; + match r { + Ok(_) => {} + Err(e) => { + println!("{} {}", gettext("Error when request:"), e); + return None; + } + } + let r = r.unwrap(); + self.handle_set_cookie(&r); + if self.verbose { + println!("{}", r.status()); + } + Some(r) + } + + pub fn _aget(&mut self, url: U, headers: H) -> RequestBuilder { let s = url.as_str(); if self.verbose { println!("GET {}", s); @@ -274,6 +314,84 @@ impl WebClient { r } + pub async fn adownload_stream + ?Sized>(file_name: &S, r: Response, opt: &OptHelper, progress_bars: Option>) -> Result<(), ()> { + let content_length = r.content_length(); + let use_progress_bar = match &content_length { + Some(_) => { opt.use_progress_bar() } + None => { false } + }; + let mut bar = if use_progress_bar { + Some(ProgressBar::new(content_length.unwrap())) + } else { + None + }; + let p = Path::new(file_name); + if bar.is_some() { + bar.as_mut().unwrap().set_style(ProgressStyle::default_bar() + .template(opt.progress_bar_template().as_ref()).unwrap() + .progress_chars("#>-")); + let tmp = p.file_name().unwrap_or(p.as_os_str()); + bar.as_mut().unwrap().set_message(gettext("Downloading \"\".").replace("", tmp.to_str().unwrap_or(""))); + if progress_bars.is_some() { + bar = Some(progress_bars.unwrap().add(bar.unwrap())); + } + } + if p.exists() { + let re = remove_file(p); + if re.is_err() { + if bar.is_none() { + println!("{} {}", gettext("Failed to remove file:"), re.unwrap_err()); + } else { + bar.as_ref().unwrap().set_message(format!("{} {}", gettext("Failed to remove file:"), re.unwrap_err())); + bar.as_ref().unwrap().abandon(); + } + return Err(()); + } + } + let f = File::create(p); + if f.is_err() { + if bar.is_none() { + println!("{} {}", gettext("Failed to create file:"), f.unwrap_err()); + } else { + bar.as_ref().unwrap().set_message(format!("{} {}", gettext("Failed to create file:"), f.unwrap_err())); + bar.as_ref().unwrap().abandon(); + } + return Err(()); + } + let mut f = f.unwrap(); + let mut stream = r.bytes_stream(); + while let Some(data) = stream.next().await { + if data.is_err() { + if bar.is_none() { + println!("{} {}", gettext("Error when downloading file:"), data.unwrap_err()); + } else { + bar.as_ref().unwrap().set_message(format!("{} {}", gettext("Error when downloading file:"), data.unwrap_err())); + bar.as_ref().unwrap().abandon(); + } + return Err(()); + } + let data = data.unwrap(); + if bar.is_some() { + bar.as_ref().unwrap().inc(data.len() as u64); + bar.as_ref().unwrap().tick(); + } + let r = f.write(&data); + if r.is_err() { + if bar.is_none() { + println!("{} {}", gettext("Failed to write file:"), r.unwrap_err()); + } else { + bar.as_ref().unwrap().set_message(format!("{} {}", gettext("Failed to write file:"), r.unwrap_err())); + bar.as_ref().unwrap().abandon(); + } + return Err(()); + } + } + if bar.is_some() { + bar.as_mut().unwrap().finish_with_message(format!("{} {}", gettext("Downloaded image:"), p.to_str().unwrap_or("(null)"))); + } + Ok(()) + } + /// Download a stream /// * `file_name` - File name /// * `r` - Response @@ -291,7 +409,7 @@ impl WebClient { }; if bar.is_some() { bar.as_mut().unwrap().set_style(ProgressStyle::default_bar() - .template(opt.progress_bar_template().as_ref()) + .template(opt.progress_bar_template().as_ref()).unwrap() .progress_chars("#>-")); } let mut downloaded = 0usize;