diff --git a/Cargo.lock b/Cargo.lock index 47b7e55..a9bee3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,15 +226,6 @@ dependencies = [ "clap_derive 4.5.47", ] -[[package]] -name = "clap-num" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822c4000301ac390e65995c62207501e3ef800a1fc441df913a5e8e4dc374816" -dependencies = [ - "num-traits", -] - [[package]] name = "clap_builder" version = "4.5.47" @@ -1171,7 +1162,6 @@ dependencies = [ "anyhow", "byteorder", "clap 4.5.47", - "clap-num", "csv", "ctrlc", "emote-psb", @@ -1253,15 +1243,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "num_cpus" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index 37cbd1d..703c93b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ exclude = [".github", "*.py"] anyhow = "1" byteorder = { version = "1.5", default-features = false, optional = true} clap = { version = "4.5", features = ["derive"] } -clap-num = "1.2" csv = "1.3" ctrlc = "3.4" emote-psb = { version = "0.5", optional = true , features = ["serde"] } @@ -85,7 +84,7 @@ yaneurao-itufuru = ["yaneurao"] # basic feature image = ["png"] image-jpg = ["mozjpeg"] -image-jxl = ["image", "jpegxl-sys"] +image-jxl = ["image", "jpegxl-sys", "utils-threadpool"] image-webp = ["webp"] lossless-audio = ["utils-pcm"] audio-flac = ["libflac-sys", "utils-pcm"] diff --git a/src/args.rs b/src/args.rs index 47c2075..44202f6 100644 --- a/src/args.rs +++ b/src/args.rs @@ -1,4 +1,6 @@ use crate::types::*; +#[allow(unused)] +use crate::utils::num_range::*; use clap::{ArgAction, ArgGroup, Parser, Subcommand}; #[cfg(feature = "flate2")] @@ -13,7 +15,7 @@ fn parse_compression_level(level: &str) -> Result { } else if lower == "fast" { return Ok(1); } - clap_num::number_range(level, 0, 9) + number_range(level, 0, 9) } #[cfg(feature = "mozjpeg")] @@ -22,7 +24,7 @@ fn parse_jpeg_quality(quality: &str) -> Result { if lower == "best" { return Ok(100); } - clap_num::number_range(quality, 0, 100) + number_range(quality, 0, 100) } #[cfg(feature = "zstd")] @@ -33,7 +35,7 @@ fn parse_zstd_compression_level(level: &str) -> Result { } else if lower == "best" { return Ok(22); } - clap_num::number_range(level, 0, 22) + number_range(level, 0, 22) } #[cfg(feature = "webp")] @@ -42,7 +44,7 @@ fn parse_webp_quality(quality: &str) -> Result { if lower == "best" { return Ok(100); } - clap_num::number_range(quality, 0, 100) + number_range(quality, 0, 100) } #[cfg(feature = "audio-flac")] @@ -55,7 +57,18 @@ fn parse_flac_compression_level(level: &str) -> Result { } else if lower == "default" { return Ok(5); } - clap_num::number_range(level, 0, 8) + number_range(level, 0, 8) +} + +#[cfg(feature = "image-jxl")] +fn parse_jxl_distance(s: &str) -> Result { + let lower = s.to_ascii_lowercase(); + if lower == "lossless" { + return Ok(0.0); + } else if lower == "visually-lossless" { + return Ok(1.0); + } + number_range(s, 0.0, 25.0) } /// Tools for export and import scripts @@ -428,6 +441,20 @@ pub struct Arg { #[arg(long, global = true, action = ArgAction::SetTrue)] /// Do not filter ascii strings in Favorite HCB script. pub favorite_hcb_no_filter_ascii: bool, + #[cfg(feature = "image-jxl")] + #[arg(long, global = true, action = ArgAction::SetTrue, alias = "jxl-no-lossless")] + /// Disable JXL lossless compression for output images + pub jxl_lossy: bool, + #[cfg(feature = "image-jxl")] + #[arg(long, global = true, default_value_t = 1.0, value_parser = parse_jxl_distance)] + /// JXL distance for output images. 0 means mathematically lossless compression. 1.0 means visually lossless compression. + /// Allowed range is 0.0-25.0. Recommended range is 0.5-3.0. Default value is 1 + pub jxl_distance: f32, + #[cfg(feature = "image-jxl")] + #[arg(long, global = true, default_value_t = 1)] + /// Workers count for encode JXL images in parallel. Default is 1. + /// Set this to 1 to disable parallel encoding. 0 means same as 1 + pub jxl_workers: usize, #[command(subcommand)] /// Command pub command: Command, diff --git a/src/main.rs b/src/main.rs index 2231b1d..f8db399 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1927,6 +1927,12 @@ fn main() { favorite_hcb_filter_ascii: !arg.favorite_hcb_no_filter_ascii, #[cfg(feature = "bgi-img")] bgi_img_workers: arg.bgi_img_workers, + #[cfg(feature = "image-jxl")] + jxl_lossless: !arg.jxl_lossy, + #[cfg(feature = "image-jxl")] + jxl_distance: arg.jxl_distance, + #[cfg(feature = "image-jxl")] + jxl_workers: arg.jxl_workers, }; match &arg.command { args::Command::Export { input, output } => { diff --git a/src/scripts/bgi/archive/dsc.rs b/src/scripts/bgi/archive/dsc.rs index e3e28de..bbe2e37 100644 --- a/src/scripts/bgi/archive/dsc.rs +++ b/src/scripts/bgi/archive/dsc.rs @@ -4,6 +4,7 @@ use crate::ext::vec::*; use crate::scripts::base::*; use crate::types::*; use crate::utils::bit_stream::*; +use crate::utils::num_range::*; use anyhow::Result; use rand::Rng; use std::collections::BinaryHeap; @@ -688,5 +689,5 @@ impl Script for Dsc { /// Parses the minimum length for LZSS compression from a string. pub fn parse_min_length(len: &str) -> Result { - clap_num::number_range(len, 2, 256) + number_range(len, 2, 256) } diff --git a/src/scripts/bgi/image/cbg.rs b/src/scripts/bgi/image/cbg.rs index 766e6c3..a3bc54b 100644 --- a/src/scripts/bgi/image/cbg.rs +++ b/src/scripts/bgi/image/cbg.rs @@ -345,7 +345,7 @@ impl<'a> CbgDecoder<'a> { has_alpha: AtomicBool::new(false), }); - let thread_pool = ThreadPool::new(self.workers, Some("cbg-decoder-worker-"))?; + let thread_pool = ThreadPool::new(self.workers, Some("cbg-decoder-worker-"), false)?; let mut dst = 0i32; for i in 0..y_blocks { @@ -359,7 +359,7 @@ impl<'a> CbgDecoder<'a> { let decoder_ref = Arc::clone(&decoder); thread_pool.execute( - move || { + move |_| { decoder_ref.unpack_block(block_offset, next_offset - block_offset, closure_dst) }, true, @@ -370,7 +370,7 @@ impl<'a> CbgDecoder<'a> { if self.info.bpp == 32 { let decoder_ref = Arc::clone(&decoder); thread_pool.execute( - move || decoder_ref.unpack_alpha(offsets[y_blocks as usize]), + move |_| decoder_ref.unpack_alpha(offsets[y_blocks as usize]), true, )?; } diff --git a/src/types.rs b/src/types.rs index e1eb1f8..60fbf02 100644 --- a/src/types.rs +++ b/src/types.rs @@ -431,6 +431,20 @@ pub struct ExtraConfig { /// Workers count for decode BGI compressed images v2 in parallel. Default is half of CPU cores. /// Set this to 1 to disable parallel decoding. 0 means same as 1. pub bgi_img_workers: usize, + #[cfg(feature = "image-jxl")] + #[default(true)] + /// Use JXL lossless compression for output images. Enabled by default. + pub jxl_lossless: bool, + #[cfg(feature = "image-jxl")] + #[default(1.0)] + /// JXL distance for output images. 0 means mathematically lossless compression. 1.0 means visually lossless compression. + /// Allowed range is 0.0-25.0. Recommended range is 0.5-3.0. Default value is 1.0. + pub jxl_distance: f32, + #[cfg(feature = "image-jxl")] + #[default(1)] + /// Workers count for encode JXL images in parallel. Default is 1. + /// Set this to 1 to disable parallel encoding. 0 means same as 1 + pub jxl_workers: usize, } #[derive(Clone, Copy, Debug, ValueEnum, PartialEq, Eq, PartialOrd, Ord)] diff --git a/src/utils/jxl.rs b/src/utils/jxl.rs index 78ab80e..13b572d 100644 --- a/src/utils/jxl.rs +++ b/src/utils/jxl.rs @@ -1,11 +1,15 @@ //! JPEG XL image support use super::img::*; +use super::num_range::*; +use super::threadpool::*; use crate::types::*; use anyhow::Result; use jpegxl_sys::common::types::*; use jpegxl_sys::decode::*; use jpegxl_sys::encoder::encode::*; use jpegxl_sys::metadata::codestream_header::*; +use jpegxl_sys::threads::parallel_runner::*; +use std::ffi::c_void; use std::io::Read; struct JxlDecoderHandle { @@ -32,6 +36,57 @@ impl Drop for JxlEncoderHandle { } } +struct ThreadPoolRunner { + thread_pool: ThreadPool<()>, +} + +impl ThreadPoolRunner { + fn new(workers: usize) -> Result { + let thread_pool = ThreadPool::new(workers, Some("jxl-thread-runner-"), true)?; + Ok(Self { thread_pool }) + } +} + +#[derive(Clone, Copy)] +struct JpegxlPointer(*mut c_void); + +unsafe impl Send for JpegxlPointer {} + +unsafe extern "C-unwind" fn thread_pool_runner( + runner_opaque: *mut c_void, + jpegxl_opaque: *mut c_void, + init: JxlParallelRunInit, + func: JxlParallelRunFunction, + start_range: u32, + end_range: u32, +) -> JxlParallelRetCode { + if runner_opaque.is_null() || jpegxl_opaque.is_null() { + return JXL_PARALLEL_RET_RUNNER_ERROR; + } + let runner = unsafe { &*(runner_opaque as *const ThreadPoolRunner) }; + let initre = unsafe { init(jpegxl_opaque, runner.thread_pool.size()) }; + if initre != JXL_PARALLEL_RET_SUCCESS { + return initre; + } + let jpegxl = JpegxlPointer(jpegxl_opaque); + for i in start_range..end_range { + let jpegxl = jpegxl; + let func = func; + match runner.thread_pool.execute( + move |thread_id| unsafe { + let jpegxl = jpegxl; + func(jpegxl.0, i, thread_id) + }, + true, + ) { + Ok(_) => {} + Err(_) => return JXL_PARALLEL_RET_RUNNER_ERROR, + } + } + runner.thread_pool.join(); + JXL_PARALLEL_RET_SUCCESS +} + fn check_decoder_status(status: JxlDecoderStatus) -> Result<()> { match status { JxlDecoderStatus::Success => Ok(()), @@ -145,12 +200,27 @@ pub fn decode_jxl(mut r: R) -> Result { } /// Encode image data to JXL format -pub fn encode_jxl(mut img: ImageData, _config: &ExtraConfig) -> Result> { +pub fn encode_jxl(mut img: ImageData, config: &ExtraConfig) -> Result> { let encoder = unsafe { JxlEncoderCreate(std::ptr::null()) }; if encoder.is_null() { return Err(anyhow::anyhow!("Failed to create JXL encoder")); } let eh = JxlEncoderHandle { handle: encoder }; + let ph = if config.jxl_workers > 1 { + let ph = ThreadPoolRunner::new(config.jxl_workers)?; + Some(ph) + } else { + None + }; + if let Some(ph) = &ph { + check_encoder_status(unsafe { + JxlEncoderSetParallelRunner( + eh.handle, + thread_pool_runner, + ph as *const _ as *mut c_void, + ) + })?; + } let mut basic_info = default_basic_info(); basic_info.xsize = img.width; basic_info.ysize = img.height; @@ -184,7 +254,14 @@ pub fn encode_jxl(mut img: ImageData, _config: &ExtraConfig) -> Result> "Failed to create JXL encoder frame settings" )); } - check_encoder_status(unsafe { JxlEncoderSetFrameLossless(options, JxlBool::True) })?; + check_encoder_status(unsafe { + JxlEncoderSetFrameLossless(options, JxlBool::from(config.jxl_lossless)) + })?; + if !config.jxl_lossless { + let distance = check_range(config.jxl_distance, 0.0, 25.0) + .map_err(|e| anyhow::anyhow!("Invalid JXL distance: {}", e))?; + check_encoder_status(unsafe { JxlEncoderSetFrameDistance(options, distance) })?; + } let format = JxlPixelFormat { num_channels: img.color_type.bpp(1) as u32, data_type: if img.depth <= 8 { diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 2b0b9e6..deeebf1 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -22,6 +22,7 @@ pub mod jxl; pub mod lossless_audio; mod macros; pub mod name_replacement; +pub mod num_range; #[cfg(feature = "utils-pcm")] pub mod pcm; #[cfg(feature = "utils-str")] diff --git a/src/utils/num_range.rs b/src/utils/num_range.rs new file mode 100644 index 0000000..f246743 --- /dev/null +++ b/src/utils/num_range.rs @@ -0,0 +1,27 @@ +//! Functions for parsing numbers within a range. +/// Check if a value is within the specified range. +pub fn check_range(val: T, min: T, max: T) -> Result +where + T: PartialOrd, + T: std::fmt::Display, +{ + if val < min { + return Err(format!("Value {} is less than minimum {}", val, min)); + } else if val > max { + return Err(format!("Value {} is greater than maximum {}", val, max)); + } + Ok(val) +} + +/// Parse a number from a string and check if it is within the specified range. +pub fn number_range(s: &str, min: T, max: T) -> Result +where + T: std::str::FromStr, + ::Err: std::fmt::Display, + T: PartialOrd, + T: std::fmt::Display, +{ + debug_assert!(min <= max, "min should be less than or equal to max"); + let val = s.parse::().map_err(|e| format!("{}", e))?; + check_range(val, min, max) +} diff --git a/src/utils/threadpool.rs b/src/utils/threadpool.rs index c9a4195..d7ffc0a 100644 --- a/src/utils/threadpool.rs +++ b/src/utils/threadpool.rs @@ -7,7 +7,7 @@ use std::sync::{ }; use std::thread::{self, JoinHandle}; -type Job = Box T + Send + 'static>; +type Job = Box T + Send + 'static>; /// A simple generic thread pool. /// @@ -61,7 +61,12 @@ impl ThreadPool { /// the channel is full, further submissions will block or return error depending on the flag. /// /// * `name` - Optional base name for worker threads. If None, "threadpool-worker-" is used. - pub fn new<'a>(size: usize, name: Option<&'a str>) -> Result { + /// * `no_result` - If true, results are not stored (saves some overhead if not needed). + pub fn new<'a>( + size: usize, + name: Option<&'a str>, + no_result: bool, + ) -> Result { if size == 0 { return Err(std::io::Error::new( std::io::ErrorKind::InvalidInput, @@ -98,8 +103,8 @@ impl ThreadPool { match job { Ok(job) => { // Execute the job and store result - let res = job(); - { + let res = job(id); + if !no_result { let mut r = results_clone.lock_blocking(); r.push(res); } @@ -135,9 +140,11 @@ impl ThreadPool { /// Execute a task. If `block_if_full` is true, this call will block when the internal /// submission channel is full (i.e. all workers busy and buffer full) until space becomes available. /// If `block_if_full` is false, this returns Err(ExecuteError::Full) when the channel is full. + /// + /// job: a closure that takes the worker id (0..size-1) and returns a T. pub fn execute(&self, job: F, block_if_full: bool) -> Result<(), ExecuteError> where - F: FnOnce() -> T + Send + 'static, + F: FnOnce(usize) -> T + Send + 'static, { let sender = match &self.sender { Some(s) => s, @@ -191,6 +198,12 @@ impl ThreadPool { } } + /// Take all results, leaving an empty results vector. + pub fn take_results(&self) -> Vec { + let mut results = self.results.lock_blocking(); + results.split_off(0) + } + /// Wait until all submitted tasks have completed, then return the results. pub fn into_results(self) -> Vec { self.join();