diff --git a/Cargo.toml b/Cargo.toml index 5adf01e..d7ef330 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,7 +124,7 @@ will-plus-img = ["will-plus", "image"] yaneurao = [] yaneurao-itufuru = ["yaneurao", "utils-xored-stream"] yuris = ["dep:chrono", "dep:hex", "utils-serde-base64bytes", "utils-xored-stream"] -yuris-arc = ["yuris", "dep:adler", "dep:crc32fast", "flate2", "dep:int-enum", "utils-murmur2", "dep:xxhash-rust", "xxhash-rust/xxh32"] +yuris-arc = ["yuris", "dep:adler", "dep:crc32fast", "flate2", "dep:int-enum", "utils-murmur2", "dep:xxhash-rust", "xxhash-rust/xxh32", "zopfli"] yuris-img = ["yuris", "image", "qoi", "webp"] # basic feature image = ["dep:png"] diff --git a/README.md b/README.md index fcf6ece..064eaa1 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ msg-tool create -t | Archive Type | Feature Name | Name | Unpack | Pack | Remarks | |---|---|---|---|---|---| -| `yuris-ypf` | `yuris-arc` | Yu-Ris Archive (.ypf) | ✔️ | ❌ | | +| `yuris-ypf` | `yuris-arc` | Yu-Ris Archive (.ypf) | ✔️ | ✔️ | | | Image Type | Feature Name | Name | Export | Import | Export Multiple | Import Multiple | Create | Remarks | |---|---|---|---|---|---|---|---|---| diff --git a/src/args.rs b/src/args.rs index ba38641..6bdc4e6 100644 --- a/src/args.rs +++ b/src/args.rs @@ -794,6 +794,22 @@ pub struct Arg { /// Print debug information for Yu-RIS archive (.ypf) when extracting archive to stdout. /// This is used to find correct configuration for Yu-RIS archives. pub yuris_debug_archive: bool, + #[cfg(feature = "yuris-arc")] + #[arg(long, global = true)] + /// Yu-RIS YPF engine version. Used when pack files into Yu-RIS archive. + pub yuris_ypf_version: Option, + #[cfg(feature = "yuris-arc")] + #[arg(long, global = true)] + /// Do not compress file when packing files into Yu-RIS archive. + pub yuris_ypf_no_compress_file: bool, + #[cfg(feature = "yuris-arc")] + #[arg(long, global = true)] + /// Use zopfli to compress files in Yu-RIS archive. + pub yuris_ypf_zopfli: bool, + #[cfg(feature = "yuris-arc")] + #[arg(long, global = true, default_value_t = num_cpus::get())] + /// Workers count for compress files in Yu-RIS archive when creating in parallel. Default is CPU cores count. + pub yuris_ypf_workers: usize, #[command(subcommand)] /// Command pub command: Command, diff --git a/src/ext/io.rs b/src/ext/io.rs index a695f8a..d2d0057 100644 --- a/src/ext/io.rs +++ b/src/ext/io.rs @@ -2803,3 +2803,38 @@ impl Drop for AlignedWriter { } } } + +pub struct HashStream<'a, T, H: std::hash::Hasher> { + inner: T, + hasher: &'a mut H, +} + +impl<'a, T, H: std::hash::Hasher> HashStream<'a, T, H> { + pub fn new(inner: T, hasher: &'a mut H) -> Self { + Self { inner, hasher } + } + + pub fn digest(&self) -> u64 { + self.hasher.finish() + } +} + +impl<'a, T: Read, H: std::hash::Hasher> Read for HashStream<'a, T, H> { + fn read(&mut self, buf: &mut [u8]) -> Result { + let readed = self.inner.read(buf)?; + std::hash::Hasher::write(self.hasher, &buf[..readed]); + Ok(readed) + } +} + +impl<'a, T: Write, H: std::hash::Hasher> Write for HashStream<'a, T, H> { + fn write(&mut self, buf: &[u8]) -> Result { + let written = self.inner.write(buf)?; + std::hash::Hasher::write(self.hasher, &buf[..written]); + Ok(written) + } + + fn flush(&mut self) -> Result<()> { + self.inner.flush() + } +} diff --git a/src/main.rs b/src/main.rs index 1afbd13..aaa0fab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3431,6 +3431,14 @@ fn main() { yuris_check_hash: arg.yuris_check_hash, #[cfg(feature = "yuris-arc")] yuris_debug_archive: arg.yuris_debug_archive, + #[cfg(feature = "yuris-arc")] + yuris_ypf_version: arg.yuris_ypf_version, + #[cfg(feature = "yuris-arc")] + yuris_ypf_compress_file: !arg.yuris_ypf_no_compress_file, + #[cfg(feature = "yuris-arc")] + yuris_ypf_zopfli: arg.yuris_ypf_zopfli, + #[cfg(feature = "yuris-arc")] + yuris_ypf_workers: arg.yuris_ypf_workers, }); match &arg.command { args::Command::Export { input, output } => { diff --git a/src/scripts/yuris/arc/ypf.rs b/src/scripts/yuris/arc/ypf.rs index 810291d..abaea00 100644 --- a/src/scripts/yuris/arc/ypf.rs +++ b/src/scripts/yuris/arc/ypf.rs @@ -1,14 +1,21 @@ //! Yu-Ris Archive (.ypf) use crate::ext::io::*; +use crate::ext::mutex::*; use crate::scripts::base::*; use crate::types::*; use crate::utils::encoding::*; use crate::utils::murmur2::*; +use crate::utils::struct_pack::*; +use crate::utils::threadpool::*; use anyhow::{Result, anyhow, bail}; use clap::ValueEnum; use int_enum::IntEnum; +use std::any::Any; +use std::collections::HashMap; use std::hash::Hasher; use std::io::{Read, Seek, SeekFrom, Write}; +use std::num::NonZeroU64; +use std::ops::DerefMut; use std::sync::{Arc, Mutex}; #[derive(Debug)] @@ -97,10 +104,24 @@ impl ScriptBuilder for YpfBuilder { fn is_archive(&self) -> bool { true } + + fn create_archive( + &self, + filename: &str, + files: &[&str], + encoding: Encoding, + config: &ExtraConfig, + ) -> Result> { + let f = std::fs::File::create(filename)?; + let writer = std::io::BufWriter::new(f); + Ok(Box::new(YPFArchiveWriter::new( + writer, files, encoding, config, + )?)) + } } #[repr(u8)] -#[derive(Debug, IntEnum)] +#[derive(Debug, IntEnum, Clone, Copy)] enum ResourceType { Default, BMP, @@ -123,8 +144,9 @@ impl Default for ResourceType { } } -#[derive(Debug)] +#[derive(Clone, Debug)] struct YPFEntry { + name_hash: u32, name: String, #[allow(unused)] typ: ResourceType, @@ -135,6 +157,67 @@ struct YPFEntry { hash: Option, } +fn get_info_as_version(info: &Option>) -> Result { + Ok(*info + .as_ref() + .ok_or_else(|| anyhow::anyhow!("info not found"))? + .downcast_ref() + .ok_or_else(|| anyhow::anyhow!("not YSTBHeader"))?) +} + +impl StructPack for YPFEntry { + fn pack( + &self, + writer: &mut W, + big: bool, + encoding: Encoding, + info: &Option>, + ) -> Result<()> { + let version = get_info_as_version(info)?; + self.name_hash.pack(writer, big, encoding, info)?; + let table = if version < 500 { + &NAME_DEFAULT_TABLE + } else { + &NAME_V500_TABLE + }; + let mut name = encode_string(encoding, &self.name, true)?; + if name.len() > 0xFF { + bail!("File name can not longer than 255 bytes."); + } + let name_len = name.len() as u8; + let name_len = (table + .iter() + .position(|s| *s == name_len) + .ok_or_else(|| anyhow!("No suitable len found in table"))? + as u8) + ^ 0xFF; + name_len.pack(writer, big, encoding, info)?; + for num in name.iter_mut() { + *num ^= match version { + 290 => 64, + 500 => 54, + _ => 0, + }; + *num = !(*num); + } + writer.write_all(&name)?; + (self.typ as u8).pack(writer, big, encoding, info)?; + self.compressed.pack(writer, big, encoding, info)?; + self.size.pack(writer, big, encoding, info)?; + self.compressed_size.pack(writer, big, encoding, info)?; + if version >= 480 { + self.offset.pack(writer, big, encoding, info)?; + } else { + (self.offset as u32).pack(writer, big, encoding, info)?; + }; + if version >= 473 { + let hash = self.hash.ok_or_else(|| anyhow!("hash not specified."))?; + hash.pack(writer, big, encoding, info)?; + } + Ok(()) + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] pub enum NameHashType { /// Crc32 @@ -312,6 +395,7 @@ impl<'b, T: Read + Seek + std::fmt::Debug + Send + Sync + 'b> YPF<'b, T> { } let name = decode_to_string(archive_encoding, &name, true)?; entries.push(YPFEntry { + name_hash: hash, name: name.clone(), typ: index .read_u8()? @@ -602,3 +686,314 @@ impl Hasher for Xxh32 { self.inner.digest() as u64 } } + +pub struct YPFArchiveWriter { + writer: Arc>, + headers: Arc>>, + version: u32, + compress: bool, + zopfli: bool, + compress_level: u32, + zopfli_iteration_count: NonZeroU64, + zopfli_iterations_without_improvement: NonZeroU64, + zopfli_maximum_block_splits: u16, + runner: ThreadPool>, + data_hash: DataHashType, + encoding: Encoding, +} + +impl YPFArchiveWriter { + /// Creates a new YPF Archive Writer. + /// + /// * `writer` - The writer to write the archive to. + /// * `files` - The list of files to include in the archive. + /// * `encoding` - The encoding used for the archive. + /// * `config` - Extra configuration options. + pub fn new( + mut writer: T, + files: &[&str], + encoding: Encoding, + config: &ExtraConfig, + ) -> Result { + writer.write_all(b"YPF\0")?; + let version = config.yuris_ypf_version.ok_or_else(|| { + anyhow!("Version is required. Use --yuris-ypf-version to specify version.") + })?; + writer.write_u32(version)?; + let file_count = files.len() as u32; + writer.write_u32(file_count)?; + writer.write_u32(0)?; // placeholder for header size + writer.write_u128(0)?; // unused + let mut headers = HashMap::new(); + let info = &Some(Box::new(version) as Box); + for file in files { + let name = encode_string(encoding, file, true)?; + let mut hasher: Box = match config.yuris_name_hash_type { + NameHashType::Crc32 => Box::new(crc32fast::Hasher::new()), + NameHashType::Murmur2 => Box::new(StreamingMurmur2::new(0, name.len() as u32)), + }; + hasher.write(&name); + let header = YPFEntry { + name_hash: hasher.finish() as u32, + name: file.to_string(), + typ: ResourceType::Default, + compressed: config.yuris_ypf_compress_file, + size: 0, + compressed_size: 0, + offset: 0, + hash: if version >= 473 { Some(0) } else { None }, + }; + header.pack(&mut writer, false, encoding, info)?; + headers.insert(file.to_string(), header); + } + let header_size = writer.stream_position()?; + writer.write_u32_at(12, header_size as u32)?; + Ok(Self { + writer: Arc::new(Mutex::new(writer)), + headers: Arc::new(Mutex::new(headers)), + version, + compress: config.yuris_ypf_compress_file, + zopfli: config.yuris_ypf_zopfli, + compress_level: config.zlib_compression_level, + zopfli_iteration_count: config.zopfli_iteration_count, + zopfli_iterations_without_improvement: config.zopfli_iterations_without_improvement, + zopfli_maximum_block_splits: config.zopfli_maximum_block_splits, + runner: ThreadPool::new( + if config.yuris_ypf_compress_file { + config.yuris_ypf_workers + } else { + 1 + }, + Some("yuris-ypf-writer"), + false, + )?, + encoding, + data_hash: config.yuris_data_hash_type, + }) + } + + fn create_hasher(&self, length: u32) -> Box { + match self.data_hash { + DataHashType::Adler32 => Box::new(adler::Adler32::new()), + DataHashType::Murmur2 => Box::new(StreamingMurmur2::new(0, length)), + DataHashType::Xxh32 => Box::new(Xxh32::new(0)), + } + } + + fn create_hasher2(&self) -> Box { + match self.data_hash { + DataHashType::Adler32 => Box::new(adler::Adler32::new()), + DataHashType::Murmur2 => Box::new(Murmur2::new(0)), + DataHashType::Xxh32 => Box::new(Xxh32::new(0)), + } + } +} + +impl Archive for YPFArchiveWriter { + fn new_file<'a>( + &'a mut self, + name: &str, + size: Option, + ) -> Result> { + let inner = self.new_file_non_seek(name, size)?; + Ok(Box::new(Writer { + inner, + mem: MemWriter::new(), + })) + } + + fn new_file_non_seek<'a>( + &'a mut self, + name: &str, + size: Option, + ) -> Result> { + let mut entry = self + .headers + .lock_blocking() + .get(name) + .ok_or_else(|| anyhow::anyhow!("File '{}' not found in archive", name))? + .clone(); + if self.compress { + let (reader, writer) = std::io::pipe()?; + let file = self.writer.clone(); + let headers = self.headers.clone(); + let compress_level = self.compress_level; + let name = name.to_owned(); + let zopfli = self.zopfli; + let iteration_count = self.zopfli_iteration_count; + let iterations_without_improvement = self.zopfli_iterations_without_improvement; + let maximum_block_splits = self.zopfli_maximum_block_splits; + let data_hash = self.data_hash; + self.runner.execute( + move |_| { + let mut tsize = 0; + let mut reader = TrackStream::new(reader, &mut tsize); + let mut data = Vec::new(); + if entry.compressed { + let mut compressed = MemWriter::new(); + compressed.write_all(b"x\xDA")?; + if zopfli { + let mut encoder = zopfli::DeflateEncoder::new( + zopfli::Options { + iteration_count, + iterations_without_improvement, + maximum_block_splits, + }, + zopfli::BlockType::Dynamic, + &mut compressed, + ); + std::io::copy(&mut reader, &mut encoder)?; + encoder.finish()?; + } else { + let mut encoder = flate2::write::DeflateEncoder::new( + &mut compressed, + flate2::Compression::new(compress_level), + ); + std::io::copy(&mut reader, &mut encoder)?; + encoder.finish()?; + } + data = compressed.into_inner(); + } else { + reader.read_to_end(&mut data)?; + } + entry.size = tsize as u32; + entry.compressed_size = data.len() as u32; + if let Some(hash) = entry.hash.as_mut() { + let mut hasher: Box = match data_hash { + DataHashType::Adler32 => Box::new(adler::Adler32::new()), + DataHashType::Murmur2 => { + Box::new(StreamingMurmur2::new(0, entry.compressed_size)) + } + DataHashType::Xxh32 => Box::new(Xxh32::new(0)), + }; + hasher.write(&data); + *hash = hasher.finish() as u32; + } + let mut writer = file.lock_blocking(); + entry.offset = writer.seek(SeekFrom::End(0))?; + writer.write_all(&data)?; + headers.lock_blocking().insert(name, entry); + Ok(()) + }, + true, + )?; + Ok(Box::new(writer)) + } else { + let mut writer = self.writer.lock_blocking(); + entry.offset = writer.seek(SeekFrom::End(0))?; + Ok(Box::new(YPFArchiveFile { + entry, + writer: self.writer.clone(), + pos: 0, + headers: self.headers.clone(), + hasher: if let Some(size) = size { + self.create_hasher(size as u32) + } else { + self.create_hasher2() + }, + })) + } + } + + fn write_header(&mut self) -> Result<()> { + self.runner.join(); + for err in self.runner.take_results() { + err?; + } + let mut writer = self.writer.lock_blocking(); + let headers = self.headers.lock_blocking(); + writer.seek(SeekFrom::Start(0x20))?; + let mut files = headers.iter().map(|(_, d)| d).collect::>(); + files.sort_by_key(|f| f.offset); + let info = &Some(Box::new(self.version) as Box); + for file in files { + file.pack(writer.deref_mut(), false, self.encoding, info)?; + } + Ok(()) + } +} + +struct YPFArchiveFile { + entry: YPFEntry, + writer: Arc>, + pos: usize, + headers: Arc>>, + hasher: Box, +} + +impl Write for YPFArchiveFile { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let mut writer = self.writer.lock().map_err(|_| { + std::io::Error::new(std::io::ErrorKind::Other, "Failed to lock the mutex") + })?; + writer.seek(SeekFrom::Start(self.entry.offset + self.pos as u64))?; + let bytes_written = writer.write(buf)?; + self.pos += bytes_written; + self.entry.size = self.entry.size.max(self.pos as u32); + self.hasher.write(&buf[..bytes_written]); + Ok(bytes_written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.writer + .lock() + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::Other, "Failed to lock the mutex") + })? + .flush() + } +} + +impl Drop for YPFArchiveFile { + fn drop(&mut self) { + self.entry.compressed_size = self.entry.size; + if let Some(hash) = self.entry.hash.as_mut() { + *hash = self.hasher.finish() as u32; + } + self.headers + .lock_blocking() + .insert(self.entry.name.clone(), self.entry.clone()); + } +} + +struct Writer<'a> { + inner: Box, + mem: MemWriter, +} + +impl std::fmt::Debug for Writer<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Writer").field("mem", &self.mem).finish() + } +} + +impl<'a> Write for Writer<'a> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.mem.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.mem.flush() + } +} + +impl<'a> Seek for Writer<'a> { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + self.mem.seek(pos) + } + + fn stream_position(&mut self) -> std::io::Result { + self.mem.stream_position() + } + + fn rewind(&mut self) -> std::io::Result<()> { + self.mem.rewind() + } +} + +impl<'a> Drop for Writer<'a> { + fn drop(&mut self) { + let _ = self.inner.write_all(&self.mem.data); + let _ = self.inner.flush(); + } +} diff --git a/src/types.rs b/src/types.rs index a198c79..7ec99dd 100644 --- a/src/types.rs +++ b/src/types.rs @@ -693,6 +693,20 @@ pub struct ExtraConfig { /// Print debug information for Yu-RIS archive (.ypf) when extracting archive to stdout. /// This is used to find correct configuration for Yu-RIS archives. pub yuris_debug_archive: bool, + #[cfg(feature = "yuris-arc")] + /// Yu-RIS YPF engine version. Used when pack files into Yu-RIS archive. + pub yuris_ypf_version: Option, + #[cfg(feature = "yuris-arc")] + #[default(true)] + /// Compress file when packing files into Yu-RIS archive. + pub yuris_ypf_compress_file: bool, + #[cfg(feature = "yuris-arc")] + /// Use zopfli to compress files in Yu-RIS archive. + pub yuris_ypf_zopfli: bool, + #[cfg(feature = "yuris-arc")] + #[default(num_cpus::get())] + /// Workers count for compress files in Yu-RIS archive when creating in parallel. Default is CPU cores count. + pub yuris_ypf_workers: usize, } #[cfg(feature = "artemis")]