From 2a6f3ab70c1861b3e2de1273d375b0512f7e9721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Sun, 14 Jul 2024 20:59:18 +0200 Subject: [PATCH 01/26] Add support for DXArchive8 encryption. --- ArcFormats/DxLib/DxKey.cs | 54 +++++++++++++++++++++++++++++++++++++++ GameRes/ArcView.cs | 3 ++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/ArcFormats/DxLib/DxKey.cs b/ArcFormats/DxLib/DxKey.cs index 445f22d1..6e7df16f 100644 --- a/ArcFormats/DxLib/DxKey.cs +++ b/ArcFormats/DxLib/DxKey.cs @@ -26,6 +26,7 @@ using System; using System.Linq; using System.Security.Cryptography; +using System.Xml.Linq; using GameRes.Utility; namespace GameRes.Formats.DxLib @@ -153,4 +154,57 @@ namespace GameRes.Formats.DxLib throw new NotSupportedException ("SHA-256 key cannot be restored."); } } + + [Serializable] + public class DxKey8 : DxKey + { + public DxKey8(string password) : base(password ?? "DXARC") + { + } + + public override byte[] GetEntryKey(string name) + { + var password = this.Password; + var path = name.Split('\\', '/'); + password += string.Join("", path.Reverse().Select(n => n.ToUpperInvariant())); + return CreateKey(password); + } + + protected override byte[] CreateKey(string keyword) + { + //from DxArchive.cpp + //first check if the keyword is too short + if (keyword.Length < 4) + { + keyword += "DXARC"; + } + string oddString, evenString; + byte[] key = new byte[7]; + oddString = string.Concat(keyword.Where((c, i) => i % 2 == 0)); + evenString = string.Concat(keyword.Where((c, i) => (i+1) % 2 == 0)); + UInt32 crc_0, crc_1; + crc_0 = Crc32.Compute(Encodings.ASCII.GetBytes(oddString), 0, oddString.Length); + crc_1 = Crc32.Compute(Encodings.ASCII.GetBytes(evenString), 0, evenString.Length); + byte[] crc_0_Bytes = BitConverter.GetBytes(crc_0),crc_1_Bytes=BitConverter.GetBytes(crc_1); + key[0] = crc_0_Bytes[0]; + key[1] = crc_0_Bytes[1]; + key[2] = crc_0_Bytes[2]; + key[3] = crc_0_Bytes[3]; + key[4] = crc_1_Bytes[0]; + key[5] = crc_1_Bytes[1]; + key[6] = crc_1_Bytes[2]; + return key; + /* + using (var sha = SHA256.Create()) + { + var bytes = Encodings.cp932.GetBytes(keyword); + return sha.ComputeHash(bytes); + } */ + } + + protected override string RestoreKey(byte[] key) + { + throw new NotSupportedException("CRC key cannot be restored."); + } + } } diff --git a/GameRes/ArcView.cs b/GameRes/ArcView.cs index d1a26bac..aa6aaa80 100644 --- a/GameRes/ArcView.cs +++ b/GameRes/ArcView.cs @@ -33,7 +33,8 @@ namespace GameRes { public static class Encodings { - public static readonly Encoding cp932 = Encoding.GetEncoding (932); + public static readonly Encoding cp932 = Encoding.GetEncoding(932); + public static readonly Encoding ASCII = Encoding.GetEncoding(32765); public static Encoding WithFatalFallback (this Encoding enc) { From ddabe6d2f45efad6bd46c408b57b69d44925425e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 15 Jul 2024 14:14:52 +0200 Subject: [PATCH 02/26] Fix broken ASCII pointer. --- GameRes/ArcView.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GameRes/ArcView.cs b/GameRes/ArcView.cs index aa6aaa80..3ec59a73 100644 --- a/GameRes/ArcView.cs +++ b/GameRes/ArcView.cs @@ -34,7 +34,7 @@ namespace GameRes public static class Encodings { public static readonly Encoding cp932 = Encoding.GetEncoding(932); - public static readonly Encoding ASCII = Encoding.GetEncoding(32765); + public static readonly Encoding ASCII = Encoding.ASCII; public static Encoding WithFatalFallback (this Encoding enc) { From 2e8057aa54f4adc46ba23d3d52398b9a638d25b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 15 Jul 2024 14:15:14 +0200 Subject: [PATCH 03/26] Update ArcDX8.cs Add anything that can be done without user input. --- ArcFormats/DxLib/ArcDX8.cs | 49 +++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index edc05b80..1a009819 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -41,27 +41,64 @@ namespace GameRes.Formats.DxLib public Dx8Opener () { - Extensions = new[] { "bin" }; + Extensions = new string[] { "dxa", "hud", "usi", "med", "dat", "bin", "bcx", "wolf" }; Signatures = new[] { 0x00085844u }; } static readonly byte[] DefaultKey = new byte[] { 0xBE, 0xC8, 0x8A, 0xF5, 0x28, 0x50, 0xC9 }; + + DxScheme DefaultScheme = new DxScheme { KnownKeys = new List() }; + + + internal struct DxHeaderV8 + { + public long BaseOffset; + public long IndexOffset; + public uint IndexSize; + public long FileTable; + public long DirTable; + public int CodePage; + public DXA8Flags Flags; + public byte HuffmanKB; + //15 bytes of padding. + } + + internal enum DXA8Flags : UInt32 + { + DXA_FLAG_NO_KEY=1, //file is not encrypted + DXA_FLAG_NO_HEAD_PRESS=1<<1, //do not compress headers + } + public override ArcFile TryOpen (ArcView file) { - var dx = new DxHeader { + var dx = new DxHeaderV8 { IndexSize = file.View.ReadUInt32 (4), BaseOffset = file.View.ReadInt64 (8), IndexOffset = file.View.ReadInt64 (0x10), - FileTable = (uint)file.View.ReadInt64 (0x18), - DirTable = (uint)file.View.ReadInt64 (0x20), + FileTable = file.View.ReadInt64 (0x18), + DirTable = file.View.ReadInt64 (0x20), CodePage = file.View.ReadInt32 (0x28), + Flags = (DXA8Flags)file.View.ReadUInt32(0x2C), + HuffmanKB = file.View.ReadByte(0x30) }; if (dx.DirTable >= dx.IndexSize || dx.FileTable >= dx.IndexSize) return null; + //at this point we cannot proceed without user input. If NO_HEAD_PRESS is set we could maybe restore the 7-byte key + //Otherwise (assuming the archive is encrypted) we have no way to continue without user input. + + //TODO: Ask for key here. + var key = DefaultKey; - var index = file.View.ReadBytes (dx.IndexOffset, dx.IndexSize); - Decrypt (index, 0, index.Length, 0, key); + if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) != 0) + { + var index = file.View.ReadBytes(dx.IndexOffset, dx.IndexSize); + Decrypt(index, 0, index.Length, 0, key); + } else + { + //input is compressed. First by huffman then by LZ. if it's also encrypted then we're stuck. + throw new NotImplementedException(); + } // decrypt-2 // decompress return null; From b3f5195a9ad93cb08bc86cb3f15f7521331b859a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 15 Jul 2024 14:20:17 +0200 Subject: [PATCH 04/26] Ensure that DXA is not "decrypted" when not needed. --- ArcFormats/DxLib/ArcDX8.cs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 1a009819..36e8eacc 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -90,14 +90,18 @@ namespace GameRes.Formats.DxLib //TODO: Ask for key here. var key = DefaultKey; - if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) != 0) + if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_KEY) == 0) { - var index = file.View.ReadBytes(dx.IndexOffset, dx.IndexSize); - Decrypt(index, 0, index.Length, 0, key); - } else - { - //input is compressed. First by huffman then by LZ. if it's also encrypted then we're stuck. - throw new NotImplementedException(); + if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) != 0) + { + var index = file.View.ReadBytes(dx.IndexOffset, dx.IndexSize); + Decrypt(index, 0, index.Length, 0, key); + } + else + { + //input is compressed. First by huffman then by LZ. if it's also encrypted then we're stuck. + throw new NotImplementedException(); + } } // decrypt-2 // decompress From 30e04eae1fd9caa58117b5b00f4d0ec25a4a1cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 15 Jul 2024 14:22:34 +0200 Subject: [PATCH 05/26] Update ArcDX8.cs --- ArcFormats/DxLib/ArcDX8.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 36e8eacc..3c707813 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -90,11 +90,12 @@ namespace GameRes.Formats.DxLib //TODO: Ask for key here. var key = DefaultKey; + + var index = file.View.ReadBytes(dx.IndexOffset, dx.IndexSize); if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_KEY) == 0) { if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) != 0) { - var index = file.View.ReadBytes(dx.IndexOffset, dx.IndexSize); Decrypt(index, 0, index.Length, 0, key); } else From 3bd697577c466ae2ea21d2cf1e001f0a822dee5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Wed, 17 Jul 2024 19:53:13 +0200 Subject: [PATCH 06/26] Multiple Changes: DXA8 now directly asks for password, as guessing it, is made improbable. Promote several private methods to `protected` status Add code for indexing DXA8 files. (not finished) --- ArcFormats/ArcFormats.csproj | 7 + ArcFormats/DxLib/ArcDX.cs | 59 +++++--- ArcFormats/DxLib/ArcDX8.cs | 166 +++++++++++++++++---- ArcFormats/DxLib/WidgetDXA.xaml | 19 +++ ArcFormats/DxLib/WidgetDXA.xaml.cs | 29 ++++ ArcFormats/Properties/Settings.Designer.cs | 22 ++- ArcFormats/Properties/Settings.settings | 3 + ArcFormats/app.config | 3 + 8 files changed, 259 insertions(+), 49 deletions(-) create mode 100644 ArcFormats/DxLib/WidgetDXA.xaml create mode 100644 ArcFormats/DxLib/WidgetDXA.xaml.cs diff --git a/ArcFormats/ArcFormats.csproj b/ArcFormats/ArcFormats.csproj index 0057045d..155e2339 100644 --- a/ArcFormats/ArcFormats.csproj +++ b/ArcFormats/ArcFormats.csproj @@ -126,6 +126,9 @@ + + WidgetDXA.xaml + @@ -1140,6 +1143,10 @@ Designer MSBuild:Compile + + Designer + MSBuild:Compile + Designer MSBuild:Compile diff --git a/ArcFormats/DxLib/ArcDX.cs b/ArcFormats/DxLib/ArcDX.cs index 12a9e263..79073490 100644 --- a/ArcFormats/DxLib/ArcDX.cs +++ b/ArcFormats/DxLib/ArcDX.cs @@ -198,7 +198,7 @@ namespace GameRes.Formats.DxLib } } - byte[] Unpack (Stream stream) + protected byte[] Unpack (Stream stream) { using (var input = new ArcView.Reader (stream)) { @@ -262,7 +262,7 @@ namespace GameRes.Formats.DxLib } } - List ReadIndex (ArcView file, int version, byte[] key) + protected List ReadIndex (ArcView file, int version, byte[] key) { DxHeader dx = null; if (version <= 4) @@ -313,12 +313,15 @@ namespace GameRes.Formats.DxLib internal static void Decrypt (byte[] data, int index, int count, long offset, byte[] key) { - int key_pos = (int)(offset % key.Length); - for (int i = 0; i < count; ++i) + if (key.Length !=0) { - data[index+i] ^= key[key_pos++]; - if (key.Length == key_pos) - key_pos = 0; + int key_pos = (int)(offset % key.Length); + for (int i = 0; i < count; ++i) + { + data[index + i] ^= key[key_pos++]; + if (key.Length == key_pos) + key_pos = 0; + } } } @@ -361,8 +364,10 @@ namespace GameRes.Formats.DxLib { if (version <= 4) return new IndexReaderV2 (header, version, input); - else if (version >= 6) + else if (version >= 6 && version < 8) return new IndexReaderV6 (header, version, input); + else if (version >=8) + return new IndexReaderV8 (header, version, input); else throw new InvalidFormatException ("Not supported DX archive version."); } @@ -545,7 +550,7 @@ namespace GameRes.Formats.DxLib : base (stream, leave_open) { m_key = key; - m_base_pos = (int)(base_position % m_key.Length); + m_base_pos = m_key.Length !=0 ?(int)(base_position % m_key.Length):0; } public override int Read (byte[] buffer, int offset, int count) @@ -559,32 +564,48 @@ namespace GameRes.Formats.DxLib public override int ReadByte () { - int key_pos = (int)((m_base_pos + Position) % m_key.Length); int b = BaseStream.ReadByte(); - if (-1 != b) + if (m_key.Length !=0) { - b ^= m_key[key_pos]; + int key_pos = (int)((m_base_pos + Position) % m_key.Length); + if (-1 != b) + { + b ^= m_key[key_pos]; + } } return b; } public override void Write (byte[] buffer, int offset, int count) { - int key_pos = (int)((m_base_pos + Position) % m_key.Length); + byte[] write_buf = new byte[count]; - for (int i = 0; i < count; ++i) + if (m_key.Length != 0) { - write_buf[i] = (byte)(buffer[offset+i] ^ m_key[key_pos++]); - if (m_key.Length == key_pos) - key_pos = 0; + int key_pos = (int)((m_base_pos + Position) % m_key.Length); + for (int i = 0; i < count; ++i) + { + write_buf[i] = (byte)(buffer[offset + i] ^ m_key[key_pos++]); + if (m_key.Length == key_pos) + key_pos = 0; + } + } else + { + write_buf = buffer; } BaseStream.Write (write_buf, 0, count); } public override void WriteByte (byte value) { - int key_pos = (int)((m_base_pos + Position) % m_key.Length); - BaseStream.WriteByte ((byte)(value ^ m_key[key_pos])); + if(m_key.Length != 0) + { + int key_pos = (int)((m_base_pos + Position) % m_key.Length); + BaseStream.WriteByte((byte)(value ^ m_key[key_pos])); + } else + { + BaseStream.WriteByte ((byte)value); + } } } } diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 3c707813..d8e80857 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -23,14 +23,21 @@ // IN THE SOFTWARE. // +using GameRes.Formats.PkWare; +using GameRes.Formats.Strings; +using NAudio.SoundFont; using System; using System.Collections.Generic; using System.ComponentModel.Composition; using System.IO; +using System.Linq; + + + namespace GameRes.Formats.DxLib { - [Export(typeof(ArchiveFormat))] + [Export(typeof(ArchiveFormat))] public class Dx8Opener : DxOpener { public override string Tag { get { return "BIN/DXLIB"; } } @@ -51,14 +58,10 @@ namespace GameRes.Formats.DxLib DxScheme DefaultScheme = new DxScheme { KnownKeys = new List() }; - internal struct DxHeaderV8 + internal class DxHeaderV8 :DxHeader { - public long BaseOffset; - public long IndexOffset; - public uint IndexSize; - public long FileTable; - public long DirTable; - public int CodePage; + new public long FileTable; + new public long DirTable; public DXA8Flags Flags; public byte HuffmanKB; //15 bytes of padding. @@ -67,7 +70,44 @@ namespace GameRes.Formats.DxLib internal enum DXA8Flags : UInt32 { DXA_FLAG_NO_KEY=1, //file is not encrypted - DXA_FLAG_NO_HEAD_PRESS=1<<1, //do not compress headers + DXA_FLAG_NO_HEAD_PRESS=1<<1, //do not compress the entire file after compressing individual entries + } + + [Serializable] + public class DXAOpts : ResourceOptions + { + public string Keyword; + } + + public override ResourceOptions GetDefaultOptions() + { + return new DXAOpts + { + Keyword = Properties.Settings.Default.DXAPassword + }; + } + + string QueryPassword(ArcView file) + { + var options = Query(arcStrings.ZIPEncryptedNotice); + return options.Keyword; + } + + public override ResourceOptions GetOptions(object widget) + { + if (widget is GUI.WidgetDXA) + { + return new DXAOpts + { + Keyword = ((GUI.WidgetDXA)widget).Password.Text + }; + } + return GetDefaultOptions(); + } + + public override object GetAccessWidget() + { + return new GUI.WidgetDXA(); } public override ArcFile TryOpen (ArcView file) @@ -84,29 +124,105 @@ namespace GameRes.Formats.DxLib }; if (dx.DirTable >= dx.IndexSize || dx.FileTable >= dx.IndexSize) return null; - //at this point we cannot proceed without user input. If NO_HEAD_PRESS is set we could maybe restore the 7-byte key - //Otherwise (assuming the archive is encrypted) we have no way to continue without user input. - - //TODO: Ask for key here. - - var key = DefaultKey; - - var index = file.View.ReadBytes(dx.IndexOffset, dx.IndexSize); - if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_KEY) == 0) + DxKey8 key = null; + + //FIXME: ReadBytes sets hard cap of filesize to 4GB. + var bodyBuffer = file.View.ReadBytes(dx.BaseOffset, (uint)(file.MaxOffset-dx.BaseOffset)); + bool isencrypted = (dx.Flags & DXA8Flags.DXA_FLAG_NO_KEY) == 0; + + if (isencrypted) { - if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) != 0) + var keyStr = Query(arcStrings.ZIPEncryptedNotice).Keyword; + key = new DxKey8(keyStr); + Decrypt(bodyBuffer, 0, bodyBuffer.Length, 0, key.Key); + + + } + //Decrypted but might be compressed + if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) == 0) + { + //IndexSize refers to uncompressed + throw new NotImplementedException(); + } + + var readyStr = new MemoryStream(bodyBuffer); + ArcView arcView = new ArcView(readyStr, "body",(uint) bodyBuffer.LongLength); + List entries; + using (var indexStr = arcView.CreateStream(dx.IndexOffset,dx.IndexSize)) + using (var reader = IndexReader.Create(dx, 8, indexStr)) + { + entries = reader.Read(); + } + return new DxArchive(arcView, this,entries ,key, 8); + //return null; + } + } + + internal sealed class IndexReaderV8 : IndexReader + { + readonly int m_entry_size; + public IndexReaderV8(DxHeader header, int version, Stream input) : base(header, version, input) + { + m_entry_size = 0x48; + } + private class DxDirectory + { + public long DirOffset; + public long ParentDirOffset; + public int FileCount; + public long FileTable; + } + + DxDirectory ReadDirEntry() + { + var dir = new DxDirectory(); + dir.DirOffset = m_input.ReadInt64(); + dir.ParentDirOffset = m_input.ReadInt64(); + dir.FileCount = (int)m_input.ReadInt64(); + dir.FileTable = m_input.ReadInt64(); + return dir; + } + + protected override void ReadFileTable(string root, long table_offset) + { + m_input.Position = m_header.DirTable + table_offset; + var dir = ReadDirEntry(); + if (dir.DirOffset != -1 && dir.ParentDirOffset != -1) + { + m_input.Position = m_header.FileTable + dir.DirOffset; + root = Path.Combine(root, ExtractFileName(m_input.ReadInt64())); + } + long current_pos = m_header.FileTable + dir.FileTable; + for (int i = 0; i < dir.FileCount; ++i) + { + m_input.Position = current_pos; + var name_offset = m_input.ReadInt64(); + uint attr = (uint)m_input.ReadInt64(); + m_input.Seek(0x18, SeekOrigin.Current); + var offset = m_input.ReadInt64(); + if (0 != (attr & 0x10)) // FILE_ATTRIBUTE_DIRECTORY { - Decrypt(index, 0, index.Length, 0, key); + if (0 == offset || table_offset == offset) + throw new InvalidFormatException("Infinite recursion in DXA directory index"); + ReadFileTable(root, offset); } else { - //input is compressed. First by huffman then by LZ. if it's also encrypted then we're stuck. - throw new NotImplementedException(); + var size = m_input.ReadInt64(); + var packed_size = m_input.ReadInt64(); + var huffman_packed_size = m_input.ReadInt64(); + var entry = FormatCatalog.Instance.Create(Path.Combine(root, ExtractFileName(name_offset))); + entry.Offset = m_header.BaseOffset + offset; + entry.UnpackedSize = (uint)size; + entry.IsPacked = (-1 != packed_size) || -1 != huffman_packed_size; + if (entry.IsPacked) + entry.Size = (uint)(huffman_packed_size!=-1 ? huffman_packed_size:packed_size); + else + entry.Size = (uint)size; + m_dir.Add(entry); } + current_pos += m_entry_size; } - // decrypt-2 - // decompress - return null; } } } diff --git a/ArcFormats/DxLib/WidgetDXA.xaml b/ArcFormats/DxLib/WidgetDXA.xaml new file mode 100644 index 00000000..25e817e7 --- /dev/null +++ b/ArcFormats/DxLib/WidgetDXA.xaml @@ -0,0 +1,19 @@ + + diff --git a/ArcFormats/DxLib/WidgetDXA.xaml.cs b/ArcFormats/DxLib/WidgetDXA.xaml.cs new file mode 100644 index 00000000..1f40f949 --- /dev/null +++ b/ArcFormats/DxLib/WidgetDXA.xaml.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Controls; +using System.Windows.Data; +using System.Windows.Documents; +using System.Windows.Input; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Navigation; +using System.Windows.Shapes; +using GameRes.Formats.DxLib; + +namespace GameRes.Formats.GUI +{ + /// + /// Logika interakcji dla klasy WidgetDXA.xaml + /// + public partial class WidgetDXA : StackPanel + { + public WidgetDXA() + { + InitializeComponent(); + } + } +} diff --git a/ArcFormats/Properties/Settings.Designer.cs b/ArcFormats/Properties/Settings.Designer.cs index 60c30c27..192f7036 100644 --- a/ArcFormats/Properties/Settings.Designer.cs +++ b/ArcFormats/Properties/Settings.Designer.cs @@ -1,10 +1,10 @@ //------------------------------------------------------------------------------ // -// This code was generated by a tool. -// Runtime Version:4.0.30319.42000 +// Ten kod został wygenerowany przez narzędzie. +// Wersja wykonawcza:4.0.30319.42000 // -// Changes to this file may cause incorrect behavior and will be lost if -// the code is regenerated. +// Zmiany w tym pliku mogą spowodować nieprawidłowe zachowanie i zostaną utracone, jeśli +// kod zostanie ponownie wygenerowany. // //------------------------------------------------------------------------------ @@ -12,7 +12,7 @@ namespace GameRes.Formats.Properties { [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "17.0.3.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "17.10.0.0")] public sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); @@ -813,5 +813,17 @@ namespace GameRes.Formats.Properties { this["AFAEncodingCP"] = value; } } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("DXARC")] + public string DXAPassword { + get { + return ((string)(this["DXAPassword"])); + } + set { + this["DXAPassword"] = value; + } + } } } diff --git a/ArcFormats/Properties/Settings.settings b/ArcFormats/Properties/Settings.settings index 4b4c6753..ebd367cb 100644 --- a/ArcFormats/Properties/Settings.settings +++ b/ArcFormats/Properties/Settings.settings @@ -200,5 +200,8 @@ 932 + + DXARC + \ No newline at end of file diff --git a/ArcFormats/app.config b/ArcFormats/app.config index dd9cbd38..e78ee4ea 100644 --- a/ArcFormats/app.config +++ b/ArcFormats/app.config @@ -202,6 +202,9 @@ 932 + + DXARC + From 6a900187989448cca7efb099fe5166afad563e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Wed, 31 Jul 2024 16:32:35 +0200 Subject: [PATCH 07/26] Apply suggestions from upstream --- ArcFormats/DxLib/ArcDX.cs | 5 ++--- ArcFormats/DxLib/DxKey.cs | 5 +++-- ArcFormats/Properties/Settings.Designer.cs | 8 ++++---- GameRes/ArcView.cs | 1 - 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX.cs b/ArcFormats/DxLib/ArcDX.cs index 79073490..f740ddd6 100644 --- a/ArcFormats/DxLib/ArcDX.cs +++ b/ArcFormats/DxLib/ArcDX.cs @@ -313,8 +313,8 @@ namespace GameRes.Formats.DxLib internal static void Decrypt (byte[] data, int index, int count, long offset, byte[] key) { - if (key.Length !=0) - { + if (key.Length == 0) + return; int key_pos = (int)(offset % key.Length); for (int i = 0; i < count; ++i) { @@ -322,7 +322,6 @@ namespace GameRes.Formats.DxLib if (key.Length == key_pos) key_pos = 0; } - } } public override ResourceScheme Scheme diff --git a/ArcFormats/DxLib/DxKey.cs b/ArcFormats/DxLib/DxKey.cs index 6e7df16f..6fdcccff 100644 --- a/ArcFormats/DxLib/DxKey.cs +++ b/ArcFormats/DxLib/DxKey.cs @@ -26,6 +26,7 @@ using System; using System.Linq; using System.Security.Cryptography; +using System.Text; using System.Xml.Linq; using GameRes.Utility; @@ -183,8 +184,8 @@ namespace GameRes.Formats.DxLib oddString = string.Concat(keyword.Where((c, i) => i % 2 == 0)); evenString = string.Concat(keyword.Where((c, i) => (i+1) % 2 == 0)); UInt32 crc_0, crc_1; - crc_0 = Crc32.Compute(Encodings.ASCII.GetBytes(oddString), 0, oddString.Length); - crc_1 = Crc32.Compute(Encodings.ASCII.GetBytes(evenString), 0, evenString.Length); + crc_0 = Crc32.Compute(Encoding.ASCII.GetBytes(oddString), 0, oddString.Length); + crc_1 = Crc32.Compute(Encoding.ASCII.GetBytes(evenString), 0, evenString.Length); byte[] crc_0_Bytes = BitConverter.GetBytes(crc_0),crc_1_Bytes=BitConverter.GetBytes(crc_1); key[0] = crc_0_Bytes[0]; key[1] = crc_0_Bytes[1]; diff --git a/ArcFormats/Properties/Settings.Designer.cs b/ArcFormats/Properties/Settings.Designer.cs index 192f7036..7844d50b 100644 --- a/ArcFormats/Properties/Settings.Designer.cs +++ b/ArcFormats/Properties/Settings.Designer.cs @@ -1,10 +1,10 @@ //------------------------------------------------------------------------------ // -// Ten kod został wygenerowany przez narzędzie. -// Wersja wykonawcza:4.0.30319.42000 +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 // -// Zmiany w tym pliku mogą spowodować nieprawidłowe zachowanie i zostaną utracone, jeśli -// kod zostanie ponownie wygenerowany. +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. // //------------------------------------------------------------------------------ diff --git a/GameRes/ArcView.cs b/GameRes/ArcView.cs index 3ec59a73..4cd3a533 100644 --- a/GameRes/ArcView.cs +++ b/GameRes/ArcView.cs @@ -34,7 +34,6 @@ namespace GameRes public static class Encodings { public static readonly Encoding cp932 = Encoding.GetEncoding(932); - public static readonly Encoding ASCII = Encoding.ASCII; public static Encoding WithFatalFallback (this Encoding enc) { From 738b2950f6a7c398c94753be54faae64a5c2785c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Wed, 31 Jul 2024 16:32:53 +0200 Subject: [PATCH 08/26] Fix logic error. --- ArcFormats/DxLib/ArcDX8.cs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index d8e80857..c5cce01a 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -70,7 +70,7 @@ namespace GameRes.Formats.DxLib internal enum DXA8Flags : UInt32 { DXA_FLAG_NO_KEY=1, //file is not encrypted - DXA_FLAG_NO_HEAD_PRESS=1<<1, //do not compress the entire file after compressing individual entries + DXA_FLAG_NO_HEAD_PRESS=1<<1, //do not compress the header after compressing individual entries } [Serializable] @@ -127,28 +127,30 @@ namespace GameRes.Formats.DxLib DxKey8 key = null; //FIXME: ReadBytes sets hard cap of filesize to 4GB. - var bodyBuffer = file.View.ReadBytes(dx.BaseOffset, (uint)(file.MaxOffset-dx.BaseOffset)); + var headerBuffer = file.View.ReadBytes(dx.IndexOffset, (uint)(file.MaxOffset-dx.IndexOffset)); bool isencrypted = (dx.Flags & DXA8Flags.DXA_FLAG_NO_KEY) == 0; if (isencrypted) { var keyStr = Query(arcStrings.ZIPEncryptedNotice).Keyword; key = new DxKey8(keyStr); - Decrypt(bodyBuffer, 0, bodyBuffer.Length, 0, key.Key); } + + Decrypt(headerBuffer, 0, headerBuffer.Length, 0, key.Key); //Decrypted but might be compressed if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) == 0) { - //IndexSize refers to uncompressed + //IndexSize refers to uncompressed size of the header (that is FileName + File + Dir buffers) throw new NotImplementedException(); } + - var readyStr = new MemoryStream(bodyBuffer); - ArcView arcView = new ArcView(readyStr, "body",(uint) bodyBuffer.LongLength); + var readyStr = new MemoryStream(headerBuffer); + ArcView arcView = new ArcView(readyStr, "hdr",(uint)headerBuffer.LongLength); List entries; - using (var indexStr = arcView.CreateStream(dx.IndexOffset,dx.IndexSize)) + using (var indexStr = arcView.CreateStream(dx.IndexOffset, dx.IndexSize)) using (var reader = IndexReader.Create(dx, 8, indexStr)) { entries = reader.Read(); From 45e7af60ea3e8d143b945d11db6d5e596e6a0c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Thu, 1 Aug 2024 11:46:41 +0200 Subject: [PATCH 09/26] Fix issues. --- ArcFormats/DxLib/ArcDX.cs | 6 +++--- ArcFormats/DxLib/ArcDX8.cs | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX.cs b/ArcFormats/DxLib/ArcDX.cs index f740ddd6..ec81a5af 100644 --- a/ArcFormats/DxLib/ArcDX.cs +++ b/ArcFormats/DxLib/ArcDX.cs @@ -335,9 +335,9 @@ namespace GameRes.Formats.DxLib { public long BaseOffset; public long IndexOffset; - public uint IndexSize; - public uint FileTable; - public uint DirTable; + public long IndexSize; + public long FileTable; + public long DirTable; public int CodePage; } diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index c5cce01a..05ed84e9 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -23,7 +23,6 @@ // IN THE SOFTWARE. // -using GameRes.Formats.PkWare; using GameRes.Formats.Strings; using NAudio.SoundFont; using System; @@ -31,12 +30,18 @@ using System.Collections.Generic; using System.ComponentModel.Composition; using System.IO; using System.Linq; +using System.Windows.Navigation; namespace GameRes.Formats.DxLib { + + internal class DXA8PackedEntry : PackedEntry { + public bool HuffmanCompressed { get; set; } + } + [Export(typeof(ArchiveFormat))] public class Dx8Opener : DxOpener { @@ -60,10 +65,8 @@ namespace GameRes.Formats.DxLib internal class DxHeaderV8 :DxHeader { - new public long FileTable; - new public long DirTable; public DXA8Flags Flags; - public byte HuffmanKB; + public byte HuffmanKB; // oddly used only in Compression process not in decompression. //15 bytes of padding. } @@ -150,7 +153,7 @@ namespace GameRes.Formats.DxLib var readyStr = new MemoryStream(headerBuffer); ArcView arcView = new ArcView(readyStr, "hdr",(uint)headerBuffer.LongLength); List entries; - using (var indexStr = arcView.CreateStream(dx.IndexOffset, dx.IndexSize)) + using (var indexStr = arcView.CreateStream(0, dx.IndexSize)) using (var reader = IndexReader.Create(dx, 8, indexStr)) { entries = reader.Read(); @@ -213,10 +216,11 @@ namespace GameRes.Formats.DxLib var size = m_input.ReadInt64(); var packed_size = m_input.ReadInt64(); var huffman_packed_size = m_input.ReadInt64(); - var entry = FormatCatalog.Instance.Create(Path.Combine(root, ExtractFileName(name_offset))); + var entry = FormatCatalog.Instance.Create(Path.Combine(root, ExtractFileName(name_offset))); entry.Offset = m_header.BaseOffset + offset; entry.UnpackedSize = (uint)size; - entry.IsPacked = (-1 != packed_size) || -1 != huffman_packed_size; + entry.IsPacked = -1 != packed_size; + entry.HuffmanCompressed = -1 != huffman_packed_size; if (entry.IsPacked) entry.Size = (uint)(huffman_packed_size!=-1 ? huffman_packed_size:packed_size); else From beaab0ecee279a7edd7e1f69ede6200d5af56017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Thu, 1 Aug 2024 11:53:08 +0200 Subject: [PATCH 10/26] Fixes --- ArcFormats/DxLib/ArcDX.cs | 6 +++--- ArcFormats/DxLib/ArcDX8.cs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX.cs b/ArcFormats/DxLib/ArcDX.cs index ec81a5af..8f25ae26 100644 --- a/ArcFormats/DxLib/ArcDX.cs +++ b/ArcFormats/DxLib/ArcDX.cs @@ -271,7 +271,7 @@ namespace GameRes.Formats.DxLib dx = ReadArcHeaderV6 (file, version, key); if (null == dx || dx.DirTable >= dx.IndexSize || dx.FileTable >= dx.IndexSize) return null; - using (var encrypted = file.CreateStream (dx.IndexOffset, dx.IndexSize)) + using (var encrypted = file.CreateStream (dx.IndexOffset, (uint)dx.IndexSize)) using (var index = new EncryptedStream (encrypted, version >= 6 ? 0 : dx.IndexOffset, key)) using (var reader = IndexReader.Create (dx, version, index)) { @@ -305,8 +305,8 @@ namespace GameRes.Formats.DxLib IndexSize = LittleEndian.ToUInt32 (header, 0), BaseOffset = LittleEndian.ToInt64 (header, 4), IndexOffset = LittleEndian.ToInt64 (header, 0x0C), - FileTable = (uint)LittleEndian.ToInt64 (header, 0x14), - DirTable = (uint)LittleEndian.ToInt64 (header, 0x1C), + FileTable = LittleEndian.ToInt64 (header, 0x14), + DirTable = LittleEndian.ToInt64 (header, 0x1C), CodePage = LittleEndian.ToInt32 (header, 0x24), }; } diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 05ed84e9..4022a8dc 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -153,7 +153,8 @@ namespace GameRes.Formats.DxLib var readyStr = new MemoryStream(headerBuffer); ArcView arcView = new ArcView(readyStr, "hdr",(uint)headerBuffer.LongLength); List entries; - using (var indexStr = arcView.CreateStream(0, dx.IndexSize)) + //TODO: Try to memmap files with over 4GB. + using (var indexStr = arcView.CreateStream(0, (uint)dx.IndexSize)) using (var reader = IndexReader.Create(dx, 8, indexStr)) { entries = reader.Read(); From 881e5f1e3fe93b0960086217682d03ebe327523f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Thu, 1 Aug 2024 14:42:40 +0200 Subject: [PATCH 11/26] Update ArcDX8.cs Start work on huffman compression. --- ArcFormats/DxLib/ArcDX8.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 4022a8dc..d67651ea 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -24,7 +24,6 @@ // using GameRes.Formats.Strings; -using NAudio.SoundFont; using System; using System.Collections.Generic; using System.ComponentModel.Composition; @@ -42,6 +41,17 @@ namespace GameRes.Formats.DxLib public bool HuffmanCompressed { get; set; } } + internal struct DXA8HuffmanNode + { + UInt64 Weight; + int bitNumber; + byte[] bitArray; //32 bytes here + int Index; + + int ParentNode; // index of parent node. + int[] ChildNode; //two children nodes, -1 if not existent. + } + [Export(typeof(ArchiveFormat))] public class Dx8Opener : DxOpener { @@ -153,7 +163,7 @@ namespace GameRes.Formats.DxLib var readyStr = new MemoryStream(headerBuffer); ArcView arcView = new ArcView(readyStr, "hdr",(uint)headerBuffer.LongLength); List entries; - //TODO: Try to memmap files with over 4GB. + //There MAY be the case where the singular file is over 4GB, but it's very rare. using (var indexStr = arcView.CreateStream(0, (uint)dx.IndexSize)) using (var reader = IndexReader.Create(dx, 8, indexStr)) { From aa2900f15b00b3677ee693d72e796cdc70bfcee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Fri, 2 Aug 2024 19:29:19 +0200 Subject: [PATCH 12/26] Implement construction of DxLib's Huffman tree --- ArcFormats/ArcFormats.csproj | 1 + ArcFormats/DxLib/ArcDX8.cs | 34 ++--- ArcFormats/DxLib/HuffmanDecoder.cs | 195 +++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 21 deletions(-) create mode 100644 ArcFormats/DxLib/HuffmanDecoder.cs diff --git a/ArcFormats/ArcFormats.csproj b/ArcFormats/ArcFormats.csproj index 155e2339..979c9b79 100644 --- a/ArcFormats/ArcFormats.csproj +++ b/ArcFormats/ArcFormats.csproj @@ -126,6 +126,7 @@ + WidgetDXA.xaml diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index d67651ea..6ceb9827 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -39,18 +39,10 @@ namespace GameRes.Formats.DxLib internal class DXA8PackedEntry : PackedEntry { public bool HuffmanCompressed { get; set; } + public uint HuffmanSize { get; set; } } - internal struct DXA8HuffmanNode - { - UInt64 Weight; - int bitNumber; - byte[] bitArray; //32 bytes here - int Index; - - int ParentNode; // index of parent node. - int[] ChildNode; //two children nodes, -1 if not existent. - } + [Export(typeof(ArchiveFormat))] public class Dx8Opener : DxOpener @@ -67,7 +59,7 @@ namespace GameRes.Formats.DxLib Signatures = new[] { 0x00085844u }; } - static readonly byte[] DefaultKey = new byte[] { 0xBE, 0xC8, 0x8A, 0xF5, 0x28, 0x50, 0xC9 }; + //static readonly byte[] DefaultKey = new byte[] { 0xBE, 0xC8, 0x8A, 0xF5, 0x28, 0x50, 0xC9 }; DxScheme DefaultScheme = new DxScheme { KnownKeys = new List() }; @@ -100,11 +92,8 @@ namespace GameRes.Formats.DxLib }; } - string QueryPassword(ArcView file) - { - var options = Query(arcStrings.ZIPEncryptedNotice); - return options.Keyword; - } + + public override ResourceOptions GetOptions(object widget) { @@ -191,11 +180,13 @@ namespace GameRes.Formats.DxLib DxDirectory ReadDirEntry() { - var dir = new DxDirectory(); - dir.DirOffset = m_input.ReadInt64(); - dir.ParentDirOffset = m_input.ReadInt64(); - dir.FileCount = (int)m_input.ReadInt64(); - dir.FileTable = m_input.ReadInt64(); + var dir = new DxDirectory + { + DirOffset = m_input.ReadInt64(), + ParentDirOffset = m_input.ReadInt64(), + FileCount = (int)m_input.ReadInt64(), + FileTable = m_input.ReadInt64() + }; return dir; } @@ -232,6 +223,7 @@ namespace GameRes.Formats.DxLib entry.UnpackedSize = (uint)size; entry.IsPacked = -1 != packed_size; entry.HuffmanCompressed = -1 != huffman_packed_size; + entry.HuffmanSize = (uint)huffman_packed_size; if (entry.IsPacked) entry.Size = (uint)(huffman_packed_size!=-1 ? huffman_packed_size:packed_size); else diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs new file mode 100644 index 00000000..5a4a9cdc --- /dev/null +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -0,0 +1,195 @@ +//! \file HuffmanDecoder.cs +//! \date 2024 Aug 2 +//! \brief Custom Huffman decoder for DXA archives. +// +// Copyright (C) 2017 by morkt - GetBits function +// Copyright (C) 2024 by MrSoup678 +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal in the Software without restriction, including without limitation the +// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +// IN THE SOFTWARE. +// + + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using GameRes.Utility; + +namespace GameRes.Formats.DxLib +{ + internal struct DXA8HuffmanNode + { + public UInt64 Weight; + public int bitNumber; + public byte[] bitArray; //32 bytes here + public int Index; + + public int ParentNode; // index of parent node. + public int[] ChildNode; //two children nodes, -1 if not existent. + } + + internal sealed class HuffmanDecoder + { + byte[] m_input; + byte[] m_output; + + int m_src; + ulong m_bits; + int m_bit_count; + + ulong m_readBytes; + ulong m_readBits; + + + DXA8HuffmanNode[] nodes; //256+255 nodes + + ulong originalSize; + ulong compressedSize; + ulong headerSize; + + //ushort token = 256; + + public HuffmanDecoder (byte[] src, byte[] dst) + { + m_input = src; + m_output = dst; + + m_src = 0; + m_bit_count = 0; + m_readBytes = 0; + m_readBits = 0; + originalSize = compressedSize = headerSize = 0; + ushort[] weights = new ushort[256]; + nodes = new DXA8HuffmanNode[256+255]; //256 "base" nodes, then 255 nodes making a pyramid. + } + + public byte[] Unpack () + { + + for (int i=0; i 1) + { + int MinNode1 = -1; + int MinNode2 = -1; + int NodeIndex = 0; + + + for (int i = 0; i < DataNum; NodeIndex++) { + //don't do anything if we already have a parent set. + if (nodes[NodeIndex].ParentNode != -1) continue; + i++; + //we need to get the two lowest numbers for parenting. + if (MinNode1 == -1 || nodes[MinNode1].Weight > nodes[NodeIndex].Weight) + { + { + MinNode2 = MinNode1; + MinNode1 = NodeIndex; + } + } else if (MinNode2 == -1 || nodes[MinNode2].Weight > nodes[NodeIndex].Weight) + { + MinNode2 = NodeIndex; + } + } + nodes[NodeNum].ParentNode = -1; + nodes[NodeNum].Weight = nodes[MinNode1].Weight + nodes[MinNode2].Weight; + nodes[NodeNum].ChildNode[0] = MinNode1; + nodes[NodeNum].ChildNode[1] = MinNode2; + nodes[MinNode1].Index = 0; + nodes[MinNode2].Index = 1; + nodes[MinNode1].ParentNode = NodeNum; + nodes[MinNode2].ParentNode = NodeNum; + + NodeNum++; + DataNum--; + } + } + + ulong GetBits (int count) + { + ulong bits = 0; + while (count --> 0) + { + if (0 == m_bit_count) + { + m_bits = LittleEndian.ToUInt64 (m_input, m_src); + m_src += 8; + m_bit_count = 64; + } + bits = bits << 1 | (m_bits & 1); + m_bits >>= 1; + --m_bit_count; + m_readBits++; + if (m_readBits ==8) + { + m_readBits = 0; + m_readBytes++; + } + } + return bits; + } + + ulong GetReadBytes() + { + return m_readBytes + (m_readBits != 0 ? 1ul : 0ul); + } + } +} From ab67cb51873cdebdb1610cd1f67d3745308795a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 15:07:56 +0200 Subject: [PATCH 13/26] Finish up implementation. Debugging next. --- ArcFormats/DxLib/ArcDX8.cs | 11 +- ArcFormats/DxLib/HuffmanDecoder.cs | 169 ++++++++++++++++++++++++++++- 2 files changed, 174 insertions(+), 6 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 6ceb9827..9def10ed 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -144,8 +144,15 @@ namespace GameRes.Formats.DxLib //Decrypted but might be compressed if ((dx.Flags & DXA8Flags.DXA_FLAG_NO_HEAD_PRESS) == 0) { - //IndexSize refers to uncompressed size of the header (that is FileName + File + Dir buffers) - throw new NotImplementedException(); + byte[] huffmanBuffer = new byte[headerBuffer.Length]; + byte[] lzBuffer; + headerBuffer.CopyTo(huffmanBuffer, 0); + huffmanBuffer = headerBuffer; + HuffmanDecoder decoder = new HuffmanDecoder(huffmanBuffer, (ulong)huffmanBuffer.LongLength); + lzBuffer = decoder.Unpack(); + MemoryStream lzStream = new MemoryStream(lzBuffer); + headerBuffer = Unpack(lzStream); + } diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 5a4a9cdc..26dd2e6e 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -24,8 +24,11 @@ // IN THE SOFTWARE. // +//Original file is Huffman.cpp. Creator: 山田 巧 Date: 2018 Dec 16 + using System; +using System.Diagnostics; using System.IO; using System.Runtime.CompilerServices; using GameRes.Utility; @@ -62,20 +65,23 @@ namespace GameRes.Formats.DxLib ulong compressedSize; ulong headerSize; + ulong srcSize; + //ushort token = 256; - public HuffmanDecoder (byte[] src, byte[] dst) + public HuffmanDecoder (byte[] src,ulong srcSize) { m_input = src; - m_output = dst; + m_output = null; + this.srcSize = srcSize; m_src = 0; m_bit_count = 0; m_readBytes = 0; m_readBits = 0; originalSize = compressedSize = headerSize = 0; ushort[] weights = new ushort[256]; - nodes = new DXA8HuffmanNode[256+255]; //256 "base" nodes, then 255 nodes making a pyramid. + nodes = new DXA8HuffmanNode[256+255]; //256 data nodes, then 255 hierarchy nodes. } public byte[] Unpack () @@ -88,8 +94,162 @@ namespace GameRes.Formats.DxLib nodes[i].ChildNode[1] = -1; } SetupWeights(); + //check if compressedSize and src size match. + if (srcSize!=compressedSize) + { + throw new FileSizeException(String.Format("Supplied srcSize does not match with compressedSize. Expected {0} got {1}",compressedSize,srcSize)); + } + m_output = new byte[originalSize]; CreateTree(); - throw new NotImplementedException(); + PopulateDataNodes(); + DoUnpack(); + return m_output; + } + + private void DoUnpack() + { + var targetSize = originalSize; + byte[] compressedData = new byte[compressedSize - headerSize]; + Array.Copy(m_input, (long)headerSize, compressedData, 0, (long)(compressedSize - headerSize)); + + int PressBitCounter=0, PressBitData=0, Index=0, NodeIndex=0; + int PressSizeCounter = 0; + ulong DestSizeCounter = 0; + int[] NodeIndexTable=new int[512]; + { + ushort[] bitMask = new ushort[9]; + for (int i = 0; i < 9; i++) + { + bitMask[i] = (ushort)((1< 9) continue; + + BitArrayFirstBatch = (ushort)(nodes[j].bitArray[0] | (nodes[j].bitNumber << 8)); + + if ((i & bitMask[nodes[j].bitNumber - 1]) == (BitArrayFirstBatch & bitMask[nodes[j].bitNumber-1])) + { + NodeIndexTable[i] = j; + break; + } + } + + } + + } + PressBitData = compressedData[PressBitCounter]; + + for (DestSizeCounter = 0;DestSizeCounter < originalSize; DestSizeCounter++) + { + if (DestSizeCounter>= originalSize - 17) + { + NodeIndex = 510; + } + else + { + if (PressBitCounter==8) + { + PressSizeCounter++; + PressBitData = compressedData[PressSizeCounter]; + PressBitCounter = 0; + } + + PressBitData = (PressBitData | (compressedData[PressSizeCounter+1]<<(8-PressBitCounter))) & 0x1ff; + NodeIndex = NodeIndexTable[PressBitData]; + PressBitCounter += nodes[NodeIndex].bitNumber; + if (PressBitCounter >= 16) + { + PressSizeCounter += 2; + PressBitCounter -= 16; + PressBitData = compressedData[PressSizeCounter] >> PressBitCounter; + } + else if (PressBitCounter >=8) + { + PressSizeCounter ++; + PressBitCounter -= 8; + PressBitData = compressedData[PressSizeCounter] >> PressBitCounter; + } + else + { + PressBitData >>= nodes[NodeIndex].bitNumber; + } + while (NodeIndex>255) + { + if (PressBitCounter==8) + { + PressSizeCounter++; + PressBitData = compressedData[PressSizeCounter]; + PressBitCounter = 0; + } + Index = PressBitData & 1; + PressBitData >>= 1; + PressSizeCounter++; + NodeIndex = nodes[NodeIndex].ChildNode[Index]; + } + } + m_output[DestSizeCounter] = (byte)NodeIndex; + } + + } + + private void PopulateDataNodes() + { + //The data which is populated is path from root to target node in bits. + byte[] ScratchSpace = new byte[32]; + int TempBitIndex, TempBitCount; + + for (int i = 0; i < 256 + 254; i++) //root node is excluded. + { + nodes[i].bitNumber = 0; + TempBitIndex = 0; + TempBitCount = 0; + ScratchSpace[TempBitIndex] = 0; + + for (int j = i; nodes[j].ParentNode!=-1;j = nodes[j].ParentNode) + { + if (TempBitCount == 8) + { + TempBitCount = 0; + TempBitIndex++; + ScratchSpace[TempBitIndex] = 0; + } + ScratchSpace[TempBitIndex] <<= 1; + ScratchSpace[TempBitIndex] |= (byte)nodes[j].Index; + TempBitCount++; + nodes[i].bitNumber++; + + } + //path is now backwards (target to root). Pupulate BitPath from root to target. + int BitIndex=0, BitCount=0; + nodes[i].bitArray[BitIndex] = 0; + while (TempBitIndex >= 0) + { + if (BitCount == 8) + { + BitCount = 0; + BitIndex++; + nodes[i].bitArray[BitIndex] = 0; + } + nodes[i].bitArray[BitIndex] |= (byte)((ScratchSpace[TempBitIndex] & 1) << BitCount); + ScratchSpace[TempBitIndex] >>= 1; + TempBitCount--; + if (TempBitCount == 0) + { + TempBitIndex--; + TempBitCount = 8; + } + BitCount++; + } + } + + } private void SetupWeights() @@ -115,6 +275,7 @@ namespace GameRes.Formats.DxLib SaveData = (ushort)GetBits(BitNum); weights[i] = (ushort)(Minus == 1 ? weights[i - 1] - SaveData : weights[i - 1] + SaveData); } + headerSize = GetReadBytes(); for (int i = 0;i < 256; i++) { nodes[i].Weight = weights[i]; From 071a33eb943b4b1e0215b2e63632d379b398ec63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 15:17:26 +0200 Subject: [PATCH 14/26] Actually init arrays. --- ArcFormats/DxLib/ArcDX8.cs | 2 +- ArcFormats/DxLib/HuffmanDecoder.cs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 9def10ed..17f7ca2f 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -147,7 +147,7 @@ namespace GameRes.Formats.DxLib byte[] huffmanBuffer = new byte[headerBuffer.Length]; byte[] lzBuffer; headerBuffer.CopyTo(huffmanBuffer, 0); - huffmanBuffer = headerBuffer; + //huffmanBuffer = headerBuffer; HuffmanDecoder decoder = new HuffmanDecoder(huffmanBuffer, (ulong)huffmanBuffer.LongLength); lzBuffer = decoder.Unpack(); MemoryStream lzStream = new MemoryStream(lzBuffer); diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 26dd2e6e..11f61d86 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -35,7 +35,7 @@ using GameRes.Utility; namespace GameRes.Formats.DxLib { - internal struct DXA8HuffmanNode + internal class DXA8HuffmanNode { public UInt64 Weight; public int bitNumber; @@ -44,6 +44,13 @@ namespace GameRes.Formats.DxLib public int ParentNode; // index of parent node. public int[] ChildNode; //two children nodes, -1 if not existent. + + DXA8HuffmanNode() + { + bitArray = new byte[32]; + ChildNode = new int[2]; + } + } internal sealed class HuffmanDecoder From 681e82dd3f738ba8400996d3802b2172efd66cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 15:37:02 +0200 Subject: [PATCH 15/26] Add ToUint64 for big endian numbers --- ArcFormats/DxLib/HuffmanDecoder.cs | 5 +++-- GameRes/Utility.cs | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 11f61d86..4ab50dcd 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -45,7 +45,7 @@ namespace GameRes.Formats.DxLib public int ParentNode; // index of parent node. public int[] ChildNode; //two children nodes, -1 if not existent. - DXA8HuffmanNode() + internal DXA8HuffmanNode() { bitArray = new byte[32]; ChildNode = new int[2]; @@ -96,6 +96,7 @@ namespace GameRes.Formats.DxLib for (int i=0; i (TArray value, int index) where TArray : IList + public static uint ToUInt32(TArray value, int index) where TArray : IList { - return (uint)(value[index] << 24 | value[index+1] << 16 | value[index+2] << 8 | value[index+3]); + return (uint)(value[index] << 24 | value[index + 1] << 16 | value[index + 2] << 8 | value[index + 3]); } - public static int ToInt32 (TArray value, int index) where TArray : IList + public static int ToInt32(TArray value, int index) where TArray : IList { - return (int)ToUInt32 (value, index); + return (int)ToUInt32(value, index); + } + public static ulong ToUInt64(TArray value, int index) where TArray : IList + { + return (ulong)(value[index]<<56|value[index+1]<<48|value[index+2]<<40|value[index+3]<<32|value[index+4] << 24 | value[index + 5] << 16 | value[index + 6] << 8 | value[index + 7]); + } + + public static long ToInt64(TArray value, int index) where TArray : IList + { + return (long)ToUInt64(value, index); } public static void Pack (ushort value, byte[] buf, int index) From e3236c0fbd7331e9aa5b23d3bc236241ae634412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 15:43:28 +0200 Subject: [PATCH 16/26] Revert "Add ToUint64 for big endian numbers" This reverts commit 681e82dd3f738ba8400996d3802b2172efd66cf2. --- ArcFormats/DxLib/HuffmanDecoder.cs | 5 ++--- GameRes/Utility.cs | 17 ++++------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 4ab50dcd..11f61d86 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -45,7 +45,7 @@ namespace GameRes.Formats.DxLib public int ParentNode; // index of parent node. public int[] ChildNode; //two children nodes, -1 if not existent. - internal DXA8HuffmanNode() + DXA8HuffmanNode() { bitArray = new byte[32]; ChildNode = new int[2]; @@ -96,7 +96,6 @@ namespace GameRes.Formats.DxLib for (int i=0; i(TArray value, int index) where TArray : IList + public static uint ToUInt32 (TArray value, int index) where TArray : IList { - return (uint)(value[index] << 24 | value[index + 1] << 16 | value[index + 2] << 8 | value[index + 3]); + return (uint)(value[index] << 24 | value[index+1] << 16 | value[index+2] << 8 | value[index+3]); } - public static int ToInt32(TArray value, int index) where TArray : IList + public static int ToInt32 (TArray value, int index) where TArray : IList { - return (int)ToUInt32(value, index); - } - public static ulong ToUInt64(TArray value, int index) where TArray : IList - { - return (ulong)(value[index]<<56|value[index+1]<<48|value[index+2]<<40|value[index+3]<<32|value[index+4] << 24 | value[index + 5] << 16 | value[index + 6] << 8 | value[index + 7]); - } - - public static long ToInt64(TArray value, int index) where TArray : IList - { - return (long)ToUInt64(value, index); + return (int)ToUInt32 (value, index); } public static void Pack (ushort value, byte[] buf, int index) From 4d5be92386c0c245ffca622e330562c83c7f34f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 15:44:55 +0200 Subject: [PATCH 17/26] Update HuffmanDecoder.cs Fix constructor visibility error; Change to big endian 32 bit. --- ArcFormats/DxLib/HuffmanDecoder.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 11f61d86..42036769 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -45,7 +45,7 @@ namespace GameRes.Formats.DxLib public int ParentNode; // index of parent node. public int[] ChildNode; //two children nodes, -1 if not existent. - DXA8HuffmanNode() + internal DXA8HuffmanNode() { bitArray = new byte[32]; ChildNode = new int[2]; @@ -338,9 +338,9 @@ namespace GameRes.Formats.DxLib { if (0 == m_bit_count) { - m_bits = LittleEndian.ToUInt64 (m_input, m_src); - m_src += 8; - m_bit_count = 64; + m_bits = BigEndian.ToUInt32 (m_input, m_src); + m_src += 4; + m_bit_count = 32; } bits = bits << 1 | (m_bits & 1); m_bits >>= 1; From 3f4c263ec5a3655d00386dcf288dfb86befd0d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 15:47:28 +0200 Subject: [PATCH 18/26] Update HuffmanDecoder.cs Since Huffman node is now a class it must be explicitely initiated. --- ArcFormats/DxLib/HuffmanDecoder.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 42036769..008bc253 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -89,6 +89,10 @@ namespace GameRes.Formats.DxLib originalSize = compressedSize = headerSize = 0; ushort[] weights = new ushort[256]; nodes = new DXA8HuffmanNode[256+255]; //256 data nodes, then 255 hierarchy nodes. + for (int i = 0; i < nodes.Length; i++) + { + nodes[i] = new DXA8HuffmanNode(); + } } public byte[] Unpack () From 2ad5095cec24aab9dfc57454ee8db7d58760c2ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 16:07:31 +0200 Subject: [PATCH 19/26] Update HuffmanDecoder.cs compressedSize does not include header. --- ArcFormats/DxLib/HuffmanDecoder.cs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 008bc253..418d28ef 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -59,11 +59,11 @@ namespace GameRes.Formats.DxLib byte[] m_output; int m_src; - ulong m_bits; + byte m_bits; int m_bit_count; ulong m_readBytes; - ulong m_readBits; + byte m_readBits; DXA8HuffmanNode[] nodes; //256+255 nodes @@ -106,9 +106,9 @@ namespace GameRes.Formats.DxLib } SetupWeights(); //check if compressedSize and src size match. - if (srcSize!=compressedSize) + if (srcSize!=(compressedSize+headerSize)) { - throw new FileSizeException(String.Format("Supplied srcSize does not match with compressedSize. Expected {0} got {1}",compressedSize,srcSize)); + throw new FileSizeException(String.Format("Supplied srcSize does not match with compressedSize+headerSize. Expected {0} got {1}",compressedSize+headerSize,srcSize)); } m_output = new byte[originalSize]; CreateTree(); @@ -120,8 +120,8 @@ namespace GameRes.Formats.DxLib private void DoUnpack() { var targetSize = originalSize; - byte[] compressedData = new byte[compressedSize - headerSize]; - Array.Copy(m_input, (long)headerSize, compressedData, 0, (long)(compressedSize - headerSize)); + byte[] compressedData = new byte[compressedSize]; + Array.Copy(m_input, (long)headerSize, compressedData, 0, (long)(compressedSize)); int PressBitCounter=0, PressBitData=0, Index=0, NodeIndex=0; int PressSizeCounter = 0; @@ -338,16 +338,15 @@ namespace GameRes.Formats.DxLib ulong GetBits (int count) { ulong bits = 0; - while (count --> 0) + for (int i = 0; i < count;i++) { if (0 == m_bit_count) { - m_bits = BigEndian.ToUInt32 (m_input, m_src); - m_src += 4; - m_bit_count = 32; + m_bits = m_input[m_src]; + m_src += 1; + m_bit_count = 8; } - bits = bits << 1 | (m_bits & 1); - m_bits >>= 1; + bits |= ((ulong)((m_bits >> (7 - m_readBits)) & 1)) <<(count-1-i); --m_bit_count; m_readBits++; if (m_readBits ==8) From 4773ff148fcc3f553f2f04018c40fa6f5916a5f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Mon, 5 Aug 2024 16:13:32 +0200 Subject: [PATCH 20/26] Update HuffmanDecoder.cs --- ArcFormats/DxLib/HuffmanDecoder.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 418d28ef..02b63ba8 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -343,9 +343,10 @@ namespace GameRes.Formats.DxLib if (0 == m_bit_count) { m_bits = m_input[m_src]; - m_src += 1; + m_src++; m_bit_count = 8; } + //bits are read backwards. bits |= ((ulong)((m_bits >> (7 - m_readBits)) & 1)) <<(count-1-i); --m_bit_count; m_readBits++; From 57749e5cacb4608c356c516d076ab83679a70678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Wed, 7 Aug 2024 17:51:43 +0200 Subject: [PATCH 21/26] Update HuffmanDecoder.cs Fix obvious bugs. --- ArcFormats/DxLib/HuffmanDecoder.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 02b63ba8..981ebba2 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -201,7 +201,7 @@ namespace GameRes.Formats.DxLib } Index = PressBitData & 1; PressBitData >>= 1; - PressSizeCounter++; + PressBitCounter++; NodeIndex = nodes[NodeIndex].ChildNode[Index]; } } @@ -237,7 +237,7 @@ namespace GameRes.Formats.DxLib nodes[i].bitNumber++; } - //path is now backwards (target to root). Pupulate BitPath from root to target. + //path is now backwards (target to root). Populate BitPath from root to target. int BitIndex=0, BitCount=0; nodes[i].bitArray[BitIndex] = 0; while (TempBitIndex >= 0) @@ -278,7 +278,7 @@ namespace GameRes.Formats.DxLib BitNum = (byte)(((int)GetBits(3) + 1) * 2); Minus = (byte)GetBits(1); SaveData = (ushort)GetBits(BitNum); - nodes[0].Weight = SaveData; + weights[0] = SaveData; for (int i = 1; i < 256; i++) { BitNum = (byte)(((int)GetBits(3) + 1) * 2); From ed0f0489d3d17002abe48fd343e70dc2112092cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Sun, 11 Aug 2024 15:41:21 +0200 Subject: [PATCH 22/26] Update HuffmanDecoder.cs Fix not so obvious errors. --- ArcFormats/DxLib/HuffmanDecoder.cs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/ArcFormats/DxLib/HuffmanDecoder.cs b/ArcFormats/DxLib/HuffmanDecoder.cs index 981ebba2..6d334ee0 100644 --- a/ArcFormats/DxLib/HuffmanDecoder.cs +++ b/ArcFormats/DxLib/HuffmanDecoder.cs @@ -143,7 +143,7 @@ namespace GameRes.Formats.DxLib ushort BitArrayFirstBatch; if (nodes[j].bitNumber > 9) continue; - BitArrayFirstBatch = (ushort)(nodes[j].bitArray[0] | (nodes[j].bitNumber << 8)); + BitArrayFirstBatch = (ushort)(nodes[j].bitArray[0] | (nodes[j].bitArray[1] << 8)); if ((i & bitMask[nodes[j].bitNumber - 1]) == (BitArrayFirstBatch & bitMask[nodes[j].bitNumber-1])) { @@ -191,19 +191,20 @@ namespace GameRes.Formats.DxLib { PressBitData >>= nodes[NodeIndex].bitNumber; } - while (NodeIndex>255) + } + + while (NodeIndex > 255) + { + if (PressBitCounter == 8) { - if (PressBitCounter==8) - { - PressSizeCounter++; - PressBitData = compressedData[PressSizeCounter]; - PressBitCounter = 0; - } - Index = PressBitData & 1; - PressBitData >>= 1; - PressBitCounter++; - NodeIndex = nodes[NodeIndex].ChildNode[Index]; + PressSizeCounter++; + PressBitData = compressedData[PressSizeCounter]; + PressBitCounter = 0; } + Index = PressBitData & 1; + PressBitData >>= 1; + PressBitCounter++; + NodeIndex = nodes[NodeIndex].ChildNode[Index]; } m_output[DestSizeCounter] = (byte)NodeIndex; } From 0817dacd997f67482b54e3e800c3f60d38d1ff06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Sun, 11 Aug 2024 18:59:30 +0200 Subject: [PATCH 23/26] Update ArcDX8.cs More fixes. This is not done yet. --- ArcFormats/DxLib/ArcDX8.cs | 117 ++++++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 16 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 17f7ca2f..9e3e8b60 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -25,11 +25,14 @@ using GameRes.Formats.Strings; using System; +using System.CodeDom; using System.Collections.Generic; using System.ComponentModel.Composition; using System.IO; using System.Linq; +using System.Runtime.Remoting.Messaging; using System.Windows.Navigation; +using static GameRes.Formats.DxLib.Dx8Opener; @@ -40,11 +43,28 @@ namespace GameRes.Formats.DxLib internal class DXA8PackedEntry : PackedEntry { public bool HuffmanCompressed { get; set; } public uint HuffmanSize { get; set; } + + public uint LZSize { get; set; } } - + internal class DxArchive8 : DxArchive + { + public byte huffmanMaxKB; - [Export(typeof(ArchiveFormat))] + public DxArchive8(ArcView arc, ArchiveFormat impl, ICollection dir, IDxKey enc, int version,byte huffmanKB) : base(arc, impl, dir, enc, version) + { + huffmanMaxKB = huffmanKB; + } + } + + internal class DxHeaderV8 : DxHeader + { + public DXA8Flags Flags; + public byte HuffmanKB; // oddly used only in Compression process not in decompression. + //15 bytes of padding. + } + + [Export(typeof(ArchiveFormat))] public class Dx8Opener : DxOpener { public override string Tag { get { return "BIN/DXLIB"; } } @@ -65,12 +85,7 @@ namespace GameRes.Formats.DxLib DxScheme DefaultScheme = new DxScheme { KnownKeys = new List() }; - internal class DxHeaderV8 :DxHeader - { - public DXA8Flags Flags; - public byte HuffmanKB; // oddly used only in Compression process not in decompression. - //15 bytes of padding. - } + internal enum DXA8Flags : UInt32 { @@ -156,16 +171,74 @@ namespace GameRes.Formats.DxLib } - var readyStr = new MemoryStream(headerBuffer); - ArcView arcView = new ArcView(readyStr, "hdr",(uint)headerBuffer.LongLength); List entries; - //There MAY be the case where the singular file is over 4GB, but it's very rare. - using (var indexStr = arcView.CreateStream(0, (uint)dx.IndexSize)) - using (var reader = IndexReader.Create(dx, 8, indexStr)) + //There MAY be the case where the singular file is over 4GB, but it's very rare. + using (var reader = IndexReader.Create(dx, 8, new MemoryStream(headerBuffer))) { - entries = reader.Read(); + entries = reader.Read(); } - return new DxArchive(arcView, this,entries ,key, 8); + return new DxArchive8(file, this,entries ,key, 8,dx.HuffmanKB); + //return null; + } + + public override Stream OpenEntry(ArcFile arc, Entry entry) + { + Stream input = arc.File.CreateStream(entry.Offset, entry.Size); + var dx_arc = arc as DxArchive8; + if (null == dx_arc) + return input; + var dx_ent = (DXA8PackedEntry)entry; + long dec_offset = dx_ent.UnpackedSize; //is this still right? + var key = dx_arc.Encryption.GetEntryKey(dx_ent.Name); + input = new EncryptedStream(input, dec_offset, key); + if (!dx_ent.HuffmanCompressed && !dx_ent.IsPacked) + return input; + //we ruled out the case in which neither compression is applied. We still have 3 cases to go. + + byte[] tmpBuffer = new byte[dx_ent.Size]; + input.Read(tmpBuffer, 0, tmpBuffer.Length); + if (dx_ent.HuffmanCompressed) + { + byte[] buffer = new byte[dx_ent.HuffmanSize]; + byte[] outBuffer = new byte[dx_ent.IsPacked ? dx_ent.LZSize : dx_ent.UnpackedSize]; + Array.Copy(tmpBuffer, buffer, dx_ent.HuffmanSize); + HuffmanDecoder decoder = new HuffmanDecoder(buffer,dx_ent.HuffmanSize); + byte[] partTmpBuffer = decoder.Unpack(); + //returned buffer might be partial. Check if this is the case. + var outBufSize = dx_ent.IsPacked ? dx_ent.LZSize : dx_ent.UnpackedSize; + if(dx_arc.huffmanMaxKB != 0xff && outBufSize > dx_arc.huffmanMaxKB * 1024 * 2) + { + //What we have here is two huffmanMaxKB KB buffers, that constitute the beginning and end of file respectively. + Array.Copy(partTmpBuffer,0, outBuffer, 0,dx_arc.huffmanMaxKB*1024); + Array.Copy(partTmpBuffer,dx_arc.huffmanMaxKB*1024,outBuffer,outBuffer.Length-dx_arc.huffmanMaxKB*1024,dx_arc.huffmanMaxKB*1024); + //uncompressed part goes into middle. + input.Read(outBuffer, dx_arc.huffmanMaxKB * 1024, outBuffer.Length-dx_arc.huffmanMaxKB*1024*2); + tmpBuffer = outBuffer; + } else + { + //that is all that needs to be done. + tmpBuffer = partTmpBuffer; + } + } + if(dx_ent.IsPacked) + { + byte[] buffer = new byte[dx_ent.LZSize]; + tmpBuffer.CopyTo(buffer, 0); + var tmpMemStream = new MemoryStream(buffer); + tmpBuffer = Unpack(tmpMemStream); + + } + return new BinMemoryStream(tmpBuffer, entry.Name); + + /* + if (!dx_ent.IsPacked) + return input; + using (input) + { + var data = Unpack(input); + return new BinMemoryStream(data, entry.Name); + } + */ //return null; } } @@ -231,7 +304,19 @@ namespace GameRes.Formats.DxLib entry.IsPacked = -1 != packed_size; entry.HuffmanCompressed = -1 != huffman_packed_size; entry.HuffmanSize = (uint)huffman_packed_size; - if (entry.IsPacked) + entry.LZSize = (uint)packed_size; + //Huffman compression: huffman_packed_size will not exceed 2*HuffmanKB KB. The rest of data is uncompressed (as far as Huffman compressor is concerned). + //Add length of uncompressed data to entry.Size + if (entry.HuffmanCompressed) + { + var outBufSize = entry.IsPacked ? packed_size : size; + var dx8_hdr = (DxHeaderV8)m_header; + if (outBufSize > dx8_hdr.HuffmanKB * 1024 * 2) + { + huffman_packed_size += outBufSize - dx8_hdr.HuffmanKB * 1024 * 2; + } + } + if (entry.IsPacked||entry.HuffmanCompressed) entry.Size = (uint)(huffman_packed_size!=-1 ? huffman_packed_size:packed_size); else entry.Size = (uint)size; From d2a22fa2fbb0687e8965bf19cbf6caf4fc1824e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Wed, 14 Aug 2024 18:30:45 +0200 Subject: [PATCH 24/26] Update ArcDX8.cs We already read the entry to the array. Don't attempt to read twice. Or in other words, just copy from the prepared array. --- ArcFormats/DxLib/ArcDX8.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 9e3e8b60..45b46528 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -178,7 +178,7 @@ namespace GameRes.Formats.DxLib entries = reader.Read(); } return new DxArchive8(file, this,entries ,key, 8,dx.HuffmanKB); - //return null; + //retu rn null; } public override Stream OpenEntry(ArcFile arc, Entry entry) @@ -188,7 +188,7 @@ namespace GameRes.Formats.DxLib if (null == dx_arc) return input; var dx_ent = (DXA8PackedEntry)entry; - long dec_offset = dx_ent.UnpackedSize; //is this still right? + long dec_offset = dx_ent.UnpackedSize; var key = dx_arc.Encryption.GetEntryKey(dx_ent.Name); input = new EncryptedStream(input, dec_offset, key); if (!dx_ent.HuffmanCompressed && !dx_ent.IsPacked) @@ -212,7 +212,7 @@ namespace GameRes.Formats.DxLib Array.Copy(partTmpBuffer,0, outBuffer, 0,dx_arc.huffmanMaxKB*1024); Array.Copy(partTmpBuffer,dx_arc.huffmanMaxKB*1024,outBuffer,outBuffer.Length-dx_arc.huffmanMaxKB*1024,dx_arc.huffmanMaxKB*1024); //uncompressed part goes into middle. - input.Read(outBuffer, dx_arc.huffmanMaxKB * 1024, outBuffer.Length-dx_arc.huffmanMaxKB*1024*2); + Array.Copy(tmpBuffer, dx_ent.HuffmanSize, outBuffer, dx_arc.huffmanMaxKB * 1024, outBufSize - dx_arc.huffmanMaxKB * 1024 * 2); tmpBuffer = outBuffer; } else { From 329589122aad3b8514ccea388cf4be6163d37f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Sun, 18 Aug 2024 10:46:41 +0200 Subject: [PATCH 25/26] Fix cases where files with non-ascii names would fail to decrypt. --- ArcFormats/DxLib/ArcDX8.cs | 5 +---- ArcFormats/DxLib/DxKey.cs | 37 ++++++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/ArcFormats/DxLib/ArcDX8.cs b/ArcFormats/DxLib/ArcDX8.cs index 45b46528..8daf5bfd 100644 --- a/ArcFormats/DxLib/ArcDX8.cs +++ b/ArcFormats/DxLib/ArcDX8.cs @@ -150,7 +150,7 @@ namespace GameRes.Formats.DxLib if (isencrypted) { var keyStr = Query(arcStrings.ZIPEncryptedNotice).Keyword; - key = new DxKey8(keyStr); + key = new DxKey8(keyStr,dx.CodePage); } @@ -191,9 +191,6 @@ namespace GameRes.Formats.DxLib long dec_offset = dx_ent.UnpackedSize; var key = dx_arc.Encryption.GetEntryKey(dx_ent.Name); input = new EncryptedStream(input, dec_offset, key); - if (!dx_ent.HuffmanCompressed && !dx_ent.IsPacked) - return input; - //we ruled out the case in which neither compression is applied. We still have 3 cases to go. byte[] tmpBuffer = new byte[dx_ent.Size]; input.Read(tmpBuffer, 0, tmpBuffer.Length); diff --git a/ArcFormats/DxLib/DxKey.cs b/ArcFormats/DxLib/DxKey.cs index 6fdcccff..5e5ba6b9 100644 --- a/ArcFormats/DxLib/DxKey.cs +++ b/ArcFormats/DxLib/DxKey.cs @@ -159,8 +159,10 @@ namespace GameRes.Formats.DxLib [Serializable] public class DxKey8 : DxKey { - public DxKey8(string password) : base(password ?? "DXARC") + private int codepage; + public DxKey8(string password,int codepage) : base(password ?? "DXARC") { + this.codepage = codepage; } public override byte[] GetEntryKey(string name) @@ -179,13 +181,26 @@ namespace GameRes.Formats.DxLib { keyword += "DXARC"; } - string oddString, evenString; - byte[] key = new byte[7]; - oddString = string.Concat(keyword.Where((c, i) => i % 2 == 0)); - evenString = string.Concat(keyword.Where((c, i) => (i+1) % 2 == 0)); + //first split string to bytes. Use original encoding as basis. Otherwise we would fail to decrypt that. + Encoding defEncoding = Encoding.UTF8; + Encoding tgtEncoding = Encoding.GetEncoding(codepage); + byte[] defBytes = defEncoding.GetBytes(keyword); + byte[] tgtBytes = Encoding.Convert(defEncoding, tgtEncoding, defBytes); + byte[] oddBuffer = new byte[tgtBytes.Length]; int oddCounter = 0; + byte[] evenBuffer = new byte[tgtBytes.Length]; int evenCounter = 0; + for (int i=0; i i % 2 == 0)); + evenString = string.Concat(keyword.Where((c, i) => (i+1) % 2 == 0)); + UInt32 crc_0, crc_1; + crc_0 = Crc32.Compute(Encoding.ASCII.GetBytes(oddString), 0, oddString.Length); + crc_1 = Crc32.Compute(Encoding.ASCII.GetBytes(evenString), 0, evenString.Length); */ /* using (var sha = SHA256.Create()) { From ad7f4f17880105a636d954de4a311b6c863527c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20=C5=9Apiewak?= Date: Tue, 20 Aug 2024 15:01:08 +0200 Subject: [PATCH 26/26] Simplify code. --- ArcFormats/DxLib/DxKey.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ArcFormats/DxLib/DxKey.cs b/ArcFormats/DxLib/DxKey.cs index 5e5ba6b9..44c57901 100644 --- a/ArcFormats/DxLib/DxKey.cs +++ b/ArcFormats/DxLib/DxKey.cs @@ -182,12 +182,10 @@ namespace GameRes.Formats.DxLib keyword += "DXARC"; } //first split string to bytes. Use original encoding as basis. Otherwise we would fail to decrypt that. - Encoding defEncoding = Encoding.UTF8; Encoding tgtEncoding = Encoding.GetEncoding(codepage); - byte[] defBytes = defEncoding.GetBytes(keyword); - byte[] tgtBytes = Encoding.Convert(defEncoding, tgtEncoding, defBytes); - byte[] oddBuffer = new byte[tgtBytes.Length]; int oddCounter = 0; - byte[] evenBuffer = new byte[tgtBytes.Length]; int evenCounter = 0; + byte[] tgtBytes = tgtEncoding.GetBytes(keyword); + byte[] oddBuffer = new byte[(tgtBytes.Length/2)+(tgtBytes.Length%2)]; int oddCounter = 0; + byte[] evenBuffer = new byte[(tgtBytes.Length/2)]; int evenCounter = 0; for (int i=0; i