From 3bbf2d6a9017c65b4f6a4e153aff510f6e649190 Mon Sep 17 00:00:00 2001 From: morkt Date: Sun, 17 Aug 2014 01:31:58 +0400 Subject: [PATCH] implemented Ren'Py archives creation. --- ArcFormats/ArcFormats.csproj | 7 + ArcFormats/ArcRPA.cs | 479 +++++++++++++++++++-- ArcFormats/CreateRPAWidget.xaml | 35 ++ ArcFormats/CreateRPAWidget.xaml.cs | 39 ++ ArcFormats/Properties/Settings.Designer.cs | 12 + ArcFormats/Properties/Settings.settings | 3 + ArcFormats/Strings/arcStrings.Designer.cs | 9 + ArcFormats/Strings/arcStrings.resx | 3 + ArcFormats/Strings/arcStrings.ru-RU.resx | 3 + ArcFormats/app.config | 3 + 10 files changed, 556 insertions(+), 37 deletions(-) create mode 100644 ArcFormats/CreateRPAWidget.xaml create mode 100644 ArcFormats/CreateRPAWidget.xaml.cs diff --git a/ArcFormats/ArcFormats.csproj b/ArcFormats/ArcFormats.csproj index 1ab9825a..561fabfc 100644 --- a/ArcFormats/ArcFormats.csproj +++ b/ArcFormats/ArcFormats.csproj @@ -80,6 +80,9 @@ CreatePDWidget.xaml + + CreateRPAWidget.xaml + CreateSGWidget.xaml @@ -158,6 +161,10 @@ Designer MSBuild:Compile + + Designer + MSBuild:Compile + Designer MSBuild:Compile diff --git a/ArcFormats/ArcRPA.cs b/ArcFormats/ArcRPA.cs index be385df8..3c8f789e 100644 --- a/ArcFormats/ArcRPA.cs +++ b/ArcFormats/ArcRPA.cs @@ -31,22 +31,30 @@ using System.Diagnostics; using System.Globalization; using System.IO; using System.Text; +using GameRes.Formats.Properties; +using GameRes.Formats.Strings; using ZLibNet; namespace GameRes.Formats.RenPy { - internal class RpaEntry : Entry + internal class RpaEntry : PackedEntry { public byte[] Header = null; } + public class RpaOptions : ResourceOptions + { + public uint Key; + } + [Export(typeof(ArchiveFormat))] public class RpaOpener : ArchiveFormat { - public override string Tag { get { return "RPA"; } } + public override string Tag { get { return "RPA"; } } public override string Description { get { return Strings.arcStrings.RPADescription; } } - public override uint Signature { get { return 0x2d415052; } } // "RPA-" - public override bool IsHierarchic { get { return true; } } + public override uint Signature { get { return 0x2d415052; } } // "RPA-" + public override bool IsHierarchic { get { return true; } } + public override bool CanCreate { get { return true; } } public override ArcFile TryOpen (ArcView file) { @@ -63,11 +71,11 @@ namespace GameRes.Formats.RenPy if (!uint.TryParse (key_str, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out key)) return null; - Hashtable dict = null; + IDictionary dict = null; using (var index = new ZLibStream (file.CreateStream (index_offset), CompressionMode.Decompress)) { var pickle = new Pickle (index); - dict = pickle.Load() as Hashtable; + dict = pickle.Load() as IDictionary; } if (null == dict) return null; @@ -75,8 +83,8 @@ namespace GameRes.Formats.RenPy foreach (DictionaryEntry item in dict) { var name_raw = item.Key as byte[]; - var value = item.Value as ArrayList; - if (null == name_raw || null == value || value.Count < 1) + var values = item.Value as IList; + if (null == name_raw || null == values || values.Count < 1) { Trace.WriteLine ("invalid index entry", "RpaOpener.TryOpen"); return null; @@ -84,7 +92,7 @@ namespace GameRes.Formats.RenPy string name = Encoding.UTF8.GetString (name_raw); if (string.IsNullOrEmpty (name)) return null; - var tuple = value[0] as ArrayList; + var tuple = values[0] as IList; if (null == tuple || tuple.Count < 2) { Trace.WriteLine ("invalid index tuple", "RpaOpener.TryOpen"); @@ -94,12 +102,16 @@ namespace GameRes.Formats.RenPy { Name = name, Type = FormatCatalog.Instance.GetTypeFromName (name), - Offset = (uint)((int)tuple[0] ^ key), - Size = (uint)((int)tuple[1] ^ key), + Offset = (uint)((int)tuple[0] ^ key), + UnpackedSize = (uint)((int)tuple[1] ^ key), }; + entry.Size = entry.UnpackedSize; if (tuple.Count > 2) + { entry.Header = tuple[2] as byte[]; - + if (null != entry.Header) + entry.Size -= (uint)entry.Header.Length; + } dir.Add (entry); } if (dir.Count > 0) @@ -109,43 +121,338 @@ namespace GameRes.Formats.RenPy public override Stream OpenEntry (ArcFile arc, Entry entry) { - var input = arc.File.CreateStream (entry.Offset, entry.Size); + Stream input; + if (0 != entry.Size) + input = arc.File.CreateStream (entry.Offset, entry.Size); + else + input = Stream.Null; var rpa_entry = entry as RpaEntry; if (null == rpa_entry || null == rpa_entry.Header) return input; return new RpaStream (rpa_entry.Header, input); } + + public override ResourceOptions GetDefaultOptions () + { + return new RpaOptions { Key = Settings.Default.RPAKey }; + } + + public override object GetCreationWidget () + { + return new GUI.CreateRPAWidget(); + } + + public override void Create (Stream output, IEnumerable list, ResourceOptions options, + EntryCallback callback) + { + var rpa_options = GetOptions (options); + int callback_count = 0; + var file_table = new Dictionary(); + long data_offset = 0x22; + output.Position = data_offset; + foreach (var entry in list) + { + if (null != callback) + callback (callback_count++, entry, arcStrings.MsgAddingFile); + + string name = entry.Name.Replace (@"\", "/"); + var rpa_entry = new RpaEntry { Name = name }; + using (var file = File.OpenRead (entry.Name)) + { + var size = file.Length; + if (size > uint.MaxValue) + throw new FileSizeException(); + int header_size = (int)Math.Min (size, 0x10); + rpa_entry.Offset = output.Position ^ rpa_options.Key; + rpa_entry.Header = new byte[header_size]; + rpa_entry.UnpackedSize = (uint)size ^ rpa_options.Key; + rpa_entry.Size = (uint)(size - header_size); + file.Read (rpa_entry.Header, 0, header_size); + file.CopyTo (output); + } + var py_name = new PyString (name); + if (file_table.ContainsKey (py_name)) + file_table[py_name].Add (rpa_entry); + else + file_table[py_name] = new ArrayList { rpa_entry }; + } + long index_pos = output.Position; + string signature = string.Format (CultureInfo.InvariantCulture, "RPA-3.0 {0:x16} {1:x8}\n", + index_pos, rpa_options.Key); + var header = Encoding.ASCII.GetBytes (signature); + if (header.Length > data_offset) + throw new ApplicationException ("Signature serialization failed."); + + if (null != callback) + callback (callback_count++, null, arcStrings.MsgWritingIndex); + + using (var index = new ZLibStream (output, CompressionMode.Compress, CompressionLevel.Level9, true)) + { + var pickle = new Pickle (index); + if (!pickle.Dump (file_table)) + throw new ApplicationException ("Archive index serialization failed."); + } + output.Position = 0; + output.Write (header, 0, header.Length); + } } public class Pickle { - Stream m_stream; + Stream m_stream; ArrayList m_stack = new ArrayList(); Stack m_marks = new Stack(); const int HIGHEST_PROTOCOL = 2; - const int PROTO = 0x80; /* identify pickle protocol */ - const int TUPLE2 = 0x86; /* build 2-tuple from two topmost stack items */ - const int TUPLE3 = 0x87; /* build 3-tuple from three topmost stack items */ - const int MARK = '('; - const int STOP = '.'; - const int BININT = 'J'; - const int BININT1 = 'K'; - const int BININT2 = 'M'; - const int SHORT_BINSTRING = 'U'; - const int EMPTY_LIST = ']'; - const int APPEND = 'a'; - const int BINPUT = 'q'; - const int LONG_BINPUT = 'r'; - const int SETITEMS = 'u'; - const int EMPTY_DICT = '}'; + const int BATCHSIZE = 1000; + const byte PROTO = 0x80; /* identify pickle protocol */ + const byte TUPLE2 = 0x86; /* build 2-tuple from two topmost stack items */ + const byte TUPLE3 = 0x87; /* build 3-tuple from three topmost stack items */ + const byte MARK = (byte)'('; + const byte STOP = (byte)'.'; + const byte INT = (byte)'I'; + const byte BININT = (byte)'J'; + const byte BININT1 = (byte)'K'; + const byte BININT2 = (byte)'M'; + const byte BINSTRING = (byte)'T'; + const byte SHORT_BINSTRING = (byte)'U'; + const byte BINUNICODE = (byte)'X'; + const byte EMPTY_LIST = (byte)']'; + const byte APPEND = (byte)'a'; + const byte APPENDS = (byte)'e'; + const byte BINPUT = (byte)'q'; + const byte LONG_BINPUT = (byte)'r'; + const byte SETITEM = (byte)'s'; + const byte TUPLE = (byte)'t'; + const byte SETITEMS = (byte)'u'; + const byte EMPTY_DICT = (byte)'}'; public Pickle (Stream stream) { m_stream = stream; } + public bool Dump (object obj) + { + m_stream.WriteByte (PROTO); + m_stream.WriteByte ((byte)HIGHEST_PROTOCOL); + if (!Save (obj)) + return false; + m_stream.WriteByte (STOP); + return true; + } + + bool Save (object obj) + { + if (null == obj) + { + Trace.WriteLine ("Null reference not serialized", "Pickle.Save"); + return false; + } + switch (Type.GetTypeCode (obj.GetType())) + { + case TypeCode.Byte: return SaveInt ((uint)(byte)obj); + case TypeCode.SByte: return SaveInt ((uint)(sbyte)obj); + case TypeCode.UInt16: return SaveInt ((uint)(ushort)obj); + case TypeCode.Int16: return SaveInt ((uint)(short)obj); + case TypeCode.Int32: return SaveInt ((uint)(int)obj); + case TypeCode.UInt32: return SaveInt ((uint)obj); + case TypeCode.Int64: return SaveLong ((long)obj); + case TypeCode.UInt64: return SaveLong ((long)(ulong)obj); + case TypeCode.Object: break; + default: + Trace.WriteLine (obj, "Object could not be serialized"); + return false; + } + if (obj is RpaEntry) + return SaveEntry (obj as RpaEntry); + if (obj is PyString) + return SaveString (obj as PyString); + if (obj is byte[]) + return SaveString (obj as byte[]); + if (obj is IDictionary) + return SaveDict (obj as IDictionary); + if (obj is IList) + return SaveList (obj as IList); + + Trace.WriteLine (obj, "Object could not be serialized"); + return false; + } + + bool SaveString (byte[] str) + { + int size = str.Length; + if (size < 256) + { + m_stream.WriteByte (SHORT_BINSTRING); + m_stream.WriteByte ((byte)size); + } + else + { + m_stream.WriteByte (BINSTRING); + PutInt (size); + } + m_stream.Write (str, 0, size); + return true; + } + + bool SaveString (PyString str) + { + if (str.IsAscii) + return SaveString (str.Bytes); + m_stream.WriteByte (BINUNICODE); + PutInt (str.Length); + m_stream.Write (str.Bytes, 0, str.Length); + return true; + } + + bool SaveEntry (RpaEntry entry) + { + byte opcode = null == entry.Header ? TUPLE2 : TUPLE3; + SaveLong (entry.Offset); + SaveInt (entry.UnpackedSize); + if (null != entry.Header) + SaveString (entry.Header); + m_stream.WriteByte (opcode); + return true; + } + + bool SaveList (IList list) + { + m_stream.WriteByte (EMPTY_LIST); + if (0 == list.Count) + return true; + return BatchList (list.GetEnumerator()); + } + + bool BatchList (IEnumerator iterator) + { + int n = 0; + do + { + if (!iterator.MoveNext()) + return false; + var first_item = iterator.Current; + if (!iterator.MoveNext()) + { + if (!Save (first_item)) + return false; + m_stream.WriteByte (APPEND); + break; + } + m_stream.WriteByte (MARK); + if (!Save (first_item)) + return false; + n = 1; + do + { + if (!Save (iterator.Current)) + return false; + if (++n == BATCHSIZE) + break; + } + while (iterator.MoveNext()); + m_stream.WriteByte (APPENDS); + } + while (n == BATCHSIZE); + return true; + } + + bool SaveInt (uint i) + { + byte[] buf = new byte[5]; + buf[1] = (byte)( i & 0xff); + buf[2] = (byte)((i >> 8) & 0xff); + buf[3] = (byte)((i >> 16) & 0xff); + buf[4] = (byte)((i >> 24) & 0xff); + int length; + if (0 == buf[4] && 0 == buf[3]) + { + if (0 == buf[2]) + { + buf[0] = BININT1; + length = 2; + } + else + { + buf[0] = BININT2; + length = 3; + } + } + else + { + buf[0] = BININT; + length = 5; + } + m_stream.Write (buf, 0, length); + return true; + } + + bool SaveLong (long l) + { + if (0 == ((l >> 32) & 0xffffffff)) + return SaveInt ((uint)l); + m_stream.WriteByte (INT); + string num = l.ToString (CultureInfo.InvariantCulture); + var num_data = Encoding.ASCII.GetBytes (num); + m_stream.Write (num_data, 0, num_data.Length); + m_stream.WriteByte (0x0a); + return true; + } + + bool SaveDict (IDictionary dict) + { + m_stream.WriteByte (EMPTY_DICT); + if (0 == dict.Count) + return true; + return BatchDict (dict); + } + + bool BatchDict (IDictionary dict) + { + int dict_size = dict.Count; + var iterator = dict.GetEnumerator(); + if (1 == dict_size) + { + if (!iterator.MoveNext()) + return false; + if (!Save (iterator.Key)) + return false; + if (!Save (iterator.Value)) + return false; + m_stream.WriteByte (SETITEM); + return true; + } + int i; + do + { + i = 0; + m_stream.WriteByte (MARK); + while (iterator.MoveNext()) + { + if (!Save (iterator.Key)) + return false; + if (!Save (iterator.Value)) + return false; + if (++i == BATCHSIZE) + break; + } + m_stream.WriteByte (SETITEMS); + } + while (i == BATCHSIZE); + return true; + } + + bool PutInt (int i) + { + m_stream.WriteByte ((byte)(i & 0xff)); + m_stream.WriteByte ((byte)((i >> 8) & 0xff)); + m_stream.WriteByte ((byte)((i >> 16) & 0xff)); + m_stream.WriteByte ((byte)((i >> 24) & 0xff)); + return true; + } + public object Load () { for (;;) @@ -183,6 +490,12 @@ namespace GameRes.Formats.RenPy break; continue; + case BINSTRING: + case BINUNICODE: + if (!LoadBinUnicode()) + break; + continue; + case EMPTY_LIST: if (!LoadEmptyList()) break; @@ -203,6 +516,11 @@ namespace GameRes.Formats.RenPy break; continue; + case INT: + if (!LoadInt()) + break; + continue; + case TUPLE2: if (!LoadCountedTuple (2)) break; @@ -218,6 +536,11 @@ namespace GameRes.Formats.RenPy break; continue; + case SETITEM: + if (!LoadSetItem()) + break; + continue; + case SETITEMS: if (!LoadSetItems()) break; @@ -232,7 +555,7 @@ namespace GameRes.Formats.RenPy return null; default: - Trace.TraceError ("Unknown Pickle serialization key {0:X2}", sym); + Trace.TraceError ("Unknown Pickle serialization opcode 0x{0:X2}", sym); return null; } break; @@ -266,7 +589,6 @@ namespace GameRes.Formats.RenPy int key = m_stream.ReadByte(); if (-1 == key || 0 == m_stack.Count) return false; -// m_memo[key] = m_stack.Peek(); return true; } @@ -275,7 +597,6 @@ namespace GameRes.Formats.RenPy int key; if (!ReadInt (4, out key) || 0 == m_stack.Count || key < 0) return false; -// m_memo[key] = m_stack.Peek(); return true; } @@ -300,6 +621,19 @@ namespace GameRes.Formats.RenPy int length = m_stream.ReadByte(); if (-1 == length) return false; + return LoadBinString (length); + } + + bool LoadBinUnicode () + { + int length; + if (!ReadInt (4, out length)) + return false; + return LoadBinString (length); + } + + bool LoadBinString (int length) + { var bytes = new byte[length]; if (length != m_stream.Read (bytes, 0, length)) return false; @@ -335,6 +669,16 @@ namespace GameRes.Formats.RenPy return true; } + bool LoadInt () + { + var num = m_stream.ReadStringUntil (0x0a, Encoding.ASCII); + long n; + if (!long.TryParse (num, NumberStyles.Integer, CultureInfo.InvariantCulture, out n)) + return false; + m_stack.Push (n); + return true; + } + bool LoadCountedTuple (int count) { if (m_stack.Count < count) @@ -353,7 +697,7 @@ namespace GameRes.Formats.RenPy bool LoadAppend () { int x = m_stack.Count - 1; - if (m_stack.Count < x || 0 == x) + if (x <= 0) { Trace.WriteLine ("Stack underflow", "LoadAppend"); return false; @@ -381,9 +725,18 @@ namespace GameRes.Formats.RenPy return list; } + bool LoadSetItem () + { + return DoSetItems (m_stack.Count-2); + } + bool LoadSetItems () { - int mark = GetMarker(); + return DoSetItems (GetMarker()); + } + + bool DoSetItems (int mark) + { if (!(m_stack.Count >= mark && mark > 0)) { Trace.WriteLine ("Stack underflow", "LoadSetItems"); @@ -408,9 +761,8 @@ namespace GameRes.Formats.RenPy { if (clearto < 0) return false; - if (clearto >= m_stack.Count) - return true; - m_stack.RemoveRange (clearto, m_stack.Count-clearto); + if (clearto < m_stack.Count) + m_stack.RemoveRange (clearto, m_stack.Count-clearto); return true; } } @@ -435,6 +787,59 @@ namespace GameRes.Formats.RenPy } } + internal class PyString : IEquatable + { + int m_hash; + byte[] m_bytes; + Lazy m_is_ascii; + + public PyString (string s) + { + m_hash = s.GetHashCode(); + m_bytes = Encoding.UTF8.GetBytes (s); + m_is_ascii = new Lazy (() => -1 == Array.FindIndex (m_bytes, x => x > 0x7f)); + } + + public PyString () : this ("") + { + } + + public bool IsAscii { get { return m_is_ascii.Value; } } + + public byte[] Bytes { get { return m_bytes; } } + + public int Length { get { return m_bytes.Length; } } + + public bool Equals (PyString other) + { + if (null == other) + return false; + if (this.m_hash != other.m_hash) + return false; + if (this.Length != other.Length) + return false; + for (var i = 0; i < m_bytes.Length; ++i) + if (m_bytes[i] != other.m_bytes[i]) + return false; + return true; + } + + public override bool Equals (object other) + { + return this.Equals (other as PyString); + } + + public override int GetHashCode () + { + return m_hash; + } + + public override string ToString () + { + return Encoding.UTF8.GetString (m_bytes); + } + } + public class RpaStream : Stream { byte[] m_header; diff --git a/ArcFormats/CreateRPAWidget.xaml b/ArcFormats/CreateRPAWidget.xaml new file mode 100644 index 00000000..c00d4f23 --- /dev/null +++ b/ArcFormats/CreateRPAWidget.xaml @@ -0,0 +1,35 @@ + + + + + + + + + diff --git a/ArcFormats/CreateRPAWidget.xaml.cs b/ArcFormats/CreateRPAWidget.xaml.cs new file mode 100644 index 00000000..ccc04015 --- /dev/null +++ b/ArcFormats/CreateRPAWidget.xaml.cs @@ -0,0 +1,39 @@ +using System.Globalization; +using System.Windows.Controls; +using System.Windows.Data; + +namespace GameRes.Formats.GUI +{ + /// + /// Interaction logic for CreateRPAWidget.xaml + /// + public partial class CreateRPAWidget : Grid + { + public CreateRPAWidget () + { + InitializeComponent (); + } + } + + [ValueConversion(typeof(uint), typeof(string))] + public class UInt32Converter : IValueConverter + { + public object Convert (object value, System.Type targetType, object parameter, CultureInfo culture) + { + if (null == value) + return ""; + uint key = (uint)value; + return key.ToString ("x"); + } + + public object ConvertBack (object value, System.Type targetType, object parameter, CultureInfo culture) + { + string strValue = value as string; + uint result_key; + if (uint.TryParse(strValue, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out result_key)) + return result_key; + else + return null; + } + } +} diff --git a/ArcFormats/Properties/Settings.Designer.cs b/ArcFormats/Properties/Settings.Designer.cs index 681b4a15..64b0d991 100644 --- a/ArcFormats/Properties/Settings.Designer.cs +++ b/ArcFormats/Properties/Settings.Designer.cs @@ -189,5 +189,17 @@ namespace GameRes.Formats.Properties { this["YPFVersion"] = value; } } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("1111638594")] + public uint RPAKey { + get { + return ((uint)(this["RPAKey"])); + } + set { + this["RPAKey"] = value; + } + } } } diff --git a/ArcFormats/Properties/Settings.settings b/ArcFormats/Properties/Settings.settings index e94deacf..638a30e5 100644 --- a/ArcFormats/Properties/Settings.settings +++ b/ArcFormats/Properties/Settings.settings @@ -44,5 +44,8 @@ 290 + + 1111638594 + \ No newline at end of file diff --git a/ArcFormats/Strings/arcStrings.Designer.cs b/ArcFormats/Strings/arcStrings.Designer.cs index cdd7cbcf..b7d8bafc 100644 --- a/ArcFormats/Strings/arcStrings.Designer.cs +++ b/ArcFormats/Strings/arcStrings.Designer.cs @@ -396,6 +396,15 @@ namespace GameRes.Formats.Strings { } } + /// + /// Looks up a localized string similar to 32-bit key. + /// + public static string RPALabelKey { + get { + return ResourceManager.GetString("RPALabelKey", resourceCulture); + } + } + /// /// Looks up a localized string similar to Amaterasu Translations Muv-Luv script file. /// diff --git a/ArcFormats/Strings/arcStrings.resx b/ArcFormats/Strings/arcStrings.resx index ea8b3257..22e4bf0f 100644 --- a/ArcFormats/Strings/arcStrings.resx +++ b/ArcFormats/Strings/arcStrings.resx @@ -231,6 +231,9 @@ predefined encryption scheme. Ren'Py game engine archive + + 32-bit key + Amaterasu Translations Muv-Luv script file diff --git a/ArcFormats/Strings/arcStrings.ru-RU.resx b/ArcFormats/Strings/arcStrings.ru-RU.resx index a3116a53..e9dadca1 100644 --- a/ArcFormats/Strings/arcStrings.ru-RU.resx +++ b/ArcFormats/Strings/arcStrings.ru-RU.resx @@ -201,6 +201,9 @@ Шифровать содержимое + + 32-битный ключ + Кодировка имён файлов diff --git a/ArcFormats/app.config b/ArcFormats/app.config index 50534fe7..760fad6a 100644 --- a/ArcFormats/app.config +++ b/ArcFormats/app.config @@ -46,6 +46,9 @@ 290 + + 1111638594 +