implemented "strong" QLIE archives encryption.

ArcFormats/ArcFormats.csproj

@@ -87,6 +87,9 @@
     <Compile Include="Qlie\ArcABMP.cs" />
     <Compile Include="Qlie\ImageDPNG.cs" />
     <Compile Include="Qlie\QlieMersenneTwister.cs" />
+    <Compile Include="Qlie\WidgetQLIE.xaml.cs">
+      <DependentUpon>WidgetQLIE.xaml</DependentUpon>
+    </Compile>
     <Compile Include="Sas5\ArcIAR.cs" />
     <Compile Include="Sas5\ArcSec5.cs" />
     <Compile Include="Sas5\ArcWAR.cs" />
@@ -393,6 +396,10 @@
       <SubType>Designer</SubType>
       <Generator>MSBuild:Compile</Generator>
     </Page>
+    <Page Include="Qlie\WidgetQLIE.xaml">
+      <SubType>Designer</SubType>
+      <Generator>MSBuild:Compile</Generator>
+    </Page>
     <Page Include="RenPy\CreateRPAWidget.xaml">
       <SubType>Designer</SubType>
       <Generator>MSBuild:Compile</Generator>

ArcFormats/Properties/Settings.Designer.cs

@@ -453,5 +453,17 @@ namespace GameRes.Formats.Properties {
                 this["RPMScheme"] = value;
             }
         }
+        
+        [global::System.Configuration.UserScopedSettingAttribute()]
+        [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]
+        [global::System.Configuration.DefaultSettingValueAttribute("")]
+        public string QLIEScheme {
+            get {
+                return ((string)(this["QLIEScheme"]));
+            }
+            set {
+                this["QLIEScheme"] = value;
+            }
+        }
     }
 }

ArcFormats/Properties/Settings.settings

@@ -110,5 +110,8 @@
     <Setting Name="RPMScheme" Type="System.String" Scope="User">
       <Value Profile="(Default)" />
     </Setting>
+    <Setting Name="QLIEScheme" Type="System.String" Scope="User">
+      <Value Profile="(Default)" />
+    </Setting>
   </Settings>
 </SettingsFile>

ArcFormats/Qlie/ArcQLIE.cs

@@ -23,12 +23,15 @@
 // IN THE SOFTWARE.
 //
 
+using System;
 using System.IO;
-using System.Text;
 using System.Collections.Generic;
 using System.ComponentModel.Composition;
-using System;
+using System.Diagnostics;
+using System.Linq;
 using GameRes.Utility;
+using GameRes.Formats.Properties;
+using GameRes.Formats.Strings;
 
 namespace GameRes.Formats.Qlie
 {
@@ -36,7 +39,46 @@ namespace GameRes.Formats.Qlie
     {
         public bool IsEncrypted;
         public uint Hash;
-        public uint Key;
+        public byte[] RawName;
+
+        /// <summary>
+        /// Data from a separate key file "key.fkey" that comes with installed game.
+        /// null if not used.
+        /// </summary>
+        public byte[] KeyFile;
+    }
+
+    internal class QlieArchive : ArcFile
+    {
+        /// <summary>
+        /// Hash generated from the key data contained within archive index.
+        /// </summary>
+        public uint Hash;
+
+        /// <summary>
+        /// Internal game data used to decrypt encrypted entries.
+        /// null if not used.
+        /// </summary>
+        public byte[] GameKeyData;
+
+        public QlieArchive (ArcView arc, ArchiveFormat impl, ICollection<Entry> dir,
+                            uint hash, byte[] key_data)
+            : base (arc, impl, dir)
+        {
+            Hash = hash;
+            GameKeyData = key_data;
+        }
+    }
+
+    internal class QlieOptions : ResourceOptions
+    {
+        public byte[] GameKeyData;
+    }
+
+    [Serializable]
+    public class QlieScheme : ResourceScheme
+    {
+        public Dictionary<string, byte[]> KnownKeys;
     }
 
     [Export(typeof(ArchiveFormat))]
@@ -53,6 +95,19 @@ namespace GameRes.Formats.Qlie
             Extensions = new string [] { "pack" };
         }
 
+        /// <summary>
+        /// Possible locations of the 'key.fkey' file relative to an archive being accessed.
+        /// </summary>
+        static readonly string[] KeyLocations = { ".", "..", @"..\DLL", "DLL" };
+
+        public static Dictionary<string, byte[]> KnownKeys = new Dictionary<string, byte[]>();
+
+        public override ResourceScheme Scheme
+        {
+            get { return new QlieScheme { KnownKeys = KnownKeys }; }
+            set { KnownKeys = ((QlieScheme)value).KnownKeys; }
+        }
+
         public override ArcFile TryOpen (ArcView file)
         {
             if (file.MaxOffset <= 0x1c)
@@ -71,15 +126,20 @@ namespace GameRes.Formats.Qlie
             if (index_offset < 0 || index_offset >= file.MaxOffset)
                 return null;
 
-            uint pack_key;
+            byte[] arc_key = null;
+            byte[] key_file = null;
+            uint name_key = 0xC4; // default name encryption key for versions 1 and 2
             if (3 == pack_version)
             {
+                key_file = FindKeyFile (file);
+                // currently, user is prompted to choose encryption scheme only if there's 'key.fkey' file found.
+                if (key_file != null)
+                    arc_key = QueryEncryption();
+
                 var key_data = new byte[0x100];
                 file.View.Read (file.MaxOffset-0x41C, key_data, 0, 0x100);
-                pack_key = GenerateKey (key_data) & 0x0FFFFFFFu;
+                name_key = GenerateKey (key_data) & 0x0FFFFFFFu;
             }
-            else
-                pack_key = 0xC4;
 
             var name_buffer = new byte[0x100];
             var dir = new List<Entry> (count);
@@ -91,12 +151,14 @@ namespace GameRes.Formats.Qlie
                 if (name_length != file.View.Read (index_offset+2, name_buffer, 0, (uint)name_length))
                     return null;
 
-                int key = name_length + ((int)pack_key ^ 0x3e);
+                int key = name_length + ((int)name_key ^ 0x3e);
                 for (int k = 0; k < name_length; ++k)
                     name_buffer[k] ^= (byte)(((k + 1) ^ key) + k + 1);
 
                 string name = Encodings.cp932.GetString (name_buffer, 0, name_length);
                 var entry = FormatCatalog.Instance.Create<QlieEntry> (name);
+                if (key_file != null)
+                    entry.RawName = name_buffer.Take (name_length).ToArray();
 
                 index_offset += 2 + name_length;
                 entry.Offset = file.View.ReadInt64 (index_offset);
@@ -107,34 +169,48 @@ namespace GameRes.Formats.Qlie
                 entry.IsPacked = 0 != file.View.ReadInt32 (index_offset+0x10);
                 entry.IsEncrypted = 0 != file.View.ReadInt32 (index_offset+0x14);
                 entry.Hash = file.View.ReadUInt32 (index_offset+0x18);
-                if (3 == pack_version)
+                entry.KeyFile = key_file;
-                    entry.Key = pack_key;
+                if (3 == pack_version && entry.Name.Contains ("pack_keyfile"))
-                else
+                {
-                    entry.Key = 0;
+                    // note that 'pack_keyfile' itself is encrypted using 'key.fkey' file contents.
+                    key_file = ReadEntryBytes (file, entry, name_key, arc_key);
+                }
                 dir.Add (entry);
                 index_offset += 0x1c;
             }
-            return new ArcFile (file, this, dir);
+            if (pack_version < 3)
+                name_key = 0;
+            return new QlieArchive (file, this, dir, name_key, arc_key);
         }
 
         public override Stream OpenEntry (ArcFile arc, Entry entry)
         {
             var qent = entry as QlieEntry;
-            if (null == qent || (!qent.IsEncrypted && !qent.IsPacked))
+            var qarc = arc as QlieArchive;
-                return arc.File.CreateStream (entry.Offset, entry.Size);
+            if (null == qent || null == qarc || (!qent.IsEncrypted && !qent.IsPacked))
-            var data = new byte[entry.Size];
-            if (entry.Size != arc.File.View.Read (entry.Offset, data, 0, entry.Size))
                 return arc.File.CreateStream (entry.Offset, entry.Size);
+            var data = ReadEntryBytes (arc.File, qent, qarc.Hash, qarc.GameKeyData);
+            return new MemoryStream (data);
+        }
 
-            if (qent.IsEncrypted)
+        private byte[] ReadEntryBytes (ArcView file, QlieEntry entry, uint hash, byte[] game_key)
-                Decrypt (data, 0, data.Length, qent.Key);
+        {
-            if (qent.IsPacked)
+            var data = new byte[entry.Size];
+            file.View.Read (entry.Offset, data, 0, entry.Size);
+            if (entry.IsEncrypted)
+            {
+                if (entry.KeyFile != null)
+                    DecryptV3 (data, 0, data.Length, entry.RawName, hash, entry.KeyFile, game_key);
+                else
+                    Decrypt (data, 0, data.Length, hash);
+            }
+            if (entry.IsPacked)
             {
                 var unpacked = Decompress (data);
                 if (null != unpacked)
                     data = unpacked;
             }
-            return new MemoryStream (data);
+            return data;
         }
 
         private void Decrypt (byte[] buffer, int offset, int length, uint key)
@@ -162,6 +238,74 @@ namespace GameRes.Formats.Qlie
             }
         }
 
+        private void DecryptV3 (byte[] data, int offset, int length, byte[] file_name,
+                                uint arc_hash, byte[] key_file, byte[] game_key)
+        {
+            // play it safe with 'unsafe' sections
+            if (offset < 0)
+                throw new ArgumentOutOfRangeException ("offset");
+            if (length > data.Length || offset > data.Length - length)
+                throw new ArgumentOutOfRangeException ("length");
+
+            if (length < 8)
+                return;
+
+            uint hash = 0x85F532;
+            uint seed = 0x33F641;
+
+            for (uint i = 0; i < file_name.Length; i++)
+            {
+                hash += (i & 0xFF) * file_name[i];
+                seed ^= hash;
+            }
+
+            seed += arc_hash ^ (7 * ((uint)data.Length & 0xFFFFFF) + (uint)data.Length
+                                + hash + (hash ^ (uint)data.Length ^ 0x8F32DCu));
+            seed = 9 * (seed & 0xFFFFFF);
+
+            if (game_key != null)
+                seed ^= 0x453A;
+
+            var mt = new QlieMersenneTwister (seed);
+            if (key_file != null)
+                mt.XorState (key_file);
+            if (game_key != null)
+                mt.XorState (game_key);
+
+            // game code fills dword[41] table, but only the first 16 qwords are used
+            ulong[] table = new ulong[16];
+            for (int i = 0; i < table.Length; ++i)
+                table[i] = mt.Rand64();
+
+            // compensate for 9 discarded dwords
+            for (int i = 0; i < 9; ++i)
+                mt.Rand();
+
+            ulong hash64 = mt.Rand64();
+            uint t = mt.Rand() & 0xF;
+            unsafe
+            {
+                fixed (byte* raw_data = &data[offset])
+                {
+                    ulong* data64 = (ulong*)raw_data;
+                    int qword_length = length / 8;
+                    for (int i = 0; i < qword_length; ++i)
+                    {
+                        hash64 = MMX.PAddD (hash64 ^ table[t], table[t]);
+
+                        ulong d = data64[i] ^ hash64;
+                        data64[i] = d;
+
+                        hash64 = MMX.PAddB (hash64, d) ^ d;
+                        hash64 = MMX.PAddW (MMX.PSllD (hash64, 1), d);
+
+                        t++;
+                        t &= 0xF;
+                    }
+                }
+            }
+        }
+
         internal static byte[] Decompress (byte[] input)
         {
             if (LittleEndian.ToUInt32 (input, 0) != 0xFF435031)
@@ -265,5 +409,52 @@ namespace GameRes.Formats.Qlie
             }
             return (uint)(key ^ (key >> 32));
         }
+
+        public override ResourceOptions GetDefaultOptions ()
+        {
+            return new QlieOptions {
+                GameKeyData = GetKeyData (Settings.Default.QLIEScheme)
+            };
+        }
+
+        public override object GetAccessWidget ()
+        {
+            return new GUI.WidgetQLIE();
+        }
+
+        byte[] QueryEncryption ()
+        {
+            var options = Query<QlieOptions> (arcStrings.ArcEncryptedNotice);
+            return options.GameKeyData;
+        }
+
+        static byte[] GetKeyData (string scheme)
+        {
+            byte[] key;
+            if (KnownKeys.TryGetValue (scheme, out key))
+                return key;
+            return null;
+        }
+
+        /// <summary>
+        /// Look for 'key.fkey' file within nearby directories specified by KeyLocations.
+        /// </summary>
+        static byte[] FindKeyFile (ArcView arc_file)
+        {
+            // QLIE archives with key could be opened at the physical file system level only
+            if (VFS.IsVirtual)
+                return null;
+            var dir_name = Path.GetDirectoryName (arc_file.Name);
+            foreach (var path in KeyLocations)
+            {
+                var name = Path.Combine (dir_name, path, "key.fkey");
+                if (File.Exists (name))
+                {
+                    Trace.WriteLine ("reading key from "+name, "[QLIE]");
+                    return File.ReadAllBytes (name);
+                }
+            }
+            return null;
+        }
     }
 }

ArcFormats/Qlie/WidgetQLIE.xaml

@@ -0,0 +1,9 @@
+<StackPanel x:Class="GameRes.Formats.GUI.WidgetQLIE"
+            xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+            xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+            xmlns:p="clr-namespace:GameRes.Formats.Properties"
+            Orientation="Vertical" MaxWidth="250">
+    <ComboBox Name="Scheme" ItemsSource="{Binding}"
+              SelectedValue="{Binding Source={x:Static p:Settings.Default}, Path=QLIEScheme, Mode=TwoWay}"
+              Width="200" Grid.Row="0"/>
+</StackPanel>

ArcFormats/Qlie/WidgetQLIE.xaml.cs

@@ -0,0 +1,22 @@
+using GameRes.Formats.Qlie;
+using GameRes.Formats.Strings;
+using System.Windows.Controls;
+using System.Linq;
+
+namespace GameRes.Formats.GUI
+{
+    /// <summary>
+    /// Interaction logic for WidgetQLIE.xaml
+    /// </summary>
+    public partial class WidgetQLIE : StackPanel
+    {
+        public WidgetQLIE ()
+        {
+            InitializeComponent ();
+            var keys = new string[] { arcStrings.QLIEDefaultScheme };
+            Scheme.ItemsSource = keys.Concat (PackOpener.KnownKeys.Keys.OrderBy (x => x));
+            if (-1 == Scheme.SelectedIndex)
+                Scheme.SelectedIndex = 0;
+        }
+    }
+}

ArcFormats/Strings/arcStrings.Designer.cs

@@ -533,6 +533,15 @@ namespace GameRes.Formats.Strings {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized string similar to Use default encryption scheme.
+        /// </summary>
+        public static string QLIEDefaultScheme {
+            get {
+                return ResourceManager.GetString("QLIEDefaultScheme", resourceCulture);
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized string similar to Choose title or enter a password.
         /// </summary>

ArcFormats/Strings/arcStrings.resx

@@ -339,4 +339,7 @@ Choose encryption scheme or enter a passphrase.</value>
 but encryption key guess failed.
 Choose appropriate encryption scheme.</value>
   </data>
+  <data name="QLIEDefaultScheme" xml:space="preserve">
+    <value>Use default encryption scheme</value>
+  </data>
 </root>

ArcFormats/Strings/arcStrings.ru-RU.resx

@@ -248,6 +248,9 @@
   <data name="PDScrambleContents" xml:space="preserve">
     <value>Шифровать содержимое</value>
   </data>
+  <data name="QLIEDefaultScheme" xml:space="preserve">
+    <value>"Старый" метод шифрования</value>
+  </data>
   <data name="RCTChoose" xml:space="preserve">
     <value>Выберите наименование или введите пароль</value>
   </data>

ArcFormats/app.config

@@ -112,6 +112,9 @@
             <setting name="RPMScheme" serializeAs="String">
                 <value />
             </setting>
+            <setting name="QLIEScheme" serializeAs="String">
+                <value />
+            </setting>
         </GameRes.Formats.Properties.Settings>
     </userSettings>
 <startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup></configuration>

supported.html

@@ -313,11 +313,13 @@ Ryoujoku Gojuusou<br/>
 <tr class="odd"><td>*.cwd</td><td><tt>cwd</tt></td><td>No</td></tr>
 <tr class="odd"><td>*.eog</td><td><tt>CRM</tt></td><td>No</td></tr>
 <tr class="odd"><td>*.zbm<br/>*.cwl</td><td><tt>SZDD</tt></td><td>No</td></tr>
-<tr><td>*.pack</td><td><tt>FilePackVer1.0</tt><br/><tt>FilePackVer2.0</tt><br/><tt>FilePackVer3.0</tt></td><td>No</td><td rowspan="2">QLIE</td><td rowspan="2">
+<tr><td>*.pack</td><td><tt>FilePackVer1.0</tt><br/><tt>FilePackVer2.0</tt><br/><tt>FilePackVer3.0</tt></td><td>No</td><td rowspan="3">QLIE</td><td rowspan="3">
 Mehime no Toriko<br/>
+Nanatsu no Fushigi no Owaru Toki<br/>
 Soshite Ashita no Sekai yori<br/>
 </td></tr>
 <tr><td>*.b</td><td><tt>ABMP7</tt><br/><tt>abmp10</tt><br/><tt>abmp11</tt></td><td>No</td></tr>
+<tr><td>*.png</td><td><tt>DPNG</tt></td><td>No</td></tr>
 <tr class="odd"><td>*.dat</td><td>-</td><td>No</td><td rowspan="3">Circus</td><td rowspan="3">
 Maid no Yakata ~Zetsubou Hen~<br/>
 </td></tr>