add restore support for leveldb

game_backuper/config.py

@@ -97,6 +97,10 @@ class BasicConfig:
             if isinstance(v, str) and len(v) > 0:
                 return v
 
+    @cached_property
+    def real_name(self) -> str:
+        return self.name if self.name else self.path
+
     @cached_property
     def path(self) -> str:
         data = getattr(self, "data", None)
@@ -151,7 +155,7 @@ class ConfigOLeveldb(BasicOption, NFBasicOption, BasicConfig):
                 dms = []
                 for i in self.data['domains']:
                     if isinstance(i, str) and len(i) > 0:
-                        dms.append(i)
+                        dms.append(i.encode())
                 if len(dms) > 0:
                     return dms
 
@@ -308,6 +312,9 @@ class Program(BasicOption, NFBasicOption):
                                 n = i['name']
                     if not relpath(name, n).startswith('..'):
                         return ConfigPath(i, self._cfg, self)
+                elif t == 'leveldb':
+                    if relpath(i['name'], name) == '.':
+                        return ConfigOLeveldb(i, self._cfg, self)
 
     @cached_property
     def name(self) -> str:

game_backuper/leveldb.py

@@ -6,7 +6,7 @@ except ImportError:
 
 
 if have_leveldb:
-    from typing import List
+    from typing import List, Union
     from hashlib import sha512
     from base64 import b85encode
     from collections import namedtuple
@@ -18,8 +18,11 @@ if have_leveldb:
     PRIMARY KEY(key)
     )'''
 
-    def list_leveldb_entries(db: str, dms: List[bytes] = None):
-        d = DB(db)
+    def list_leveldb_entries(db: Union[str, DB], dms: List[bytes] = None):
+        if isinstance(db, str):
+            d = DB(db)
+        else:
+            d = db
         r = []
         for i in d.iterator(include_value=False):
             if isinstance(i, bytes):
@@ -38,6 +41,8 @@ if have_leveldb:
                         if dm in dms:
                             r.append(i)
         r.sort()
+        if isinstance(db, str):
+            d.close()
         return r
 
     def leveldb_stats(db: str, entries: List[bytes]) -> LeveldbStats:
@@ -67,3 +72,33 @@ if have_leveldb:
         s.commit()
         d.close()
         s.close()
+
+    def sqlite_to_leveldb(db: str, dest: str, dms: List[bytes]):
+        s = connect(db)
+        s.text_factory = bytes
+        d = DB(dest, create_if_missing=True)
+        try:
+            ents = list_leveldb_entries(d, dms)
+            for i in ents:
+                d.delete(i)
+            cur = s.execute('SELECT * FROM map;')
+            for i in cur:
+                if dms is None:
+                    d.put(i[0], i[1])
+                else:
+                    if i[0] == b'VERSION':
+                        d.put(i[0], i[1])
+                    elif i[0].startswith(b'META:'):
+                        dm = i[0][5:]
+                        if dm in dms:
+                            d.put(i[0], i[1])
+                    elif i[0].startswith(b'_'):
+                        dmi = i[0].find(b'\x00\x01')
+                        dm = i[0][1:dmi]
+                        if dm in dms:
+                            d.put(i[0], i[1])
+        except Exception:
+            from traceback import print_exc
+            print_exc()
+        d.close()
+        s.close()

game_backuper/restorer.py

@@ -1,5 +1,5 @@
 from threading import Thread
-from game_backuper.config import Config, Program, ConfigPath
+from game_backuper.config import Config, Program, ConfigPath, ConfigOLeveldb
 from game_backuper.db import Db
 from os.path import join, relpath, isabs, isfile, isdir, exists
 from game_backuper.file import (
@@ -7,8 +7,10 @@ from game_backuper.file import (
     copy_file,
     remove_dirs,
     new_file,
+    mkdir_for_file,
 )
 from os import remove
+from game_backuper.filetype import FileType
 
 
 class RestoreTask(Thread):
@@ -30,7 +32,7 @@ class RestoreTask(Thread):
             if isinstance(r, ConfigPath):
                 if f.type is not None:
                     raise ValueError('Type dismatched.')
-                nam = r.name if r.name else r.path
+                nam = r.real_name
                 src = join(self.cfg.dest, prog, fn)
                 tmp = relpath(fn, nam)
                 if isabs(r.path):
@@ -41,12 +43,46 @@ class RestoreTask(Thread):
                     dest = join(dest, tmp)
                 if dest in pl:
                     pl.remove(dest)
+                if not exists(src):
+                    print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})')  # noqa: E501
+                    continue
                 if exists(dest):
                     tf = new_file(dest, nam, prog)
                     if tf.size == f.size and tf.hash == f.hash:
-                        print(f'{prog}: Skip {f.file}')
+                        print(f'{prog}: Skip {fn}')
                         continue
                 copy_file(src, dest, nam, prog)
+            elif isinstance(r, ConfigOLeveldb):
+                from game_backuper.leveldb import have_leveldb
+                if not have_leveldb:
+                    raise NotImplementedError('Leveldb is not supported.')
+                if f.type != FileType.LEVELDB:
+                    raise ValueError('Type dismatched.')
+                nam = r.real_name
+                src = join(self.cfg.dest, prog, fn + '.db')
+                if isabs(r.path):
+                    dest = r.path
+                else:
+                    dest = join(b, r.path)
+                if dest in pl:
+                    pl.remove(dest)
+                if not exists(src):
+                    print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})')  # noqa: E501
+                    continue
+                from game_backuper.leveldb import (
+                    sqlite_to_leveldb,
+                    leveldb_stats,
+                    list_leveldb_entries,
+                )
+                if exists(dest):
+                    ents = list_leveldb_entries(dest, r.domains)
+                    stat = leveldb_stats(dest, ents)
+                    if f.size == stat.size and f.hash == stat.hash:
+                        print(f'{prog}: Skip {fn}')
+                        continue
+                mkdir_for_file(dest)
+                sqlite_to_leveldb(src, dest, r.domains)
+                print(f'{prog}: Covert leveldb done. {src}({fn}) -> {dest}')
         for i in pl:
             if isfile(i):
                 remove(i)