add leveldb support

game_backuper/backuper.py

@@ -8,11 +8,9 @@ from game_backuper.config import (
 from game_backuper.cml import Opts, OptAction
 from threading import Thread
 from os.path import exists, join
-from os import mkdir
-from game_backuper.file import new_file, copy_file
-from game_backuper.leveldb import have_leveldb
-if have_leveldb:
-    from game_backuper.leveldb import list_leveldb_entries
+from os import mkdir, remove
+from game_backuper.file import new_file, copy_file, File, mkdir_for_file
+from game_backuper.filetype import FileType
 
 
 class BackupTask(Thread):
@@ -46,9 +44,38 @@ class BackupTask(Thread):
                         copy_file(f[1], de, f[0], prog)
                         self.db.add_file(nf)
             elif isinstance(f, ConfigLeveldb):
+                from game_backuper.leveldb import have_leveldb
                 if not have_leveldb:
                     raise ValueError('Leveldb is not supported.')
-                print(list_leveldb_entries(f.full_path, f.domains))
+                from game_backuper.leveldb import (
+                    list_leveldb_entries,
+                    leveldb_stats,
+                    leveldb_to_sqlite,
+                )
+                ent = list_leveldb_entries(f.full_path, f.domains)
+                stats = leveldb_stats(f.full_path, ent)
+                ori = self.db.get_file(prog, f.name)
+                de = join(bp, f.name + ".db")
+                if ori is not None:
+                    if ori.size == stats.size and ori.hash == stats.hash:
+                        continue
+                    if ori.type is None or ori.type != FileType.LEVELDB:
+                        pp = join(bp, ori.file)
+                        if exists(pp):
+                            remove(pp)
+                        self.db.remove_file(ori)
+                        ori = None
+                if exists(de):
+                    remove(de)
+                mkdir_for_file(de)
+                leveldb_to_sqlite(f.full_path, de, ent)
+                print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {de}')  # noqa: E501
+                if ori is None:
+                    nf = File(None, f.name, stats.size, prog, stats.hash,
+                              FileType.LEVELDB)
+                    self.db.add_file(nf)
+                else:
+                    self.db.set_file(ori.id, stats.size, stats.hash)
 
 
 class Backuper:

game_backuper/db.py

@@ -1,6 +1,6 @@
 from sqlite3 import connect
 from os.path import join
-from typing import List
+from typing import List, Union
 from threading import Lock
 from game_backuper.file import File
 from game_backuper.filetype import FileType
@@ -107,10 +107,25 @@ class Db:
                 'SELECT files.*, filetype.type FROM files LEFT JOIN filetype ON files.id=filetype.id WHERE program=? AND file=?;',  # noqa: E501
                 (prog, file))
             for i in cur:
-                if i[5] is not None:
-                    i[5] = FileType(i[5])
                 return File(*i)
 
+    def remove_file(self, id: Union[int, File]):
+        with self._lock:
+            ft = None
+            if isinstance(id, File):
+                iid = id.id
+                ft = id.type
+            else:
+                cur = self.db.execute('SELECT type FROM filetype WHERE id=?;',
+                                      (id,))
+                for i in cur:
+                    ft = FileType(i[0])
+                iid = id
+            self.db.execute('DELETE FROM files WHERE id=?;', (iid,))
+            if ft is not None:
+                self.db.execute('DELETE FROM filetype WHERE id=?;', (iid,))
+            self.db.commit()
+
     def set_file(self, id: int, size: int, hash: str):
         with self._lock:
             self.db.execute('UPDATE files SET size=?, hash=? WHERE id=?;',

game_backuper/file.py

@@ -9,10 +9,14 @@ from game_backuper.filetype import FileType
 File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type'])
 
 
-def copy_file(loc: str, dest: str, name: str, prog: str):
-    d = dirname(abspath(dest))
+def mkdir_for_file(p: str):
+    d = dirname(abspath(p))
     if not exists(d):
         makedirs(d)
+
+
+def copy_file(loc: str, dest: str, name: str, prog: str):
+    mkdir_for_file(dest)
     r = copy2(loc, dest)
     print(f'{prog}: Copyed {loc}({name}) -> {r}')
 

game_backuper/leveldb.py

@@ -2,11 +2,22 @@ try:
     from plyvel import DB
     have_leveldb = True
     from typing import List
+    from hashlib import sha512
+    from base64 import b85encode
+    from collections import namedtuple
+    from sqlite3 import connect
 except ImportError:
     have_leveldb = False
 
 
 if have_leveldb:
+    LeveldbStats = namedtuple('LeveldbStats', ['hash', 'size'])
+    MAP_TABLE = '''CREATE TABLE map (
+    key TEXT,
+    value TEXT,
+    PRIMARY KEY(key)
+    )'''
+
     def list_leveldb_entries(db: str, dms: List[bytes] = None):
         d = DB(db)
         r = []
@@ -28,3 +39,31 @@ if have_leveldb:
                             r.append(i)
         r.sort()
         return r
+
+    def leveldb_stats(db: str, entries: List[bytes]) -> LeveldbStats:
+        d = DB(db)
+        h = sha512()
+        le = 0
+        for e in entries:
+            v = d.get(e)
+            if v is not None:
+                h.update(e)
+                h.update(v)
+                le += len(e) + len(v)
+        d.close()
+        return LeveldbStats(b85encode(h.digest()).decode(), le)
+
+    def leveldb_to_sqlite(db: str, dest: str, entries: List[bytes]):
+        d = DB(db)
+        s = connect(dest)
+        s.text_factory = bytes
+        s.execute(MAP_TABLE)
+        for e in entries:
+            v = d.get(e)
+            if v is not None:
+                s.execute('INSERT INTO map VALUES (?, ?);', (e, v))
+        s.commit()
+        s.execute('VACUUM;')
+        s.commit()
+        d.close()
+        s.close()