From d0257d6de71838253c0d8bef70b6d83a9033f06a Mon Sep 17 00:00:00 2001 From: lifegpc Date: Mon, 6 Sep 2021 14:14:28 +0800 Subject: [PATCH] add leveldb support --- game_backuper/backuper.py | 39 +++++++++++++++++++++++++++++++++------ game_backuper/db.py | 21 ++++++++++++++++++--- game_backuper/file.py | 8 ++++++-- game_backuper/leveldb.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 11 deletions(-) diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py index c8d70ba..1c0aad8 100644 --- a/game_backuper/backuper.py +++ b/game_backuper/backuper.py @@ -8,11 +8,9 @@ from game_backuper.config import ( from game_backuper.cml import Opts, OptAction from threading import Thread from os.path import exists, join -from os import mkdir -from game_backuper.file import new_file, copy_file -from game_backuper.leveldb import have_leveldb -if have_leveldb: - from game_backuper.leveldb import list_leveldb_entries +from os import mkdir, remove +from game_backuper.file import new_file, copy_file, File, mkdir_for_file +from game_backuper.filetype import FileType class BackupTask(Thread): @@ -46,9 +44,38 @@ class BackupTask(Thread): copy_file(f[1], de, f[0], prog) self.db.add_file(nf) elif isinstance(f, ConfigLeveldb): + from game_backuper.leveldb import have_leveldb if not have_leveldb: raise ValueError('Leveldb is not supported.') - print(list_leveldb_entries(f.full_path, f.domains)) + from game_backuper.leveldb import ( + list_leveldb_entries, + leveldb_stats, + leveldb_to_sqlite, + ) + ent = list_leveldb_entries(f.full_path, f.domains) + stats = leveldb_stats(f.full_path, ent) + ori = self.db.get_file(prog, f.name) + de = join(bp, f.name + ".db") + if ori is not None: + if ori.size == stats.size and ori.hash == stats.hash: + continue + if ori.type is None or ori.type != FileType.LEVELDB: + pp = join(bp, ori.file) + if exists(pp): + remove(pp) + self.db.remove_file(ori) + ori = None + if exists(de): + remove(de) + mkdir_for_file(de) + leveldb_to_sqlite(f.full_path, de, ent) + print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {de}') # noqa: E501 + if ori is None: + nf = File(None, f.name, stats.size, prog, stats.hash, + FileType.LEVELDB) + self.db.add_file(nf) + else: + self.db.set_file(ori.id, stats.size, stats.hash) class Backuper: diff --git a/game_backuper/db.py b/game_backuper/db.py index 59f527e..7f6ed1c 100644 --- a/game_backuper/db.py +++ b/game_backuper/db.py @@ -1,6 +1,6 @@ from sqlite3 import connect from os.path import join -from typing import List +from typing import List, Union from threading import Lock from game_backuper.file import File from game_backuper.filetype import FileType @@ -107,10 +107,25 @@ class Db: 'SELECT files.*, filetype.type FROM files LEFT JOIN filetype ON files.id=filetype.id WHERE program=? AND file=?;', # noqa: E501 (prog, file)) for i in cur: - if i[5] is not None: - i[5] = FileType(i[5]) return File(*i) + def remove_file(self, id: Union[int, File]): + with self._lock: + ft = None + if isinstance(id, File): + iid = id.id + ft = id.type + else: + cur = self.db.execute('SELECT type FROM filetype WHERE id=?;', + (id,)) + for i in cur: + ft = FileType(i[0]) + iid = id + self.db.execute('DELETE FROM files WHERE id=?;', (iid,)) + if ft is not None: + self.db.execute('DELETE FROM filetype WHERE id=?;', (iid,)) + self.db.commit() + def set_file(self, id: int, size: int, hash: str): with self._lock: self.db.execute('UPDATE files SET size=?, hash=? WHERE id=?;', diff --git a/game_backuper/file.py b/game_backuper/file.py index c8847bd..76407d5 100644 --- a/game_backuper/file.py +++ b/game_backuper/file.py @@ -9,10 +9,14 @@ from game_backuper.filetype import FileType File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type']) -def copy_file(loc: str, dest: str, name: str, prog: str): - d = dirname(abspath(dest)) +def mkdir_for_file(p: str): + d = dirname(abspath(p)) if not exists(d): makedirs(d) + + +def copy_file(loc: str, dest: str, name: str, prog: str): + mkdir_for_file(dest) r = copy2(loc, dest) print(f'{prog}: Copyed {loc}({name}) -> {r}') diff --git a/game_backuper/leveldb.py b/game_backuper/leveldb.py index e740162..0991c94 100644 --- a/game_backuper/leveldb.py +++ b/game_backuper/leveldb.py @@ -2,11 +2,22 @@ try: from plyvel import DB have_leveldb = True from typing import List + from hashlib import sha512 + from base64 import b85encode + from collections import namedtuple + from sqlite3 import connect except ImportError: have_leveldb = False if have_leveldb: + LeveldbStats = namedtuple('LeveldbStats', ['hash', 'size']) + MAP_TABLE = '''CREATE TABLE map ( + key TEXT, + value TEXT, + PRIMARY KEY(key) + )''' + def list_leveldb_entries(db: str, dms: List[bytes] = None): d = DB(db) r = [] @@ -28,3 +39,31 @@ if have_leveldb: r.append(i) r.sort() return r + + def leveldb_stats(db: str, entries: List[bytes]) -> LeveldbStats: + d = DB(db) + h = sha512() + le = 0 + for e in entries: + v = d.get(e) + if v is not None: + h.update(e) + h.update(v) + le += len(e) + len(v) + d.close() + return LeveldbStats(b85encode(h.digest()).decode(), le) + + def leveldb_to_sqlite(db: str, dest: str, entries: List[bytes]): + d = DB(db) + s = connect(dest) + s.text_factory = bytes + s.execute(MAP_TABLE) + for e in entries: + v = d.get(e) + if v is not None: + s.execute('INSERT INTO map VALUES (?, ?);', (e, v)) + s.commit() + s.execute('VACUUM;') + s.commit() + d.close() + s.close()