From eed35713ea84605fbca31a9c06da109c4982bd01 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Fri, 10 Sep 2021 12:09:02 +0800 Subject: [PATCH] add restore support for leveldb --- game_backuper/config.py | 9 ++++++++- game_backuper/leveldb.py | 41 +++++++++++++++++++++++++++++++++++--- game_backuper/restorer.py | 42 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/game_backuper/config.py b/game_backuper/config.py index 0c9c61d..c6d63a5 100644 --- a/game_backuper/config.py +++ b/game_backuper/config.py @@ -97,6 +97,10 @@ class BasicConfig: if isinstance(v, str) and len(v) > 0: return v + @cached_property + def real_name(self) -> str: + return self.name if self.name else self.path + @cached_property def path(self) -> str: data = getattr(self, "data", None) @@ -151,7 +155,7 @@ class ConfigOLeveldb(BasicOption, NFBasicOption, BasicConfig): dms = [] for i in self.data['domains']: if isinstance(i, str) and len(i) > 0: - dms.append(i) + dms.append(i.encode()) if len(dms) > 0: return dms @@ -308,6 +312,9 @@ class Program(BasicOption, NFBasicOption): n = i['name'] if not relpath(name, n).startswith('..'): return ConfigPath(i, self._cfg, self) + elif t == 'leveldb': + if relpath(i['name'], name) == '.': + return ConfigOLeveldb(i, self._cfg, self) @cached_property def name(self) -> str: diff --git a/game_backuper/leveldb.py b/game_backuper/leveldb.py index a7ef6ba..8bcf384 100644 --- a/game_backuper/leveldb.py +++ b/game_backuper/leveldb.py @@ -6,7 +6,7 @@ except ImportError: if have_leveldb: - from typing import List + from typing import List, Union from hashlib import sha512 from base64 import b85encode from collections import namedtuple @@ -18,8 +18,11 @@ if have_leveldb: PRIMARY KEY(key) )''' - def list_leveldb_entries(db: str, dms: List[bytes] = None): - d = DB(db) + def list_leveldb_entries(db: Union[str, DB], dms: List[bytes] = None): + if isinstance(db, str): + d = DB(db) + else: + d = db r = [] for i in d.iterator(include_value=False): if isinstance(i, bytes): @@ -38,6 +41,8 @@ if have_leveldb: if dm in dms: r.append(i) r.sort() + if isinstance(db, str): + d.close() return r def leveldb_stats(db: str, entries: List[bytes]) -> LeveldbStats: @@ -67,3 +72,33 @@ if have_leveldb: s.commit() d.close() s.close() + + def sqlite_to_leveldb(db: str, dest: str, dms: List[bytes]): + s = connect(db) + s.text_factory = bytes + d = DB(dest, create_if_missing=True) + try: + ents = list_leveldb_entries(d, dms) + for i in ents: + d.delete(i) + cur = s.execute('SELECT * FROM map;') + for i in cur: + if dms is None: + d.put(i[0], i[1]) + else: + if i[0] == b'VERSION': + d.put(i[0], i[1]) + elif i[0].startswith(b'META:'): + dm = i[0][5:] + if dm in dms: + d.put(i[0], i[1]) + elif i[0].startswith(b'_'): + dmi = i[0].find(b'\x00\x01') + dm = i[0][1:dmi] + if dm in dms: + d.put(i[0], i[1]) + except Exception: + from traceback import print_exc + print_exc() + d.close() + s.close() diff --git a/game_backuper/restorer.py b/game_backuper/restorer.py index 34cc98b..dbee9cc 100644 --- a/game_backuper/restorer.py +++ b/game_backuper/restorer.py @@ -1,5 +1,5 @@ from threading import Thread -from game_backuper.config import Config, Program, ConfigPath +from game_backuper.config import Config, Program, ConfigPath, ConfigOLeveldb from game_backuper.db import Db from os.path import join, relpath, isabs, isfile, isdir, exists from game_backuper.file import ( @@ -7,8 +7,10 @@ from game_backuper.file import ( copy_file, remove_dirs, new_file, + mkdir_for_file, ) from os import remove +from game_backuper.filetype import FileType class RestoreTask(Thread): @@ -30,7 +32,7 @@ class RestoreTask(Thread): if isinstance(r, ConfigPath): if f.type is not None: raise ValueError('Type dismatched.') - nam = r.name if r.name else r.path + nam = r.real_name src = join(self.cfg.dest, prog, fn) tmp = relpath(fn, nam) if isabs(r.path): @@ -41,12 +43,46 @@ class RestoreTask(Thread): dest = join(dest, tmp) if dest in pl: pl.remove(dest) + if not exists(src): + print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})') # noqa: E501 + continue if exists(dest): tf = new_file(dest, nam, prog) if tf.size == f.size and tf.hash == f.hash: - print(f'{prog}: Skip {f.file}') + print(f'{prog}: Skip {fn}') continue copy_file(src, dest, nam, prog) + elif isinstance(r, ConfigOLeveldb): + from game_backuper.leveldb import have_leveldb + if not have_leveldb: + raise NotImplementedError('Leveldb is not supported.') + if f.type != FileType.LEVELDB: + raise ValueError('Type dismatched.') + nam = r.real_name + src = join(self.cfg.dest, prog, fn + '.db') + if isabs(r.path): + dest = r.path + else: + dest = join(b, r.path) + if dest in pl: + pl.remove(dest) + if not exists(src): + print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})') # noqa: E501 + continue + from game_backuper.leveldb import ( + sqlite_to_leveldb, + leveldb_stats, + list_leveldb_entries, + ) + if exists(dest): + ents = list_leveldb_entries(dest, r.domains) + stat = leveldb_stats(dest, ents) + if f.size == stat.size and f.hash == stat.hash: + print(f'{prog}: Skip {fn}') + continue + mkdir_for_file(dest) + sqlite_to_leveldb(src, dest, r.domains) + print(f'{prog}: Covert leveldb done. {src}({fn}) -> {dest}') for i in pl: if isfile(i): remove(i)