From 493af568b506dd23d2a431da7e12a577e388d636 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Thu, 9 Sep 2021 18:51:22 +0800 Subject: [PATCH] add support to restore normal files --- game_backuper/backuper.py | 5 ++ game_backuper/config.py | 106 ++++++++++++++++++++++++++++++++++++++ game_backuper/file.py | 34 +++++++++++- game_backuper/restorer.py | 56 ++++++++++++++++++++ setup.py | 8 ++- 5 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 game_backuper/restorer.py diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py index 9d4cc12..0c529b1 100644 --- a/game_backuper/backuper.py +++ b/game_backuper/backuper.py @@ -11,6 +11,7 @@ from os.path import exists, join, isdir from os import mkdir, remove from game_backuper.file import new_file, copy_file, File, mkdir_for_file from game_backuper.filetype import FileType +from game_backuper.restorer import RestoreTask class BackupTask(Thread): @@ -114,6 +115,10 @@ class Backuper: t.start() elif self.opts.action == OptAction.LIST: print(prog.name) + elif self.opts.action == OptAction.RESTORE: + t = RestoreTask(prog, self.db, self.conf) + self.tasks.append(t) + t.start() def run(self): if self.opts.action == OptAction.LIST_LEVELDB_KEY: diff --git a/game_backuper/config.py b/game_backuper/config.py index add80ae..0c9c61d 100644 --- a/game_backuper/config.py +++ b/game_backuper/config.py @@ -83,6 +83,79 @@ class NFBasicOption: del v +# pylint: disable=unsupported-membership-test, unsubscriptable-object +class BasicConfig: + def __repr__(self) -> str: + data = getattr(self, "data", None) + return f"{self.__class__.__name__}<{data}>" + + @cached_property + def name(self) -> str: + data = getattr(self, "data", None) + if isinstance(data, dict) and 'name' in data: + v = data['name'] + if isinstance(v, str) and len(v) > 0: + return v + + @cached_property + def path(self) -> str: + data = getattr(self, "data", None) + if isinstance(data, dict) and 'path' in data: + v = data['path'] + if isinstance(v, str) and len(v) > 0: + return v + raise ValueError('Path not found.') + + @cached_property + def type(self) -> str: + data = getattr(self, "data", None) + if isinstance(data, dict) and 'type' in data: + v = data['type'] + if isinstance(v, str) and len(v) > 0: + return v + raise ValueError('Type not found.') +# pylint: enable=unsupported-membership-test, unsubscriptable-object + + +class ConfigPath(BasicOption, NFBasicOption, BasicConfig): + def __init__(self, data, cfg, prog): + NFBasicOption.__init__(self, cfg, prog) + if isinstance(data, str): + self.data = {"path": data, "type": "path"} + elif isinstance(data, dict): + self.data = data + else: + raise TypeError('Must be str or dict.') + self.parse_all() + self.parse_all_nf() + + +class ConfigOLeveldb(BasicOption, NFBasicOption, BasicConfig): + def __init__(self, data, cfg, prog): + NFBasicOption.__init__(self, cfg, prog) + if isinstance(data, dict): + self.data = data + else: + raise TypeError('Must be dict.') + self.parse_all() + self.parse_all_nf() + + @cached_property + def ignore_hidden_files(self): + True + + @cached_property + def domains(self) -> List[str]: + if 'domains' in self.data: + if isinstance(self.data['domains'], list): + dms = [] + for i in self.data['domains']: + if isinstance(i, str) and len(i) > 0: + dms.append(i) + if len(dms) > 0: + return dms + + def namedtuple_bo(typename, field_names): a = namedtuple(typename, field_names) return type(typename, (a, BasicOption), {}) @@ -91,6 +164,7 @@ def namedtuple_bo(typename, field_names): ConfigNormalFile = namedtuple_bo('ConfigNormalFile', ['name', 'full_path']) ConfigLeveldb = namedtuple_bo('ConfigLeveldb', ['name', 'full_path', 'domains']) # noqa: E501 ConfigResult = Union[ConfigNormalFile, ConfigLeveldb] +ConfigOriginResult = Union[ConfigPath, ConfigOLeveldb] class Program(BasicOption, NFBasicOption): @@ -101,6 +175,20 @@ class Program(BasicOption, NFBasicOption): self.parse_all() self.parse_all_nf() + @cached_property + def all_configs(self) -> List[ConfigOriginResult]: + r = [] + for i in self.data['files']: + if isinstance(i, str): + r.append(ConfigPath(i, self._cfg, self)) + elif isinstance(i, dict): + t = i['type'] + if t == 'path': + r.append(ConfigPath(i, self._cfg, self)) + elif t == 'leveldb': + r.append(ConfigOLeveldb(i, self._cfg, self)) + return r + @cached_property def base(self) -> str: if 'base' in self.data: @@ -203,6 +291,24 @@ class Program(BasicOption, NFBasicOption): i._prog = self return r + def get_config(self, name: str) -> ConfigOriginResult: + for i in self.data['files']: + if isinstance(i, str): + if not relpath(name, i).startswith('..'): + return ConfigPath(i, self._cfg, self) + elif isinstance(i, dict): + t = i['type'] + if t == 'path': + if isabs(i['path']): + n = i['name'] + else: + n = i['path'] + if 'name' in i and isinstance(i['name'], str): + if i['name'] != '': + n = i['name'] + if not relpath(name, n).startswith('..'): + return ConfigPath(i, self._cfg, self) + @cached_property def name(self) -> str: if 'name' in self.data: diff --git a/game_backuper/file.py b/game_backuper/file.py index a140c49..57ec0d7 100644 --- a/game_backuper/file.py +++ b/game_backuper/file.py @@ -1,9 +1,10 @@ from collections import namedtuple -from os.path import exists, dirname, abspath, isfile, isdir, join +from os.path import exists, dirname, abspath, isfile, isdir, join, isabs from os import stat, makedirs, listdir from game_backuper.hashl import sha512 from shutil import copy2 from game_backuper.filetype import FileType +from os import remove File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type']) @@ -36,9 +37,40 @@ def listdirs(loc: str, ignore_hidden_files: bool = True): return r +def list_all_paths(base: str, cli): + from game_backuper.config import ConfigPath, ConfigOLeveldb + r = [] + for c in cli: + if isinstance(c, ConfigPath): + if isabs(c.path): + bp = c.path + else: + bp = join(base, c.path) + if isfile(bp): + r.append(bp) + elif isdir(bp): + r += listdirs(bp, c.ignore_hidden_files) + elif isinstance(c, ConfigOLeveldb): + r.append(c.path if isabs(c.path) else join(base, c.path)) + return r + + def new_file(loc: str, name: str, prog: str, type: FileType = None) -> File: if exists(loc): fs = stat(loc).st_size with open(loc, 'rb') as f: hs = sha512(f) return File(None, name, fs, prog, hs, type) + + +def remove_dirs(loc: str): + bl = listdirs(loc, False) + for i in bl: + if isfile(i): + remove(i) + elif isdir(i): + try: + remove_dirs(i) + except Exception: + remove_dirs(i) + remove(loc) diff --git a/game_backuper/restorer.py b/game_backuper/restorer.py new file mode 100644 index 0000000..34cc98b --- /dev/null +++ b/game_backuper/restorer.py @@ -0,0 +1,56 @@ +from threading import Thread +from game_backuper.config import Config, Program, ConfigPath +from game_backuper.db import Db +from os.path import join, relpath, isabs, isfile, isdir, exists +from game_backuper.file import ( + list_all_paths, + copy_file, + remove_dirs, + new_file, +) +from os import remove + + +class RestoreTask(Thread): + def __init__(self, prog: Program, db: Db, cfg: Config): + Thread.__init__(self, name=f"Restore_{prog.name}") + self.cfg = cfg + self.prog = prog + self.db = db + + def run(self): + b = self.prog.base + prog = self.prog.name + li = self.db.get_file_list(prog) + cli = self.prog.all_configs + pl = list_all_paths(b, cli) + for fn in li: + f = self.db.get_file(prog, fn) + r = self.prog.get_config(fn) + if isinstance(r, ConfigPath): + if f.type is not None: + raise ValueError('Type dismatched.') + nam = r.name if r.name else r.path + src = join(self.cfg.dest, prog, fn) + tmp = relpath(fn, nam) + if isabs(r.path): + dest = r.path + else: + dest = join(b, r.path) + if not tmp.startswith('.'): + dest = join(dest, tmp) + if dest in pl: + pl.remove(dest) + if exists(dest): + tf = new_file(dest, nam, prog) + if tf.size == f.size and tf.hash == f.hash: + print(f'{prog}: Skip {f.file}') + continue + copy_file(src, dest, nam, prog) + for i in pl: + if isfile(i): + remove(i) + print(f'{prog}: Removed {i}') + elif isdir(i): + remove_dirs(i) + print(f'{prog}: Removed {i}') diff --git a/setup.py b/setup.py index 15b4dc5..5e0ac3b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # flake8: noqa import sys from game_backuper import __version__ -if len(sys.argv) == 2 and sys.argv[1] == "py2exe": +if "py2exe" in sys.argv: from distutils.core import setup import py2exe params = { @@ -29,7 +29,11 @@ else: "install_requires": ["pyyaml"], 'entry_points': { 'console_scripts': ['game-backuper = game_backuper:start'] - } + }, + "extras_require": { + "leveldb": "plyvel" + }, + "python_requires": ">=3.6" } setup( name="game-backuper",