From 824947375503ff2a43c86d5c630d98b7ea9e6a9f Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 12 Sep 2021 12:13:59 +0800 Subject: [PATCH] add bzip2 compress support for path type files --- example.yaml | 12 +++- game_backuper/backuper.py | 27 +++++++-- game_backuper/compress.py | 123 ++++++++++++++++++++++++++++++++++++++ game_backuper/config.py | 27 +++++++++ game_backuper/file.py | 13 ++++ game_backuper/leveldb.py | 4 ++ game_backuper/restorer.py | 9 ++- 7 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 game_backuper/compress.py diff --git a/example.yaml b/example.yaml index afa5596..fe9b8f7 100644 --- a/example.yaml +++ b/example.yaml @@ -2,20 +2,26 @@ dest: /path/to/store/backup/files # The programs will store database and backup enable_pcre2: false # Optional. Default value: false. Try to use PCRE2 first. PCRE2 may be a little slower than internal regex library. remove_old_files: true # Optional. Default value: true. Remove unneeded backup files which already deleted in source tree when backuping files. ignore_hidden_files: true # Optional. Default value: true. Whether to ignore files which its name starts with ".". Only effect folder which type is "path". +compress_method: "bzip2" # Optional. Default value: null. Supported value: "bzip2" +compress_level: 6 # Optional. Default value: null. bzip2 support 1-9 (Default: 9). programs: - name: Your program name # This name is used to identify different application. base: /path/to/save/path # Must be absoulte path. enable_pcre2: false # Optional. remove_old_files: true # Optional. ignore_hidden_files: true # Optional. + compress_method: null # Optional. + compress_level: null # Optional. files: - BGI.gdb # path to a file/folder. All subfolders will include if it is a folder. Must be relative path. - type: path path: folder # path to a file/folder. All subfolders will include if it is a folder. Must be relative path if name not found. name: folder2 # optional. path to the backup files. Shoule be a relative path - enable_pcre2: false + enable_pcre2: false # Optional. remove_old_files: true # Optional. ignore_hidden_files: true # Optional. + compress_method: null # Optional. + compress_level: null # Optional. excludes: # Optional. Exculde some files. Only effected when path is a folder. - data.db # Releative path - /path/to/data.db # Absolute path @@ -33,7 +39,9 @@ programs: - type: leveldb # module plyvel is needed to support this type. This will store leveldb database to a single file database (sqlite3) path: leveldb # path to leveldb. Must be relative path. name: dest # optional. path to the backup files. Shoule be a relative path - enable_pcre2: false + enable_pcre2: false # Optional. remove_old_files: true # Optional. + compress_method: null # Optional. + compress_level: null # Optional. domains: # optional. Just backup minor domains in localstorage database. Only chromium is tested. - some domain diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py index 0c529b1..0fa3028 100644 --- a/game_backuper/backuper.py +++ b/game_backuper/backuper.py @@ -12,6 +12,8 @@ from os import mkdir, remove from game_backuper.file import new_file, copy_file, File, mkdir_for_file from game_backuper.filetype import FileType from game_backuper.restorer import RestoreTask +from game_backuper.file import remove_compress_files +from game_backuper.compress import compress class BackupTask(Thread): @@ -33,6 +35,7 @@ class BackupTask(Thread): if exists(f[1]): if f.name in fl: fl.remove(f.name) + c = f.compress_config ori = self.db.get_file(prog, f[0]) nf = new_file(f[1], f[0], prog) if nf is None: @@ -40,12 +43,28 @@ class BackupTask(Thread): de = join(bp, f[0]) if ori is not None: if ori.size == nf.size and ori.hash == nf.hash: - print(f'{prog}: Skip {f[0]}.') - continue - copy_file(f[1], de, f[0], prog) + if c is None: + if exists(de): + print(f'{prog}: Skip {f[0]}.') + remove_compress_files(de, prog, f.name) + continue + else: + if exists(de + c.ext): + print(f'{prog}: Skip {f.name}.') + remove_compress_files(de, prog, f.name, c.ext) # noqa: E501 + continue + if c is None: + copy_file(f[1], de, f[0], prog) + remove_compress_files(de, prog, f.name) + else: + compress(f[1], de, c, f.name, prog) self.db.set_file(ori.id, nf.size, nf.hash) else: - copy_file(f[1], de, f[0], prog) + if c is None: + copy_file(f[1], de, f[0], prog) + remove_compress_files(de, prog, f.name) + else: + compress(f[1], de, c, f.name, prog) self.db.add_file(nf) elif isinstance(f, ConfigLeveldb): from game_backuper.leveldb import have_leveldb diff --git a/game_backuper/compress.py b/game_backuper/compress.py new file mode 100644 index 0000000..3f66c19 --- /dev/null +++ b/game_backuper/compress.py @@ -0,0 +1,123 @@ +try: + from bz2 import BZ2File + have_bz2 = True +except ImportError: + have_bz2 = False +from enum import IntEnum +try: + from functools import cached_property +except ImportError: + cached_property = property +from os.path import exists, isfile, getsize +from os import remove + + +class CompressMethod(IntEnum): + BZIP2 = 0 + + @staticmethod + def from_str(v: str) -> IntEnum: + if isinstance(v, str): + t = v.lower() + if t == 'bzip2': + return CompressMethod.BZIP2 + else: + raise TypeError('Must be str.') + + +class CompressConfig: + def __init__(self, method: str, level: int = None): + self._method = CompressMethod.from_str(method) + if self._method is None: + raise ValueError('Unknown compress method.') + if self._method == CompressMethod.BZIP2: + if not have_bz2: + raise NotImplementedError("bzip2 not supported.") + if level is None: + self._level = 9 + else: + if isinstance(level, int) and level >= 1 and level <= 9: + self._level = level + else: + raise ValueError('bzip2: compress_level should be 1-9.') + self._ext = ".bz2" + self._chunk_size = 1048576 + + def __repr__(self): + t = type(self) + return f"<{t.__module__}.{t.__qualname__} object at {hex(id(self))}; method={repr(self._method)}, level={repr(self._level)}, ext={repr(self._ext)}>" # noqa: E501 + + @cached_property + def chunk_size(self) -> int: + return self._chunk_size + + @cached_property + def ext(self) -> str: + return self._ext + + @cached_property + def level(self) -> int: + return self._level + + @cached_property + def method(self) -> CompressMethod: + return self._method + + +supported_exts = [] +if have_bz2: + supported_exts.append('.bz2') + + +def sizeof_fmt(num, suffix='B'): + for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', ' Ei', 'Zi']: + if abs(num) < 1024.0: + return "%3.1f%s%s" % (num, unit, suffix) + num /= 1024.0 + return "%.1f%s%s" % (num, 'Yi', suffix) + + +def compress_info(ori: int, re: int): + return f"{sizeof_fmt(ori)} -> {sizeof_fmt(re)} ({re/ori*100:.2f}%)" + + +def compress(src: str, dest: str, c: CompressConfig, name: str, prog: str): + exts = [''] + supported_exts.copy() + exts.remove(c.ext) + fn = dest + c.ext + cs = c.chunk_size + if exists(fn): + remove(fn) + if c.method == CompressMethod.BZIP2: + with open(src, 'rb') as t: + with BZ2File(fn, 'wb', compresslevel=c.level) as f: + a = t.read(cs) + while a != b'': + f.write(a) + a = t.read(cs) + del a + i = compress_info(getsize(src), getsize(fn)) + print(f'{prog}: Compressed {src}({name}) -> {fn} ({i})') + del i + for i in exts: + f = dest + i + if exists(f) and isfile(f): + remove(f) + print(f'{prog}: Removed {f}({name})') + + +def decompress(src: str, dest: str, c: CompressConfig, name: str, prog: str): + fn = src + c.ext + if exists(dest): + remove(dest) + cs = c.chunk_size + if c.method == CompressMethod.BZIP2: + with BZ2File(fn, 'rb') as f: + with open(dest, 'wb') as t: + a = f.read(cs) + while a != b'': + t.write(a) + a = f.read(cs) + del a + i = compress_info(getsize(dest), getsize(fn)) + print(f'{prog}: Decompressed {fn}({name}) -> {dest} ({i})') diff --git a/game_backuper/config.py b/game_backuper/config.py index 7573aaf..3db9df9 100644 --- a/game_backuper/config.py +++ b/game_backuper/config.py @@ -12,12 +12,28 @@ try: except ImportError: cached_property = property from game_backuper.regexp import Regex, wildcards_to_regex +from game_backuper.compress import CompressConfig class BasicOption: '''Basic options which is included in config, program and files.''' _remove_old_files = None _enable_pcre2 = None + _compress_config = None + + @property + def compress_config(self) -> CompressConfig: + if self._compress_config is not None: + return self._compress_config + prog = getattr(self, "_prog", None) + if prog is not None: + if prog._compress_config is not None: + return prog._compress_config + cfg = getattr(self, "_cfg", None) + if cfg is not None: + if cfg._compress_config is not None: + return cfg._compress_config + return None @cached_property def enable_pcre2(self) -> bool: @@ -48,9 +64,20 @@ class BasicOption: return True def parse_all(self, data=None): + self.parse_compress_config(data) self.parse_remove_old_files(data) self.parse_enable_pcre2(data) + def parse_compress_config(self, data=None): + if data is None: + data = getattr(self, 'data') + if 'compress_method' in data: + v = data['compress_method'] + if isinstance(v, str): + self._compress_config = CompressConfig(v, data.get("compress_level")) # noqa: E501 + elif v is not None: + raise TypeError('compress_method option should be str or None.') # noqa: E501 + def parse_enable_pcre2(self, data=None): if data is None: data = getattr(self, 'data') diff --git a/game_backuper/file.py b/game_backuper/file.py index 57ec0d7..59bddd9 100644 --- a/game_backuper/file.py +++ b/game_backuper/file.py @@ -5,6 +5,7 @@ from game_backuper.hashl import sha512 from shutil import copy2 from game_backuper.filetype import FileType from os import remove +from game_backuper.compress import supported_exts File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type']) @@ -74,3 +75,15 @@ def remove_dirs(loc: str): except Exception: remove_dirs(i) remove(loc) + + +def remove_compress_files(loc: str, prog: str, name: str, ext: str = None): + exts = supported_exts.copy() + if ext is not None: + exts.remove(ext) + exts.append('') + for i in exts: + f = loc + i + if exists(f) and isfile(f): + remove(f) + print(f'{prog}: Removed {f}({name})') diff --git a/game_backuper/leveldb.py b/game_backuper/leveldb.py index 8bcf384..762fddb 100644 --- a/game_backuper/leveldb.py +++ b/game_backuper/leveldb.py @@ -11,6 +11,8 @@ if have_leveldb: from base64 import b85encode from collections import namedtuple from sqlite3 import connect + from os.path import exists + from os import remove LeveldbStats = namedtuple('LeveldbStats', ['hash', 'size']) MAP_TABLE = '''CREATE TABLE map ( key TEXT, @@ -60,6 +62,8 @@ if have_leveldb: def leveldb_to_sqlite(db: str, dest: str, entries: List[bytes]): d = DB(db) + if exists(dest): + remove(dest) s = connect(dest) s.text_factory = bytes s.execute(MAP_TABLE) diff --git a/game_backuper/restorer.py b/game_backuper/restorer.py index dbee9cc..69fd70c 100644 --- a/game_backuper/restorer.py +++ b/game_backuper/restorer.py @@ -11,6 +11,7 @@ from game_backuper.file import ( ) from os import remove from game_backuper.filetype import FileType +from game_backuper.compress import decompress class RestoreTask(Thread): @@ -34,6 +35,7 @@ class RestoreTask(Thread): raise ValueError('Type dismatched.') nam = r.real_name src = join(self.cfg.dest, prog, fn) + c = r.compress_config tmp = relpath(fn, nam) if isabs(r.path): dest = r.path @@ -43,7 +45,7 @@ class RestoreTask(Thread): dest = join(dest, tmp) if dest in pl: pl.remove(dest) - if not exists(src): + if (c is None and not exists(src)) or (c is not None and not exists(src + c.ext)): # noqa: E501 print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})') # noqa: E501 continue if exists(dest): @@ -51,7 +53,10 @@ class RestoreTask(Thread): if tf.size == f.size and tf.hash == f.hash: print(f'{prog}: Skip {fn}') continue - copy_file(src, dest, nam, prog) + if c is None: + copy_file(src, dest, nam, prog) + else: + decompress(src, dest, c, nam, prog) elif isinstance(r, ConfigOLeveldb): from game_backuper.leveldb import have_leveldb if not have_leveldb: