lifegpc/game-backuper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

add bzip2 compress support for path type files

Browse Source
This commit is contained in:
lifegpc
2021-09-12 12:13:59 +08:00
parent 875e39a2c9
commit 8249473755
7 changed files with 207 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
example.yaml
View File

@@ -2,20 +2,26 @@ dest: /path/to/store/backup/files # The programs will store database and backup
enable_pcre2: false # Optional. Default value: false. Try to use PCRE2 first. PCRE2 may be a little slower than internal regex library.
remove_old_files: true # Optional. Default value: true. Remove unneeded backup files which already deleted in source tree when backuping files.
ignore_hidden_files: true # Optional. Default value: true. Whether to ignore files which its name starts with ".". Only effect folder which type is "path".
compress_method: "bzip2" # Optional. Default value: null. Supported value: "bzip2"
compress_level: 6 # Optional. Default value: null. bzip2 support 1-9 (Default: 9).
programs:
- name: Your program name # This name is used to identify different application.
base: /path/to/save/path # Must be absoulte path.
enable_pcre2: false # Optional.
remove_old_files: true # Optional.
ignore_hidden_files: true # Optional.
compress_method: null # Optional.
compress_level: null # Optional.
files:
- BGI.gdb # path to a file/folder. All subfolders will include if it is a folder. Must be relative path.
- type: path
path: folder # path to a file/folder. All subfolders will include if it is a folder. Must be relative path if name not found.
name: folder2 # optional. path to the backup files. Shoule be a relative path
enable_pcre2: false
enable_pcre2: false # Optional.
remove_old_files: true # Optional.
ignore_hidden_files: true # Optional.
compress_method: null # Optional.
compress_level: null # Optional.
excludes: # Optional. Exculde some files. Only effected when path is a folder.
- data.db # Releative path
- /path/to/data.db # Absolute path
@@ -33,7 +39,9 @@ programs:
- type: leveldb # module plyvel is needed to support this type. This will store leveldb database to a single file database (sqlite3)
path: leveldb # path to leveldb. Must be relative path.
name: dest # optional. path to the backup files. Shoule be a relative path
enable_pcre2: false
enable_pcre2: false # Optional.
remove_old_files: true # Optional.
compress_method: null # Optional.
compress_level: null # Optional.
domains: # optional. Just backup minor domains in localstorage database. Only chromium is tested.
- some domain

27
game_backuper/backuper.py
View File

@@ -12,6 +12,8 @@ from os import mkdir, remove
from game_backuper.file import new_file, copy_file, File, mkdir_for_file
from game_backuper.filetype import FileType
from game_backuper.restorer import RestoreTask
from game_backuper.file import remove_compress_files
from game_backuper.compress import compress
class BackupTask(Thread):
@@ -33,6 +35,7 @@ class BackupTask(Thread):
if exists(f[1]):
if f.name in fl:
fl.remove(f.name)
c = f.compress_config
ori = self.db.get_file(prog, f[0])
nf = new_file(f[1], f[0], prog)
if nf is None:
@@ -40,12 +43,28 @@ class BackupTask(Thread):
de = join(bp, f[0])
if ori is not None:
if ori.size == nf.size and ori.hash == nf.hash:
print(f'{prog}: Skip {f[0]}.')
continue
copy_file(f[1], de, f[0], prog)
if c is None:
if exists(de):
print(f'{prog}: Skip {f[0]}.')
remove_compress_files(de, prog, f.name)
continue
else:
if exists(de + c.ext):
print(f'{prog}: Skip {f.name}.')
remove_compress_files(de, prog, f.name, c.ext) # noqa: E501
continue
if c is None:
copy_file(f[1], de, f[0], prog)
remove_compress_files(de, prog, f.name)
else:
compress(f[1], de, c, f.name, prog)
self.db.set_file(ori.id, nf.size, nf.hash)
else:
copy_file(f[1], de, f[0], prog)
if c is None:
copy_file(f[1], de, f[0], prog)
remove_compress_files(de, prog, f.name)
else:
compress(f[1], de, c, f.name, prog)
self.db.add_file(nf)
elif isinstance(f, ConfigLeveldb):
from game_backuper.leveldb import have_leveldb

123
game_backuper/compress.py Normal file
View File

@@ -0,0 +1,123 @@
try:
from bz2 import BZ2File
have_bz2 = True
except ImportError:
have_bz2 = False
from enum import IntEnum
try:
from functools import cached_property
except ImportError:
cached_property = property
from os.path import exists, isfile, getsize
from os import remove
class CompressMethod(IntEnum):
BZIP2 = 0
@staticmethod
def from_str(v: str) -> IntEnum:
if isinstance(v, str):
t = v.lower()
if t == 'bzip2':
return CompressMethod.BZIP2
else:
raise TypeError('Must be str.')
class CompressConfig:
def __init__(self, method: str, level: int = None):
self._method = CompressMethod.from_str(method)
if self._method is None:
raise ValueError('Unknown compress method.')
if self._method == CompressMethod.BZIP2:
if not have_bz2:
raise NotImplementedError("bzip2 not supported.")
if level is None:
self._level = 9
else:
if isinstance(level, int) and level >= 1 and level <= 9:
self._level = level
else:
raise ValueError('bzip2: compress_level should be 1-9.')
self._ext = ".bz2"
self._chunk_size = 1048576
def __repr__(self):
t = type(self)
return f"<{t.__module__}.{t.__qualname__} object at {hex(id(self))}; method={repr(self._method)}, level={repr(self._level)}, ext={repr(self._ext)}>" # noqa: E501
@cached_property
def chunk_size(self) -> int:
return self._chunk_size
@cached_property
def ext(self) -> str:
return self._ext
@cached_property
def level(self) -> int:
return self._level
@cached_property
def method(self) -> CompressMethod:
return self._method
supported_exts = []
if have_bz2:
supported_exts.append('.bz2')
def sizeof_fmt(num, suffix='B'):
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', ' Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
def compress_info(ori: int, re: int):
return f"{sizeof_fmt(ori)} -> {sizeof_fmt(re)} ({re/ori*100:.2f}%)"
def compress(src: str, dest: str, c: CompressConfig, name: str, prog: str):
exts = [''] + supported_exts.copy()
exts.remove(c.ext)
fn = dest + c.ext
cs = c.chunk_size
if exists(fn):
remove(fn)
if c.method == CompressMethod.BZIP2:
with open(src, 'rb') as t:
with BZ2File(fn, 'wb', compresslevel=c.level) as f:
a = t.read(cs)
while a != b'':
f.write(a)
a = t.read(cs)
del a
i = compress_info(getsize(src), getsize(fn))
print(f'{prog}: Compressed {src}({name}) -> {fn} ({i})')
del i
for i in exts:
f = dest + i
if exists(f) and isfile(f):
remove(f)
print(f'{prog}: Removed {f}({name})')
def decompress(src: str, dest: str, c: CompressConfig, name: str, prog: str):
fn = src + c.ext
if exists(dest):
remove(dest)
cs = c.chunk_size
if c.method == CompressMethod.BZIP2:
with BZ2File(fn, 'rb') as f:
with open(dest, 'wb') as t:
a = f.read(cs)
while a != b'':
t.write(a)
a = f.read(cs)
del a
i = compress_info(getsize(dest), getsize(fn))
print(f'{prog}: Decompressed {fn}({name}) -> {dest} ({i})')

27
game_backuper/config.py
View File

@@ -12,12 +12,28 @@ try:
except ImportError:
cached_property = property
from game_backuper.regexp import Regex, wildcards_to_regex
from game_backuper.compress import CompressConfig
class BasicOption:
'''Basic options which is included in config, program and files.'''
_remove_old_files = None
_enable_pcre2 = None
_compress_config = None
@property
def compress_config(self) -> CompressConfig:
if self._compress_config is not None:
return self._compress_config
prog = getattr(self, "_prog", None)
if prog is not None:
if prog._compress_config is not None:
return prog._compress_config
cfg = getattr(self, "_cfg", None)
if cfg is not None:
if cfg._compress_config is not None:
return cfg._compress_config
return None
@cached_property
def enable_pcre2(self) -> bool:
@@ -48,9 +64,20 @@ class BasicOption:
return True
def parse_all(self, data=None):
self.parse_compress_config(data)
self.parse_remove_old_files(data)
self.parse_enable_pcre2(data)
def parse_compress_config(self, data=None):
if data is None:
data = getattr(self, 'data')
if 'compress_method' in data:
v = data['compress_method']
if isinstance(v, str):
self._compress_config = CompressConfig(v, data.get("compress_level")) # noqa: E501
elif v is not None:
raise TypeError('compress_method option should be str or None.') # noqa: E501
def parse_enable_pcre2(self, data=None):
if data is None:
data = getattr(self, 'data')

13
game_backuper/file.py
View File

@@ -5,6 +5,7 @@ from game_backuper.hashl import sha512
from shutil import copy2
from game_backuper.filetype import FileType
from os import remove
from game_backuper.compress import supported_exts
File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type'])
@@ -74,3 +75,15 @@ def remove_dirs(loc: str):
except Exception:
remove_dirs(i)
remove(loc)
def remove_compress_files(loc: str, prog: str, name: str, ext: str = None):
exts = supported_exts.copy()
if ext is not None:
exts.remove(ext)
exts.append('')
for i in exts:
f = loc + i
if exists(f) and isfile(f):
remove(f)
print(f'{prog}: Removed {f}({name})')

4
game_backuper/leveldb.py
View File

@@ -11,6 +11,8 @@ if have_leveldb:
from base64 import b85encode
from collections import namedtuple
from sqlite3 import connect
from os.path import exists
from os import remove
LeveldbStats = namedtuple('LeveldbStats', ['hash', 'size'])
MAP_TABLE = '''CREATE TABLE map (
key TEXT,
@@ -60,6 +62,8 @@ if have_leveldb:
def leveldb_to_sqlite(db: str, dest: str, entries: List[bytes]):
d = DB(db)
if exists(dest):
remove(dest)
s = connect(dest)
s.text_factory = bytes
s.execute(MAP_TABLE)

9
game_backuper/restorer.py
View File

@@ -11,6 +11,7 @@ from game_backuper.file import (
)
from os import remove
from game_backuper.filetype import FileType
from game_backuper.compress import decompress
class RestoreTask(Thread):
@@ -34,6 +35,7 @@ class RestoreTask(Thread):
raise ValueError('Type dismatched.')
nam = r.real_name
src = join(self.cfg.dest, prog, fn)
c = r.compress_config
tmp = relpath(fn, nam)
if isabs(r.path):
dest = r.path
@@ -43,7 +45,7 @@ class RestoreTask(Thread):
dest = join(dest, tmp)
if dest in pl:
pl.remove(dest)
if not exists(src):
if (c is None and not exists(src)) or (c is not None and not exists(src + c.ext)): # noqa: E501
print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})') # noqa: E501
continue
if exists(dest):
@@ -51,7 +53,10 @@ class RestoreTask(Thread):
if tf.size == f.size and tf.hash == f.hash:
print(f'{prog}: Skip {fn}')
continue
copy_file(src, dest, nam, prog)
if c is None:
copy_file(src, dest, nam, prog)
else:
decompress(src, dest, c, nam, prog)
elif isinstance(r, ConfigOLeveldb):
from game_backuper.leveldb import have_leveldb
if not have_leveldb: