diff --git a/example.yaml b/example.yaml index 966ad63..cfe4464 100644 --- a/example.yaml +++ b/example.yaml @@ -2,8 +2,10 @@ dest: /path/to/store/backup/files # The programs will store database and backup enable_pcre2: false # Optional. Default value: false. Try to use PCRE2 first. PCRE2 may be a little slower than internal regex library. remove_old_files: true # Optional. Default value: true. Remove unneeded backup files which already deleted in source tree when backuping files. ignore_hidden_files: true # Optional. Default value: true. Whether to ignore files which its name starts with ".". Only effect folder which type is "path". -compress_method: "bzip2" # Optional. Default value: null. Supported value: "bzip2", "gzip", "lzma", "lzip" -compress_level: 6 # Optional. Default value: null. bzip2 support 1-9 (Default: 9). gzip support 0-9 (Default: 9). lzma or lzip support 0-9 (Default: 6). +compress_method: "bzip2" # Optional. Default value: null. Supported value: "bzip2", "gzip", "lzma", "lzip", "zstd" +# Optional. Default value: null. bzip2 support 1-9 (Default: 9). gzip support 0-9 (Default: 9). lzma or lzip support 0-9 (Default: 6). +# zstd support 0-22 (Default: 3). +compress_level: 6 programs: - name: Your program name # This name is used to identify different application. base: /path/to/save/path # Must be absoulte path. diff --git a/game_backuper/compress.py b/game_backuper/compress.py index 5122fd3..92dc616 100644 --- a/game_backuper/compress.py +++ b/game_backuper/compress.py @@ -21,6 +21,14 @@ try: have_lzip = True except ImportError: have_lzip = False +try: + from game_backuper.zstd import ( + ZSTDFile, + MAX_COMPRESS_LEVEL as ZSTD_MAX + ) + have_zstd = True +except ImportError: + have_zstd = False from enum import IntEnum, unique try: from functools import cached_property @@ -36,6 +44,7 @@ class CompressMethod(IntEnum): GZIP = 1 LZMA = 2 LZIP = 3 + ZSTD = 4 @staticmethod def from_str(v: str) -> IntEnum: @@ -49,6 +58,8 @@ class CompressMethod(IntEnum): return CompressMethod.LZMA elif t == "lzip": return CompressMethod.LZIP + elif t == "zstd": + return CompressMethod.ZSTD else: raise TypeError('Must be str.') @@ -102,6 +113,17 @@ class CompressConfig: else: raise ValueError('lzip: compress_level should be 0-9.') self._ext = ".lz" + elif self._method == CompressMethod.ZSTD: + if not have_zstd: + raise NotImplementedError("zstd not supported.") + if level is None: + self._level = 3 + else: + if isinstance(level, int) and level >= 0 and level <= ZSTD_MAX: + self._level = level + else: + raise ValueError(f'zstd: compress_level should be 0-{ZSTD_MAX}.') # noqa: E501 + self._ext = ".zst" self._chunk_size = 131072 def __repr__(self): @@ -134,6 +156,8 @@ if have_lzma: supported_exts.append('.xz') if have_lzip: supported_exts.append('.lz') +if have_zstd: + supported_exts.append('.zst') def sizeof_fmt(num, suffix='B'): @@ -187,6 +211,14 @@ def compress(src: str, dest: str, c: CompressConfig, name: str, prog: str): f.compress(a) a = t.read(cs) del a + elif c.method == CompressMethod.ZSTD: + with open(src, 'rb') as t: + with ZSTDFile(fn, 'wb', compresslevel=c.level) as f: + a = t.read(cs) + while a != b'': + f.write(a) + a = t.read(cs) + del a i = compress_info(getsize(src), getsize(fn)) print(f'{prog}: Compressed {src}({name}) -> {fn} ({i})') del i @@ -232,5 +264,13 @@ def decompress(src: str, dest: str, c: CompressConfig, name: str, prog: str): for a in f: t.write(a) del a + elif c.method == CompressMethod.ZSTD: + with ZSTDFile(fn, 'rb') as f: + with open(dest, 'wb') as t: + a = f.read(cs) + while a != b'': + t.write(a) + a = f.read(cs) + del a i = compress_info(getsize(dest), getsize(fn)) print(f'{prog}: Decompressed {fn}({name}) -> {dest} ({i})') diff --git a/game_backuper/zstd.py b/game_backuper/zstd.py new file mode 100644 index 0000000..ca836f1 --- /dev/null +++ b/game_backuper/zstd.py @@ -0,0 +1,177 @@ +try: + from game_backuper._zstd import ( + ZSTDCompressor, + ZSTDDecompressor, + maxCLevel, + ) + have_zstd = True +except ImportError: + have_zstd = False +if have_zstd: + from builtins import open as _builtin_open + from _compression import BaseStream, DecompressReader + import os + import io + + _MODE_CLOSED = 0 + _MODE_READ = 1 + _MODE_WRITE = 3 + MAX_COMPRESS_LEVEL = maxCLevel() + + class ZSTDFile(BaseStream): + def __init__(self, filename, mode="r", *, compresslevel=3): + self._fp = None + self._closefp = False + self._mode = _MODE_CLOSED + if not (1 <= compresslevel <= MAX_COMPRESS_LEVEL): + raise ValueError(f"compresslevel must be between 1 and {MAX_COMPRESS_LEVEL}") # noqa: E501 + if mode in ("", "r", "rb"): + mode = "rb" + mode_code = _MODE_READ + elif mode in ("w", "wb"): + mode = "wb" + mode_code = _MODE_WRITE + self._compressor = ZSTDCompressor(compresslevel) + elif mode in ("x", "xb"): + mode = "xb" + mode_code = _MODE_WRITE + self._compressor = ZSTDCompressor(compresslevel) + elif mode in ("a", "ab"): + mode = "ab" + mode_code = _MODE_WRITE + self._compressor = ZSTDCompressor(compresslevel) + else: + raise ValueError("Invalid mode: %r" % (mode,)) + if isinstance(filename, (str, bytes, os.PathLike)): + self._fp = _builtin_open(filename, mode) + self._closefp = True + self._mode = mode_code + else: + raise TypeError("filename must be a str, bytes, file or PathLike object") # noqa: E501 + if self._mode == _MODE_READ: + raw = DecompressReader(self._fp, ZSTDDecompressor, + trailing_error=OSError) + self._buffer = io.BufferedReader(raw) + else: + self._pos = 0 + + def close(self): + if self._mode == _MODE_CLOSED: + return + try: + if self._mode == _MODE_READ: + self._buffer.close() + elif self._mode == _MODE_WRITE: + self._fp.write(self._compressor.flush()) + self._compressor = None + finally: + try: + if self._closefp: + self._fp.close() + finally: + self._fp = None + self._closefp = False + self._mode = _MODE_CLOSED + self._buffer = None + + @property + def closed(self): + return self._mode == _MODE_CLOSED + + def fileno(self): + self._check_not_closed() + return self._fp.fileno() + + def seekable(self): + return self.readable() and self._buffer.seekable() + + def readable(self): + self._check_not_closed() + return self._mode == _MODE_READ + + def writable(self): + self._check_not_closed() + return self._mode == _MODE_WRITE + + def peek(self, n=0): + self._check_can_read() + return self._buffer.peek(n) + + def read(self, size=-1): + self._check_can_read() + return self._buffer.read(size) + + def read1(self, size=-1): + self._check_can_read() + if size < 0: + size = io.DEFAULT_BUFFER_SIZE + return self._buffer.read1(size) + + def readinto(self, b): + self._check_can_read() + return self._buffer.readinto(b) + + def readline(self, size=-1): + if not isinstance(size, int): + if not hasattr(size, "__index__"): + raise TypeError("Integer argument expected") + size = size.__index__() + self._check_can_read() + return self._buffer.readline(size) + + def __iter__(self): + self._check_can_read() + return self._buffer.__iter__() + + def readlines(self, size=-1): + if not isinstance(size, int): + if not hasattr(size, "__index__"): + raise TypeError("Integer argument expected") + size = size.__index__() + self._check_can_read() + return self._buffer.readlines(size) + + def write(self, data): + self._check_can_write() + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + data = memoryview(data) + length = data.nbytes + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += length + return length + + def writelines(self, seq): + return BaseStream.writelines(self, seq) + + def seek(self, offset, whence=io.SEEK_SET): + self._check_can_seek() + return self._buffer.seek(offset, whence) + + def tell(self): + self._check_not_closed() + if self._mode == _MODE_READ: + return self._buffer.tell() + return self._pos + + def open(filename, mode="rb", compresslevel=3, encoding=None, errors=None, newline=None): # noqa: E501 + if "t" in mode: + if "b" in mode: + raise ValueError("Invalid mode: %r" % (mode,)) + else: + if encoding is not None: + raise ValueError("Argument 'encoding' not supported in binary mode") # noqa: E501 + if errors is not None: + raise ValueError("Argument 'errors' not supported in binary mode") # noqa: E501 + if newline is not None: + raise ValueError("Argument 'newline' not supported in binary mode") # noqa: E501 + + bz_mode = mode.replace("t", "") + binary_file = ZSTDFile(filename, bz_mode, compresslevel=compresslevel) + + if "t" in mode: + return io.TextIOWrapper(binary_file, encoding, errors, newline) + else: + return binary_file