Fix sqlite3.dump module not found

fix bug in zstd compression warp code
fix bug when showing compression inforamtion

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 .pyre/
 
 .vscode/
+
+# Cython generated files
+*.c

example.yaml

@@ -1,21 +1,55 @@
 dest: /path/to/store/backup/files  # The programs will store database and backup files in this location
+enable_pcre2: false  # Optional. Default value: false. Try to use PCRE2 first. PCRE2 may be a little slower than internal regex library.
 remove_old_files: true  # Optional. Default value: true. Remove unneeded backup files which already deleted in source tree when backuping files.
 ignore_hidden_files: true  # Optional. Default value: true. Whether to ignore files which its name starts with ".". Only effect folder which type is "path".
+compress_method: "bzip2"  # Optional. Default value: null. Supported value: "bzip2", "gzip", "lzma", "lzip", "zstd", "snappy", "brotli"
+# Optional. Default value: null. bzip2 support 1-9 (Default: 9). gzip support 0-9 (Default: 9). lzma or lzip support 0-9 (Default: 6).
+# zstd support 0-22 (Default: 3). brotli support 0-11 (Default: unset).
+compress_level: 6
+encrypt_db: false  # Optional. Default value: false. Encrypt the database. Warning: The default python sqlite library don't support encrypt, it just ignore encrypt phases.
+db_password: "Password"  # Specify the password of the encryped database.
+encrypt_files: false  # Optional. Default value: false. Encrypt backup files. The key information will stored in database.
 programs:
   - name: Your program name  # This name is used to identify different application.
     base: /path/to/save/path  # Must be absoulte path.
+    enable_pcre2: false  # Optional.
     remove_old_files: true  # Optional.
     ignore_hidden_files: true  # Optional.
+    compress_method: null  # Optional.
+    compress_level: null  # Optional.
+    encrypt_files: false  # Optional
     files:
       - BGI.gdb  # path to a file/folder. All subfolders will include if it is a folder. Must be relative path.
       - type: path
         path: folder  # path to a file/folder. All subfolders will include if it is a folder. Must be relative path if name not found.
         name: folder2  # optional. path to the backup files. Shoule be a relative path
+        enable_pcre2: false  # Optional.
         remove_old_files: true  # Optional.
         ignore_hidden_files: true  # Optional.
+        compress_method: null  # Optional.
+        compress_level: null  # Optional.
+        encrypt_files: false  # Optional.
+        excludes:  # Optional. Exculde some files. Only effected when path is a folder.
+          - data.db  # Releative path
+          - /path/to/data.db  # Absolute path
+          - type: wildcards
+            rule: "*/*.db"
+          - type: regex
+            rule: "\\d+\\.db"
+        includes:  # Optional. Only include some files. Only effected when path is a folder.
+          - data.db
+          - /path/to/data.db
+          - type: wildcards
+            rule: "*/*.db"
+          - type: regex
+            rule: "\\d+\\.db"
       - type: leveldb  # module plyvel is needed to support this type. This will store leveldb database to a single file database (sqlite3)
         path: leveldb  # path to leveldb. Must be relative path.
         name: dest  # optional. path to the backup files. Shoule be a relative path
+        enable_pcre2: false  # Optional.
         remove_old_files: true  # Optional.
+        compress_method: null  # Optional.
+        compress_level: null  # Optional.
+        encrypt_files: false  # Optional.
         domains:  # optional. Just backup minor domains in localstorage database. Only chromium is tested.
           - some domain

game_backuper/_Python.pxd

@@ -0,0 +1,5 @@
+cdef extern from "Python.h":
+    const char* PyUnicode_AsUTF8(object unicode)
+    const char* PyUnicode_AsUTF8AndSize(object unicode, Py_ssize_t *size)
+    int PyBytes_AsStringAndSize(object obj, char **buff, Py_ssize_t *length)
+    object PyBytes_FromStringAndSize(const char* v, Py_ssize_t le)

game_backuper/_pcre2.h

@@ -0,0 +1,2 @@
+#define PCRE2_CODE_UNIT_WIDTH 8
+#include "pcre2.h"

game_backuper/_pcre2.pxd

@@ -0,0 +1,35 @@
+from libc.stddef cimport size_t
+from libc.stdint cimport uint8_t, uint32_t
+
+
+cdef extern from "_pcre2.h":
+    ctypedef uint8_t PCRE2_UCHAR
+    ctypedef const uint8_t* PCRE2_SPTR8
+    ctypedef PCRE2_SPTR8 PCRE2_SPTR
+    ctypedef size_t PCRE2_SIZE
+    ctypedef struct pcre2_compile_context:
+        pass
+    ctypedef struct pcre2_code:
+        pass
+    ctypedef struct pcre2_match_data:
+        pass
+    ctypedef struct pcre2_general_context:
+        pass
+    ctypedef struct pcre2_match_context:
+        pass
+
+    pcre2_code *pcre2_compile(PCRE2_SPTR pattern, PCRE2_SIZE length, uint32_t options, int *errorcode, PCRE2_SIZE *erroroffset, pcre2_compile_context *ccontext)
+    void pcre2_code_free(pcre2_code *code)
+    int pcre2_get_error_message(int errorcode, PCRE2_UCHAR *buffer, PCRE2_SIZE bufflen)
+    PCRE2_SPTR pcre2_get_mark(pcre2_match_data *match_data)
+    uint32_t pcre2_get_ovector_count(pcre2_match_data *match_data)
+    PCRE2_SIZE *pcre2_get_ovector_pointer(pcre2_match_data *match_data)
+    int pcre2_match(const pcre2_code *code, PCRE2_SPTR subject, PCRE2_SIZE length, PCRE2_SIZE startoffset, uint32_t options, pcre2_match_data *match_data, pcre2_match_context *mcontext)
+    int pcre2_pattern_info(const pcre2_code *code, uint32_t what, void *where)
+    pcre2_match_data *pcre2_match_data_create_from_pattern(const pcre2_code *code, pcre2_general_context *gcontext)
+    void pcre2_match_data_free(pcre2_match_data *match_data)
+    int pcre2_substring_get_bynumber(pcre2_match_data *match_data, uint32_t number, PCRE2_UCHAR **bufferptr, PCRE2_SIZE *bufflen)
+    void pcre2_substring_free(PCRE2_UCHAR *buffer)
+    int pcre2_substring_length_bynumber(pcre2_match_data *match_data, uint32_t number, PCRE2_SIZE *length)
+    void pcre2_substring_list_free(PCRE2_SPTR *list)
+    int pcre2_substring_list_get(pcre2_match_data *match_data, PCRE2_UCHAR ***listptr, PCRE2_SIZE **lengthsptr)

game_backuper/_pcre2.pyx

@@ -0,0 +1,436 @@
+from ._pcre2 cimport *
+from ._Python cimport *
+from enum import IntFlag
+from libc.stdlib cimport malloc, free
+from libc.string cimport strcpy
+from cpython.mem cimport PyMem_Free
+try:
+    from functools import cached_property
+except ImportError:
+    cached_property = property
+
+
+cdef extern from "_pcre2.h":
+    const uint32_t _PCRE2_ANCHORED "PCRE2_ANCHORED"
+    const uint32_t _PCRE2_ALLOW_EMPTY_CLASS "PCRE2_ALLOW_EMPTY_CLASS"
+    const uint32_t _PCRE2_ALT_BSUX "PCRE2_ALT_BSUX"
+    const uint32_t _PCRE2_ALT_CIRCUMFLEX "PCRE2_ALT_CIRCUMFLEX"
+    const uint32_t _PCRE2_ALT_VERBNAMES "PCRE2_ALT_VERBNAMES"
+    const uint32_t _PCRE2_AUTO_CALLOUT "PCRE2_AUTO_CALLOUT"
+    const uint32_t _PCRE2_CASELESS "PCRE2_CASELESS"
+    const uint32_t _PCRE2_COPY_MATCHED_SUBJECT "PCRE2_COPY_MATCHED_SUBJECT"
+    const uint32_t _PCRE2_DOLLAR_ENDONLY "PCRE2_DOLLAR_ENDONLY"
+    const uint32_t _PCRE2_DOTALL "PCRE2_DOTALL"
+    const uint32_t _PCRE2_DUPNAMES "PCRE2_DUPNAMES"
+    const uint32_t _PCRE2_ENDANCHORED "PCRE2_ENDANCHORED"
+    const uint32_t _PCRE2_EXTENDED "PCRE2_EXTENDED"
+    const uint32_t _PCRE2_FIRSTLINE "PCRE2_FIRSTLINE"
+    const uint32_t _PCRE2_LITERAL "PCRE2_LITERAL"
+    const uint32_t _PCRE2_MATCH_INVALID_UTF "PCRE2_MATCH_INVALID_UTF"
+    const uint32_t _PCRE2_MATCH_UNSET_BACKREF "PCRE2_MATCH_UNSET_BACKREF"
+    const uint32_t _PCRE2_MULTILINE "PCRE2_MULTILINE"
+    const uint32_t _PCRE2_NEVER_BACKSLASH_C "PCRE2_NEVER_BACKSLASH_C"
+    const uint32_t _PCRE2_NEVER_UCP "PCRE2_NEVER_UCP"
+    const uint32_t _PCRE2_NEVER_UTF "PCRE2_NEVER_UTF"
+    const uint32_t _PCRE2_NOTBOL "PCRE2_NOTBOL"
+    const uint32_t _PCRE2_NOTEOL "PCRE2_NOTEOL"
+    const uint32_t _PCRE2_NOTEMPTY "PCRE2_NOTEMPTY"
+    const uint32_t _PCRE2_NOTEMPTY_ATSTART "PCRE2_NOTEMPTY_ATSTART"
+    const uint32_t _PCRE2_NO_AUTO_CAPTURE "PCRE2_NO_AUTO_CAPTURE"
+    const uint32_t _PCRE2_NO_AUTO_POSSESS "PCRE2_NO_AUTO_POSSESS"
+    const uint32_t _PCRE2_NO_DOTSTAR_ANCHOR "PCRE2_NO_DOTSTAR_ANCHOR"
+    const uint32_t _PCRE2_NO_JIT "PCRE2_NO_JIT"
+    const uint32_t _PCRE2_NO_START_OPTIMIZE "PCRE2_NO_START_OPTIMIZE"
+    const uint32_t _PCRE2_NO_UTF_CHECK "PCRE2_NO_UTF_CHECK"
+    const uint32_t _PCRE2_PARTIAL_HARD "PCRE2_PARTIAL_HARD"
+    const uint32_t _PCRE2_PARTIAL_SOFT "PCRE2_PARTIAL_SOFT"
+    const uint32_t _PCRE2_UCP "PCRE2_UCP"
+    const uint32_t _PCRE2_UNGREEDY "PCRE2_UNGREEDY"
+    const uint32_t _PCRE2_USE_OFFSET_LIMIT "PCRE2_USE_OFFSET_LIMIT"
+    const uint32_t _PCRE2_UTF "PCRE2_UTF"
+    const size_t PCRE2_ZERO_TERMINATED
+    const int PCRE2_ERROR_NOMEMORY
+    const int PCRE2_ERROR_UNSET
+    const int PCRE2_INFO_NAMECOUNT
+    const int PCRE2_INFO_NAMEENTRYSIZE
+    const int PCRE2_INFO_NAMETABLE
+
+
+class Option(IntFlag):
+    PCRE2_ANCHORED = _PCRE2_ANCHORED
+    PCRE2_ALLOW_EMPTY_CLASS = _PCRE2_ALLOW_EMPTY_CLASS
+    PCRE2_ALT_BSUX = _PCRE2_ALT_BSUX
+    PCRE2_ALT_CIRCUMFLEX = _PCRE2_ALT_CIRCUMFLEX
+    PCRE2_ALT_VERBNAMES = _PCRE2_ALT_VERBNAMES
+    PCRE2_AUTO_CALLOUT = _PCRE2_AUTO_CALLOUT
+    PCRE2_CASELESS = _PCRE2_CASELESS
+    PCRE2_DOLLAR_ENDONLY = _PCRE2_DOLLAR_ENDONLY
+    PCRE2_DOTALL = _PCRE2_DOTALL
+    PCRE2_DUPNAMES = _PCRE2_DUPNAMES
+    PCRE2_ENDANCHORED = _PCRE2_ENDANCHORED
+    PCRE2_EXTENDED = _PCRE2_EXTENDED
+    PCRE2_FIRSTLINE = _PCRE2_FIRSTLINE
+    PCRE2_LITERAL = _PCRE2_LITERAL
+    PCRE2_MATCH_INVALID_UTF = _PCRE2_MATCH_INVALID_UTF
+    PCRE2_MATCH_UNSET_BACKREF = _PCRE2_MATCH_UNSET_BACKREF
+    PCRE2_MULTILINE = _PCRE2_MULTILINE
+    PCRE2_NEVER_BACKSLASH_C = _PCRE2_NEVER_BACKSLASH_C
+    PCRE2_NEVER_UCP = _PCRE2_NEVER_UCP
+    PCRE2_NEVER_UTF = _PCRE2_NEVER_UTF
+    PCRE2_NO_AUTO_CAPTURE = _PCRE2_NO_AUTO_CAPTURE
+    PCRE2_NO_AUTO_POSSESS = _PCRE2_NO_AUTO_POSSESS
+    PCRE2_NO_DOTSTAR_ANCHOR = _PCRE2_NO_DOTSTAR_ANCHOR
+    PCRE2_NO_START_OPTIMIZE = _PCRE2_NO_START_OPTIMIZE
+    PCRE2_NO_UTF_CHECK = _PCRE2_NO_UTF_CHECK
+    PCRE2_UCP = _PCRE2_UCP
+    PCRE2_UNGREEDY = _PCRE2_UNGREEDY
+    PCRE2_USE_OFFSET_LIMIT = _PCRE2_USE_OFFSET_LIMIT
+    PCRE2_UTF = _PCRE2_UTF
+
+
+class MatchOption(IntFlag):
+    PCRE2_ANCHORED = _PCRE2_ANCHORED
+    PCRE2_COPY_MATCHED_SUBJECT = _PCRE2_COPY_MATCHED_SUBJECT
+    PCRE2_ENDANCHORED = _PCRE2_ENDANCHORED
+    PCRE2_NOTBOL = _PCRE2_NOTBOL
+    PCRE2_NOTEOL = _PCRE2_NOTEOL
+    PCRE2_NOTEMPTY = _PCRE2_NOTEMPTY
+    PCRE2_NOTEMPTY_ATSTART = _PCRE2_NOTEMPTY_ATSTART
+    PCRE2_NO_JIT = _PCRE2_NO_JIT
+    PCRE2_NO_UTF_CHECK = _PCRE2_NO_UTF_CHECK
+    PCRE2_PARTIAL_HARD = _PCRE2_PARTIAL_HARD
+    PCRE2_PARTIAL_SOFT = _PCRE2_PARTIAL_SOFT
+
+
+cdef class Match:
+    cdef pcre2_match_data* data
+    cdef object inp
+    cdef object r
+    def __cinit__(self):
+        self.data = NULL
+
+    def __dealloc__(self):
+        if self.data != NULL:
+            pcre2_match_data_free(self.data)
+            self.data = NULL
+
+    def __getitem__(self, uint32_t i):
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        if i >= count:
+            raise IndexError(u'No such group')
+        cdef PCRE2_UCHAR *buf
+        cdef PCRE2_SIZE le
+        cdef PCRE2_UCHAR* errbuf
+        cdef size_t errsize = sizeof(PCRE2_UCHAR) * 1024
+        cdef int re = pcre2_substring_get_bynumber(self.data, i, &buf, &le)
+        if re == 0:
+            s = buf.decode()
+            pcre2_substring_free(buf)
+            return s
+        elif re == PCRE2_ERROR_UNSET:
+            return None
+        elif re == PCRE2_ERROR_NOMEMORY:
+            raise MemoryError()
+        else:
+            errbuf = <PCRE2_UCHAR*> malloc(errsize)
+            if errbuf == NULL:
+                raise MemoryError()
+            s = None
+            if pcre2_get_error_message(re, errbuf, errsize) > 0:
+                s = errbuf.decode()
+            free(errbuf)
+            raise ValueError(s if s else u'Can not get substring.')
+
+    def __init__(self, unicode inp, r):
+        self.inp = inp
+        self.r = r
+
+    def end(self) -> int:
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_SIZE* vect = pcre2_get_ovector_pointer(self.data)
+        return vect[1]
+
+    @cached_property
+    def endpos(self) -> int:
+        return self.end()
+
+    def group(self) -> str:
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_UCHAR *buf
+        cdef PCRE2_SIZE le
+        cdef PCRE2_UCHAR* errbuf
+        cdef size_t errsize = sizeof(PCRE2_UCHAR) * 1024
+        cdef int re = pcre2_substring_get_bynumber(self.data, 0, &buf, &le)
+        if re == 0:
+            s = buf.decode()
+            pcre2_substring_free(buf)
+            return s
+        elif re == PCRE2_ERROR_NOMEMORY:
+            raise MemoryError()
+        else:
+            errbuf = <PCRE2_UCHAR*> malloc(errsize)
+            if errbuf == NULL:
+                raise MemoryError()
+            s = None
+            if pcre2_get_error_message(re, errbuf, errsize) > 0:
+                s = errbuf.decode()
+            free(errbuf)
+            raise ValueError(s if s else u'Can not get substring.')
+
+    def groupdict(self):
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_UCHAR **li
+        cdef re = pcre2_substring_list_get(self.data, &li, NULL)
+        if re == PCRE2_ERROR_NOMEMORY:
+            raise MemoryError()
+        elif re != 0:
+            raise ValueError(u'Unexpected error')
+        d = {}
+        cdef size_t i = 1
+        t = self.r.namedtable()
+        while li[i] != NULL:
+            if i not in t:
+                i += 1
+                continue
+            if li[i][0] == 0 and pcre2_substring_length_bynumber(self.data, i, NULL) != 0:
+                if t[i] not in d:
+                    d[t[i]] = None
+            else:
+                if t[i] not in d or d[t[i]] is None:
+                    d[t[i]] = li[i].decode()
+            i += 1
+        while i < count:
+            if i in t:
+                if t[i] not in d:
+                    d[t[i]] = None
+            i += 1
+        pcre2_substring_list_free(li)
+        return d
+
+    def groups(self):
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_UCHAR **li
+        cdef re = pcre2_substring_list_get(self.data, &li, NULL)
+        if re == PCRE2_ERROR_NOMEMORY:
+            raise MemoryError()
+        elif re != 0:
+            raise ValueError(u'Unexpected error')
+        cdef size_t i = 1
+        l = []
+        while li[i] != NULL:
+            if li[i][0] == 0 and pcre2_substring_length_bynumber(self.data, i, NULL) != 0:
+                l.append(None)
+            else:
+                l.append(li[i].decode())
+            i += 1
+        while i < count:
+            l.append(None)
+            i += 1
+        pcre2_substring_list_free(li)
+        return tuple(l)
+
+    @cached_property
+    def lastgroup(self) -> str:
+        regs = self.regs
+        cdef uint32_t le = len(regs)
+        cdef uint32_t i = le - 1
+        while i >= 0:
+            if regs[i][0] == -1:
+                i -= 1
+                continue
+            break
+        if i == 0:
+            return None
+        t = self.r.namedtable()
+        if i in t:
+            return t[i]
+
+    @cached_property
+    def lastindex(self) -> int:
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        return count - 1
+
+    @cached_property
+    def pos(self) -> int:
+        return self.start()
+
+    @cached_property
+    def re(self):
+        return self.r
+
+    @cached_property
+    def regs(self):
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_SIZE* vect = pcre2_get_ovector_pointer(self.data)
+        l = []
+        cdef uint32_t i = 0
+        while i < count:
+            if vect[i * 2] == <PCRE2_SIZE> -1:
+                l.append((-1, -1))
+            else:
+                l.append((vect[i * 2], vect[i * 2 + 1]))
+            i += 1
+        return tuple(l)
+
+    def span(self) -> (int, int):
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_SIZE* vect = pcre2_get_ovector_pointer(self.data)
+        return (vect[0], vect[1])
+
+    def start(self) -> int:
+        if self.data == NULL:
+            raise ValueError(u'No matched data.')
+        cdef uint32_t count = pcre2_get_ovector_count(self.data)
+        if count <= 0:
+            raise ValueError(u'No match')
+        cdef PCRE2_SIZE* vect = pcre2_get_ovector_pointer(self.data)
+        return vect[0]
+
+    @cached_property
+    def string(self) -> str:
+        return self.inp
+
+    cdef set_data(self, pcre2_match_data* data):
+        if data == NULL:
+            raise ValueError(u'data is NULL.')
+        self.data = data
+
+
+cdef class PCRE2:
+    cdef pcre2_code* code
+    def __cinit__(self):
+        self.code = NULL
+
+    def __dealloc__(self):
+        if self.code != NULL:
+            pcre2_code_free(self.code)
+            self.code = NULL
+
+    def __init__(self, unicode inp, opt: Option = None):
+        if inp is None:
+            raise ValueError(u'Empty pattern.')
+        cdef uint32_t opts = _PCRE2_UTF | _PCRE2_ALT_BSUX
+        cdef int err
+        cdef PCRE2_SIZE erroffset
+        cdef PCRE2_UCHAR* errbuf
+        cdef size_t errsize = sizeof(PCRE2_UCHAR) * 1024
+        if isinstance(opt, Option):
+            opts = opt.value
+        elif isinstance(opt, int):
+            opts = opt
+        cdef pcre2_code* re = pcre2_compile(<PCRE2_SPTR>PyUnicode_AsUTF8(inp),
+                                            PCRE2_ZERO_TERMINATED, opts, &err,
+                                            &erroffset, NULL)
+        if re is NULL:
+            errbuf = <PCRE2_UCHAR*> malloc(errsize)
+            if errbuf == NULL:
+                raise MemoryError()
+            s = None
+            if pcre2_get_error_message(err, errbuf, errsize) > 0:
+                s = u"Error at offset %d: %s" % (erroffset, errbuf.decode())
+            free(errbuf)
+            raise ValueError(s if s else u'Not invalid')
+        self.code = re
+
+    def match(self, unicode inp, opt: MatchOption = None, PCRE2_SIZE startoffset = 0, int search_only = 0) -> Match:
+        if inp is None:
+            raise ValueError(u'Empty input.')
+        if self.code == NULL:
+            raise ValueError(u'pattern is NULL.')
+        cdef uint32_t opts = 0
+        if isinstance(opt, MatchOption):
+            opts = opt.value
+        elif isinstance(opt, int):
+            opts = opt
+        cdef pcre2_match_data* data = pcre2_match_data_create_from_pattern(self.code, NULL)
+        cdef PCRE2_UCHAR* errbuf
+        cdef size_t errsize = sizeof(PCRE2_UCHAR) * 1024
+        if data == NULL:
+            raise MemoryError()
+        cdef int re = pcre2_match(self.code, <PCRE2_SPTR>PyUnicode_AsUTF8(inp),
+                                  PCRE2_ZERO_TERMINATED, startoffset, opts, data, NULL)
+        if re <= 0:
+            pcre2_match_data_free(data)
+            data = NULL
+            if re == -1:  # No match
+                if search_only:
+                    return False
+                else:
+                    return None
+            elif re == 0:
+                raise ValueError(u'The vector of offsets is too small')
+            else:
+                errbuf = <PCRE2_UCHAR*> malloc(errsize)
+                if errbuf == NULL:
+                    raise MemoryError()
+                s = None
+                if pcre2_get_error_message(re, errbuf, errsize) > 0:
+                    s = errbuf.decode()
+                free(errbuf)
+                raise ValueError(s if s else u'Can not match')
+        if search_only:
+            pcre2_match_data_free(data)
+            return True
+        m = Match(inp, self)
+        m.set_data(data)
+        return m
+
+    def namedtable(self):
+        if self.code == NULL:
+            raise ValueError(u'pattern is NULL.')
+        cdef uint32_t count
+        cdef uint32_t ensize
+        if pcre2_pattern_info(self.code, PCRE2_INFO_NAMECOUNT, &count) != 0:
+            raise ValueError(u'Can not get namedtable')
+        if pcre2_pattern_info(self.code, PCRE2_INFO_NAMEENTRYSIZE, &ensize) != 0:
+            raise ValueError(u'Can not get namedtable')
+        if count <= 0 or ensize <= 0:
+            raise ValueError(u'Can not get namedtable')
+        cdef PCRE2_SPTR buf
+        if pcre2_pattern_info(self.code, PCRE2_INFO_NAMETABLE, &buf) != 0:
+            raise ValueError(u'Can not get namedtable')
+        cdef uint32_t i = 0
+        cdef size_t ind = 0
+        cdef char* tmp = <char*> malloc(ensize)
+        if tmp == NULL:
+            raise MemoryError()
+        d = {}
+        while i < count:
+            ind = i * ensize
+            strcpy(tmp, <char*>buf + ind + 2)
+            d[buf[ind] * 256 + buf[ind + 1]] = tmp.decode()
+            i += 1
+        free(tmp)
+        return d

game_backuper/_zstd.pxd

@@ -0,0 +1,46 @@
+from libc.stddef cimport size_t
+
+
+cdef extern from "zstd.h":
+    ctypedef struct ZSTD_CCtx:
+        pass
+    ctypedef ZSTD_CCtx ZSTD_CStream
+    ctypedef enum ZSTD_EndDirective:
+        ZSTD_e_continue = 0
+        ZSTD_e_flush = 1
+        ZSTD_e_end = 2
+    cdef struct ZSTD_inBuffer_s:
+        const void* src
+        size_t size
+        size_t pos
+    ctypedef ZSTD_inBuffer_s ZSTD_inBuffer
+    cdef struct ZSTD_outBuffer_s:
+        void* dst
+        size_t size
+        size_t pos
+    ctypedef ZSTD_outBuffer_s ZSTD_outBuffer
+    ctypedef enum ZSTD_cParameter:
+        ZSTD_c_compressionLevel
+        ZSTD_c_checksumFlag
+    ctypedef struct ZSTD_DCtx:
+        pass
+    ctypedef ZSTD_DCtx ZSTD_DStream
+
+    const char* ZSTD_versionString()
+    unsigned ZSTD_isError(size_t code)
+    const char* ZSTD_getErrorName(size_t code)
+    int ZSTD_maxCLevel()
+    ZSTD_CCtx* ZSTD_createCCtx()
+    size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
+    ZSTD_CStream* ZSTD_createCStream()
+    size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
+    size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
+    size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
+    size_t ZSTD_CStreamInSize()
+    size_t ZSTD_CStreamOutSize()
+    size_t ZSTD_DStreamInSize()
+    size_t ZSTD_DStreamOutSize()
+    size_t ZSTD_compressStream2(ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* inp, ZSTD_EndDirective endOp)
+    ZSTD_DCtx* ZSTD_createDCtx()
+    size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx)
+    size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* inp)

game_backuper/_zstd.pyi

@@ -0,0 +1,15 @@
+def version() -> str: ...
+def maxCLevel() -> int: ...
+class ZSTDCompressor:  # noqa: E302
+    def __init__(self, compresslevel: int = 3): ...
+    def compress(self, inp: bytes) -> bytes: ...
+    def flush(self) -> bytes: ...
+class ZSTDDecompressor:  # noqa: E302
+    def __init__(self): ...
+    def decompress(self, data: bytes, max_length: int = -1) -> bytes: ...
+    @property
+    def eof(self) -> bool: ...
+    @property
+    def unused_data(self) -> bytes: ...
+    @property
+    def needs_input(self) -> bool: ...

game_backuper/_zstd.pyx

@@ -0,0 +1,194 @@
+from ._zstd cimport *
+from ._Python cimport *
+from libc.stdlib cimport malloc, free
+
+
+cdef void CHECK_ZSTD(size_t i) except *:
+    if ZSTD_isError(i):
+        raise ValueError(ZSTD_getErrorName(i).decode())
+
+
+def version():
+    cdef const char* v = ZSTD_versionString()
+    return v.decode()
+
+
+def maxCLevel():
+    return ZSTD_maxCLevel()
+
+
+cdef class ZSTDCompressor:
+    cdef ZSTD_CCtx* cctx
+    cdef void* buffOut
+    cdef size_t buffOutSize
+    cdef int finish
+
+    def __cinit__(self):
+        self.cctx = NULL
+        self.buffOut = NULL
+        self.buffOutSize = 0
+
+    def __dealloc__(self):
+        if self.cctx != NULL:
+            ZSTD_freeCCtx(self.cctx)
+        if self.buffOut != NULL:
+            free(self.buffOut)
+
+    def __init__(self, int compresslevel = 3):
+        if compresslevel < 1 or compresslevel > ZSTD_maxCLevel():
+            raise ValueError(u'unsupported compresslevel')
+        self.finish = 0
+        self.cctx = ZSTD_createCCtx()
+        if self.cctx == NULL:
+            raise MemoryError()
+        CHECK_ZSTD(ZSTD_CCtx_setParameter(self.cctx, ZSTD_c_compressionLevel, compresslevel))
+        CHECK_ZSTD(ZSTD_CCtx_setParameter(self.cctx, ZSTD_c_checksumFlag, 1))
+
+    def compress(self, bytes inp):
+        if self.finish:
+            raise ValueError('Compressor has been flushed')
+        if self.buffOut == NULL:
+            self.buffOutSize = ZSTD_CStreamOutSize()
+            self.buffOut = malloc(self.buffOutSize)
+            if self.buffOut == NULL:
+                raise MemoryError()
+        cdef int finished = 0
+        cdef ZSTD_outBuffer out
+        cdef ZSTD_inBuffer i
+        cdef Py_ssize_t si
+        cdef size_t remaining
+        cdef char* obuf
+        if PyBytes_AsStringAndSize(inp, <char**>&i.src, &si) == -1:
+            raise ValueError(u'Can not convert object to void*.')
+        i.size = si
+        i.pos = 0
+        b = b''
+        while not finished:
+            out.dst = self.buffOut
+            out.size = self.buffOutSize
+            out.pos = 0
+            remaining = ZSTD_compressStream2(self.cctx, &out, &i, ZSTD_e_continue)
+            CHECK_ZSTD(remaining)
+            obuf = <char*> out.dst
+            b += PyBytes_FromStringAndSize(obuf, out.pos)
+            finished = i.pos == i.size
+        return b
+
+    def flush(self):
+        if self.finish:
+            raise ValueError('Repeated call to flush()')
+        if self.buffOut == NULL:
+            self.finish = 1
+            return b''
+        cdef int finished = 0
+        cdef ZSTD_outBuffer out
+        cdef ZSTD_inBuffer i
+        cdef char* obuf
+        i.src = NULL
+        i.size = 0
+        i.pos = 0
+        b = b''
+        while not finished:
+            out.dst = self.buffOut
+            out.size = self.buffOutSize
+            out.pos = 0
+            remaining = ZSTD_compressStream2(self.cctx, &out, &i, ZSTD_e_end)
+            CHECK_ZSTD(remaining)
+            obuf = <char*> out.dst
+            b += PyBytes_FromStringAndSize(obuf, out.pos)
+            finished = remaining == 0
+        self.finish = 1
+        return b
+
+
+cdef class ZSTDDecompressor:
+    cdef ZSTD_DCtx* dctx
+    cdef int finish
+    cdef object _buff
+    cdef int need_inp
+    cdef void* buffOut
+    cdef size_t buffOutSize
+    cdef object _unused_data
+
+    def __cinit__(self):
+        self.dctx = NULL
+        self._buff = b''
+        self._unused_data = b''
+        self.buffOut = NULL
+        self.buffOutSize = 0
+
+    def __dealloc__(self):
+        if self.dctx != NULL:
+            ZSTD_freeDCtx(self.dctx)
+        if self.buffOut != NULL:
+            free(self.buffOut)
+
+    def __init__(self):
+        self.dctx = ZSTD_createDCtx()
+        if self.dctx == NULL:
+            raise MemoryError()
+        self.finish = 0
+        self.need_inp = 1
+
+    def decompress(self, bytes data, Py_ssize_t max_length = -1):
+        if not self.need_inp:
+            self.need_inp = 1
+            tmp = self._buff
+            self._buff = b''
+            if self.finish:
+                print(data)
+                self._unused_data += data
+            return tmp
+        if self.finish:
+            raise EOFError('End of stream already reached')
+        if self.buffOut == NULL:
+            self.buffOutSize = ZSTD_DStreamOutSize()
+            self.buffOut = malloc(self.buffOutSize)
+            if self.buffOut == NULL:
+                raise MemoryError()
+        if self.need_inp:
+            b = b''
+        else:
+            b = self._buff
+            self._buff = b''
+        cdef int finished = 0
+        cdef ZSTD_inBuffer i
+        cdef ZSTD_outBuffer out
+        cdef size_t ret
+        cdef Py_ssize_t si
+        cdef char* obuf
+        if PyBytes_AsStringAndSize(data, <char**>&i.src, &si) == -1:
+            raise ValueError(u'Can not convert object to void*.')
+        i.size = si
+        i.pos = 0
+        while not finished:
+            out.dst = self.buffOut
+            out.size = self.buffOutSize
+            out.pos = 0
+            ret = ZSTD_decompressStream(self.dctx, &out, &i)
+            CHECK_ZSTD(ret)
+            obuf = <char*> out.dst
+            b += PyBytes_FromStringAndSize(obuf, out.pos)
+            self.finish = ret == 0
+            finished = out.pos < out.size
+        if self.finish and i.pos < i.size:
+            obuf = (<char*> i.src) + i.pos
+            self._unused_data = PyBytes_FromStringAndSize(obuf, i.size - i.pos)
+        if max_length == -1 or len(b) <= max_length:
+            return b
+        else:
+            self._buff = b[max_length:]
+            self.need_inp = 0
+            return b[:max_length]
+
+    @property
+    def eof(self):
+        return True if self.finish else False
+
+    @property
+    def unused_data(self):
+        return self._unused_data
+
+    @property
+    def needs_input(self):
+        return True if self.need_inp else False

game_backuper/backuper.py

@@ -8,10 +8,14 @@ from game_backuper.config import (
 from game_backuper.cml import Opts, OptAction
 from threading import Thread
 from os.path import exists, join, isdir
-from os import mkdir, remove
+from os import mkdir, remove, close
 from game_backuper.file import new_file, copy_file, File, mkdir_for_file
 from game_backuper.filetype import FileType
 from game_backuper.restorer import RestoreTask
+from game_backuper.file import remove_compress_files, remove_unencryped_files
+from game_backuper.compress import compress
+from game_backuper.enc import encrypt_file
+from tempfile import mkstemp
 
 
 class BackupTask(Thread):
@@ -25,6 +29,7 @@ class BackupTask(Thread):
         self.prog.clear_cache()
         prog = self.prog.name
         bp = join(self.cfg.dest, prog)
+        ebp = join(self.cfg.dest, '.encrypt', prog)
         if not exists(bp):
             mkdir(bp)
         fl = self.db.get_file_list(prog)
@@ -33,19 +38,59 @@ class BackupTask(Thread):
                 if exists(f[1]):
                     if f.name in fl:
                         fl.remove(f.name)
+                    c = f.compress_config
                     ori = self.db.get_file(prog, f[0])
                     nf = new_file(f[1], f[0], prog)
                     if nf is None:
                         continue
-                    de = join(bp, f[0])
+                    de = join(ebp if f.encrypt_files else bp, f[0])
                     if ori is not None:
                         if ori.size == nf.size and ori.hash == nf.hash:
-                            print(f'{prog}: Skip {f[0]}.')
-                            continue
-                        copy_file(f[1], de, f[0], prog)
+                            if c is None:
+                                if exists(de) and not f.encrypt_files:
+                                    print(f'{prog}: Skip {f[0]}.')
+                                    remove_compress_files(de, prog, f.name)
+                                    self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori)  # noqa: E501
+                                    continue
+                                elif exists(de) and f.encrypt_files and not ori.compressed:  # noqa: E501
+                                    print(f'{prog}: Skip {f[0]}.')
+                                    remove_unencryped_files(join(bp, f[0]), prog, f.name)  # noqa: E501
+                                    continue
+                            else:
+                                if not f.encrypt_files and exists(de + c.ext):
+                                    print(f'{prog}: Skip {f.name}.')
+                                    remove_compress_files(de, prog, f.name, c.ext)  # noqa: E501
+                                    self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori)  # noqa: E501
+                                    continue
+                                elif f.encrypt_files and ori.compressed_type == c.method:  # noqa: E501
+                                    print(f'{prog}: Skip {f.name}.')
+                                    remove_unencryped_files(join(bp, f.name), prog, f.name)  # noqa: E501
+                                    continue
+                        stats = None
+                        if f.encrypt_files:
+                            stats = encrypt_file(f[1], de, nf, f.name, prog, c)
+                            remove_unencryped_files(join(bp, f[0]), prog, f.name)  # noqa: E501
+                        elif c is None:
+                            copy_file(f[1], de, f[0], prog)
+                            remove_compress_files(de, prog, f.name)
+                            self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori)  # noqa: E501
+                        else:
+                            compress(f[1], de, c, f.name, prog)
+                            self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori)  # noqa: E501
                         self.db.set_file(ori.id, nf.size, nf.hash)
+                        self.db.set_file_encrypt_information(ori.id, stats)
                     else:
-                        copy_file(f[1], de, f[0], prog)
+                        if f.encrypt_files:
+                            s = encrypt_file(f[1], de, nf, f.name, prog, c)
+                            nf = File.from_encrypt_stats(s, nf)
+                            remove_unencryped_files(join(bp, f[0]), prog, f.name)  # noqa: E501
+                        elif c is None:
+                            copy_file(f[1], de, f[0], prog)
+                            remove_compress_files(de, prog, f.name)
+                            self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori)  # noqa: E501
+                        else:
+                            compress(f[1], de, c, f.name, prog)
+                            self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori)  # noqa: E501
                         self.db.add_file(nf)
             elif isinstance(f, ConfigLeveldb):
                 from game_backuper.leveldb import have_leveldb
@@ -63,28 +108,69 @@ class BackupTask(Thread):
                 ent = list_leveldb_entries(f.full_path, f.domains)
                 stats = leveldb_stats(f.full_path, ent)
                 ori = self.db.get_file(prog, f.name)
-                de = join(bp, f.name + ".db")
+                c = f.compress_config
+                de = join(ebp if f.encrypt_files else bp, f.name + ".db")
                 if ori is not None:
-                    if ori.size == stats.size and ori.hash == stats.hash:
-                        print(f'{prog}: Skip {f[0]}')
-                        continue
                     if ori.type is None or ori.type != FileType.LEVELDB:
                         pp = join(bp, ori.file)
                         if exists(pp):
                             remove(pp)
+                        remove_compress_files(pp, prog, f.name)
+                        self.remove_encrypted_file(join(ebp, ori.file), prog, f.name, ori)  # noqa: E501
                         self.db.remove_file(ori)
                         ori = None
-                if exists(de):
-                    remove(de)
+                if ori is not None:
+                    if ori.size == stats.size and ori.hash == stats.hash:
+                        if c is None:
+                            if exists(de) and not f.encrypt_files:
+                                print(f'{prog}: Skip {f[0]}.')
+                                remove_compress_files(de, prog, f.name)
+                                self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori)  # noqa: E501
+                                continue
+                            elif exists(de) and f.encrypt_files and not ori.compressed:  # noqa: E501
+                                print(f'{prog}: Skip {f[0]}.')
+                                remove_unencryped_files(join(bp, f[0] + '.db'), prog, f.name)  # noqa: E501
+                                continue
+                        else:
+                            if not f.encrypt_files and exists(de + c.ext):
+                                print(f'{prog}: Skip {f.name}.')
+                                remove_compress_files(de, prog, f.name, c.ext)
+                                self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori)  # noqa: E501
+                                continue
+                            elif f.encrypt_files and ori.compressed_type == c.method:  # noqa: E501
+                                print(f'{prog}: Skip {f.name}.')
+                                remove_unencryped_files(join(bp, f.name + '.db'), prog, f.name)  # noqa: E501
+                                continue
                 mkdir_for_file(de)
-                leveldb_to_sqlite(f.full_path, de, ent)
-                print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {de}')  # noqa: E501
+                st = None
+                if c is None:
+                    leveldb_to_sqlite(f.full_path, de, ent)
+                    print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {de}')  # noqa: E501
+                    remove_compress_files(de, prog, f.name)
+                    self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori)  # noqa: E501
+                else:
+                    tmp = mkstemp()
+                    close(tmp[0])
+                    tmp = tmp[1]
+                    leveldb_to_sqlite(f.full_path, tmp, ent)
+                    print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {tmp}')  # noqa: E501
+                    if f.encrypt_files:
+                        st = encrypt_file(tmp, de, File.from_leveldb_stats(stats), f.name, prog, c)  # noqa: E501
+                        remove_unencryped_files(join(bp, f[0] + '.db'), prog, f.name)  # noqa: E501
+                    else:
+                        compress(tmp, de, c, f.name, prog)
+                        self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori)  # noqa: E501
+                    remove(tmp)
+                    print(f'{prog}: Removed tempfile {tmp}')
                 if ori is None:
                     nf = File(None, f.name, stats.size, prog, stats.hash,
-                              FileType.LEVELDB)
+                              FileType.LEVELDB, None, None, None, None, None)
+                    if st:
+                        nf = File.from_encrypt_stats(st, nf)
                     self.db.add_file(nf)
                 else:
                     self.db.set_file(ori.id, stats.size, stats.hash)
+                    self.db.set_file_encrypt_information(ori.id, st)
         for fn in fl:
             f = self.db.get_file(prog, fn)
             if f.type is None:
@@ -92,14 +178,23 @@ class BackupTask(Thread):
                 if exists(de):
                     remove(de)
                     print(f'{prog}: Remove {de}({fn})')
+                remove_compress_files(de, prog, fn)
+                self.remove_encrypted_file(join(ebp, fn), prog, fn, f)
                 self.db.remove_file(f)
             if f.type == FileType.LEVELDB:
                 de = join(bp, fn + '.db')
                 if exists(de):
                     remove(de)
                     print(f'{prog}: Remove {de}({fn})')
+                remove_compress_files(de, prog, fn + '.db')
+                self.remove_encrypted_file(join(ebp, fn + '.db'), prog, fn, f)
                 self.db.remove_file(f)
 
+    def remove_encrypted_file(self, loc: str, prog: str, name: str, f: File):
+        if exists(loc):
+            remove(loc)
+            print(f'{prog}: Removed {loc}({name})')
+
 
 class Backuper:
     def __init__(self, db: Db, config: Config, opts: Opts):

game_backuper/cfapi.py

@@ -0,0 +1,35 @@
+from ctypes import HRESULT, byref, c_uint, windll
+from ctypes.wintypes import (
+    DWORD, HANDLE, LARGE_INTEGER, LPCWSTR, LPVOID, PHANDLE
+)
+
+
+dll = windll.CldApi
+CF_OPEN_FILE_FLAG_NONE = 0
+CF_OPEN_FILE_FLAG_EXCLUSIVE = 1
+CF_OPEN_FILE_FLAG_WRITE_ACCESS = 2
+CF_OPEN_FILE_FLAG_DELETE_ACCESS = 3
+CF_OPEN_FILE_FLAG_FOREGROUND = 4
+CfOpenFileWithOplock = dll.CfOpenFileWithOplock
+CfOpenFileWithOplock.argtypes = [LPCWSTR, c_uint, PHANDLE]
+CfOpenFileWithOplock.restype = HRESULT
+CfHydratePlaceholder = dll.CfHydratePlaceholder
+CfHydratePlaceholder.argtypes = [HANDLE, LARGE_INTEGER, LARGE_INTEGER, c_uint, LPVOID]  # noqa: E501
+CfHydratePlaceholder.restype = HRESULT
+CfCloseHandle = dll.CfCloseHandle
+CfCloseHandle.argtypes = [HANDLE]
+ERROR_INVALID_FUNCTION = 1
+GetLastError = windll.Kernel32.GetLastError
+GetLastError.restype = DWORD
+
+
+def hydrate_file(s: str):
+    h = HANDLE()
+    try:
+        CfOpenFileWithOplock(s, CF_OPEN_FILE_FLAG_NONE, byref(h))
+        CfHydratePlaceholder(h, 0, -1, 0, LPVOID())
+    except OSError as e:
+        if GetLastError() != ERROR_INVALID_FUNCTION:
+            CfCloseHandle(h)
+            raise e
+    CfCloseHandle(h)

game_backuper/cml.py

@@ -36,10 +36,13 @@ class Opts:
     config_file: str = DEFAULT_CONFIG
     action = OptAction.BACKUP
     programs_list = None
+    optimize_db = False
+    change_key = False
 
     def __init__(self, cml: List[str]):
         try:
-            r = getopt(cml, 'hc:', ['help', 'config='])
+            r = getopt(cml, 'hc:', ['help', 'config=', 'optimize-db',
+                                    'change-key'])
             for i in r[0]:
                 if i[0] == '-h' or i[0] == '--help':
                     self.print_help()
@@ -47,6 +50,10 @@ class Opts:
                     sys.exit(0)
                 elif i[0] == '-c' or i[0] == '--config':
                     self.config_file = i[1]
+                elif i[0] == '--optimize-db':
+                    self.optimize_db = True
+                elif i[0] == '--change-key':
+                    self.change_key = True
             if len(r[1]) > 0:
                 cm = r[1]
                 re = OptAction.from_str(cm[0])
@@ -74,4 +81,6 @@ game-backuper [options] list
 game-backuper [options] list_leveldb_key [<db_path> [...]]
 Options:
     -h, --help          Print help message.
-    -c, --config <path> Set config file.''')
+    -c, --config <path> Set config file.
+    --optimize-db       Optimize the sqlite3 database
+    --change-key        Change encrypt password''')

game_backuper/compress.py

@@ -0,0 +1,505 @@
+try:
+    from bz2 import BZ2File, BZ2Compressor, BZ2Decompressor
+    have_bz2 = True
+except ImportError:
+    have_bz2 = False
+try:
+    from gzip import GzipFile
+    have_gzip = True
+except ImportError:
+    have_gzip = False
+try:
+    from lzma import LZMAFile, LZMACompressor, LZMADecompressor
+    have_lzma = True
+except ImportError:
+    have_lzma = False
+try:
+    from lzip import (
+        FileEncoder as LZIPFileEncoder,
+        decompress_file_iter as LZIP_decompress_file_iter,
+        level_to_dictionary_size_and_match_len_limit as lzip_level_dict,
+    )
+    from lzip_extension import (
+        Decoder as LZIPDecoder,
+        Encoder as LZIPEncoder,
+    )
+    have_lzip = True
+except ImportError:
+    have_lzip = False
+try:
+    from game_backuper.zstd import (
+        ZSTDCompressor,
+        ZSTDDecompressor,
+        ZSTDFile,
+        MAX_COMPRESS_LEVEL as ZSTD_MAX,
+    )
+    have_zstd = True
+except ImportError:
+    have_zstd = False
+try:
+    from snappy import (
+        StreamCompressor as Snappy_Compressor,
+        StreamDecompressor as Snappy_Decompressor,
+    )
+    have_snappy = True
+except ImportError:
+    have_snappy = False
+try:
+    from brotli import (
+        Compressor as BrotliCompressor,
+        Decompressor as BrotliDecompressor,
+    )
+    have_brotli = True
+except ImportError:
+    have_brotli = False
+from enum import IntEnum, unique
+try:
+    from functools import cached_property
+except ImportError:
+    cached_property = property
+from os.path import exists, isfile, getsize
+from os import remove
+from game_backuper.file import mkdir_for_file, hydrate_file_if_needed
+
+
+if have_gzip:
+    class GZCompressor:
+        def __init__(self, compress_level: int = 9, file_obj=None):
+            self.write_to_file = True
+            self._fileobj = file_obj
+            self._level = compress_level
+            self._f = None
+
+        def close(self):
+            self._f.close()
+
+        def compress(self, data: bytes) -> bytes:
+            if self._f is None:
+                self._f = GzipFile(mode="wb", compresslevel=self._level, fileobj=self._fileobj)  # noqa: E501
+            self._f.write(data)
+            return b''
+
+        def flush(self) -> bytes:
+            self._f.flush()
+            return b''
+
+    class GZDecompressor:
+        def __init__(self, file_obj=None):
+            self.write_to_file = True
+            self._fileobj = file_obj
+            self._f = None
+
+        def close(self):
+            self._f.close()
+
+        def read(self, len: int) -> bytes:
+            if self._f is None:
+                self._f = GzipFile(mode="rb", fileobj=self._fileobj)
+            return self._f.read(len)
+
+if have_lzip:
+    class LZIPCompressor:
+        def __init__(self, level: int = 6):
+            d = lzip_level_dict[level]
+            self._encoder = LZIPEncoder(d[0], d[1], 1 << 51)
+
+        def compress(self, data: bytes) -> bytes:
+            return self._encoder.compress(data)
+
+        def flush(self) -> bytes:
+            return self._encoder.finish()
+
+    class LZIPDecompressor:
+        def __init__(self):
+            self._decoder = LZIPDecoder(1)
+
+        def decompress(self, data: bytes) -> bytes:
+            return self._decoder.decompress(data)
+
+        @property
+        def eof(self):
+            return False
+
+if have_brotli:
+    class BrotliCompressor2:
+        def __init__(self, *k, **kw):
+            self._c = BrotliCompressor(*k, **kw)
+
+        def compress(self, data: bytes) -> bytes:
+            return self._c.process(data)
+
+        def flush(self) -> bytes:
+            return self._c.finish()
+
+    class BrotliDecompressor2:
+        def __init__(self, *k, **kw) -> None:
+            self._c = BrotliDecompressor(*k, **kw)
+
+        def decompress(self, data: bytes) -> bytes:
+            return self._c.process(data)
+
+        @property
+        def eof(self):
+            return self._c.is_finished()
+
+
+@unique
+class CompressMethod(IntEnum):
+    BZIP2 = 0
+    GZIP = 1
+    LZMA = 2
+    LZIP = 3
+    ZSTD = 4
+    SNAPPY = 5
+    BROTLI = 6
+
+    @staticmethod
+    def from_str(v: str) -> IntEnum:
+        if isinstance(v, str):
+            t = v.lower()
+            if t == 'bzip2':
+                return CompressMethod.BZIP2
+            elif t == "gzip":
+                return CompressMethod.GZIP
+            elif t == "lzma":
+                return CompressMethod.LZMA
+            elif t == "lzip":
+                return CompressMethod.LZIP
+            elif t == "zstd":
+                return CompressMethod.ZSTD
+            elif t == "snappy":
+                return CompressMethod.SNAPPY
+            elif t == "brotli":
+                return CompressMethod.BROTLI
+        else:
+            raise TypeError('Must be str.')
+
+    def to_str(self) -> str:
+        return {
+            CompressMethod.BZIP2: 'bzip2',
+            CompressMethod.GZIP: 'gzip',
+            CompressMethod.LZMA: 'lzma',
+            CompressMethod.LZIP: 'lzip',
+            CompressMethod.ZSTD: 'zstd',
+            CompressMethod.SNAPPY: 'snappy',
+            CompressMethod.BROTLI: 'brotli',
+        }[self]
+
+
+class CompressConfig:
+    def __init__(self, method: str, level: int = None):
+        self._method = CompressMethod.from_str(method)
+        if self._method is None:
+            raise ValueError('Unknown compress method.')
+        if self._method == CompressMethod.BZIP2:
+            if not have_bz2:
+                raise NotImplementedError("bzip2 not supported.")
+            if level is None:
+                self._level = 9
+            else:
+                if isinstance(level, int) and level >= 1 and level <= 9:
+                    self._level = level
+                else:
+                    raise ValueError('bzip2: compress_level should be 1-9.')
+            self._ext = ".bz2"
+        elif self._method == CompressMethod.GZIP:
+            if not have_gzip:
+                raise NotImplementedError("gzip not supported.")
+            if level is None:
+                self._level = 9
+            else:
+                if isinstance(level, int) and level >= 0 and level <= 9:
+                    self._level = level
+                else:
+                    raise ValueError('gzip: compress_level should be 0-9.')
+            self._ext = '.gz'
+        elif self._method == CompressMethod.LZMA:
+            if not have_lzma:
+                raise NotImplementedError("lzma not supported.")
+            if level is None:
+                self._level = 6
+            else:
+                if isinstance(level, int) and level >= 0 and level <= 9:
+                    self._level = level
+                else:
+                    raise ValueError('lzma: compress_level should be 0-9.')
+            self._ext = ".xz"
+        elif self._method == CompressMethod.LZIP:
+            if not have_lzip:
+                raise NotImplementedError("lzip not supported.")
+            if level is None:
+                self._level = 6
+            else:
+                if isinstance(level, int) and level >= 0 and level <= 9:
+                    self._level = level
+                else:
+                    raise ValueError('lzip: compress_level should be 0-9.')
+            self._ext = ".lz"
+        elif self._method == CompressMethod.ZSTD:
+            if not have_zstd:
+                raise NotImplementedError("zstd not supported.")
+            if level is None:
+                self._level = 3
+            else:
+                if isinstance(level, int) and level >= 0 and level <= ZSTD_MAX:
+                    self._level = level
+                else:
+                    raise ValueError(f'zstd: compress_level should be 0-{ZSTD_MAX}.')  # noqa: E501
+            self._ext = ".zst"
+        elif self._method == CompressMethod.SNAPPY:
+            if not have_snappy:
+                raise NotImplementedError("snappy not supported.")
+            self._level = None
+            self._ext = ".snappy"
+        elif self._method == CompressMethod.BROTLI:
+            if not have_brotli:
+                raise NotImplementedError("brotli not supported.")
+            if level is None:
+                self._level = None
+            else:
+                if isinstance(level, int) and level >= 0 and level <= 11:
+                    self._level = level
+                else:
+                    raise ValueError('brotli: compress_level should be 0-11.')
+            self._ext = '.br'
+        self._chunk_size = 131072
+
+    def __repr__(self):
+        t = type(self)
+        return f"<{t.__module__}.{t.__qualname__} object at {hex(id(self))}; method={repr(self._method)}, level={repr(self._level)}, ext={repr(self._ext)}>"  # noqa: E501
+
+    def compressor(self, fileobj):
+        if self._method == CompressMethod.BZIP2:
+            return BZ2Compressor(self._level)
+        elif self._method == CompressMethod.GZIP:
+            return GZCompressor(self._level, fileobj)
+        elif self._method == CompressMethod.LZMA:
+            return LZMACompressor(preset=self._level)
+        elif self._method == CompressMethod.LZIP:
+            return LZIPCompressor(self._level)
+        elif self._method == CompressMethod.ZSTD:
+            return ZSTDCompressor(self._level)
+        elif self._method == CompressMethod.SNAPPY:
+            return Snappy_Compressor()
+        elif self._method == CompressMethod.BROTLI:
+            c = {}
+            if self._level is not None:
+                c['quality'] = self._level
+            return BrotliCompressor2(**c)
+
+    def decompressor(self, fileobj):
+        if self._method == CompressMethod.BZIP2:
+            return BZ2Decompressor()
+        elif self._method == CompressMethod.GZIP:
+            return GZDecompressor(fileobj)
+        elif self._method == CompressMethod.LZMA:
+            return LZMADecompressor()
+        elif self._method == CompressMethod.LZIP:
+            return LZIPDecompressor()
+        elif self._method == CompressMethod.ZSTD:
+            return ZSTDDecompressor()
+        elif self._method == CompressMethod.SNAPPY:
+            return Snappy_Decompressor()
+        elif self._method == CompressMethod.BROTLI:
+            return BrotliDecompressor2()
+
+    @cached_property
+    def chunk_size(self) -> int:
+        return self._chunk_size
+
+    @cached_property
+    def ext(self) -> str:
+        return self._ext
+
+    @cached_property
+    def level(self) -> int:
+        return self._level
+
+    @cached_property
+    def method(self) -> CompressMethod:
+        return self._method
+
+
+supported_exts = []
+if have_bz2:
+    supported_exts.append('.bz2')
+if have_gzip:
+    supported_exts.append('.gz')
+if have_lzma:
+    supported_exts.append('.xz')
+if have_lzip:
+    supported_exts.append('.lz')
+if have_zstd:
+    supported_exts.append('.zst')
+if have_snappy:
+    supported_exts.append('.snappy')
+if have_brotli:
+    supported_exts.append('.br')
+
+
+def sizeof_fmt(num, suffix='B'):
+    for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', ' Ei', 'Zi']:
+        if abs(num) < 1024.0:
+            return "%3.1f%s%s" % (num, unit, suffix)
+        num /= 1024.0
+    return "%.1f%s%s" % (num, 'Yi', suffix)
+
+
+def compress_info(ori: int, re: int):
+    if ori == 0:
+        return f"{sizeof_fmt(ori)} -> {sizeof_fmt(re)} ({float('inf')})"
+    return f"{sizeof_fmt(ori)} -> {sizeof_fmt(re)} ({re/ori*100:.2f}%)"
+
+
+def compress(src: str, dest: str, c: CompressConfig, name: str, prog: str):
+    exts = [''] + supported_exts.copy()
+    exts.remove(c.ext)
+    fn = dest + c.ext
+    cs = c.chunk_size
+    if exists(fn):
+        remove(fn)
+    mkdir_for_file(fn)
+    if c.method == CompressMethod.BZIP2:
+        with open(src, 'rb') as t:
+            with BZ2File(fn, 'wb', compresslevel=c.level) as f:
+                a = t.read(cs)
+                while a != b'':
+                    f.write(a)
+                    a = t.read(cs)
+                del a
+    elif c.method == CompressMethod.GZIP:
+        with open(src, 'rb') as t:
+            with GzipFile(fn, 'wb', compresslevel=c.level) as f:
+                a = t.read(cs)
+                while a != b'':
+                    f.write(a)
+                    a = t.read(cs)
+                del a
+    elif c.method == CompressMethod.LZMA:
+        with open(src, 'rb') as t:
+            with LZMAFile(fn, 'wb', preset=c.level) as f:
+                a = t.read(cs)
+                while a != b'':
+                    f.write(a)
+                    a = t.read(cs)
+                del a
+    elif c.method == CompressMethod.LZIP:
+        with open(src, 'rb') as t:
+            with LZIPFileEncoder(fn, c.level) as f:
+                a = t.read(cs)
+                while a != b'':
+                    f.compress(a)
+                    a = t.read(cs)
+                del a
+    elif c.method == CompressMethod.ZSTD:
+        with open(src, 'rb') as t:
+            with ZSTDFile(fn, 'wb', compresslevel=c.level) as f:
+                a = t.read(cs)
+                while a != b'':
+                    f.write(a)
+                    a = t.read(cs)
+                del a
+    elif c.method == CompressMethod.SNAPPY:
+        with open(src, 'rb') as t:
+            with open(fn, 'wb') as f:
+                o = Snappy_Compressor()
+                a = t.read(cs)
+                while a != b'':
+                    b = o.compress(a)
+                    f.write(b)
+                    a = t.read(cs)
+                del a, b, o
+    elif c.method == CompressMethod.BROTLI:
+        k = {}
+        if c.level is not None:
+            k['quality'] = c.level
+        with open(src, 'rb') as t:
+            with open(fn, 'wb') as f:
+                o = BrotliCompressor(**k)
+                a = t.read(cs)
+                while a != b'':
+                    b = o.process(a)
+                    f.write(b)
+                    a = t.read(cs)
+                f.write(o.finish())
+                del a, b, o
+        del k
+    i = compress_info(getsize(src), getsize(fn))
+    print(f'{prog}: Compressed {src}({name}) -> {fn} ({i})')
+    del i
+    for i in exts:
+        f = dest + i
+        if exists(f) and isfile(f):
+            remove(f)
+            print(f'{prog}: Removed {f}({name})')
+
+
+def decompress(src: str, dest: str, c: CompressConfig, name: str, prog: str):
+    fn = src + c.ext
+    if exists(dest):
+        remove(dest)
+    cs = c.chunk_size
+    hydrate_file_if_needed(fn)
+    if c.method == CompressMethod.BZIP2:
+        with BZ2File(fn, 'rb') as f:
+            with open(dest, 'wb') as t:
+                a = f.read(cs)
+                while a != b'':
+                    t.write(a)
+                    a = f.read(cs)
+                del a
+    elif c.method == CompressMethod.GZIP:
+        with GzipFile(fn, 'rb') as f:
+            with open(dest, 'wb') as t:
+                a = f.read(cs)
+                while a != b'':
+                    t.write(a)
+                    a = f.read(cs)
+                del a
+    elif c.method == CompressMethod.LZMA:
+        with LZMAFile(fn, 'rb') as f:
+            with open(dest, 'wb') as t:
+                a = f.read(cs)
+                while a != b'':
+                    t.write(a)
+                    a = f.read(cs)
+                del a
+    elif c.method == CompressMethod.LZIP:
+        with open(dest, 'wb') as t:
+            f = LZIP_decompress_file_iter(fn, chunk_size=cs)
+            for a in f:
+                t.write(a)
+            del a
+    elif c.method == CompressMethod.ZSTD:
+        with ZSTDFile(fn, 'rb') as f:
+            with open(dest, 'wb') as t:
+                a = f.read(cs)
+                while a != b'':
+                    t.write(a)
+                    a = f.read(cs)
+                del a
+    elif c.method == CompressMethod.SNAPPY:
+        with open(fn, 'rb') as f:
+            with open(dest, 'wb') as t:
+                o = Snappy_Decompressor()
+                a = f.read(cs)
+                while a != b'':
+                    b = o.decompress(a)
+                    t.write(b)
+                    a = f.read(cs)
+                o.flush()
+                del a, b, o
+    elif c.method == CompressMethod.BROTLI:
+        with open(fn, 'rb') as f:
+            with open(dest, 'wb') as t:
+                o = BrotliDecompressor()
+                a = f.read(cs)
+                while a != b'':
+                    b = o.process(a)
+                    t.write(b)
+                    a = f.read(cs)
+                if not o.is_finished():
+                    raise ValueError('Read all datas from file but seems not finished.')  # noqa: E501
+                del a, b, o
+    i = compress_info(getsize(dest), getsize(fn))
+    print(f'{prog}: Decompressed {fn}({name}) -> {dest} ({i})')

game_backuper/config.py

@@ -3,7 +3,7 @@ try:
     from yaml import CSafeLoader as SafeLoader
 except Exception:
     from yaml import SafeLoader
-from os.path import join, relpath, isfile, isdir, isabs
+from os.path import join, relpath, isfile, isdir, isabs, abspath
 from typing import List, Union
 from game_backuper.file import listdirs
 from collections import namedtuple
@@ -11,11 +11,58 @@ try:
     from functools import cached_property
 except ImportError:
     cached_property = property
+from game_backuper.regexp import Regex, wildcards_to_regex
+from game_backuper.compress import CompressConfig
 
 
 class BasicOption:
     '''Basic options which is included in config, program and files.'''
     _remove_old_files = None
+    _enable_pcre2 = None
+    _encrypt_files = None
+    _compress_config = None
+
+    @property
+    def compress_config(self) -> CompressConfig:
+        if self._compress_config is not None:
+            return self._compress_config
+        prog = getattr(self, "_prog", None)
+        if prog is not None:
+            if prog._compress_config is not None:
+                return prog._compress_config
+        cfg = getattr(self, "_cfg", None)
+        if cfg is not None:
+            if cfg._compress_config is not None:
+                return cfg._compress_config
+        return None
+
+    @cached_property
+    def enable_pcre2(self) -> bool:
+        if self._enable_pcre2 is not None:
+            return self._enable_pcre2
+        prog = getattr(self, "_prog", None)
+        if prog is not None:
+            if prog._enable_pcre2 is not None:
+                return prog._enable_pcre2
+        cfg = getattr(self, "_cfg", None)
+        if cfg is not None:
+            if cfg._enable_pcre2 is not None:
+                return cfg._enable_pcre2
+        return False
+
+    @cached_property
+    def encrypt_files(self) -> bool:
+        if self._encrypt_files is not None:
+            return self._encrypt_files
+        prog = getattr(self, "_prog", None)
+        if prog is not None:
+            if prog._encrypt_files is not None:
+                return prog._encrypt_files
+        cfg = getattr(self, "_cfg", None)
+        if cfg is not None:
+            if cfg._encrypt_files is not None:
+                return cfg._encrypt_files
+        return False
 
     @cached_property
     def remove_old_files(self) -> bool:
@@ -32,7 +79,42 @@ class BasicOption:
         return True
 
     def parse_all(self, data=None):
+        self.parse_compress_config(data)
         self.parse_remove_old_files(data)
+        self.parse_enable_pcre2(data)
+        self.parse_encrypt_files(data)
+
+    def parse_compress_config(self, data=None):
+        if data is None:
+            data = getattr(self, 'data')
+        if 'compress_method' in data:
+            v = data['compress_method']
+            if isinstance(v, str):
+                self._compress_config = CompressConfig(v, data.get("compress_level"))  # noqa: E501
+            elif v is not None:
+                raise TypeError('compress_method option should be str or None.')  # noqa: E501
+
+    def parse_enable_pcre2(self, data=None):
+        if data is None:
+            data = getattr(self, 'data')
+        if 'enable_pcre2' in data:
+            v = data['enable_pcre2']
+            if isinstance(v, bool):
+                self._enable_pcre2 = v
+            else:
+                raise TypeError('enable_pcre2 option must be a boolean.')
+            del v
+
+    def parse_encrypt_files(self, data=None):
+        if data is None:
+            data = getattr(self, 'data')
+        if 'encrypt_files' in data:
+            v = data['encrypt_files']
+            if isinstance(v, bool):
+                self._encrypt_files = v
+            else:
+                raise TypeError('encrypt_files option must be a boolean.')
+            del v
 
     def parse_remove_old_files(self, data=None):
         if data is None:
@@ -121,6 +203,20 @@ class BasicConfig:
 # pylint: enable=unsupported-membership-test, unsubscriptable-object
 
 
+def parse_ex_or_in_cludes(li: list, enable_pcre2) -> List[Union[str, Regex]]:
+    r = []
+    for i in li:
+        if isinstance(i, str):
+            r.append(i)
+        elif isinstance(i, dict):
+            t = i['type']
+            if t == 'wildcards':
+                r.append(wildcards_to_regex(i['rule'], use_pcre2=enable_pcre2))
+            elif t == "regex":
+                r.append(Regex(i['rule'], use_pcre2=enable_pcre2))
+    return r
+
+
 class ConfigPath(BasicOption, NFBasicOption, BasicConfig):
     def __init__(self, data, cfg, prog):
         NFBasicOption.__init__(self, cfg, prog)
@@ -133,6 +229,61 @@ class ConfigPath(BasicOption, NFBasicOption, BasicConfig):
         self.parse_all()
         self.parse_all_nf()
 
+    @property
+    def excludes(self) -> List[Union[str, Regex]]:
+        t = getattr(self, "__excludes", None)
+        if t is not None:
+            return t
+        del t
+        if 'excludes' in self.data:
+            if isinstance(self.data['excludes'], list):
+                r = parse_ex_or_in_cludes(self.data["excludes"], self.enable_pcre2)  # noqa: E501
+                self.__excludes = r
+                return r
+
+    @property
+    def includes(self) -> List[Union[str, Regex]]:
+        t = getattr(self, "__includes", None)
+        if t is not None:
+            return t
+        del t
+        if 'includes' in self.data:
+            if isinstance(self.data['includes'], list):
+                r = parse_ex_or_in_cludes(self.data["includes"], self.enable_pcre2)  # noqa: E501
+                self.__includes = r
+                return r
+
+    def is_ex_or_in_clude(self, b: str, loc: str, exclude: bool) -> bool:
+        e = self.excludes if exclude else self.includes
+        if e is None:
+            return False if exclude else True
+        if isabs(loc):
+            bl = abspath(loc)
+            rl = relpath(loc, b)
+        else:
+            bl = abspath(join(b, loc))
+            rl = relpath(join(b, loc), b)
+        for i in e:
+            if isinstance(i, str):
+                if isabs(i):
+                    if abspath(i) == bl:
+                        return True
+                else:
+                    if relpath(join(b, i), b) == rl:
+                        return True
+            elif isinstance(i, Regex):
+                if i.match_only(rl):
+                    return True
+                elif bl != loc and i.match_only(bl):
+                    return True
+        return False
+
+    def is_exclude(self, b: str, loc: str) -> bool:
+        return self.is_ex_or_in_clude(b, loc, True)
+
+    def is_include(self, b: str, loc: str) -> bool:
+        return self.is_ex_or_in_clude(b, loc, False)
+
 
 class ConfigOLeveldb(BasicOption, NFBasicOption, BasicConfig):
     def __init__(self, data, cfg, prog):
@@ -218,78 +369,46 @@ class Program(BasicOption, NFBasicOption):
             return self._files.copy()
         r = []
         self._files = r.copy()
-        for i in self.data[ke]:
+        for i in self.all_configs:
             b = self.base
-            if isinstance(i, str):
-                if isabs(i):
-                    raise ValueError('Absolute path must need a name.')
-                bp = join(b, i)
+            if isinstance(i, ConfigPath):
+                if isabs(i.path):
+                    bp = i.path
+                else:
+                    bp = join(b, i.path)
+                name = i.real_name
                 if isfile(bp):
-                    tname = relpath(join(b, i), b)
-                    r.append(ConfigNormalFile(tname, bp))
+                    tname = relpath(join(b, name), b)
+                    tmp = ConfigNormalFile(tname, bp)
                     del tname
+                    tmp.parse_all(i.data)
+                    r.append(tmp)
                 elif isdir(bp):
                     top = NFBasicOption(self._cfg, self)
+                    top.parse_ignore_hidden_files(i.data)
                     ll = listdirs(bp, top.ignore_hidden_files)
                     del top
                     for ii in ll:
-                        r.append(ConfigNormalFile(relpath(ii, b), ii))
-            elif isinstance(i, dict):
-                t = i['type']
-                if t == 'path':
-                    if isabs(i['path']):
-                        if 'name' not in i or not isinstance(i['name'], str) or i['name'] == '':  # noqa: E501
-                            raise ValueError('Absolute path must need a name.')
-                        bp = i['path']
-                        name = i['name']
-                    else:
-                        bp = join(b, i['path'])
-                        name = i['path']
-                        if 'name' in i and isinstance(i['name'], str):
-                            if i['name'] != '':
-                                name = i['name']
-                    if isfile(bp):
-                        tname = relpath(join(b, name), b)
-                        tmp = ConfigNormalFile(tname, bp)
+                        if i.is_exclude(bp, ii):
+                            continue
+                        if not i.is_include(bp, ii):
+                            continue
+                        tname = relpath(join(b, join(name, relpath(ii, bp))), b)  # noqa: E501
+                        tmp = ConfigNormalFile(tname, ii)
                         del tname
-                        tmp.parse_all(i)
+                        tmp.parse_all(i.data)
                         r.append(tmp)
-                    elif isdir(bp):
-                        top = NFBasicOption(self._cfg, self)
-                        top.parse_ignore_hidden_files(i)
-                        ll = listdirs(bp, top.ignore_hidden_files)
-                        del top
-                        for ii in ll:
-                            tname = relpath(join(b, join(name, relpath(ii, bp))), b)  # noqa: E501
-                            tmp = ConfigNormalFile(tname, ii)
-                            del tname
-                            tmp.parse_all(i)
-                            r.append(tmp)
-                elif t == 'leveldb':
-                    if isabs(i['path']):
-                        if 'name' not in i or not isinstance(i['name'], str) or i['name'] == '':  # noqa: E501
-                            raise ValueError('Absolute path must need a name.')
-                        p = i['path']
-                        n = i['name']
-                    else:
-                        p = join(b, i['path'])
-                        n = i['path']
-                        if 'name' in i and isinstance(i['name'], str):
-                            if i['name'] != '':
-                                n = i['name']
-                    dms = None
-                    if 'domains' in i and isinstance(i['domains'], list):
-                        dms = []
-                        for ii in i['domains']:
-                            if isinstance(ii, str) and len(ii) > 0:
-                                dms.append(ii.encode())
-                        if len(dms) == 0:
-                            dms = None
-                    tname = relpath(join(b, n), b)
-                    tmp = ConfigLeveldb(tname, p, dms)
-                    del tname
-                    tmp.parse_all(i)
-                    r.append(tmp)
+            elif isinstance(i, ConfigOLeveldb):
+                if isabs(i.path):
+                    p = i.path
+                else:
+                    p = join(b, i.path)
+                name = i.real_name
+                tname = relpath(join(b, name), b)
+                tmp = ConfigLeveldb(tname, p, i.domains)
+                del tname
+                tmp.parse_all(i.data)
+                r.append(tmp)
         for i in r:
             i._cfg = self._cfg
             i._prog = self
@@ -311,7 +430,13 @@ class Program(BasicOption, NFBasicOption):
                             if i['name'] != '':
                                 n = i['name']
                     if not relpath(name, n).startswith('..'):
-                        return ConfigPath(i, self._cfg, self)
+                        r = ConfigPath(i, self._cfg, self)
+                        tmp = relpath(name, n)
+                        if r.is_exclude(n, tmp):
+                            continue
+                        if not r.is_include(n, tmp):
+                            continue
+                        return r
                 elif t == 'leveldb':
                     if relpath(i['name'], name) == '.':
                         return ConfigOLeveldb(i, self._cfg, self)
@@ -326,6 +451,8 @@ class Program(BasicOption, NFBasicOption):
 
 class Config(BasicOption, NFBasicOption):
     dest = ''
+    encrypt_db = False
+    db_password = None
     progs = []
     progs_name = []
 
@@ -339,6 +466,16 @@ class Config(BasicOption, NFBasicOption):
         if 'dest' not in t or not isinstance(t['dest'], str):
             raise ValueError("Config file don't have dest or dest is not str.")
         self.dest = t['dest']
+        if 'encrypt_db' in t:
+            if not isinstance(t['encrypt_db'], bool):
+                raise ValueError('encrypt_db should be true or false.')
+            self.encrypt_db = t['encrypt_db']
+        else:
+            self.encrypt_db = False
+        if 'db_password' in t:
+            if not isinstance(t['db_password'], str):
+                raise ValueError('db_password should be a string.')
+            self.db_password = t['db_password']
         if 'programs' not in t:
             raise ValueError("No programs found.")
         self.parse_all(t)

game_backuper/db.py

@@ -1,8 +1,15 @@
-from sqlite3 import connect
+from getpass import getpass as _getpass
+from os import close
 from os.path import join
-from typing import List, Union
+from shutil import move
+from sqlite3 import connect, Connection, DatabaseError
+from tempfile import mkstemp
 from threading import Lock
-from game_backuper.file import File
+from typing import List, Union
+from game_backuper.cml import Opts
+from game_backuper.config import Config
+from game_backuper.enc import EncryptStats
+from game_backuper.file import File, hydrate_file_if_needed
 from game_backuper.filetype import FileType
 
 
@@ -27,10 +34,25 @@ id INT,
 type INT,
 PRIMARY KEY(id)
 );'''
+ENCRYPTED_FILES_TABLE = '''CREATE TABLE encrypted_files (
+id INTEGER,
+key TEXT,
+iv TEXT,
+crc32 TEXT,
+compressed INT,
+compressed_size INT,
+PRIMARY KEY(id)
+);'''
+
+
+def getpass(prompt, cfg: Config) -> str:
+    if cfg.db_password is not None:
+        return cfg.db_password
+    return _getpass(prompt)
 
 
 class Db:
-    VERSION = [1, 0, 0, 1]
+    VERSION = [1, 0, 0, 2]
 
     def __check_database(self) -> bool:
         self.__updateExistsTable()
@@ -40,6 +62,8 @@ class Db:
         if v < self.VERSION:
             if v < [1, 0, 0, 1]:
                 self.db.execute(FILETYPE_TABLE)
+            if v < [1, 0, 0, 2]:
+                self.db.execute(ENCRYPTED_FILES_TABLE)
             self.__write_version()
         if v > self.VERSION:
             raise ValueError(
@@ -54,17 +78,68 @@ class Db:
             self.db.execute(FILES_TABLE)
         if 'filetype' not in self._exist_table:
             self.db.execute(FILETYPE_TABLE)
+        if 'encrypted_files' not in self._exist_table:
+            self.db.execute(ENCRYPTED_FILES_TABLE)
         self.db.commit()
 
-    def __init__(self, loc: str):
-        fn = join(loc, "data.db")
+    def __init__(self, config: Config, opts: Opts):
+        self._cfg = config
+        self._opt = opts
+        fn = join(config.dest, "data.db")
+        hydrate_file_if_needed(fn)
         self.db = connect(fn, check_same_thread=False)
-        self.db.execute('VACUUM;')
-        self.db.commit()
+        if config.encrypt_db:
+            passpharse = getpass('Please input the password of the database:', config)  # noqa: E501
+            if not self.encrypted:
+                tfn = mkstemp()
+                close(tfn[0])
+                tfn = tfn[1]
+                db = connect(tfn)
+                self.__set_encrypt_key(passpharse, db)
+                for q in self.db.iterdump():
+                    db.execute(q)
+                self.db.close()
+                db.close()
+                move(tfn, fn)
+                self.db = connect(fn, check_same_thread=False)
+            elif opts.change_key:
+                self.__set_encrypt_key(passpharse)
+                passpharse = getpass('Please input new password of the database:', config)  # noqa: E501
+                tfn = mkstemp()
+                close(tfn[0])
+                tfn = tfn[1]
+                db = connect(tfn)
+                self.__set_encrypt_key(passpharse, db)
+                for q in self.db.iterdump():
+                    db.execute(q)
+                self.db.close()
+                db.close()
+                move(tfn, fn)
+                self.db = connect(fn, check_same_thread=False)
+            self.__set_encrypt_key(passpharse)
+        else:
+            if self.encrypted:
+                passpharse = getpass('Please input the password of the database:', config)  # noqa: E501
+                self.__set_encrypt_key(passpharse)
+                tfn = mkstemp()
+                close(tfn[0])
+                tfn = tfn[1]
+                db = connect(tfn)
+                for q in self.db.iterdump():
+                    db.execute(q)
+                self.db.close()
+                db.close()
+                move(tfn, fn)
+                self.db = connect(fn, check_same_thread=False)
+        if opts.optimize_db:
+            self.db.execute('VACUUM;')
+            self.db.commit()
         ok = self.__check_database()
         if not ok:
             self.__create_table()
         self._lock = Lock()
+        if config.encrypt_db and not self.encrypted:
+            print('Warning: Current library do not support encryption.')
 
     def __read_version(self) -> List[int]:
         if 'version' not in self._exist_table:
@@ -73,6 +148,12 @@ class Db:
         for i in cur:
             return [k for k in i if isinstance(k, int)]
 
+    def __set_encrypt_key(self, key: str, db: Connection = None):
+        if db is None:
+            db = self.db
+        db.execute('PRAGMA cipher_salt = "x\'2d506b1d2c3e7b075518f9db81039657\'";')  # noqa: E501
+        db.execute('PRAGMA key = \'%s\';' % (key.replace("'", "\\'")))
+
     def __updateExistsTable(self):
         cur = self.db.execute('SELECT * FROM main.sqlite_master;')
         self._exist_table = {}
@@ -101,12 +182,29 @@ class Db:
                 for i in cur:
                     self.db.execute('INSERT INTO filetype VALUES (?, ?);',
                                     (i[0], f.type))
+            if f.encrypted:
+                cur = self.db.execute(
+                    'SELECT * FROM files WHERE program=? AND file=?;',
+                    (f.program, f.file))
+                for i in cur:
+                    self.db.execute('INSERT INTO encrypted_files VALUES (?, ?, ?, ?, ?, ?);', (i[0], f.key, f.iv, f.crc32, f.x_compress_type, f.compressed_size))  # noqa: E501
             self.db.commit()
 
+    @property
+    def encrypted(self):
+        try:
+            con = connect(join(self._cfg.dest, 'data.db'))
+            con.execute('SELECT count(*) FROM sqlite_master;')
+            con.close()
+            return False
+        except DatabaseError:
+            con.close()
+            return True
+
     def get_file(self, prog: str, file: str) -> File:
         with self._lock:
             cur = self.db.execute(
-                'SELECT files.*, filetype.type FROM files LEFT JOIN filetype ON files.id=filetype.id WHERE program=? AND file=?;',  # noqa: E501
+                'SELECT files.*, filetype.type, encrypted_files.key, encrypted_files.iv, encrypted_files.crc32, encrypted_files.compressed, encrypted_files.compressed_size FROM files LEFT JOIN filetype ON files.id=filetype.id LEFT JOIN encrypted_files ON files.id=encrypted_files.id WHERE program=? AND file=?;',  # noqa: E501
                 (prog, file))
             for i in cur:
                 return File(*i)
@@ -135,6 +233,10 @@ class Db:
             self.db.execute('DELETE FROM files WHERE id=?;', (iid,))
             if ft is not None:
                 self.db.execute('DELETE FROM filetype WHERE id=?;', (iid,))
+            cur = self.db.execute('SELECT * FROM encrypted_files WHERE id=?;', (iid,))  # noqa: E501
+            for i in cur:
+                self.db.execute('DELETE FROM encrypted_files WHERE id=?;', (iid,))  # noqa: E501
+                break
             self.db.commit()
 
     def set_file(self, id: int, size: int, hash: str):
@@ -142,3 +244,20 @@ class Db:
             self.db.execute('UPDATE files SET size=?, hash=? WHERE id=?;',
                             (size, hash, id))
             self.db.commit()
+
+    def set_file_encrypt_information(self, id: int, stats: EncryptStats):
+        if stats is not None and not isinstance(stats, EncryptStats):
+            raise TypeError(f"Expected EncryptStats, got {type(stats)}")
+        with self._lock:
+            if stats is None:
+                self.db.execute('DELETE FROM encrypted_files WHERE id=?;', (id,))  # noqa: E501
+            else:
+                cur = self.db.execute('SELECT * FROM encrypted_files WHERE id=?;', (id,))  # noqa: E501
+                have_data = False
+                for _ in cur:
+                    have_data = True
+                if have_data:
+                    self.db.execute('UPDATE encrypted_files SET key=?, iv=?, crc32=?, compressed=?, compressed_size=? WHERE id=?;', (stats.key, stats.iv, stats.crc32, stats.compress_type.value if stats.compressed else None, stats.compressed_size if stats.compressed else None, id))  # noqa: E501
+                else:
+                    self.db.execute('INSERT INTO encrypted_files VALUES (?, ?, ?, ?, ?, ?);', (id, stats.key, stats.iv, stats.crc32, stats.compress_type.value if stats.compressed else None, stats.compressed_size if stats.compressed else None))  # noqa: E501
+            self.db.commit()

game_backuper/enc.py

@@ -0,0 +1,370 @@
+from base64 import b85decode, b85encode
+from collections import namedtuple
+from io import RawIOBase
+from os import PathLike, remove, urandom
+from os.path import exists, getsize
+from typing import Union
+from zlib import crc32
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from game_backuper.compress import (
+    CompressConfig,
+    CompressMethod,
+    compress_info,
+)
+from game_backuper.file import File, hydrate_file_if_needed, mkdir_for_file
+
+
+_MODE_CLOSED = 0
+_MODE_READ = 1
+_MODE_WRITE = 3
+_EncrpytStats = namedtuple('EncryptStats', ['key', 'iv', 'crc32', 'x_compress_type', 'compressed_size'])  # noqa: E501
+
+
+class EncryptStats(_EncrpytStats):
+    @property
+    def compressed(self):
+        return self.x_compress_type is not None
+
+    @property
+    def compress_type(self):
+        return CompressMethod(self.x_compress_type)
+
+
+class DecryptException(Exception):
+    pass
+
+
+class EncFile(RawIOBase):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, val, tb):
+        if not self.closed:
+            self.close()
+
+    def __init__(self, fn, mode: str, salt: Union[bytes, str],
+                 key: Union[bytes, str] = None, iv: Union[bytes, str] = None,
+                 length: int = None, crc32: Union[str, int] = None,
+                 compress: CompressConfig = None):
+        self._fp = None
+        self._mode = _MODE_CLOSED
+        if mode in ["", "r", "rb"]:
+            mode = "rb"
+            mode_code = _MODE_READ
+        elif mode in ["w", "wb"]:
+            mode = "wb"
+            mode_code = _MODE_WRITE
+        elif mode in ["x", "xb"]:
+            mode = "xb"
+            mode_code = _MODE_WRITE
+        elif mode in ["a", "ab"]:
+            mode = "ab"
+            mode_code = _MODE_WRITE
+        if isinstance(fn, (str, bytes, PathLike)):
+            self._fp = open(fn, mode)
+            self._mode = mode_code
+        else:
+            raise TypeError('filename must be str, bytes or PathLike.')
+        if self._mode == _MODE_WRITE:
+            self._key = urandom(32)
+            self._iv = urandom(16)
+            self._cipher = Cipher(algorithms.AES(self._key), modes.CBC(self._iv))  # noqa: E501
+            self._enc = self._cipher.encryptor()
+            self._crc32 = 0
+            if compress is not None:
+                self._compressor = compress.compressor(self)
+                if self._compressor is None:
+                    raise NotImplementedError('Unsupported compression type.')
+            else:
+                self._compressor = None
+        if isinstance(salt, str):
+            self._salt = b85decode(salt)
+        else:
+            self._salt = salt
+        if self._mode == _MODE_READ:
+            if isinstance(key, str):
+                key = b85decode(key)
+            if isinstance(iv, str):
+                iv = b85decode(iv)
+            if key is None or len(key) != 32:
+                raise ValueError('A 256-bit key is required.')
+            self._key = key
+            if iv is None or len(iv) != 16:
+                raise ValueError('A 128-bit initialization_vector is required.')  # noqa: E501
+            self._iv = iv
+            self._cipher = Cipher(algorithms.AES(self.key), modes.CBC(self._iv))  # noqa: E501
+            self._dec = self._cipher.decryptor()
+            if length is None or not isinstance(length, int):
+                raise ValueError("data's length is needed.")
+            self._length = length
+            self._buf = b''
+            self._eof = False
+            if crc32:
+                if isinstance(crc32, str):
+                    self._crc32 = int(crc32, 16)
+                else:
+                    self._crc32 = crc32
+                self._crc32_checked = False
+            else:
+                self._crc32 = None
+            self._decompressor = None
+            if compress:
+                self._decompressor = compress.decompressor(self)
+                if self._decompressor is None:
+                    raise NotImplementedError('Unsupported compression type.')
+                self._debuf = b''
+        self._pos = 0
+        self._crc_size = algorithms.AES.block_size * 8
+        self._flushing = False
+
+    def _check_not_closed(self):
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+    def close(self):
+        if self._mode == _MODE_CLOSED:
+            return
+        try:
+            if self._mode == _MODE_READ:
+                if hasattr(self._decompressor, 'close'):
+                    self._decompressor.write_to_file = False
+                    self._decompressor.close()
+                    self._decompressor.write_to_file = True
+            if self._mode == _MODE_WRITE:
+                if hasattr(self._compressor, 'close'):
+                    self._compressor.write_to_file = False
+                    self._compressor.close()
+                    self._compressor.write_to_file = True
+                elif self._compressor:
+                    d = self._compressor.flush()
+                    if d:
+                        length = len(d)
+                        if self._pos < self._crc_size:
+                            le = min(length, self._crc_size - self._pos)
+                            self._crc32 = crc32(d[:le], self._crc32)
+                        self._fp.write(self._enc.update(d))
+                        self._pos += length
+                if self._pos % algorithms.AES.block_size != 0:
+                    self._fp.write(self._enc.update(b"\x00" * (algorithms.AES.block_size - (self._pos % algorithms.AES.block_size))))  # noqa: E501
+                self._fp.write(self._enc.finalize())
+        finally:
+            self._fp.close()
+            self._dec = None
+            self._enc = None
+            self._cipher = None
+            self._fp = None
+            self._mode = _MODE_CLOSED
+
+    @property
+    def closed(self):
+        return self._mode == _MODE_CLOSED
+
+    def check_crc32(self):
+        while len(self._buf) < self._crc_size:
+            if not self.__read():
+                break
+        if crc32(self._buf[:min(self._crc_size, self._length)]) == self._crc32:
+            return True
+        else:
+            return False
+
+    @property
+    def crc32(self):
+        return '{:08x}'.format(self._crc32)
+
+    def __decompress(self):
+        if len(self._buf) == 0:
+            return False
+        if self._decompressor:
+            le = min(len(self._buf), self._length - self._pos)
+            if le == 0 or (hasattr(self._decompressor, "eof") and self._decompressor.eof):  # noqa: E501
+                return False
+            self._debuf += self._decompressor.decompress(self._buf[:le])
+            self._pos += le
+            self._buf = self._buf[le:]
+            return True
+        return False
+
+    @property
+    def eof(self):
+        return self._pos >= self._length
+
+    def fileno(self):
+        self._check_not_closed()
+        return self._fp.fileno()
+
+    def flush(self) -> None:
+        if self._flushing:
+            return
+        self._flushing = True
+        self._check_not_closed()
+        if hasattr(self._compressor, 'write_to_file'):
+            self._compressor.write_to_file = False
+            self._compressor.flush()
+            self._compressor.write_to_file = True
+        self._fp.flush()
+        self._flushing = False
+
+    def readable(self):
+        self._check_not_closed()
+        return self._mode == _MODE_READ
+
+    def __read(self):
+        data = self._fp.read(algorithms.AES.block_size)
+        if not data:
+            if not self._eof:
+                self._buf += self._dec.finalize()
+                self._eof = True
+            else:
+                return False
+        else:
+            self._buf += self._dec.update(data)
+        return True
+
+    def read(self, size: int = -1):
+        if not self.readable():
+            raise ValueError('File is not readable.')
+        if self._crc32 is not None and not self._crc32_checked:
+            if not self.check_crc32():
+                raise DecryptException("crc32 check failed.")
+            self._crc32_checked = True
+        if size < 0:
+            return self.readall()
+        if self._decompressor and hasattr(self._decompressor, 'write_to_file') and self._decompressor.write_to_file:  # noqa: E501
+            self._decompressor.write_to_file = False
+            d = self._decompressor.read(size)
+            self._decompressor.write_to_file = True
+            return d
+        if self._decompressor and not hasattr(self._decompressor, 'write_to_file'):  # noqa: E501
+            if not size or (self.eof and len(self._debuf) == 0):
+                return b""
+            while True:
+                if not self.__read():
+                    self.__decompress()
+                    break
+                if not self.__decompress():
+                    break
+                if size <= len(self._debuf):
+                    data = self._debuf[:size]
+                    self._debuf = self._debuf[size:]
+                    return data
+            d = self._debuf[:size]
+            self._debuf = self._debuf[size:]
+            return d
+        if not size or self.eof:
+            return b""
+        size = min(size, self._length - self._pos)
+        if size <= len(self._buf):
+            data = self._buf[:size]
+            self._buf = self._buf[size:]
+            self._pos += size
+            return data
+        while True:
+            if not self.__read():
+                break
+            if size <= len(self._buf):
+                data = self._buf[:size]
+                self._buf = self._buf[size:]
+                self._pos += size
+                return data
+        self._pos += min(len(self._buf), size)
+        d = self._buf[:size]
+        self._buf = self._buf[size:]
+        return d
+
+    def readinto(self, b):
+        with memoryview(b) as view, view.cast("B") as byte_view:
+            data = self.read(len(byte_view))
+            byte_view[:len(data)] = data
+        return len(data)
+
+    def tell(self):
+        return self._pos
+
+    def writable(self):
+        self._check_not_closed()
+        return self._mode == _MODE_WRITE
+
+    def write(self, data):
+        if not self.writable():
+            raise ValueError("File was not opened for writing")
+        if isinstance(data, bytes):
+            length = len(data)
+        elif isinstance(data, str):
+            data = data.encode()
+            length = len(data)
+        elif isinstance(data, bytearray):
+            data = bytes(data)
+            length = len(data)
+        else:
+            data = memoryview(data)
+            length = data.nbytes
+            data = data.tobytes()
+        if self._compressor:
+            if hasattr(self._compressor, 'write_to_file'):
+                if self._compressor.write_to_file:
+                    self._compressor.write_to_file = False
+                    self._compressor.compress(data)
+                    self._compressor.write_to_file = True
+                    return
+            else:
+                data = self._compressor.compress(data)
+                length = len(data)
+        if self._pos < self._crc_size:
+            le = min(length, self._crc_size - self._pos)
+            self._crc32 = crc32(data[:le], self._crc32)
+        self._fp.write(self._enc.update(data))
+        self._pos += length
+
+    @property
+    def key(self):
+        if len(self._salt) < 32:
+            self._salt = self._salt + b'\x00' * (32 - len(self._salt))
+        return bytes(a ^ b for a, b in zip(self._salt, self._key))
+
+    @property
+    def iv(self):
+        return self._iv
+
+
+def encrypt_file(src: str, dest: str, f: File, name: str, prog: str, c: CompressConfig = None):  # noqa: E501
+    if exists(dest):
+        remove(dest)
+    mkdir_for_file(dest)
+    cs = 4096 if c is None else c.chunk_size
+    with open(src, 'rb') as s:
+        with EncFile(dest, 'wb', f.hash, compress=c) as t:
+            a = s.read(cs)
+            while a != b'':
+                t.write(a)
+                a = s.read(cs)
+            del a
+        stats = EncryptStats(b85encode(t.key).decode(), b85encode(t.iv).decode(), t.crc32, c._method.value if c else None, t.tell() if c else None)  # noqa: E501
+    i = compress_info(f.size, getsize(dest))
+    if c is None:
+        print(f'{prog}: Encrypted {src}({name}) -> {dest} ({i})')
+    else:
+        print(f'{prog}: Compressed and encrypted {src}({name}) -> {dest} ({i})')  # noqa: E501
+    return stats
+
+
+def decrypt_file(src: str, dest: str, f: File, name: str, prog: str, c: CompressConfig = None):  # noqa: E501
+    if not f.encrypted:
+        raise ValueError('File is not encrypted.')
+    hydrate_file_if_needed(src)
+    if exists(dest):
+        remove(dest)
+    mkdir_for_file(dest)
+    cs = 4096 if c is None else c.chunk_size
+    with EncFile(src, 'rb', f.hash, f.key, f.iv, f.encrypt_file_size, f.crc32, c) as s:  # noqa: E501
+        with open(dest, 'wb') as t:
+            a = s.read(cs)
+            while a != b'':
+                t.write(a)
+                a = s.read(cs)
+            del a
+    i = compress_info(f.size, getsize(src))
+    if c is None:
+        print(f'{prog}: Decrypted {src}({name}) -> {dest} ({i})')
+    else:
+        print(f'{prog}: Decrypted and decompressed {src}({name}) -> {dest} ({i})')  # noqa: E501

game_backuper/file.py

@@ -1,13 +1,56 @@
 from collections import namedtuple
 from os.path import exists, dirname, abspath, isfile, isdir, join, isabs
-from os import stat, makedirs, listdir
+from os import stat, makedirs, listdir, remove
 from game_backuper.hashl import sha512
 from shutil import copy2
 from game_backuper.filetype import FileType
-from os import remove
+from platform import system
+if system() == "Windows":
+    try:
+        from game_backuper.cfapi import hydrate_file
+        have_cfapi = True
+    except Exception:
+        have_cfapi = False
+else:
+    have_cfapi = False
 
 
-File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type'])
+_File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type', 'key', 'iv', 'crc32', 'x_compress_type', 'compressed_size'])  # noqa: E501
+
+
+class File(_File):
+    @property
+    def encrypted(self):
+        return bool(self.key and self.iv and self.crc32)
+
+    @property
+    def compressed(self):
+        return self.x_compress_type is not None
+
+    @property
+    def compressed_type(self):
+        from game_backuper.compress import CompressMethod
+        return CompressMethod(self.x_compress_type) if self.compressed else None  # noqa: E501
+
+    @property
+    def encrypt_file_size(self):
+        return self.compressed_size if self.compressed_size else self.size
+
+    @classmethod
+    def from_encrypt_stats(cls, stats, file):
+        from game_backuper.enc import EncryptStats
+        if not isinstance(stats, EncryptStats):
+            raise TypeError(f'Expected EncryptStats, got {type(stats)}')
+        if not isinstance(file, (File, _File)):
+            raise TypeError(f'Expected File, got {type(file)}')
+        return cls(file.id, file.file, file.size, file.program, file.hash, file.type, stats.key, stats.iv, stats.crc32, stats.x_compress_type, stats.compressed_size)  # noqa: E501
+
+    @classmethod
+    def from_leveldb_stats(cls, stats):
+        from game_backuper.leveldb import LeveldbStats
+        if not isinstance(stats, LeveldbStats):
+            raise TypeError(f'Expected LeveldbStats, got {type(stats)}')
+        return cls(None, None, stats.size, None, stats.hash, None, None, None, None, None, None)  # noqa: E501
 
 
 def mkdir_for_file(p: str):
@@ -49,7 +92,13 @@ def list_all_paths(base: str, cli):
             if isfile(bp):
                 r.append(bp)
             elif isdir(bp):
-                r += listdirs(bp, c.ignore_hidden_files)
+                re = listdirs(bp, c.ignore_hidden_files)
+                for ii in re:
+                    if c.is_exclude(bp, ii):
+                        continue
+                    if not c.is_include(bp, ii):
+                        continue
+                    r.append(ii)
         elif isinstance(c, ConfigOLeveldb):
             r.append(c.path if isabs(c.path) else join(base, c.path))
     return r
@@ -60,7 +109,7 @@ def new_file(loc: str, name: str, prog: str, type: FileType = None) -> File:
         fs = stat(loc).st_size
         with open(loc, 'rb') as f:
             hs = sha512(f)
-        return File(None, name, fs, prog, hs, type)
+        return File(None, name, fs, prog, hs, type, None, None, None, None, None)  # noqa: E501
 
 
 def remove_dirs(loc: str):
@@ -74,3 +123,30 @@ def remove_dirs(loc: str):
             except Exception:
                 remove_dirs(i)
     remove(loc)
+
+
+def remove_compress_files(loc: str, prog: str, name: str, ext: str = None):
+    from game_backuper.compress import supported_exts
+    exts = supported_exts.copy()
+    if ext is not None:
+        exts.remove(ext)
+        exts.append('')
+    for i in exts:
+        f = loc + i
+        if exists(f) and isfile(f):
+            remove(f)
+            print(f'{prog}: Removed {f}({name})')
+
+
+def hydrate_file_if_needed(fn: str):
+    if not have_cfapi:
+        return
+    if exists(fn):
+        hydrate_file(fn)
+
+
+def remove_unencryped_files(loc: str, prog: str, name: str):
+    if exists(loc):
+        remove(loc)
+        print(f'{prog}: Removed {loc}({name})')
+    remove_compress_files(loc, prog, name)

game_backuper/leveldb.py

@@ -11,6 +11,8 @@ if have_leveldb:
     from base64 import b85encode
     from collections import namedtuple
     from sqlite3 import connect
+    from os.path import exists
+    from os import remove
     LeveldbStats = namedtuple('LeveldbStats', ['hash', 'size'])
     MAP_TABLE = '''CREATE TABLE map (
     key TEXT,
@@ -60,6 +62,8 @@ if have_leveldb:
 
     def leveldb_to_sqlite(db: str, dest: str, entries: List[bytes]):
         d = DB(db)
+        if exists(dest):
+            remove(dest)
         s = connect(dest)
         s.text_factory = bytes
         s.execute(MAP_TABLE)

game_backuper/main.py

@@ -14,6 +14,6 @@ def main(cm=None):
     cfg = Config(cml.config_file)
     if not exists(cfg.dest):
         makedirs(cfg.dest)
-    db = Db(cfg.dest)
+    db = Db(cfg, cml)
     bk = Backuper(db, cfg, cml)
     return bk.run()

game_backuper/regexp.py

@@ -0,0 +1,51 @@
+try:
+    from game_backuper._pcre2 import PCRE2, Option as PCRE2Option, MatchOption
+    have_pcre2 = True
+except ImportError:
+    have_pcre2 = False
+from enum import IntFlag
+from re import I as REI, compile as re_comp
+
+
+class RegexFlag(IntFlag):
+    I = 1  # noqa: E741
+    IGNORECASE = 1
+
+
+class Regex:
+    def __init__(self, r: str, flags: RegexFlag = 0, use_pcre2: bool = False):
+        if have_pcre2 and use_pcre2:
+            opt = 0
+            if flags & RegexFlag.I:
+                opt = opt | PCRE2Option.PCRE2_CASELESS
+            self._re = PCRE2(r)
+            self._use_pcre2 = True
+        else:
+            if use_pcre2:
+                from sys import stderr
+                stderr.write("Can not load pcre2.\n")
+            self._use_pcre2 = False
+            opt = 0
+            if flags & RegexFlag.I:
+                opt = opt | REI
+            self._re = re_comp(r)
+
+    def match(self, s: str, startpos: int = 0):
+        if self._use_pcre2:
+            return self._re.match(s, MatchOption.PCRE2_ANCHORED, startpos)
+        else:
+            return self._re.match(s, startpos)
+
+    def match_only(self, s: str, startpos: int = 0) -> bool:
+        if self._use_pcre2:
+            return self._re.match(s, MatchOption.PCRE2_ANCHORED, startpos, True)  # noqa: E501
+        else:
+            return False if self._re.match(s, startpos) is None else True
+
+
+def wildcards_to_regex(s: str, **k):
+    for i in ['\\', '$', '(', ')', '+', '.', '[', '^', '{', '|']:
+        s = s.replace(i, f"\\{i}")
+    s = s.replace("*", ".*")
+    s = s.replace("?", ".")
+    return Regex(s, **k)

game_backuper/restorer.py

@@ -8,9 +8,13 @@ from game_backuper.file import (
     remove_dirs,
     new_file,
     mkdir_for_file,
+    hydrate_file_if_needed,
 )
-from os import remove
+from os import remove, close
 from game_backuper.filetype import FileType
+from game_backuper.compress import CompressConfig, decompress
+from game_backuper.enc import decrypt_file
+from tempfile import mkstemp
 
 
 class RestoreTask(Thread):
@@ -33,7 +37,11 @@ class RestoreTask(Thread):
                 if f.type is not None:
                     raise ValueError('Type dismatched.')
                 nam = r.real_name
-                src = join(self.cfg.dest, prog, fn)
+                if f.encrypted:
+                    src = join(self.cfg.dest, '.encrypt', prog, fn)
+                else:
+                    src = join(self.cfg.dest, prog, fn)
+                c = r.compress_config
                 tmp = relpath(fn, nam)
                 if isabs(r.path):
                     dest = r.path
@@ -43,15 +51,24 @@ class RestoreTask(Thread):
                     dest = join(dest, tmp)
                 if dest in pl:
                     pl.remove(dest)
-                if not exists(src):
+                if not f.encrypted and ((c is None and not exists(src)) or (c is not None and not exists(src + c.ext))):  # noqa: E501
+                    print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})')  # noqa: E501
+                    continue
+                elif f.encrypted and not exists(src):
                     print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})')  # noqa: E501
                     continue
                 if exists(dest):
-                    tf = new_file(dest, nam, prog)
+                    tf = new_file(dest, fn, prog)
                     if tf.size == f.size and tf.hash == f.hash:
                         print(f'{prog}: Skip {fn}')
                         continue
-                copy_file(src, dest, nam, prog)
+                if f.encrypted:
+                    decrypt_file(src, dest, f, fn, prog, CompressConfig(f.compressed_type.to_str()) if f.compressed else None)  # noqa: E501
+                elif c is None:
+                    hydrate_file_if_needed(src)
+                    copy_file(src, dest, fn, prog)
+                else:
+                    decompress(src, dest, c, fn, prog)
             elif isinstance(r, ConfigOLeveldb):
                 from game_backuper.leveldb import have_leveldb
                 if not have_leveldb:
@@ -59,14 +76,21 @@ class RestoreTask(Thread):
                 if f.type != FileType.LEVELDB:
                     raise ValueError('Type dismatched.')
                 nam = r.real_name
-                src = join(self.cfg.dest, prog, fn + '.db')
+                if f.encrypted:
+                    src = join(self.cfg.dest, '.encrypt', prog, fn + '.db')
+                else:
+                    src = join(self.cfg.dest, prog, fn + '.db')
+                c = r.compress_config
                 if isabs(r.path):
                     dest = r.path
                 else:
                     dest = join(b, r.path)
                 if dest in pl:
                     pl.remove(dest)
-                if not exists(src):
+                if not f.encrypted and ((c is None and not exists(src)) or (c is not None and not exists(src + c.ext))):  # noqa: E501
+                    print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})')  # noqa: E501
+                    continue
+                elif f.encrypted and not exists(src):
                     print(f'{prog}: Warn: Can not find backup files: "{src}"({fn})')  # noqa: E501
                     continue
                 from game_backuper.leveldb import (
@@ -81,8 +105,28 @@ class RestoreTask(Thread):
                         print(f'{prog}: Skip {fn}')
                         continue
                 mkdir_for_file(dest)
-                sqlite_to_leveldb(src, dest, r.domains)
-                print(f'{prog}: Covert leveldb done. {src}({fn}) -> {dest}')
+                if f.encrypted:
+                    tmp = mkstemp()
+                    close(tmp[0])
+                    tmp = tmp[1]
+                    decrypt_file(src, tmp, f, fn, prog, CompressConfig(f.compressed_type.to_str()) if f.compressed else None)  # noqa: E501
+                    sqlite_to_leveldb(tmp, dest, r.domains)
+                    print(f'{prog}: Convert leveldb done. {tmp}({fn}) -> {dest}')  # noqa: E501
+                    remove(tmp)
+                    print(f'{prog}: Removed temp file {tmp}')
+                elif c is None:
+                    hydrate_file_if_needed(src)
+                    sqlite_to_leveldb(src, dest, r.domains)
+                    print(f'{prog}: Convert leveldb done. {src}({fn}) -> {dest}')  # noqa: E501
+                else:
+                    tmp = mkstemp()
+                    close(tmp[0])
+                    tmp = tmp[1]
+                    decompress(src, tmp, c, fn, prog)
+                    sqlite_to_leveldb(tmp, dest, r.domains)
+                    print(f'{prog}: Convert leveldb done. {tmp}({fn}) -> {dest}')  # noqa: E501
+                    remove(tmp)
+                    print(f'{prog}: Removed tempfile {tmp}')
         for i in pl:
             if isfile(i):
                 remove(i)

game_backuper/zstd.py

@@ -0,0 +1,177 @@
+try:
+    from game_backuper._zstd import (
+        ZSTDCompressor,
+        ZSTDDecompressor,
+        maxCLevel,
+    )
+    have_zstd = True
+except ImportError:
+    have_zstd = False
+if have_zstd:
+    from builtins import open as _builtin_open
+    from _compression import BaseStream, DecompressReader
+    import os
+    import io
+
+    _MODE_CLOSED = 0
+    _MODE_READ = 1
+    _MODE_WRITE = 3
+    MAX_COMPRESS_LEVEL = maxCLevel()
+
+    class ZSTDFile(BaseStream):
+        def __init__(self, filename, mode="r", *, compresslevel=3):
+            self._fp = None
+            self._closefp = False
+            self._mode = _MODE_CLOSED
+            if not (1 <= compresslevel <= MAX_COMPRESS_LEVEL):
+                raise ValueError(f"compresslevel must be between 1 and {MAX_COMPRESS_LEVEL}")  # noqa: E501
+            if mode in ("", "r", "rb"):
+                mode = "rb"
+                mode_code = _MODE_READ
+            elif mode in ("w", "wb"):
+                mode = "wb"
+                mode_code = _MODE_WRITE
+                self._compressor = ZSTDCompressor(compresslevel)
+            elif mode in ("x", "xb"):
+                mode = "xb"
+                mode_code = _MODE_WRITE
+                self._compressor = ZSTDCompressor(compresslevel)
+            elif mode in ("a", "ab"):
+                mode = "ab"
+                mode_code = _MODE_WRITE
+                self._compressor = ZSTDCompressor(compresslevel)
+            else:
+                raise ValueError("Invalid mode: %r" % (mode,))
+            if isinstance(filename, (str, bytes, os.PathLike)):
+                self._fp = _builtin_open(filename, mode)
+                self._closefp = True
+                self._mode = mode_code
+            else:
+                raise TypeError("filename must be a str, bytes, file or PathLike object")  # noqa: E501
+            if self._mode == _MODE_READ:
+                raw = DecompressReader(self._fp, ZSTDDecompressor,
+                                       trailing_error=OSError)
+                self._buffer = io.BufferedReader(raw)
+            else:
+                self._pos = 0
+
+        def close(self):
+            if self._mode == _MODE_CLOSED:
+                return
+            try:
+                if self._mode == _MODE_READ:
+                    self._buffer.close()
+                elif self._mode == _MODE_WRITE:
+                    self._fp.write(self._compressor.flush())
+                    self._compressor = None
+            finally:
+                try:
+                    if self._closefp:
+                        self._fp.close()
+                finally:
+                    self._fp = None
+                    self._closefp = False
+                    self._mode = _MODE_CLOSED
+                    self._buffer = None
+
+        @property
+        def closed(self):
+            return self._mode == _MODE_CLOSED
+
+        def fileno(self):
+            self._check_not_closed()
+            return self._fp.fileno()
+
+        def seekable(self):
+            return self.readable() and self._buffer.seekable()
+
+        def readable(self):
+            self._check_not_closed()
+            return self._mode == _MODE_READ
+
+        def writable(self):
+            self._check_not_closed()
+            return self._mode == _MODE_WRITE
+
+        def peek(self, n=0):
+            self._check_can_read()
+            return self._buffer.peek(n)
+
+        def read(self, size=-1):
+            self._check_can_read()
+            return self._buffer.read(size)
+
+        def read1(self, size=-1):
+            self._check_can_read()
+            if size < 0:
+                size = io.DEFAULT_BUFFER_SIZE
+            return self._buffer.read1(size)
+
+        def readinto(self, b):
+            self._check_can_read()
+            return self._buffer.readinto(b)
+
+        def readline(self, size=-1):
+            if not isinstance(size, int):
+                if not hasattr(size, "__index__"):
+                    raise TypeError("Integer argument expected")
+                size = size.__index__()
+            self._check_can_read()
+            return self._buffer.readline(size)
+
+        def __iter__(self):
+            self._check_can_read()
+            return self._buffer.__iter__()
+
+        def readlines(self, size=-1):
+            if not isinstance(size, int):
+                if not hasattr(size, "__index__"):
+                    raise TypeError("Integer argument expected")
+                size = size.__index__()
+            self._check_can_read()
+            return self._buffer.readlines(size)
+
+        def write(self, data):
+            self._check_can_write()
+            if isinstance(data, (bytes, bytearray)):
+                length = len(data)
+            else:
+                data = memoryview(data)
+                length = data.nbytes
+            compressed = self._compressor.compress(data)
+            self._fp.write(compressed)
+            self._pos += length
+            return length
+
+        def writelines(self, seq):
+            return BaseStream.writelines(self, seq)
+
+        def seek(self, offset, whence=io.SEEK_SET):
+            self._check_can_seek()
+            return self._buffer.seek(offset, whence)
+
+        def tell(self):
+            self._check_not_closed()
+            if self._mode == _MODE_READ:
+                return self._buffer.tell()
+            return self._pos
+
+    def open(filename, mode="rb", compresslevel=3, encoding=None, errors=None, newline=None):  # noqa: E501
+        if "t" in mode:
+            if "b" in mode:
+                raise ValueError("Invalid mode: %r" % (mode,))
+        else:
+            if encoding is not None:
+                raise ValueError("Argument 'encoding' not supported in binary mode")  # noqa: E501
+            if errors is not None:
+                raise ValueError("Argument 'errors' not supported in binary mode")  # noqa: E501
+            if newline is not None:
+                raise ValueError("Argument 'newline' not supported in binary mode")  # noqa: E501
+
+        bz_mode = mode.replace("t", "")
+        binary_file = ZSTDFile(filename, bz_mode, compresslevel=compresslevel)
+
+        if "t" in mode:
+            return io.TextIOWrapper(binary_file, encoding, errors, newline)
+        else:
+            return binary_file

requirements.txt

@@ -1,2 +1,6 @@
+cryptography
 pyyaml
 plyvel[leveldb]
+lzip[lzip]
+python-snappy[snappy]
+brotli[brotli]

setup.py

@@ -1,6 +1,23 @@
 # flake8: noqa
 import sys
-from game_backuper import __version__
+from version import version, dversion
+from setuptools import Extension
+try:
+    from Cython.Build import cythonize
+except ImportError:
+    def cythonize(li):
+        return []
+
+ext_modules = []
+if '--without-pcre2' in sys.argv:
+    sys.argv.remove('--without-pcre2')
+else:
+    ext_modules.append(Extension("game_backuper._pcre2", ["game_backuper/_pcre2.pyx"], libraries=["pcre2-8"]))
+if '--without-zstd' in sys.argv:
+    sys.argv.remove('--without-zstd')
+else:
+    ext_modules.append(Extension("game_backuper._zstd", ["game_backuper/_zstd.pyx"], libraries=["zstd"]))
+
 if "py2exe" in sys.argv:
     from distutils.core import setup
     import py2exe
@@ -8,9 +25,9 @@ if "py2exe" in sys.argv:
         "console": [{
             'script': "game_backuper/__main__.py",
             "dest_base": 'game-backuper',
-            'version': __version__,
+            'version': version,
             'product_name': 'game-backuper',
-            'product_version': __version__,
+            'product_version': dversion,
             'company_name': 'lifegpc',
             'description': 'A game backuper',
         }],
@@ -18,7 +35,8 @@ if "py2exe" in sys.argv:
             "py2exe": {
                 "optimize": 2,
                 "compressed": 1,
-                "excludes": ["pydoc", "unittest"]
+                "excludes": ["pydoc", "unittest"],
+                "includes": ["cryptography.utils", "_cffi_backend", "sqlite3.dump"]
             }
         },
         "zipfile": None,
@@ -31,13 +49,16 @@ else:
             'console_scripts': ['game-backuper = game_backuper:start']
         },
         "extras_require": {
-            "leveldb": "plyvel"
+            "leveldb": "plyvel",
+            "lzip": "lzip",
+            "snappy": "python-snappy",
+            "brotli": "brotli",
         },
         "python_requires": ">=3.6"
     }
 setup(
     name="game-backuper",
-    version=__version__,
+    version=version,
     url="https://github.com/lifegpc/game-backuper",
     author="lifegpc",
     author_email="[email protected]",
@@ -50,5 +71,6 @@ setup(
     long_description="A game backuper",
     keywords="backup",
     packages=["game_backuper"],
+    ext_modules=cythonize(ext_modules, compiler_directives={'language_level': "3"}),
     **params
 )

testenc.py

@@ -0,0 +1,100 @@
+from game_backuper.compress import CompressConfig
+from game_backuper.enc import DecryptException, EncFile
+from os import urandom, remove
+from hashlib import sha512
+from zlib import crc32
+
+
+datalen = 4096
+data = urandom(datalen)
+a = sha512()
+a.update(data)
+with EncFile('a.txt', 'wb', a.digest()) as f:
+    f.write(data)
+    key = f.key
+    iv = f.iv
+    crc = f.crc32
+    print(crc)
+    crc2 = '{:08x}'.format(crc32(data[:1024]))
+    print(crc2)
+    assert crc == crc2
+with EncFile('a.txt', 'rb', a.digest(), key, iv, datalen, crc) as f:
+    d = f.read()
+    assert d == data
+with EncFile('a.txt', 'rb', b'', key, iv, datalen, crc) as f:
+    try:
+        d = f.read()
+        assert False
+    except DecryptException:
+        pass
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('gzip', 9)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, compress=CompressConfig('gzip')) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('bzip2', 1)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, CompressConfig('bzip2', 1)) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('lzma', 1)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, CompressConfig('lzma', 1)) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('lzip', 1)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, CompressConfig('lzip', 1)) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('zstd', 1)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, CompressConfig('zstd', 1)) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('snappy', 1)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, CompressConfig('snappy', 1)) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')
+with EncFile('a.txt', 'wb', b'', compress=CompressConfig('brotli', 1)) as f:
+    f.write(data)
+    f.flush()
+    key = f.key
+    iv = f.iv
+le = f.tell()
+crc = f.crc32
+with EncFile('a.txt', 'rb', b'', key, iv, le, crc, CompressConfig('brotli', 1)) as f:  # noqa: E501
+    assert data == f.read()
+remove('a.txt')

version.py

@@ -0,0 +1,60 @@
+import sys
+from os.path import exists, dirname, abspath, join
+from subprocess import Popen, DEVNULL, PIPE
+
+
+def check_git():
+    p = Popen(['git', '--help'], stdout=DEVNULL, stderr=DEVNULL)
+    p.wait()
+    return True if p.poll() == 0 else False
+
+
+def get_git_desc():
+    p = Popen(['git', 'describe', '--tags', '--long', '--dirty'], stdout=PIPE,
+              stderr=DEVNULL)
+    p.wait()
+    if p.poll() == 0:
+        return p.stdout.read()
+
+
+def normalize(s: str):
+    li = s.split("-")
+    nv = li[0].split('.')
+    nv = [int(i) for i in nv if i.isnumeric()]
+    if len(li) == 1:
+        nv = [str(i) for i in nv]
+        return '.'.join(nv)
+    if li[1].isnumeric():
+        if len(nv) >= 4:
+            nv[-1] += int(li[1])
+        else:
+            while len(nv) <= 2:
+                nv += [0]
+            nv += [int(li[1])]
+    nv = [str(i) for i in nv]
+    return '.'.join(nv)
+
+
+default_version = "1.0.0"
+use_git = True
+if '--no-git-version' in sys.argv:
+    use_git = False
+    sys.argv.remove("--no-git-version")
+d = abspath(join(dirname(abspath(__file__)), ".git"))
+if not exists(d):
+    use_git = False
+if use_git and not check_git():
+    use_git = False
+if use_git:
+    d = get_git_desc()
+    if d is None:
+        use_git = False
+        version = default_version
+        dversion = version
+    else:
+        d = d.decode().splitlines(False)[0]
+        dversion = d[1:]
+        version = normalize(dversion)
+else:
+    version = default_version
+    dversion = version