diff --git a/game_backuper/cfapi.py b/game_backuper/cfapi.py new file mode 100644 index 0000000..145656f --- /dev/null +++ b/game_backuper/cfapi.py @@ -0,0 +1,35 @@ +from ctypes import HRESULT, byref, c_uint, windll +from ctypes.wintypes import ( + DWORD, HANDLE, LARGE_INTEGER, LPCWSTR, LPVOID, PHANDLE +) + + +dll = windll.CldApi +CF_OPEN_FILE_FLAG_NONE = 0 +CF_OPEN_FILE_FLAG_EXCLUSIVE = 1 +CF_OPEN_FILE_FLAG_WRITE_ACCESS = 2 +CF_OPEN_FILE_FLAG_DELETE_ACCESS = 3 +CF_OPEN_FILE_FLAG_FOREGROUND = 4 +CfOpenFileWithOplock = dll.CfOpenFileWithOplock +CfOpenFileWithOplock.argtypes = [LPCWSTR, c_uint, PHANDLE] +CfOpenFileWithOplock.restype = HRESULT +CfHydratePlaceholder = dll.CfHydratePlaceholder +CfHydratePlaceholder.argtypes = [HANDLE, LARGE_INTEGER, LARGE_INTEGER, c_uint, LPVOID] # noqa: E501 +CfHydratePlaceholder.restype = HRESULT +CfCloseHandle = dll.CfCloseHandle +CfCloseHandle.argtypes = [HANDLE] +ERROR_INVALID_FUNCTION = 1 +GetLastError = windll.Kernel32.GetLastError +GetLastError.restype = DWORD + + +def hydrate_file(s: str): + h = HANDLE() + try: + CfOpenFileWithOplock(s, CF_OPEN_FILE_FLAG_NONE, byref(h)) + CfHydratePlaceholder(h, 0, -1, 0, LPVOID()) + except OSError as e: + if GetLastError() != ERROR_INVALID_FUNCTION: + CfCloseHandle(h) + raise e + CfCloseHandle(h) diff --git a/game_backuper/cml.py b/game_backuper/cml.py index 92204f5..0dd7d8f 100644 --- a/game_backuper/cml.py +++ b/game_backuper/cml.py @@ -36,10 +36,11 @@ class Opts: config_file: str = DEFAULT_CONFIG action = OptAction.BACKUP programs_list = None + optimize_db = False def __init__(self, cml: List[str]): try: - r = getopt(cml, 'hc:', ['help', 'config=']) + r = getopt(cml, 'hc:', ['help', 'config=', 'optimize-db']) for i in r[0]: if i[0] == '-h' or i[0] == '--help': self.print_help() @@ -47,6 +48,8 @@ class Opts: sys.exit(0) elif i[0] == '-c' or i[0] == '--config': self.config_file = i[1] + elif i[0] == '--optimize-db': + self.optimize_db = True if len(r[1]) > 0: cm = r[1] re = OptAction.from_str(cm[0]) @@ -74,4 +77,5 @@ game-backuper [options] list game-backuper [options] list_leveldb_key [ [...]] Options: -h, --help Print help message. - -c, --config Set config file.''') + -c, --config Set config file. + --optimize-db Optimize the sqlite3 database''') diff --git a/game_backuper/compress.py b/game_backuper/compress.py index 63561c5..d1ae0b5 100644 --- a/game_backuper/compress.py +++ b/game_backuper/compress.py @@ -52,7 +52,7 @@ except ImportError: cached_property = property from os.path import exists, isfile, getsize from os import remove -from game_backuper.file import mkdir_for_file +from game_backuper.file import mkdir_for_file, hydrate_file_if_needed @unique @@ -303,6 +303,7 @@ def decompress(src: str, dest: str, c: CompressConfig, name: str, prog: str): if exists(dest): remove(dest) cs = c.chunk_size + hydrate_file_if_needed(fn) if c.method == CompressMethod.BZIP2: with BZ2File(fn, 'rb') as f: with open(dest, 'wb') as t: diff --git a/game_backuper/db.py b/game_backuper/db.py index 67db77f..37a6941 100644 --- a/game_backuper/db.py +++ b/game_backuper/db.py @@ -2,7 +2,7 @@ from sqlite3 import connect from os.path import join from typing import List, Union from threading import Lock -from game_backuper.file import File +from game_backuper.file import File, hydrate_file_if_needed from game_backuper.filetype import FileType @@ -56,11 +56,13 @@ class Db: self.db.execute(FILETYPE_TABLE) self.db.commit() - def __init__(self, loc: str): + def __init__(self, loc: str, optimize_db: bool = False): fn = join(loc, "data.db") + hydrate_file_if_needed(fn) self.db = connect(fn, check_same_thread=False) - self.db.execute('VACUUM;') - self.db.commit() + if optimize_db: + self.db.execute('VACUUM;') + self.db.commit() ok = self.__check_database() if not ok: self.__create_table() diff --git a/game_backuper/file.py b/game_backuper/file.py index 371d10f..d810fdd 100644 --- a/game_backuper/file.py +++ b/game_backuper/file.py @@ -1,10 +1,15 @@ from collections import namedtuple from os.path import exists, dirname, abspath, isfile, isdir, join, isabs -from os import stat, makedirs, listdir +from os import stat, makedirs, listdir, remove from game_backuper.hashl import sha512 from shutil import copy2 from game_backuper.filetype import FileType -from os import remove +from platform import system +if system() == "Windows": + from game_backuper.cfapi import hydrate_file + have_cfapi = True +else: + have_cfapi = False File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type']) @@ -93,3 +98,10 @@ def remove_compress_files(loc: str, prog: str, name: str, ext: str = None): if exists(f) and isfile(f): remove(f) print(f'{prog}: Removed {f}({name})') + + +def hydrate_file_if_needed(fn: str): + if not have_cfapi: + return + if exists(fn): + hydrate_file(fn) diff --git a/game_backuper/main.py b/game_backuper/main.py index b752e82..d44b83e 100644 --- a/game_backuper/main.py +++ b/game_backuper/main.py @@ -14,6 +14,6 @@ def main(cm=None): cfg = Config(cml.config_file) if not exists(cfg.dest): makedirs(cfg.dest) - db = Db(cfg.dest) + db = Db(cfg.dest, cml.optimize_db) bk = Backuper(db, cfg, cml) return bk.run() diff --git a/game_backuper/restorer.py b/game_backuper/restorer.py index e903b8e..34a116c 100644 --- a/game_backuper/restorer.py +++ b/game_backuper/restorer.py @@ -8,6 +8,7 @@ from game_backuper.file import ( remove_dirs, new_file, mkdir_for_file, + hydrate_file_if_needed, ) from os import remove, close from game_backuper.filetype import FileType @@ -55,6 +56,7 @@ class RestoreTask(Thread): print(f'{prog}: Skip {fn}') continue if c is None: + hydrate_file_if_needed(src) copy_file(src, dest, nam, prog) else: decompress(src, dest, c, nam, prog) @@ -89,6 +91,7 @@ class RestoreTask(Thread): continue mkdir_for_file(dest) if c is None: + hydrate_file_if_needed(src) sqlite_to_leveldb(src, dest, r.domains) print(f'{prog}: Covert leveldb done. {src}({fn}) -> {dest}') # noqa: E501 else: