From d4dedcb3160a21dc5faa5912ec382927a268b0af Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sat, 19 Aug 2023 13:09:09 +0800 Subject: [PATCH] Add new settings unpin_file --- example.yaml | 4 ++ game_backuper/backuper.py | 23 +++++++++ game_backuper/cfapi.py | 102 ++++++++++++++++++++++++++++++++++++-- game_backuper/config.py | 27 ++++++++++ game_backuper/file.py | 9 +++- 5 files changed, 160 insertions(+), 5 deletions(-) diff --git a/example.yaml b/example.yaml index 297aacf..6810ea4 100644 --- a/example.yaml +++ b/example.yaml @@ -10,6 +10,7 @@ encrypt_db: false # Optional. Default value: false. Encrypt the database. Warni db_password: "Password" # Specify the password of the encryped database. encrypt_files: false # Optional. Default value: false. Encrypt backup files. The key information will stored in database. protect_filename: false # Optional. Default value: false. Use id in database as file name. Only works when encrypt_files is true. +unpin_file: false # Optional. Default value: false. Notifiy sync provider to dehydrate file data. programs: - name: Your program name # This name is used to identify different application. base: /path/to/save/path # Must be absoulte path. @@ -20,6 +21,7 @@ programs: compress_level: null # Optional. encrypt_files: false # Optional protect_filename: false # Optional + unpin_file: false # Optional. files: - BGI.gdb # path to a file/folder. All subfolders will include if it is a folder. Must be relative path. - type: path @@ -32,6 +34,7 @@ programs: compress_level: null # Optional. encrypt_files: false # Optional. protect_filename: false # Optional + unpin_file: false # Optional. excludes: # Optional. Exculde some files. Only effected when path is a folder. - data.db # Releative path - /path/to/data.db # Absolute path @@ -55,5 +58,6 @@ programs: compress_level: null # Optional. encrypt_files: false # Optional. protect_filename: false # Optional + unpin_file: false # Optional. domains: # optional. Just backup minor domains in localstorage database. Only chromium is tested. - some domain diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py index 5fe6f14..55db58b 100644 --- a/game_backuper/backuper.py +++ b/game_backuper/backuper.py @@ -16,6 +16,7 @@ from game_backuper.restorer import RestoreTask from game_backuper.file import remove_compress_files, remove_unencryped_files from game_backuper.compress import compress from game_backuper.enc import encrypt_file +from game_backuper.file import unpin_file_if_needed from tempfile import mkstemp @@ -64,10 +65,14 @@ class BackupTask(Thread): remove_compress_files(de, prog, f.name) self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501 self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de) continue elif exists(de) and f.encrypt_files and not ori.compressed: # noqa: E501 print(f'{prog}: Skip {f[0]}.') remove_unencryped_files(join(bp, f[0]), prog, f.name) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de) continue else: if not f.encrypt_files and exists(de + c.ext): @@ -75,10 +80,14 @@ class BackupTask(Thread): remove_compress_files(de, prog, f.name, c.ext) # noqa: E501 self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501 self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de + c.ext) continue elif f.encrypt_files and ori.compressed_type == c.method: # noqa: E501 print(f'{prog}: Skip {f.name}.') remove_unencryped_files(join(bp, f.name), prog, f.name) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de) continue stats = None if f.encrypt_files: @@ -95,6 +104,8 @@ class BackupTask(Thread): self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501 self.db.set_file(ori.id, nf.size, nf.hash) self.db.set_file_encrypt_information(ori.id, stats) + if f.unpin_file: + unpin_file_if_needed(de) else: if f.protect_filename: self.db.add_file(nf, False) @@ -115,6 +126,8 @@ class BackupTask(Thread): self.db.set_file_encrypt_information(tmpori.id, s) else: self.db.add_file(nf) + if f.unpin_file: + unpin_file_if_needed(de) elif isinstance(f, ConfigLeveldb): from game_backuper.leveldb import have_leveldb if not have_leveldb: @@ -163,10 +176,14 @@ class BackupTask(Thread): remove_compress_files(de, prog, f.name) self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori) # noqa: E501 self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de) continue elif exists(de) and f.encrypt_files and not ori.compressed: # noqa: E501 print(f'{prog}: Skip {f[0]}.') remove_unencryped_files(join(bp, f[0] + '.db'), prog, f.name) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de) continue else: if not f.encrypt_files and exists(de + c.ext): @@ -174,10 +191,14 @@ class BackupTask(Thread): remove_compress_files(de, prog, f.name, c.ext) self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori) # noqa: E501 self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de + c.ext) continue elif f.encrypt_files and ori.compressed_type == c.method: # noqa: E501 print(f'{prog}: Skip {f.name}.') remove_unencryped_files(join(bp, f.name + '.db'), prog, f.name) # noqa: E501 + if f.unpin_file: + unpin_file_if_needed(de) continue if f.protect_filename: ori = self.db.get_file(prog, f[0]) @@ -218,6 +239,8 @@ class BackupTask(Thread): else: self.db.set_file(ori.id, stats.size, stats.hash) self.db.set_file_encrypt_information(ori.id, st) + if f.unpin_file: + unpin_file_if_needed(de) for fn in fl: f = self.db.get_file(prog, fn) if f.type is None: diff --git a/game_backuper/cfapi.py b/game_backuper/cfapi.py index 145656f..047ae74 100644 --- a/game_backuper/cfapi.py +++ b/game_backuper/cfapi.py @@ -1,15 +1,69 @@ -from ctypes import HRESULT, byref, c_uint, windll -from ctypes.wintypes import ( - DWORD, HANDLE, LARGE_INTEGER, LPCWSTR, LPVOID, PHANDLE -) +from ctypes import HRESULT, POINTER, Structure, Union, byref, c_uint, sizeof, windll # noqa: E501 +from ctypes.wintypes import BYTE, DWORD, HANDLE, LARGE_INTEGER, LPCWSTR, LPVOID, PDWORD, PHANDLE, ULONG, WPARAM # noqa: E501 +ULONG_PTR = WPARAM +PVOID = LPVOID dll = windll.CldApi +CF_HYDRATE_FLAG_NONE = 0 +CF_IN_SYNC_STATE_NOT_IN_SYNC = 0 +CF_IN_SYNC_STATE_IN_SYNC = 1 CF_OPEN_FILE_FLAG_NONE = 0 CF_OPEN_FILE_FLAG_EXCLUSIVE = 1 CF_OPEN_FILE_FLAG_WRITE_ACCESS = 2 CF_OPEN_FILE_FLAG_DELETE_ACCESS = 3 CF_OPEN_FILE_FLAG_FOREGROUND = 4 +CF_PIN_STATE_UNSPECIFIED = 0 +CF_PIN_STATE_PINNED = 1 +CF_PIN_STATE_UNPINNED = 2 +CF_PIN_STATE_EXCLUDED = 3 +CF_PIN_STATE_INHERIT = 4 +CF_PLACEHOLDER_INFO_BASIC = 0 +CF_PLACEHOLDER_INFO_STANDARD = 1 +CF_SET_PIN_FLAG_NONE = 0x00000000 +CF_SET_PIN_FLAG_RECURSE = 0x00000001 +CF_SET_PIN_FLAG_RECURSE_ONLY = 0x00000002 +CF_SET_PIN_FLAG_RECURSE_STOP_ON_ERROR = 0x00000004 + + +class CF_PLACEHOLDER_BASIC_INFO(Structure): + _fields_ = [("PinState", c_uint), + ("InSyncState", c_uint), + ("FileId", LARGE_INTEGER), + ("SyncRootFileId", LARGE_INTEGER), + ("FileIdentityLength", ULONG), + ("FileIdentity", BYTE)] + + +class CF_PLACEHOLDER_STANDARD_INFO(Structure): + _fields_ = [("OnDiskDataSize", LARGE_INTEGER), + ("ValidatedDataSize", LARGE_INTEGER), + ("ModifiedDataSize", LARGE_INTEGER), + ("PropertiesSize", LARGE_INTEGER), + ("PinState", c_uint), + ("InSyncState", c_uint), + ("FileId", LARGE_INTEGER), + ("SyncRootFileId", LARGE_INTEGER), + ("FileIdentityLength", ULONG), + ("FileIdentity", BYTE)] + + +class DUMMYSTRUCTNAME(Structure): + _fields_ = [("Offset", DWORD), ("OffsetHigh", DWORD)] + + +class DUMMYUNIONNAME(Union): + _fields_ = [("DUMMYSTRUCTNAME", DUMMYSTRUCTNAME), ("Pointer", PVOID)] + + +class OVERLAPPED(Structure): + _fields_ = [("Internal", ULONG_PTR), + ("InternalHigh", ULONG_PTR), + ("DUMMYUNIONNAME", DUMMYUNIONNAME), + ("hEvent", HANDLE)] + + +LPOVERLAPPED = POINTER(OVERLAPPED) CfOpenFileWithOplock = dll.CfOpenFileWithOplock CfOpenFileWithOplock.argtypes = [LPCWSTR, c_uint, PHANDLE] CfOpenFileWithOplock.restype = HRESULT @@ -18,7 +72,14 @@ CfHydratePlaceholder.argtypes = [HANDLE, LARGE_INTEGER, LARGE_INTEGER, c_uint, L CfHydratePlaceholder.restype = HRESULT CfCloseHandle = dll.CfCloseHandle CfCloseHandle.argtypes = [HANDLE] +CfGetPlaceholderInfo = dll.CfGetPlaceholderInfo +CfGetPlaceholderInfo.argtypes = [HANDLE, c_uint, PVOID, DWORD, PDWORD] # noqa: E501 +CfGetPlaceholderInfo.restype = HRESULT +CfSetPinState = dll.CfSetPinState +CfSetPinState.argtypes = [HANDLE, c_uint, c_uint, LPOVERLAPPED] +CfSetPinState.restype = HRESULT ERROR_INVALID_FUNCTION = 1 +ERROR_MORE_DATA = 234 GetLastError = windll.Kernel32.GetLastError GetLastError.restype = DWORD @@ -33,3 +94,36 @@ def hydrate_file(s: str): CfCloseHandle(h) raise e CfCloseHandle(h) + + +def get_info(s: str, standard: bool = False): + h = HANDLE() + i = CF_PLACEHOLDER_STANDARD_INFO() if standard else CF_PLACEHOLDER_BASIC_INFO() # noqa: E501 + t = CF_PLACEHOLDER_INFO_STANDARD if standard else CF_PLACEHOLDER_INFO_BASIC + le = DWORD() + si = DWORD(sizeof(i)) + try: + CfOpenFileWithOplock(s, CF_OPEN_FILE_FLAG_FOREGROUND, byref(h)) + CfGetPlaceholderInfo(h, t, byref(i), si, byref(le)) + except OSError as e: + ee = GetLastError() + if ee == ERROR_MORE_DATA: + CfCloseHandle(h) + return i + if ee != ERROR_INVALID_FUNCTION: + CfCloseHandle(h) + raise e + CfCloseHandle(h) + return i + + +def unpin_file(s: str): + h = HANDLE() + try: + CfOpenFileWithOplock(s, CF_OPEN_FILE_FLAG_FOREGROUND, byref(h)) + CfSetPinState(h, CF_PIN_STATE_UNPINNED, CF_SET_PIN_FLAG_NONE, None) + except OSError as e: + if GetLastError() != ERROR_INVALID_FUNCTION: + CfCloseHandle(h) + raise e + CfCloseHandle(h) diff --git a/game_backuper/config.py b/game_backuper/config.py index ef238dc..72dd364 100644 --- a/game_backuper/config.py +++ b/game_backuper/config.py @@ -23,6 +23,7 @@ class BasicOption: _compress_config = None _disable_compress = False _protect_filename = None + _unpin_file = None @property def compress_config(self) -> CompressConfig: @@ -96,12 +97,27 @@ class BasicOption: return cfg._remove_old_files return True + @cached_property + def unpin_file(self) -> bool: + if self._unpin_file is not None: + return self._unpin_file + prog = getattr(self, "_prog", None) + if prog is not None: + if prog._unpin_file is not None: + return prog._unpin_file + cfg = getattr(self, "_cfg", None) + if cfg is not None: + if cfg._unpin_file is not None: + return cfg._unpin_file + return True + def parse_all(self, data=None): self.parse_compress_config(data) self.parse_remove_old_files(data) self.parse_enable_pcre2(data) self.parse_encrypt_files(data) self.parse_protect_filename(data) + self.parse_unpin_file(data) def parse_compress_config(self, data=None): if data is None: @@ -159,6 +175,17 @@ class BasicOption: raise TypeError('remove_old_files option must be a boolean.') del v + def parse_unpin_file(self, data=None): + if data is None: + data = getattr(self, 'data') + if 'unpin_file' in data: + v = data['unpin_file'] + if isinstance(v, bool): + self._unpin_file = v + else: + raise TypeError('unpin_file option must be a boolean.') + del v + class NFBasicOption: """Basic options which is included in config, program.""" diff --git a/game_backuper/file.py b/game_backuper/file.py index 7934a7c..586d8ee 100644 --- a/game_backuper/file.py +++ b/game_backuper/file.py @@ -7,7 +7,7 @@ from game_backuper.filetype import FileType from platform import system if system() == "Windows": try: - from game_backuper.cfapi import hydrate_file + from game_backuper.cfapi import hydrate_file, unpin_file have_cfapi = True except Exception: have_cfapi = False @@ -150,3 +150,10 @@ def remove_unencryped_files(loc: str, prog: str, name: str): remove(loc) print(f'{prog}: Removed {loc}({name})') remove_compress_files(loc, prog, name) + + +def unpin_file_if_needed(fn: str): + if not have_cfapi: + return + if exists(fn): + unpin_file(fn)