Add support for cloud placeholder on Windows (Such as OneDrive)

game_backuper/cfapi.py

@@ -0,0 +1,35 @@
+from ctypes import HRESULT, byref, c_uint, windll
+from ctypes.wintypes import (
+    DWORD, HANDLE, LARGE_INTEGER, LPCWSTR, LPVOID, PHANDLE
+)
+
+
+dll = windll.CldApi
+CF_OPEN_FILE_FLAG_NONE = 0
+CF_OPEN_FILE_FLAG_EXCLUSIVE = 1
+CF_OPEN_FILE_FLAG_WRITE_ACCESS = 2
+CF_OPEN_FILE_FLAG_DELETE_ACCESS = 3
+CF_OPEN_FILE_FLAG_FOREGROUND = 4
+CfOpenFileWithOplock = dll.CfOpenFileWithOplock
+CfOpenFileWithOplock.argtypes = [LPCWSTR, c_uint, PHANDLE]
+CfOpenFileWithOplock.restype = HRESULT
+CfHydratePlaceholder = dll.CfHydratePlaceholder
+CfHydratePlaceholder.argtypes = [HANDLE, LARGE_INTEGER, LARGE_INTEGER, c_uint, LPVOID]  # noqa: E501
+CfHydratePlaceholder.restype = HRESULT
+CfCloseHandle = dll.CfCloseHandle
+CfCloseHandle.argtypes = [HANDLE]
+ERROR_INVALID_FUNCTION = 1
+GetLastError = windll.Kernel32.GetLastError
+GetLastError.restype = DWORD
+
+
+def hydrate_file(s: str):
+    h = HANDLE()
+    try:
+        CfOpenFileWithOplock(s, CF_OPEN_FILE_FLAG_NONE, byref(h))
+        CfHydratePlaceholder(h, 0, -1, 0, LPVOID())
+    except OSError as e:
+        if GetLastError() != ERROR_INVALID_FUNCTION:
+            CfCloseHandle(h)
+            raise e
+    CfCloseHandle(h)

game_backuper/cml.py

@@ -36,10 +36,11 @@ class Opts:
     config_file: str = DEFAULT_CONFIG
     action = OptAction.BACKUP
     programs_list = None
+    optimize_db = False
 
     def __init__(self, cml: List[str]):
         try:
-            r = getopt(cml, 'hc:', ['help', 'config='])
+            r = getopt(cml, 'hc:', ['help', 'config=', 'optimize-db'])
             for i in r[0]:
                 if i[0] == '-h' or i[0] == '--help':
                     self.print_help()
@@ -47,6 +48,8 @@ class Opts:
                     sys.exit(0)
                 elif i[0] == '-c' or i[0] == '--config':
                     self.config_file = i[1]
+                elif i[0] == '--optimize-db':
+                    self.optimize_db = True
             if len(r[1]) > 0:
                 cm = r[1]
                 re = OptAction.from_str(cm[0])
@@ -74,4 +77,5 @@ game-backuper [options] list
 game-backuper [options] list_leveldb_key [<db_path> [...]]
 Options:
     -h, --help          Print help message.
-    -c, --config <path> Set config file.''')
+    -c, --config <path> Set config file.
+    --optimize-db       Optimize the sqlite3 database''')

game_backuper/compress.py

@@ -52,7 +52,7 @@ except ImportError:
     cached_property = property
 from os.path import exists, isfile, getsize
 from os import remove
-from game_backuper.file import mkdir_for_file
+from game_backuper.file import mkdir_for_file, hydrate_file_if_needed
 
 
 @unique
@@ -303,6 +303,7 @@ def decompress(src: str, dest: str, c: CompressConfig, name: str, prog: str):
     if exists(dest):
         remove(dest)
     cs = c.chunk_size
+    hydrate_file_if_needed(fn)
     if c.method == CompressMethod.BZIP2:
         with BZ2File(fn, 'rb') as f:
             with open(dest, 'wb') as t:

game_backuper/db.py

@@ -2,7 +2,7 @@ from sqlite3 import connect
 from os.path import join
 from typing import List, Union
 from threading import Lock
-from game_backuper.file import File
+from game_backuper.file import File, hydrate_file_if_needed
 from game_backuper.filetype import FileType
 
 
@@ -56,11 +56,13 @@ class Db:
             self.db.execute(FILETYPE_TABLE)
         self.db.commit()
 
-    def __init__(self, loc: str):
+    def __init__(self, loc: str, optimize_db: bool = False):
         fn = join(loc, "data.db")
+        hydrate_file_if_needed(fn)
         self.db = connect(fn, check_same_thread=False)
-        self.db.execute('VACUUM;')
+        if optimize_db:
-        self.db.commit()
+            self.db.execute('VACUUM;')
+            self.db.commit()
         ok = self.__check_database()
         if not ok:
             self.__create_table()

game_backuper/file.py

@@ -1,10 +1,15 @@
 from collections import namedtuple
 from os.path import exists, dirname, abspath, isfile, isdir, join, isabs
-from os import stat, makedirs, listdir
+from os import stat, makedirs, listdir, remove
 from game_backuper.hashl import sha512
 from shutil import copy2
 from game_backuper.filetype import FileType
-from os import remove
+from platform import system
+if system() == "Windows":
+    from game_backuper.cfapi import hydrate_file
+    have_cfapi = True
+else:
+    have_cfapi = False
 
 
 File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type'])
@@ -93,3 +98,10 @@ def remove_compress_files(loc: str, prog: str, name: str, ext: str = None):
         if exists(f) and isfile(f):
             remove(f)
             print(f'{prog}: Removed {f}({name})')
+
+
+def hydrate_file_if_needed(fn: str):
+    if not have_cfapi:
+        return
+    if exists(fn):
+        hydrate_file(fn)

game_backuper/main.py

@@ -14,6 +14,6 @@ def main(cm=None):
     cfg = Config(cml.config_file)
     if not exists(cfg.dest):
         makedirs(cfg.dest)
-    db = Db(cfg.dest)
+    db = Db(cfg.dest, cml.optimize_db)
     bk = Backuper(db, cfg, cml)
     return bk.run()

game_backuper/restorer.py

@@ -8,6 +8,7 @@ from game_backuper.file import (
     remove_dirs,
     new_file,
     mkdir_for_file,
+    hydrate_file_if_needed,
 )
 from os import remove, close
 from game_backuper.filetype import FileType
@@ -55,6 +56,7 @@ class RestoreTask(Thread):
                         print(f'{prog}: Skip {fn}')
                         continue
                 if c is None:
+                    hydrate_file_if_needed(src)
                     copy_file(src, dest, nam, prog)
                 else:
                     decompress(src, dest, c, nam, prog)
@@ -89,6 +91,7 @@ class RestoreTask(Thread):
                         continue
                 mkdir_for_file(dest)
                 if c is None:
+                    hydrate_file_if_needed(src)
                     sqlite_to_leveldb(src, dest, r.domains)
                     print(f'{prog}: Covert leveldb done. {src}({fn}) -> {dest}')  # noqa: E501
                 else: