diff --git a/example.yaml b/example.yaml index 051c3f2..1608adb 100644 --- a/example.yaml +++ b/example.yaml @@ -1,14 +1,18 @@ dest: /path/to/store/backup/files # The programs will store database and backup files in this location +remove_old_files: true # Optional. Default value: true. Remove unneeded backup files which already deleted in source tree when backuping files. programs: - name: Your program name # This name is used to identify different application. base: /path/to/save/path # Must be absoulte path. + remove_old_files: true # Optional. files: - BGI.gdb # path to a file/folder. All subfolders will include if it is a folder. Must be relative path. - type: path path: folder # path to a file/folder. All subfolders will include if it is a folder. Must be relative path if name not found. name: folder2 # optional. path to the backup files. Shoule be a relative path + remove_old_files: true # Optional. - type: leveldb # module plyvel is needed to support this type. This will store leveldb database to a single file database (sqlite3) path: leveldb # path to leveldb. Must be relative path. name: dest # optional. path to the backup files. Shoule be a relative path + remove_old_files: true # Optional. domains: # optional. Just backup minor domains in localstorage database. Only chromium is tested. - some domain diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py index 04e3482..9d4cc12 100644 --- a/game_backuper/backuper.py +++ b/game_backuper/backuper.py @@ -26,9 +26,12 @@ class BackupTask(Thread): bp = join(self.cfg.dest, prog) if not exists(bp): mkdir(bp) + fl = self.db.get_file_list(prog) for f in self.prog.files: if isinstance(f, ConfigNormalFile): if exists(f[1]): + if f.name in fl: + fl.remove(f.name) ori = self.db.get_file(prog, f[0]) nf = new_file(f[1], f[0], prog) if nf is None: @@ -49,6 +52,8 @@ class BackupTask(Thread): raise NotImplementedError('Leveldb is not supported.') if not exists(f.full_path): continue + if f.name in fl: + fl.remove(f.name) from game_backuper.leveldb import ( list_leveldb_entries, leveldb_stats, @@ -79,6 +84,20 @@ class BackupTask(Thread): self.db.add_file(nf) else: self.db.set_file(ori.id, stats.size, stats.hash) + for fn in fl: + f = self.db.get_file(prog, fn) + if f.type is None: + de = join(bp, fn) + if exists(de): + remove(de) + print(f'{prog}: Remove {de}({fn})') + self.db.remove_file(f) + if f.type == FileType.LEVELDB: + de = join(bp, fn + '.db') + if exists(de): + remove(de) + print(f'{prog}: Remove {de}({fn})') + self.db.remove_file(f) class Backuper: diff --git a/game_backuper/config.py b/game_backuper/config.py index 50133c9..6e337e4 100644 --- a/game_backuper/config.py +++ b/game_backuper/config.py @@ -7,19 +7,62 @@ from os.path import join, relpath, isfile, isdir, isabs from typing import List, Union from game_backuper.file import listdirs from collections import namedtuple +try: + from functools import cached_property +except ImportError: + cached_property = property -ConfigNormalFile = namedtuple('ConfigNormalFile', ['name', 'full_path']) -ConfigLeveldb = namedtuple('ConfigLeveldb', ['name', 'full_path', 'domains']) +class BasicOption: + _remove_old_files = None + + @cached_property + def remove_old_files(self) -> bool: + if self._remove_old_files is not None: + return self._remove_old_files + prog = getattr(self, "_prog", None) + if prog is not None: + if prog._remove_old_files is not None: + return prog._remove_old_files + cfg = getattr(self, "_cfg", None) + if cfg is not None: + if cfg._remove_old_files is not None: + return cfg._remove_old_files + return True + + def parse_all(self, data=None): + self.parse_remove_old_files(data) + + def parse_remove_old_files(self, data=None): + if data is None: + data = getattr(self, 'data') + if 'remove_old_files' in data: + v = data['remove_old_files'] + if isinstance(v, bool): + self._remove_old_files = v + else: + raise TypeError('remove_old_files option must be a boolean.') + del v + + +def namedtuple_bo(typename, field_names): + a = namedtuple(typename, field_names) + return type(typename, (a, BasicOption), {}) + + +ConfigNormalFile = namedtuple_bo('ConfigNormalFile', ['name', 'full_path']) +ConfigLeveldb = namedtuple_bo('ConfigLeveldb', ['name', 'full_path', 'domains']) # noqa: E501 ConfigResult = Union[ConfigNormalFile, ConfigLeveldb] -class Program: - def __init__(self, data: dict): +class Program(BasicOption): + def __init__(self, data: dict, cfg): self.data = data self._files = None + self._cfg = cfg + self.parse_all() - @property + @cached_property def base(self) -> str: if 'base' in self.data: v = self.data['base'] @@ -71,11 +114,15 @@ class Program: if i['name'] != '': name = i['name'] if isfile(bp): - r.append(ConfigNormalFile(name, bp)) + tmp = ConfigNormalFile(name, bp) + tmp.parse_all(i) + r.append(tmp) elif isdir(bp): ll = listdirs(bp) for ii in ll: - r.append(ConfigNormalFile(join(name, relpath(ii, bp)), ii)) # noqa: E501 + tmp = ConfigNormalFile(join(name, relpath(ii, bp)), ii) # noqa: E501 + tmp.parse_all(i) + r.append(tmp) elif t == 'leveldb': if isabs(i['path']): if 'name' not in i or not isinstance(i['name'], str) or i['name'] == '': # noqa: E501 @@ -96,10 +143,14 @@ class Program: dms.append(ii.encode()) if len(dms) == 0: dms = None - r.append(ConfigLeveldb(n, p, dms)) + tmp = ConfigLeveldb(n, p, dms) + tmp.parse_all(i) + r.append(tmp) + for i in r: + i._prog = self return r - @property + @cached_property def name(self) -> str: if 'name' in self.data: v = self.data['name'] @@ -107,7 +158,7 @@ class Program: return v -class Config: +class Config(BasicOption): dest = '' progs = [] progs_name = [] @@ -124,11 +175,12 @@ class Config: self.dest = t['dest'] if 'programs' not in t: raise ValueError("No programs found.") + self.parse_all(t) progs = t['programs'] if not isinstance(progs, list): raise ValueError("programs should be list.") for prog in progs: - p = Program(prog) + p = Program(prog, self) if not p.check(): raise ValueError('Config error: program information error') if p.name not in self.progs_name: diff --git a/game_backuper/db.py b/game_backuper/db.py index 73df339..67db77f 100644 --- a/game_backuper/db.py +++ b/game_backuper/db.py @@ -111,6 +111,15 @@ class Db: for i in cur: return File(*i) + def get_file_list(self, prog: str) -> List[str]: + with self._lock: + cur = self.db.execute('SELECT file FROM files WHERE program=?;', + (prog,)) + li = [] + for i in cur: + li.append(i[0]) + return li + def remove_file(self, id: Union[int, File]): with self._lock: ft = None