diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py index 3acb22c..339c092 100644 --- a/game_backuper/backuper.py +++ b/game_backuper/backuper.py @@ -1,10 +1,18 @@ from game_backuper.db import Db -from game_backuper.config import Config, Program +from game_backuper.config import ( + Config, + Program, + ConfigNormalFile, + ConfigLeveldb, +) from game_backuper.cml import Opts from threading import Thread from os.path import exists, join from os import mkdir from game_backuper.file import new_file, copy_file +from game_backuper.leveldb import have_leveldb +if have_leveldb: + from game_backuper.leveldb import list_leveldb_entries class BackupTask(Thread): @@ -21,21 +29,26 @@ class BackupTask(Thread): if not exists(bp): mkdir(bp) for f in self.prog.files: - if exists(f[1]): - ori = self.db.get_file(prog, f[0]) - nf = new_file(f[1], f[0], prog) - if nf is None: - continue - de = join(bp, f[0]) - if ori is not None: - if ori.size == nf.size and ori.hash == nf.hash: - print(f'{prog}: Skip {f[0]}({f[1]}).') + if isinstance(f, ConfigNormalFile): + if exists(f[1]): + ori = self.db.get_file(prog, f[0]) + nf = new_file(f[1], f[0], prog) + if nf is None: continue - copy_file(f[1], de, f[0], prog) - self.db.set_file(ori.id, nf.size, nf.hash) - else: - copy_file(f[1], de, f[0], prog) - self.db.add_file(nf) + de = join(bp, f[0]) + if ori is not None: + if ori.size == nf.size and ori.hash == nf.hash: + print(f'{prog}: Skip {f[0]}({f[1]}).') + continue + copy_file(f[1], de, f[0], prog) + self.db.set_file(ori.id, nf.size, nf.hash) + else: + copy_file(f[1], de, f[0], prog) + self.db.add_file(nf) + elif isinstance(f, ConfigLeveldb): + if not have_leveldb: + raise ValueError('Leveldb is not supported.') + print(list_leveldb_entries(f.full_path, f.domains)) class Backuper: diff --git a/game_backuper/config.py b/game_backuper/config.py index 6769928..87236e0 100644 --- a/game_backuper/config.py +++ b/game_backuper/config.py @@ -4,8 +4,14 @@ try: except Exception: from yaml import SafeLoader from os.path import join, relpath -from typing import List +from typing import List, Union from game_backuper.file import listdirs +from collections import namedtuple + + +ConfigNormalFile = namedtuple('ConfigNormalFile', ['name', 'full_path']) +ConfigLeveldb = namedtuple('ConfigLeveldb', ['name', 'full_path', 'domains']) +ConfigResult = Union[ConfigNormalFile, ConfigLeveldb] class Program: @@ -30,7 +36,7 @@ class Program: self._files = None @property - def files(self) -> List[str]: + def files(self) -> List[ConfigResult]: ke = 'files' if ke not in self.data or not isinstance(self.data[ke], list): raise ValueError('Files is needed and should be a list.') @@ -41,14 +47,25 @@ class Program: for i in self.data[ke]: b = self.base if isinstance(i, str): - r.append((i, join(b, i))) + r.append(ConfigNormalFile(i, join(b, i))) elif isinstance(i, dict): t = i['type'] if t == 'path': bp = join(b, i['path']) ll = listdirs(bp) for ii in ll: - r.append((relpath(ii, b), ii)) + r.append(ConfigNormalFile(relpath(ii, b), ii)) + elif t == 'leveldb': + p = join(b, i['path']) + dms = None + if 'domains' in i and isinstance(i['domains'], list): + dms = [] + for ii in i['domains']: + if isinstance(ii, str) and len(ii) > 0: + dms.append(ii.encode()) + if len(dms) == 0: + dms = None + r.append(ConfigLeveldb(i['path'], p, dms)) return r @property diff --git a/game_backuper/db.py b/game_backuper/db.py index 6f02324..59f527e 100644 --- a/game_backuper/db.py +++ b/game_backuper/db.py @@ -3,6 +3,7 @@ from os.path import join from typing import List from threading import Lock from game_backuper.file import File +from game_backuper.filetype import FileType VERSION_TABLE = '''CREATE TABLE version ( @@ -21,16 +22,25 @@ program TEXT, hash TEXT, PRIMARY KEY(id) );''' +FILETYPE_TABLE = '''CREATE TABLE filetype ( +id INT, +type INT, +PRIMARY KEY(id) +);''' class Db: - VERSION = [1, 0, 0, 0] + VERSION = [1, 0, 0, 1] def __check_database(self) -> bool: self.__updateExistsTable() v = self.__read_version() if v is None: return False + if v < self.VERSION: + if v < [1, 0, 0, 1]: + self.db.execute(FILETYPE_TABLE) + self.__write_version() if v > self.VERSION: raise ValueError( 'Database version is higher. Please update program.') @@ -82,14 +92,23 @@ class Db: with self._lock: self.db.execute('INSERT INTO files (file, size, program, hash) VALUES (?, ?, ?, ?);', # noqa: E501 (f.file, f.size, f.program, f.hash)) + if f.type is not None: + cur = self.db.execute( + 'SELECT * FROM files WHERE program=? AND file=?;', + (f.program, f.file)) + for i in cur: + self.db.execute('INSERT INTO filetype VALUES (?, ?);', + (i[0], f.type)) self.db.commit() def get_file(self, prog: str, file: str) -> File: with self._lock: cur = self.db.execute( - 'SELECT * FROM files WHERE program=? AND file=?;', (prog, - file)) + 'SELECT files.*, filetype.type FROM files LEFT JOIN filetype ON files.id=filetype.id WHERE program=? AND file=?;', # noqa: E501 + (prog, file)) for i in cur: + if i[5] is not None: + i[5] = FileType(i[5]) return File(*i) def set_file(self, id: int, size: int, hash: str): diff --git a/game_backuper/file.py b/game_backuper/file.py index 474c62d..c8847bd 100644 --- a/game_backuper/file.py +++ b/game_backuper/file.py @@ -3,9 +3,10 @@ from os.path import exists, dirname, abspath, isfile, isdir, join from os import stat, makedirs, listdir from game_backuper.hashl import sha512 from shutil import copy2 +from game_backuper.filetype import FileType -File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash']) +File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type']) def copy_file(loc: str, dest: str, name: str, prog: str): @@ -30,9 +31,9 @@ def listdirs(loc: str): return r -def new_file(loc: str, name: str, prog: str) -> File: +def new_file(loc: str, name: str, prog: str, type: FileType = None) -> File: if exists(loc): fs = stat(loc).st_size with open(loc, 'rb') as f: hs = sha512(f) - return File(None, name, fs, prog, hs) + return File(None, name, fs, prog, hs, type) diff --git a/game_backuper/filetype.py b/game_backuper/filetype.py new file mode 100644 index 0000000..9e46bc9 --- /dev/null +++ b/game_backuper/filetype.py @@ -0,0 +1,6 @@ +from enum import IntEnum, unique + + +@unique +class FileType(IntEnum): + LEVELDB = 0 diff --git a/game_backuper/leveldb.py b/game_backuper/leveldb.py new file mode 100644 index 0000000..e740162 --- /dev/null +++ b/game_backuper/leveldb.py @@ -0,0 +1,30 @@ +try: + from plyvel import DB + have_leveldb = True + from typing import List +except ImportError: + have_leveldb = False + + +if have_leveldb: + def list_leveldb_entries(db: str, dms: List[bytes] = None): + d = DB(db) + r = [] + for i in d.iterator(include_value=False): + if isinstance(i, bytes): + if dms is None: + r.append(i) + else: + if i == b'VERSION': + r.append(i) + elif i.startswith(b'META:'): + dm = i[5:] + if dm in dms: + r.append(i) + elif i.startswith(b'_'): + dmi = i.find(b'\x00\x01') + dm = i[1:dmi] + if dm in dms: + r.append(i) + r.sort() + return r diff --git a/requirements.txt b/requirements.txt index c3726e8..498b0a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pyyaml +plyvel[leveldb]