lifegpc/game-backuper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

add basic support for leveldb

Browse Source
This commit is contained in:
lifegpc
2021-09-06 08:16:20 +08:00
parent 4ff3f7e1a3
commit 3c6dbfdd9d
7 changed files with 112 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
game_backuper/backuper.py
View File

@@ -1,10 +1,18 @@
from game_backuper.db import Db
from game_backuper.config import Config, Program
from game_backuper.config import (
Config,
Program,
ConfigNormalFile,
ConfigLeveldb,
)
from game_backuper.cml import Opts
from threading import Thread
from os.path import exists, join
from os import mkdir
from game_backuper.file import new_file, copy_file
from game_backuper.leveldb import have_leveldb
if have_leveldb:
from game_backuper.leveldb import list_leveldb_entries
class BackupTask(Thread):
@@ -21,21 +29,26 @@ class BackupTask(Thread):
if not exists(bp):
mkdir(bp)
for f in self.prog.files:
if exists(f[1]):
ori = self.db.get_file(prog, f[0])
nf = new_file(f[1], f[0], prog)
if nf is None:
continue
de = join(bp, f[0])
if ori is not None:
if ori.size == nf.size and ori.hash == nf.hash:
print(f'{prog}: Skip {f[0]}({f[1]}).')
if isinstance(f, ConfigNormalFile):
if exists(f[1]):
ori = self.db.get_file(prog, f[0])
nf = new_file(f[1], f[0], prog)
if nf is None:
continue
copy_file(f[1], de, f[0], prog)
self.db.set_file(ori.id, nf.size, nf.hash)
else:
copy_file(f[1], de, f[0], prog)
self.db.add_file(nf)
de = join(bp, f[0])
if ori is not None:
if ori.size == nf.size and ori.hash == nf.hash:
print(f'{prog}: Skip {f[0]}({f[1]}).')
continue
copy_file(f[1], de, f[0], prog)
self.db.set_file(ori.id, nf.size, nf.hash)
else:
copy_file(f[1], de, f[0], prog)
self.db.add_file(nf)
elif isinstance(f, ConfigLeveldb):
if not have_leveldb:
raise ValueError('Leveldb is not supported.')
print(list_leveldb_entries(f.full_path, f.domains))
class Backuper:

25
game_backuper/config.py
View File

@@ -4,8 +4,14 @@ try:
except Exception:
from yaml import SafeLoader
from os.path import join, relpath
from typing import List
from typing import List, Union
from game_backuper.file import listdirs
from collections import namedtuple
ConfigNormalFile = namedtuple('ConfigNormalFile', ['name', 'full_path'])
ConfigLeveldb = namedtuple('ConfigLeveldb', ['name', 'full_path', 'domains'])
ConfigResult = Union[ConfigNormalFile, ConfigLeveldb]
class Program:
@@ -30,7 +36,7 @@ class Program:
self._files = None
@property
def files(self) -> List[str]:
def files(self) -> List[ConfigResult]:
ke = 'files'
if ke not in self.data or not isinstance(self.data[ke], list):
raise ValueError('Files is needed and should be a list.')
@@ -41,14 +47,25 @@ class Program:
for i in self.data[ke]:
b = self.base
if isinstance(i, str):
r.append((i, join(b, i)))
r.append(ConfigNormalFile(i, join(b, i)))
elif isinstance(i, dict):
t = i['type']
if t == 'path':
bp = join(b, i['path'])
ll = listdirs(bp)
for ii in ll:
r.append((relpath(ii, b), ii))
r.append(ConfigNormalFile(relpath(ii, b), ii))
elif t == 'leveldb':
p = join(b, i['path'])
dms = None
if 'domains' in i and isinstance(i['domains'], list):
dms = []
for ii in i['domains']:
if isinstance(ii, str) and len(ii) > 0:
dms.append(ii.encode())
if len(dms) == 0:
dms = None
r.append(ConfigLeveldb(i['path'], p, dms))
return r
@property

25
game_backuper/db.py
View File

@@ -3,6 +3,7 @@ from os.path import join
from typing import List
from threading import Lock
from game_backuper.file import File
from game_backuper.filetype import FileType
VERSION_TABLE = '''CREATE TABLE version (
@@ -21,16 +22,25 @@ program TEXT,
hash TEXT,
PRIMARY KEY(id)
);'''
FILETYPE_TABLE = '''CREATE TABLE filetype (
id INT,
type INT,
PRIMARY KEY(id)
);'''
class Db:
VERSION = [1, 0, 0, 0]
VERSION = [1, 0, 0, 1]
def __check_database(self) -> bool:
self.__updateExistsTable()
v = self.__read_version()
if v is None:
return False
if v < self.VERSION:
if v < [1, 0, 0, 1]:
self.db.execute(FILETYPE_TABLE)
self.__write_version()
if v > self.VERSION:
raise ValueError(
'Database version is higher. Please update program.')
@@ -82,14 +92,23 @@ class Db:
with self._lock:
self.db.execute('INSERT INTO files (file, size, program, hash) VALUES (?, ?, ?, ?);', # noqa: E501
(f.file, f.size, f.program, f.hash))
if f.type is not None:
cur = self.db.execute(
'SELECT * FROM files WHERE program=? AND file=?;',
(f.program, f.file))
for i in cur:
self.db.execute('INSERT INTO filetype VALUES (?, ?);',
(i[0], f.type))
self.db.commit()
def get_file(self, prog: str, file: str) -> File:
with self._lock:
cur = self.db.execute(
'SELECT * FROM files WHERE program=? AND file=?;', (prog,
file))
'SELECT files.*, filetype.type FROM files LEFT JOIN filetype ON files.id=filetype.id WHERE program=? AND file=?;', # noqa: E501
(prog, file))
for i in cur:
if i[5] is not None:
i[5] = FileType(i[5])
return File(*i)
def set_file(self, id: int, size: int, hash: str):

7
game_backuper/file.py
View File

@@ -3,9 +3,10 @@ from os.path import exists, dirname, abspath, isfile, isdir, join
from os import stat, makedirs, listdir
from game_backuper.hashl import sha512
from shutil import copy2
from game_backuper.filetype import FileType
File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash'])
File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash', 'type'])
def copy_file(loc: str, dest: str, name: str, prog: str):
@@ -30,9 +31,9 @@ def listdirs(loc: str):
return r
def new_file(loc: str, name: str, prog: str) -> File:
def new_file(loc: str, name: str, prog: str, type: FileType = None) -> File:
if exists(loc):
fs = stat(loc).st_size
with open(loc, 'rb') as f:
hs = sha512(f)
return File(None, name, fs, prog, hs)
return File(None, name, fs, prog, hs, type)

6
game_backuper/filetype.py Normal file
View File

@@ -0,0 +1,6 @@
from enum import IntEnum, unique
@unique
class FileType(IntEnum):
LEVELDB = 0

30
game_backuper/leveldb.py Normal file
View File

@@ -0,0 +1,30 @@
try:
from plyvel import DB
have_leveldb = True
from typing import List
except ImportError:
have_leveldb = False
if have_leveldb:
def list_leveldb_entries(db: str, dms: List[bytes] = None):
d = DB(db)
r = []
for i in d.iterator(include_value=False):
if isinstance(i, bytes):
if dms is None:
r.append(i)
else:
if i == b'VERSION':
r.append(i)
elif i.startswith(b'META:'):
dm = i[5:]
if dm in dms:
r.append(i)
elif i.startswith(b'_'):
dmi = i.find(b'\x00\x01')
dm = i[1:dmi]
if dm in dms:
r.append(i)
r.sort()
return r

1
requirements.txt
View File

@@ -1 +1,2 @@
pyyaml
plyvel[leveldb]