322 lines
17 KiB
Python
322 lines
17 KiB
Python
from game_backuper.db import Db
|
|
from game_backuper.config import (
|
|
Config,
|
|
Program,
|
|
ConfigNormalFile,
|
|
ConfigLeveldb,
|
|
)
|
|
from game_backuper.cml import Opts, OptAction
|
|
from threading import Thread
|
|
from os.path import exists, join, isdir
|
|
from os import remove, close
|
|
from shutil import move
|
|
from game_backuper.file import new_file, copy_file, File, mkdir_for_file
|
|
from game_backuper.filetype import FileType
|
|
from game_backuper.restorer import RestoreTask
|
|
from game_backuper.file import remove_compress_files, remove_unencryped_files
|
|
from game_backuper.compress import compress
|
|
from game_backuper.enc import encrypt_file
|
|
from game_backuper.file import unpin_file_if_needed
|
|
from tempfile import mkstemp
|
|
|
|
|
|
class BackupTask(Thread):
|
|
def __init__(self, prog: Program, db: Db, cfg: Config):
|
|
Thread.__init__(self, name=f"Backup_{prog.name}")
|
|
self.cfg = cfg
|
|
self.prog = prog
|
|
self.db = db
|
|
|
|
def run(self):
|
|
self.prog.clear_cache()
|
|
prog = self.prog.name
|
|
bp = join(self.cfg.dest, prog)
|
|
ebp = join(self.cfg.dest, '.encrypt', prog)
|
|
ebpi = join(self.cfg.dest, '.encrypt', '.id')
|
|
fl = self.db.get_file_list(prog)
|
|
for f in self.prog.files:
|
|
if isinstance(f, ConfigNormalFile):
|
|
if exists(f[1]):
|
|
if f.name in fl:
|
|
fl.remove(f.name)
|
|
c = f.compress_config
|
|
ori = self.db.get_file(prog, f[0])
|
|
nf = new_file(f[1], f[0], prog)
|
|
if nf is None:
|
|
continue
|
|
de = join(ebp if f.encrypt_files else bp, f[0])
|
|
if ori is not None:
|
|
de2 = join(ebpi if f.encrypt_files else bp, str(ori.id)) # noqa: E501
|
|
if f.protect_filename:
|
|
if not exists(de2) and exists(de):
|
|
mkdir_for_file(de2)
|
|
move(de, de2)
|
|
print(f'{prog}: Renamed {de} -> {de2}.')
|
|
de = de2
|
|
else:
|
|
if not exists(de) and exists(de2):
|
|
mkdir_for_file(de)
|
|
move(de2, de)
|
|
print(f'{prog}: Renamed {de2} -> {de}.')
|
|
if ori.size == nf.size and ori.hash == nf.hash:
|
|
if c is None:
|
|
if exists(de) and not f.encrypt_files:
|
|
print(f'{prog}: Skip {f[0]}.')
|
|
remove_compress_files(de, prog, f.name)
|
|
self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
continue
|
|
elif exists(de) and f.encrypt_files and not ori.compressed: # noqa: E501
|
|
print(f'{prog}: Skip {f[0]}.')
|
|
remove_unencryped_files(join(bp, f[0]), prog, f.name) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
continue
|
|
else:
|
|
if not f.encrypt_files and exists(de + c.ext):
|
|
print(f'{prog}: Skip {f.name}.')
|
|
remove_compress_files(de, prog, f.name, c.ext) # noqa: E501
|
|
self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de + c.ext)
|
|
continue
|
|
elif f.encrypt_files and ori.compressed_type == c.method: # noqa: E501
|
|
print(f'{prog}: Skip {f.name}.')
|
|
remove_unencryped_files(join(bp, f.name), prog, f.name) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
continue
|
|
stats = None
|
|
if f.encrypt_files:
|
|
stats = encrypt_file(f[1], de, nf, f.name, prog, c)
|
|
remove_unencryped_files(join(bp, f[0]), prog, f.name) # noqa: E501
|
|
elif c is None:
|
|
copy_file(f[1], de, f[0], prog)
|
|
remove_compress_files(de, prog, f.name)
|
|
self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
else:
|
|
compress(f[1], de, c, f.name, prog)
|
|
self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
self.db.set_file(ori.id, nf.size, nf.hash)
|
|
self.db.set_file_encrypt_information(ori.id, stats)
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
else:
|
|
if f.protect_filename:
|
|
self.db.add_file(nf, False)
|
|
tmpori = self.db.get_file(prog, f[0])
|
|
de = join(ebpi if f.encrypt_files else bp, str(tmpori.id)) # noqa: E501
|
|
if f.encrypt_files:
|
|
s = encrypt_file(f[1], de, nf, f.name, prog, c)
|
|
nf = File.from_encrypt_stats(s, nf)
|
|
remove_unencryped_files(join(bp, f[0]), prog, f.name) # noqa: E501
|
|
elif c is None:
|
|
copy_file(f[1], de, f[0], prog)
|
|
remove_compress_files(de, prog, f.name)
|
|
self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501
|
|
else:
|
|
compress(f[1], de, c, f.name, prog)
|
|
self.remove_encrypted_file(join(ebp, f[0]), prog, f.name, ori) # noqa: E501
|
|
if f.protect_filename:
|
|
self.db.set_file_encrypt_information(tmpori.id, s)
|
|
else:
|
|
self.db.add_file(nf)
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
elif isinstance(f, ConfigLeveldb):
|
|
from game_backuper.leveldb import have_leveldb
|
|
if not have_leveldb:
|
|
raise NotImplementedError('Leveldb is not supported.')
|
|
if not exists(f.full_path):
|
|
continue
|
|
if f.name in fl:
|
|
fl.remove(f.name)
|
|
from game_backuper.leveldb import (
|
|
list_leveldb_entries,
|
|
leveldb_stats,
|
|
leveldb_to_sqlite,
|
|
)
|
|
ent = list_leveldb_entries(f.full_path, f.domains)
|
|
stats = leveldb_stats(f.full_path, ent)
|
|
ori = self.db.get_file(prog, f.name)
|
|
c = f.compress_config
|
|
de = join(ebp if f.encrypt_files else bp, f.name + ".db")
|
|
if ori is not None:
|
|
de2 = join(ebpi if f.encrypt_files else bp, str(ori.id)) # noqa: E501
|
|
if f.protect_filename:
|
|
if not exists(de2) and exists(de):
|
|
mkdir_for_file(de2)
|
|
move(de, de2)
|
|
print(f'{prog}: Renamed {de} -> {de2}.')
|
|
de = de2
|
|
else:
|
|
if not exists(de) and exists(de2):
|
|
mkdir_for_file(de)
|
|
move(de2, de)
|
|
print(f'{prog}: Renamed {de2} -> {de}.')
|
|
if ori.type is None or ori.type != FileType.LEVELDB:
|
|
pp = join(bp, ori.file)
|
|
if exists(pp):
|
|
remove(pp)
|
|
remove_compress_files(pp, prog, f.name)
|
|
self.remove_encrypted_file(join(ebp, ori.file), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
self.db.remove_file(ori)
|
|
ori = None
|
|
if ori is not None:
|
|
if ori.size == stats.size and ori.hash == stats.hash:
|
|
if c is None:
|
|
if exists(de) and not f.encrypt_files:
|
|
print(f'{prog}: Skip {f[0]}.')
|
|
remove_compress_files(de, prog, f.name)
|
|
self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
continue
|
|
elif exists(de) and f.encrypt_files and not ori.compressed: # noqa: E501
|
|
print(f'{prog}: Skip {f[0]}.')
|
|
remove_unencryped_files(join(bp, f[0] + '.db'), prog, f.name) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
continue
|
|
else:
|
|
if not f.encrypt_files and exists(de + c.ext):
|
|
print(f'{prog}: Skip {f.name}.')
|
|
remove_compress_files(de, prog, f.name, c.ext)
|
|
self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de + c.ext)
|
|
continue
|
|
elif f.encrypt_files and ori.compressed_type == c.method: # noqa: E501
|
|
print(f'{prog}: Skip {f.name}.')
|
|
remove_unencryped_files(join(bp, f.name + '.db'), prog, f.name) # noqa: E501
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
continue
|
|
if f.protect_filename:
|
|
ori = self.db.get_file(prog, f[0])
|
|
if ori is None:
|
|
nf = File(None, f.name, 0, prog, None, FileType.LEVELDB, None, None, None, None, None) # noqa: E501
|
|
self.db.add_file(nf, False)
|
|
ori = self.db.get_file(prog, f[0])
|
|
de = join(ebpi if f.encrypt_files else bp, str(ori.id)) # noqa: E501
|
|
mkdir_for_file(de)
|
|
st = None
|
|
if c is None and not f.encrypt_files:
|
|
leveldb_to_sqlite(f.full_path, de, ent)
|
|
print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {de}') # noqa: E501
|
|
remove_compress_files(de, prog, f.name)
|
|
self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
else:
|
|
tmp = mkstemp()
|
|
close(tmp[0])
|
|
tmp = tmp[1]
|
|
leveldb_to_sqlite(f.full_path, tmp, ent)
|
|
print(f'{prog}: Covert leveldb done. {f.full_path}({f.name}) -> {tmp}') # noqa: E501
|
|
if f.encrypt_files:
|
|
st = encrypt_file(tmp, de, File.from_leveldb_stats(stats), f.name, prog, c) # noqa: E501
|
|
remove_unencryped_files(join(bp, f[0] + '.db'), prog, f.name) # noqa: E501
|
|
else:
|
|
compress(tmp, de, c, f.name, prog)
|
|
self.remove_encrypted_file(join(ebp, f[0] + '.db'), prog, f.name, ori) # noqa: E501
|
|
self.remove_encrypted_file(join(ebpi, str(ori.id)), prog, f.name, ori) # noqa: E501
|
|
remove(tmp)
|
|
print(f'{prog}: Removed tempfile {tmp}')
|
|
if ori is None:
|
|
nf = File(None, f.name, stats.size, prog, stats.hash,
|
|
FileType.LEVELDB, None, None, None, None, None)
|
|
if st:
|
|
nf = File.from_encrypt_stats(st, nf)
|
|
self.db.add_file(nf)
|
|
else:
|
|
self.db.set_file(ori.id, stats.size, stats.hash)
|
|
self.db.set_file_encrypt_information(ori.id, st)
|
|
if f.unpin_file:
|
|
unpin_file_if_needed(de)
|
|
for fn in fl:
|
|
f = self.db.get_file(prog, fn)
|
|
if f.type is None:
|
|
de = join(bp, fn)
|
|
if exists(de):
|
|
remove(de)
|
|
print(f'{prog}: Remove {de}({fn})')
|
|
remove_compress_files(de, prog, fn)
|
|
self.remove_encrypted_file(join(ebp, fn), prog, fn, f)
|
|
self.remove_encrypted_file(join(ebpi, str(f.id)), prog, fn, f)
|
|
self.db.remove_file(f)
|
|
if f.type == FileType.LEVELDB:
|
|
de = join(bp, fn + '.db')
|
|
if exists(de):
|
|
remove(de)
|
|
print(f'{prog}: Remove {de}({fn})')
|
|
remove_compress_files(de, prog, fn + '.db')
|
|
self.remove_encrypted_file(join(ebp, fn + '.db'), prog, fn, f)
|
|
self.remove_encrypted_file(join(ebpi, str(f.id)), prog, fn, ori) # noqa: E501
|
|
self.db.remove_file(f)
|
|
|
|
def remove_encrypted_file(self, loc: str, prog: str, name: str, f: File):
|
|
if exists(loc):
|
|
remove(loc)
|
|
print(f'{prog}: Removed {loc}({name})')
|
|
|
|
|
|
class Backuper:
|
|
def __init__(self, db: Db, config: Config, opts: Opts):
|
|
self.db = db
|
|
self.conf = config
|
|
self.opts = opts
|
|
self.tasks = []
|
|
|
|
def deal_prog(self, prog: Program):
|
|
if self.opts.action == OptAction.BACKUP:
|
|
t = BackupTask(prog, self.db, self.conf)
|
|
self.tasks.append(t)
|
|
t.start()
|
|
elif self.opts.action == OptAction.LIST:
|
|
print(prog.name)
|
|
elif self.opts.action == OptAction.RESTORE:
|
|
t = RestoreTask(prog, self.db, self.conf)
|
|
self.tasks.append(t)
|
|
t.start()
|
|
|
|
def run(self):
|
|
if self.opts.action == OptAction.LIST_LEVELDB_KEY:
|
|
from game_backuper.leveldb import have_leveldb
|
|
if not have_leveldb:
|
|
raise NotImplementedError('Leveldb is not supported.')
|
|
from game_backuper.leveldb import list_leveldb_entries
|
|
for db in self.opts.programs_list:
|
|
if exists(db):
|
|
if isdir(db):
|
|
print(f'Keys in "{db}":')
|
|
for i in list_leveldb_entries(db):
|
|
print(i)
|
|
else:
|
|
raise FileExistsError(f'"{db}" should be a directory.')
|
|
else:
|
|
raise FileNotFoundError(f'Can not find "{db}"')
|
|
elif self.opts.programs_list is None:
|
|
for prog in self.conf.progs:
|
|
self.deal_prog(prog)
|
|
else:
|
|
for n in self.opts.programs_list:
|
|
if n not in self.conf.progs_name:
|
|
raise ValueError(f'Can not find "{n}" in config file.')
|
|
for n in self.opts.programs_list:
|
|
prog = self.conf.progs[self.conf.progs_name.index(n)]
|
|
self.deal_prog(prog)
|
|
self.wait()
|
|
return 0
|
|
|
|
def wait(self):
|
|
for task in self.tasks:
|
|
task.join()
|