From 25dbdcdb24b259b2ef5bc66f68c0ad5f7a7426c9 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 5 Sep 2021 15:56:53 +0800 Subject: [PATCH] very simply version --- .flake8 | 2 + .gitignore | 2 + game_backuper/__init__.py | 0 game_backuper/__main__.py | 23 +++++++++ game_backuper/backuper.py | 58 +++++++++++++++++++++++ game_backuper/cml.py | 31 ++++++++++++ game_backuper/config.py | 88 ++++++++++++++++++++++++++++++++++ game_backuper/db.py | 99 +++++++++++++++++++++++++++++++++++++++ game_backuper/file.py | 24 ++++++++++ game_backuper/hashl.py | 12 +++++ requirements.txt | 1 + setup.py | 33 +++++++++++++ 12 files changed, 373 insertions(+) create mode 100644 .flake8 create mode 100644 game_backuper/__init__.py create mode 100644 game_backuper/__main__.py create mode 100644 game_backuper/backuper.py create mode 100644 game_backuper/cml.py create mode 100644 game_backuper/config.py create mode 100644 game_backuper/db.py create mode 100644 game_backuper/file.py create mode 100644 game_backuper/hashl.py create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..4fad93c --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +exclude=build diff --git a/.gitignore b/.gitignore index b6e4761..e72a38f 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,5 @@ dmypy.json # Pyre type checker .pyre/ + +.vscode/ diff --git a/game_backuper/__init__.py b/game_backuper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/game_backuper/__main__.py b/game_backuper/__main__.py new file mode 100644 index 0000000..505e5b5 --- /dev/null +++ b/game_backuper/__main__.py @@ -0,0 +1,23 @@ +from game_backuper.config import Config +from game_backuper.cml import Opts +from game_backuper.db import Db +from game_backuper.backuper import Backuper + + +def main(cm=None): + if cm is None: + import sys + cm = sys.argv[1:] + cml = Opts(cm) + cfg = Config(cml.config_file) + db = Db(cfg.dest) + bk = Backuper(db, cfg, cml) + return bk.run() + + +if __name__ == "__main__": + import sys + try: + sys.exit(main()) + except Exception: + sys.exit(-1) diff --git a/game_backuper/backuper.py b/game_backuper/backuper.py new file mode 100644 index 0000000..3acb22c --- /dev/null +++ b/game_backuper/backuper.py @@ -0,0 +1,58 @@ +from game_backuper.db import Db +from game_backuper.config import Config, Program +from game_backuper.cml import Opts +from threading import Thread +from os.path import exists, join +from os import mkdir +from game_backuper.file import new_file, copy_file + + +class BackupTask(Thread): + def __init__(self, prog: Program, db: Db, cfg: Config): + Thread.__init__(self, name=f"Backup_{prog.name}") + self.cfg = cfg + self.prog = prog + self.db = db + + def run(self): + self.prog.clear_cache() + prog = self.prog.name + bp = join(self.cfg.dest, prog) + if not exists(bp): + mkdir(bp) + for f in self.prog.files: + if exists(f[1]): + ori = self.db.get_file(prog, f[0]) + nf = new_file(f[1], f[0], prog) + if nf is None: + continue + de = join(bp, f[0]) + if ori is not None: + if ori.size == nf.size and ori.hash == nf.hash: + print(f'{prog}: Skip {f[0]}({f[1]}).') + continue + copy_file(f[1], de, f[0], prog) + self.db.set_file(ori.id, nf.size, nf.hash) + else: + copy_file(f[1], de, f[0], prog) + self.db.add_file(nf) + + +class Backuper: + def __init__(self, db: Db, config: Config, opts: Opts): + self.db = db + self.conf = config + self.opts = opts + self.tasks = [] + + def run(self): + for prog in self.conf.progs: + t = BackupTask(prog, self.db, self.conf) + self.tasks.append(t) + t.start() + self.wait() + return 0 + + def wait(self): + for task in self.tasks: + task.join() diff --git a/game_backuper/cml.py b/game_backuper/cml.py new file mode 100644 index 0000000..ff8f162 --- /dev/null +++ b/game_backuper/cml.py @@ -0,0 +1,31 @@ +from getopt import getopt, GetoptError +from typing import List +from platform import system +if system() == "Windows": + import os + DEFAULT_CONFIG = f'{os.environ["APPDATA"]}\\game-backuper.yaml' +else: + DEFAULT_CONFIG = '/etc/game-backuper.yaml' + + +class Opts: + config_file: str = DEFAULT_CONFIG + + def __init__(self, cml: List[str]): + try: + r = getopt(cml, 'hc:', []) + for i in r[0]: + if i[0] == '-h': + self.print_help() + import sys + sys.exit(0) + elif i[0] == '-c': + self.config_file = i[1] + except GetoptError: + from traceback import print_exc + print_exc() + import sys + sys.exit(-1) + + def print_help(self): + print('''game-backuper [options] [game names]''') diff --git a/game_backuper/config.py b/game_backuper/config.py new file mode 100644 index 0000000..3514cca --- /dev/null +++ b/game_backuper/config.py @@ -0,0 +1,88 @@ +from yaml import load +try: + from yaml import CSafeLoader as SafeLoader +except Exception: + from yaml import SafeLoader +from os.path import join, relpath +from os import listdir +from typing import List + + +class Program: + def __init__(self, data: dict): + self.data = data + self._files = None + + @property + def base(self) -> str: + if 'base' in self.data: + v = self.data['base'] + if isinstance(v, str) and len(v) > 0: + return v + + def check(self) -> bool: + if self.name is None or self.base is None: + return False + self.files + return True + + def clear_cache(self): + self._files = None + + @property + def files(self) -> List[str]: + ke = 'files' + if ke not in self.data or not isinstance(self.data[ke], list): + raise ValueError('Files is needed and should be a list.') + if self._files is not None: + return self._files.copy() + r = [] + self._files = r.copy() + for i in self.data[ke]: + b = self.base + if isinstance(i, str): + r.append((i, join(b, i))) + elif isinstance(i, dict): + t = i['type'] + if t == 'path': + bp = join(b, i['path']) + ll = listdir(bp) + for ii in ll: + if ii.startswith('.'): + continue + pa = join(bp, ii) + r.append((relpath(pa, b), pa)) + return r + + @property + def name(self) -> str: + if 'name' in self.data: + v = self.data['name'] + if isinstance(v, str) and len(v) > 0: + return v + + +class Config: + dest = '' + progs = [] + + def __init__(self, fn: str): + with open(fn, 'r', encoding='UTF-8') as f: + t = load(f.read(), SafeLoader) + if t is None: + raise ValueError('Can not load config file.') + if not isinstance(t, dict): + raise ValueError('Config file error.') + if 'dest' not in t or not isinstance(t['dest'], str): + raise ValueError("Config file don't have dest or dest is not str.") + self.dest = t['dest'] + if 'programs' not in t: + raise ValueError("No programs found.") + progs = t['programs'] + if not isinstance(progs, list): + raise ValueError("programs should be list.") + for prog in progs: + p = Program(prog) + if not p.check(): + raise ValueError('Config error: program information error') + self.progs.append(p) diff --git a/game_backuper/db.py b/game_backuper/db.py new file mode 100644 index 0000000..1624013 --- /dev/null +++ b/game_backuper/db.py @@ -0,0 +1,99 @@ +from sqlite3 import connect +from os.path import join +from typing import List +from threading import Lock +from game_backuper.file import File + + +VERSION_TABLE = '''CREATE TABLE version ( +id TEXT, +v1 INT, +v2 INT, +v3 INT, +v4 INT, +PRIMARY KEY(id) +);''' +FILES_TABLE = '''CREATE TABLE files ( +id INTEGER, +file TEXT, +size INT, +program TEXT, +hash TEXT, +PRIMARY KEY(id) +);''' + + +class Db: + VERSION = [1, 0, 0, 0] + + def __check_database(self) -> bool: + self.__updateExistsTable() + v = self.__read_version() + if v is None: + return False + if v > self.VERSION: + raise ValueError( + 'Database version is higher. Please update program.') + return True + + def __create_table(self): + if 'version' not in self._exist_table: + self.db.execute(VERSION_TABLE) + self.__write_version() + if 'files' not in self._exist_table: + self.db.execute(FILES_TABLE) + self.db.commit() + + def __init__(self, loc: str): + fn = join(loc, "data.db") + self.db = connect(fn, check_same_thread=False) + self.db.execute('VACUUM;') + self.db.commit() + ok = self.__check_database() + if not ok: + self.__create_table() + self._lock = Lock() + + def __read_version(self) -> List[int]: + if 'version' not in self._exist_table: + return None + cur = self.db.execute("SELECT * FROM version WHERE id='main';") + for i in cur: + return [k for k in i if isinstance(k, int)] + + def __updateExistsTable(self): + cur = self.db.execute('SELECT * FROM main.sqlite_master;') + self._exist_table = {} + for i in cur: + if i[0] == 'table': + self._exist_table[i[1]] = i + + def __write_version(self): + if self.__read_version() is None: + self.db.execute('INSERT INTO version VALUES (?, ?, ?, ?, ?);', + tuple(['main'] + self.VERSION)) + else: + self.db.execute( + "UPDATE version SET v1=?, v2=?, v3=?, v4=? WHERE id='main';", + tuple(self.VERSION)) + self.db.commit() + + def add_file(self, f: File): + with self._lock: + self.db.execute('INSERT INTO files (file, size, program, hash) VALUES (?, ?, ?, ?);', # noqa: E501 + (f.file, f.size, f.program, f.hash)) + self.db.commit() + + def get_file(self, prog: str, file: str) -> File: + with self._lock: + cur = self.db.execute( + 'SELECT * FROM files WHERE program=? AND file=?;', (prog, + file)) + for i in cur: + return File(*i) + + def set_file(self, id: int, size: int, hash: str): + with self._lock: + self.db.execute('UPDATE files SET size=?, hash=? WHERE id=?;', + (id,)) + self.db.commit() diff --git a/game_backuper/file.py b/game_backuper/file.py new file mode 100644 index 0000000..71e828e --- /dev/null +++ b/game_backuper/file.py @@ -0,0 +1,24 @@ +from collections import namedtuple +from os.path import exists, dirname, abspath +from os import stat, makedirs +from game_backuper.hashl import sha512 +from shutil import copy2 + + +File = namedtuple('File', ['id', 'file', 'size', 'program', 'hash']) + + +def copy_file(loc: str, dest: str, name: str, prog: str): + d = dirname(abspath(dest)) + if not exists(d): + makedirs(d) + r = copy2(loc, dest) + print(f'{prog}: Copyed {loc}({name}) -> {r}') + + +def new_file(loc: str, name: str, prog: str) -> File: + if exists(loc): + fs = stat(loc).st_size + with open(loc, 'rb') as f: + hs = sha512(f) + return File(None, name, fs, prog, hs) diff --git a/game_backuper/hashl.py b/game_backuper/hashl.py new file mode 100644 index 0000000..f16b3c4 --- /dev/null +++ b/game_backuper/hashl.py @@ -0,0 +1,12 @@ +from hashlib import sha512 as _sha512 +from base64 import b85encode +from typing import BinaryIO + + +def sha512(b: BinaryIO): + s = _sha512() + t = b.read(1024) + while len(t) > 0: + s.update(t) + t = b.read(1024) + return b85encode(s.digest()).decode() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c3726e8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pyyaml diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c06b238 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +# flake8: noqa +import sys +if len(sys.argv) == 2 and sys.argv[1] == "py2exe": + from distutils.core import setup + import py2exe + params = { + "console": [{ + 'script': "game_backuper/__main__.py", + "dest_base": 'game-backuper' + }] + } +else: + from setuptools import setup + params = { + "install_requires": ["pyyaml"] + } +setup( + name="game-backuper", + version="1.0.0", + url="https://github.com/lifegpc/game-backuper", + author="lifegpc", + author_email="g1710431395@gmail.com", + classifiers=[ + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Programming Language :: Python :: 3.7", + ], + license="GNU General Public License v3 or later", + description="A game backuper", + long_description="A game backuper", + keywords="backup", + packages=["game_backuper"], + **params +)