From 27413613bdac064eb917f14986512634c5bab0b6 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Thu, 7 Jan 2021 20:37:43 +0800 Subject: [PATCH] update --- RSSEntry.py | 10 +++++++ database.py | 61 ++++++++++++++++++++++++++++++++++++------- readset.py | 11 +++++--- rssbot.py | 3 +++ rsschecker.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 rsschecker.py diff --git a/RSSEntry.py b/RSSEntry.py index 8690e7a..d5ef325 100644 --- a/RSSEntry.py +++ b/RSSEntry.py @@ -91,6 +91,16 @@ class HashEntries: r.append(i) return r + def has(self, d: HashEntry) -> bool: + if d.hash is None or d.id is None: + return False + for v in self.__list: + if v.hash == d.hash and v.id == d.id: + if d.time > v.time: + v.time = d.time + return True + return False + def setMaxCount(self, maxCount: int): self.__maxCount = maxCount if maxCount >= 1 else 100 self.__removeMax() diff --git a/database.py b/database.py index 28a3a77..511b739 100644 --- a/database.py +++ b/database.py @@ -20,6 +20,7 @@ from typing import List from enum import Enum, unique from threading import Lock from hashl import sha256WithBase64 +from time import time def dealtext(s: str): @@ -115,15 +116,15 @@ PRIMARY KEY (hash) cur = self._db.execute( f'SELECT * FROM RSSList WHERE id="{hashd}"') has_data = False - for i in cur: + for i in cur: # pylint: disable=unused-variable has_data = True break if has_data: self._db.execute( - f"UPDATE RSSList SET title='{dealtext(title)}', ttl={ttl if ttl is not None else 'null'} WHERE id='{hashd}'") + f"UPDATE RSSList SET title='{dealtext(title)}', interval={ttl if ttl is not None else 'null'} WHERE id='{hashd}'") else: self._db.execute( - f"INSERT INTO RSSList VALUES ('{dealtext(title)}', '{dealtext(url)}', {ttl if ttl is not None else 'null'}, null, '{hashd}')") + f"INSERT INTO RSSList VALUES ('{dealtext(title)}', '{dealtext(url)}', {ttl if ttl is not None else 'null'}, {int(time())}, '{hashd}')") cur = self._db.execute( f'SELECT * FROM chatList WHERE id="{hashd}" AND chatId={chatId}') has_data2 = False @@ -144,7 +145,7 @@ PRIMARY KEY (hash) break if has_data3: self._db.execute( - f"DELETE FROM hashList WHERE ID='{hashd}'") + f"DELETE FROM hashList WHERE id='{hashd}'") for v in hashEntries.getList(): self._db.execute( f"INSERT INTO hashList VALUES ('{v.id}', '{v.hash}', {v.time})") @@ -174,16 +175,27 @@ PRIMARY KEY (hash) r.append(temp) return r - def getUserStatus(self, userId: int) -> (userStatus, str): + def getRSSListByChatId(self, chatId: int) -> List[RSSEntry]: with self._value_lock: cur = self._db.execute( - f'SELECT * FROM userStatus WHERE userId={userId}') + f"SELECT * FROM chatList WHERE chatId={chatId}") + RSSEntries = [] for i in cur: - try: + chatEntry = ChatEntry(i) + cur = self._db.execute( + f"SELECT * FROM RSSList WHERE id='{chatEntry.id}'") + for k in cur: + rssEntry = RSSEntry(k, self._main._setting._maxCount) + + def getUserStatus(self, userId: int) -> (userStatus, str): + with self._value_lock: + try: + cur = self._db.execute( + f'SELECT * FROM userStatus WHERE userId={userId}') + for i in cur: return userStatus(i[1]), i[2] - except: - pass - return userStatus.normalStatus, '' + except: + return userStatus.normalStatus, '' def setUserStatus(self, userId: int, status: userStatus = userStatus.normalStatus, hashd: str = '') -> bool: with self._value_lock: @@ -213,3 +225,32 @@ PRIMARY KEY (hash) return True except: return False + + def updateRSS(self, title: str, url: str, lastupdatetime: int, hashEntries: HashEntries, ttl: int = None): + with self._value_lock: + try: + hashd = sha256WithBase64(url) + cur = self._db.execute( + f'SELECT * FROM RSSList WHERE id="{hashd}"') + has_data = False + for i in cur: # pylint: disable=unused-variable + has_data = True + break + if not has_data: + return False + self._db.execute( + f"UPDATE RSSList SET title='{dealtext(title)}', interval={ttl if ttl is not None else 'null'}, lastupdatetime={lastupdatetime} WHERE id='{hashd}'") + cur = self._db.execute( + f"SELECT * FROM hashList WHERE id='{hashd}'") + has_data2 = False + for i in cur: + has_data2 = True + break + if has_data2: + self._db.execute( + f"DELETE FROM hashList WHERE id='{hashd}'") + for v in hashEntries.getList(): + self._db.execute( + f"INSERT INTO hashList VALUES ('{v.id}', '{v.hash}', {v.time})") + except: + return False diff --git a/readset.py b/readset.py index 864feba..ad0ce2d 100644 --- a/readset.py +++ b/readset.py @@ -14,10 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . class settings: - def __init__(self, fn: str=None): + def __init__(self, fn: str = None): if fn is not None: self.parse(fn) - + def parse(self, fn: str): d = {} with open(fn, 'r', encoding='utf8') as f: @@ -27,4 +27,9 @@ class settings: if len(l) == 2: d[l[0]] = l[1] self._token = d['token'] if 'token' in d else None - self._maxCount = int(d['maxCount']) if 'maxCount' in d and d['maxCount'].isnumeric() else 100 + self._maxCount = int( + d['maxCount']) if 'maxCount' in d and d['maxCount'].isnumeric() else 100 + self._minTTL = int(d['minTTL']) if 'minTTL' in d and d['minTTL'].isnumeric( + ) and int(d['minTTL']) >= 1 else 5 + self._maxTTL = int(d['maxTTL']) if 'maxTTL' in d and d['maxTTL'].isnumeric( + ) and int(d['maxTTL']) >= self._minTTL else max(1440, self._minTTL) diff --git a/rssbot.py b/rssbot.py index 4393efe..173780b 100644 --- a/rssbot.py +++ b/rssbot.py @@ -29,6 +29,7 @@ from rsstempdict import rssMetaInfo, rssMetaList from random import randrange from textc import textc from re import search, I +from rsschecker import RSSCheckerThread def getMediaInfo(m: dict, config: RSSConfig = RSSConfig()) -> str: @@ -251,6 +252,8 @@ class main: self._upi = None self._updateThread = updateThread(self) self._updateThread.start() + self._rssCheckerThread = RSSCheckerThread(self) + self._rssCheckerThread.start() class updateThread(Thread): diff --git a/rsschecker.py b/rsschecker.py new file mode 100644 index 0000000..c7b7631 --- /dev/null +++ b/rsschecker.py @@ -0,0 +1,72 @@ +# (C) 2021 lifegpc +# This file is part of rssbot. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from threading import Thread +from time import sleep, time +from RSSEntry import RSSEntry, calHash, ChatEntry +from traceback import format_exc +from rssparser import RSSParser + + +class RSSCheckerThread(Thread): + def __loop(self): + for rss in self._main._db.getAllRSSList(): + if self.__needUpdate(rss): + try: + p = RSSParser() + p.parse(rss.url) + updateTime = int(time()) + if p.check(): + meta = p.m + itemList = p.itemList[:self._main._setting._maxCount] + itemList.reverse() + for item in itemList: + hashEntry = calHash(rss.url, item) + if not rss.hashList.has(hashEntry): + rss.hashList.add(hashEntry) + for info in rss.chatList: + chatEntry: ChatEntry = info + try: + self._main._sendMessage( + chatEntry.chatId, meta, item, chatEntry.config) + except: + print(format_exc()) + self._main._db.updateRSS( + rss.title, rss.url, updateTime, rss.hashList, p.ttl) + except: + print(format_exc()) + + def __init__(self, m): + Thread.__init__(self) + from rssbot import main + self._main: main = m + + def __needUpdate(self, rss: RSSEntry): + if rss.lastupdatetime is None: + return True + TTL = self._main._setting._minTTL + if rss.interval is not None: + TTL = max(min(rss.interval, self._main._setting._maxTTL), + self._main._setting._minTTL) + TTL = TTL * 60 + return True if int(time()) > rss.lastupdatetime + TTL else False + + def run(self): + while True: + try: + self.__loop() + except: + print(format_exc()) + sleep(1)