diff --git a/README.md b/README.md index c257eb8..881a72b 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ rssbotLib=rssbot.dll 已弃用。 新版本的[RSSBotLib](https://github.com/lifegpc/ffmpeg-study/tree/master/rssbotlib)采用Python Extension Module模式，将动态库放置于Module Path内即可加载。加载后默认启用以下功能： - 发送视频时附带时长，视频分辨率大小信息（在视频文件大于10MB时非常有用）。 +- 支持Pixiv的[动图](https://www.pixiv.help/hc/zh-cn/articles/235584628-动图是什么-)（对HTML标签有要求，详见[wiki](https://github.com/lifegpc/rssbot/wiki/Pixiv动图自定义HTML标签：-ugoira-)）。 ### databaseLocation 可选参数。数据库位置。默认值为`data.db`。 ### retryTTL diff --git a/config.py b/config.py index f62cc80..789f007 100644 --- a/config.py +++ b/config.py @@ -26,10 +26,11 @@ class RSSConfig: self.display_entry_link = False self.send_img_as_file = False self.send_origin_file_name = False + self.send_ugoira_with_origin_pix_fmt = False self.update(d) def toJson(self): - return dumps({'disable_web_page_preview': self.disable_web_page_preview, 'show_RSS_title': self.show_RSS_title, 'show_Content_title': self.show_Content_title, 'show_content': self.show_content, 'send_media': self.send_media, 'display_entry_link': self.display_entry_link, 'send_img_as_file': self.send_img_as_file}, ensure_ascii=False) + return dumps({'disable_web_page_preview': self.disable_web_page_preview, 'show_RSS_title': self.show_RSS_title, 'show_Content_title': self.show_Content_title, 'show_content': self.show_content, 'send_media': self.send_media, 'display_entry_link': self.display_entry_link, 'send_img_as_file': self.send_img_as_file, 'send_ugoira_with_origin_pix_fmt': self.send_ugoira_with_origin_pix_fmt}, ensure_ascii=False) def update(self, d: dict): if d is not None: diff --git a/fileEntry.py b/fileEntry.py index 0f2ee1b..f9b1070 100644 --- a/fileEntry.py +++ b/fileEntry.py @@ -19,7 +19,7 @@ from time import time_ns from random import randint from requests import get from os import remove as removeFile, mkdir, listdir, removedirs -from typing import List +from typing import Dict, List from threading import Lock from config import RSSConfig @@ -39,6 +39,40 @@ def remove(s: str): remove(s) +class SubFileEntry: + def __init__(self, path: str) -> None: + self._path = path + self._abspath = path + self._fileExist = True if exists(self._path) else False + if self._fileExist: + self._fileSize = getsize(self._path) + self._localURI = f"file://{self._path}" if self._path[0] == '/' else f"file:///{self._path}" + self._f = None + + def delete(self): + if not self._fileExist: + return + if self._f is not None and not self._f.closed: + self._f.close() + try: + remove(self._path) + self._fileExist = False + except: + pass + + def open(self) -> bool: + if not self._fileExist: + return False + if self._f is not None and not self._f.closed: + self._f.seek(0, 0) + return True + try: + self._f = open(self._path, 'rb') + return True + except: + return False + + class FileEntry: def __init__(self, url: str, m, config: RSSConfig): if not exists('Temp'): @@ -98,8 +132,26 @@ class FileEntry: self._fileSize = getsize(self._abspath) self._localURI = f"file://{self._abspath}" if self._abspath[0] == '/' else f"file:///{self._abspath}" self._f = None + self._subFileDict: Dict[str, SubFileEntry] = {} + + def addSubFile(self, name: str, ext: str): + if not isinstance(name, str) or len(name) == 0: + raise ValueError('At least 1 char.') + if not isinstance(ext, str) or len(ext) == 0: + ext = 'temp' + na = f"{name}.{ext}" + if na in self._subFileDict: + return False + p = self.getSubPath(name, ext) + if not exists(p): + raise FileNotFoundError(p) + self._subFileDict[na] = SubFileEntry(p) + return True def delete(self): + for key in self._subFileDict: + self._subFileDict[key].delete() + self._subFileDict = {} if not self._fileExist: return if self._f is not None and not self._f.closed: @@ -112,6 +164,22 @@ class FileEntry: except: pass + def getSubFile(self, name: str, ext: str): + if not isinstance(name, str) or len(name) == 0: + raise ValueError('At least 1 char.') + if not isinstance(ext, str) or len(ext) == 0: + ext = 'temp' + na = f"{name}.{ext}" + if na in self._subFileDict: + return self._subFileDict[na] + + def getSubPath(self, name: str, ext: str): + if not isinstance(name, str) or len(name) == 0: + raise ValueError('At least 1 char.') + if not isinstance(ext, str) or len(ext) == 0: + ext = 'temp' + return splitext(self._abspath)[0] + name + '.' + ext + def open(self) -> bool: if not self._fileExist: return False diff --git a/rssbot.py b/rssbot.py index 43bcb5c..dc12f66 100644 --- a/rssbot.py +++ b/rssbot.py @@ -35,7 +35,7 @@ from usercheck import checkUserPermissionsInChat, UserPermissionsInChatCheckResu import sys from fileEntry import FileEntries, remove from dictdeal import json2data -from rssbotlib import loadRSSBotLib, AddVideoInfoResult +from rssbotlib import loadRSSBotLib, AddVideoInfoResult, have_rssbotlib from time import sleep, time from miraiDatabase import MiraiDatabase from mirai import Mirai @@ -73,6 +73,8 @@ def getMediaInfo(m: dict, config: RSSConfig = RSSConfig()) -> str: s = f"{s}\n发送媒体：{config.send_media}" s = f"{s}\n单独一行显示链接：{config.display_entry_link}" s += f"\n发送图片为文件：{config.send_img_as_file}" + if have_rssbotlib: + s += f"\n发送原始像素格式的Pixiv动图：{config.send_ugoira_with_origin_pix_fmt}" s += f"\nRSS全局设置：" s += f"\n发送时使用原文件名：{config.send_origin_file_name}" return s @@ -95,6 +97,7 @@ class InlineKeyBoardCallBack(Enum): SendImgAsFile = 12 GlobalSettingsPage = 13 SendOriginFileName = 14 + SendUgoiraWithOriginPixFmt = 15 def getInlineKeyBoardWhenRSS(hashd: str, m: dict, isOwn: bool) -> dict: @@ -158,6 +161,9 @@ def getInlineKeyBoardWhenRSS2(hashd: str, config: RSSConfig) -> str: i += 1 temp = '禁用发送图片为文件' if config.send_img_as_file else '启用发送图片为文件' d[i].append({'text': temp, 'callback_data': f'0,{hashd},{InlineKeyBoardCallBack.SendImgAsFile.value}'}) + if have_rssbotlib: + temp = f"{'禁用' if config.send_ugoira_with_origin_pix_fmt else '启用'}发送原始像素格式的Pixiv动图" + d[i].append({'text': temp, 'callback_data': f'0,{hashd},{InlineKeyBoardCallBack.SendUgoiraWithOriginPixFmt.value}'}) d.append([]) i += 1 d[i].append( @@ -230,7 +236,7 @@ class main: if key not in content or content[key] is None: return 0 return len(content[key]) - if not config.send_media or (getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 0): + if not config.send_media or (getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 0 and getListCount(content, 'ugoiraList') == 0): if config.disable_web_page_preview: di['disable_web_page_preview'] = True while len(text) > 0: @@ -249,7 +255,7 @@ class main: else: return False sleep(5) - elif getListCount(content, 'imgList') == 1 and getListCount(content, 'videoList') == 0: + elif getListCount(content, 'imgList') == 1 and getListCount(content, 'videoList') == 0 and getListCount(content, 'ugoiraList') == 0: f = True while len(text) > 0 or f: if f: @@ -305,7 +311,7 @@ class main: else: return False sleep(5) - elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 1: + elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 1 and getListCount(content, 'ugoiraList') == 0: f = True while len(text) > 0 or f: if f: @@ -402,6 +408,98 @@ class main: else: return False sleep(5) + elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 0 and getListCount(content, 'ugoiraList') == 1: + f = True + while len(text) > 0 or f: + if f: + di['caption'] = text.tostr(1024) + else: + di['text'] = text.tostr() + di['parse_mode'] = 'HTML' + for i in range(self._setting.maxRetryCount + 1): + if f: + if self._setting.downloadMediaFile and not self._setting.sendFileURLScheme: + di2 = {} + if not self._setting.downloadMediaFile: + di['photo'] = content['ugoiraList'][0]['poster'] + else: + fileEntry = self._tempFileEntries.add( + content['ugoiraList'][0]['poster'], config) + if not fileEntry: + continue + if self._setting.sendFileURLScheme: + di['thumb'] = fileEntry._localURI + else: + fileEntry.open() + di2['thumb'] = ( + fileEntry._fullfn, fileEntry._f) + z = self._tempFileEntries.add(content['ugoiraList'][0]['src'], config) + force_yuv420p = not config.send_ugoira_with_origin_pix_fmt + mp4_ok = z.ok and self._rssbotLib is not None and self._rssbotLib.convert_ugoira_to_mp4(z, content['ugoiraList'][0]['frames'], force_yuv420p) + if mp4_ok: + mp4 = z.getSubFile('_yuv420p' if force_yuv420p else '_origin', 'mp4') + # TODO: Generate a better thumb + if self._setting.sendFileURLScheme: + del di['thumb'] + di['animation'] = mp4._localURI + else: + del di2['thumb'] + mp4.open() + di2['animation'] = (mp4._path, mp4._f) + self._rssbotLib.addVideoInfo(mp4._path, di) + if self._setting.sendFileURLScheme: + re = self._request('sendAnimation', 'post', json=di) + else: + re = self._request('sendAnimation', 'post', json=di, files=di2) + else: + should_use_file = False if fileEntry._fileSize < MAX_PHOTO_SIZE and not config.send_img_as_file else True + if self._setting.sendFileURLScheme: + if not should_use_file: + di['photo'] = di['thumb'] + re = self._request('sendPhoto', 'post', json=di) + else: + di['document'] = di['thumb'] + re = self._request('sendDocument', 'post', json=di) + else: + if not should_use_file: + di2['photo'] = di2['thumb'] + re = self._request('sendPhoto', 'post', json=di, files=di2) + else: + di2['document'] = di2['thumb'] + re = self._request('sendDocument', 'post', json=di, files=di2) + else: + re = self._request('sendMessage', 'post', json=di) + if re is not None and 'ok' in re and re['ok']: + di['reply_to_message_id'] = re['result']['message_id'] + if f: + if 'photo' in di: + del di['photo'] + if 'document' in di: + del di['document'] + if 'animation' in di: + del di['video'] + if 'thumb' in di: + del di['thumb'] + if 'caption' in di: + del di['caption'] + if 'duration' in di: + del di['duration'] + if 'width' in di: + del di['width'] + if 'height' in di: + del di['height'] + if config.disable_web_page_preview: + di['disable_web_page_preview'] = True + f = False + break + if i == self._setting.maxRetryCount: + if returnError and re is not None and 'description' in re: + return False, re['description'] + elif returnError: + return False, '' + else: + return False + sleep(5) else: ind = 0 if self._setting.downloadMediaFile and not self._setting.sendFileURLScheme: @@ -1277,7 +1375,7 @@ class callbackQueryHandle(Thread): self._main._request("editMessageText", "post", json=di) self.answer() return - elif self._inlineKeyBoardCommand in [InlineKeyBoardCallBack.DisableWebPagePreview, InlineKeyBoardCallBack.ShowRSSTitle, InlineKeyBoardCallBack.ShowContentTitle, InlineKeyBoardCallBack.ShowContent, InlineKeyBoardCallBack.SendMedia, InlineKeyBoardCallBack.DisplayEntryLink, InlineKeyBoardCallBack.SendImgAsFile]: + elif self._inlineKeyBoardCommand in [InlineKeyBoardCallBack.DisableWebPagePreview, InlineKeyBoardCallBack.ShowRSSTitle, InlineKeyBoardCallBack.ShowContentTitle, InlineKeyBoardCallBack.ShowContent, InlineKeyBoardCallBack.SendMedia, InlineKeyBoardCallBack.DisplayEntryLink, InlineKeyBoardCallBack.SendImgAsFile, InlineKeyBoardCallBack.SendUgoiraWithOriginPixFmt]: if self._inlineKeyBoardCommand == InlineKeyBoardCallBack.DisableWebPagePreview: self._rssMeta.config.disable_web_page_preview = not self._rssMeta.config.disable_web_page_preview elif self._inlineKeyBoardCommand == InlineKeyBoardCallBack.ShowRSSTitle: @@ -1292,6 +1390,8 @@ class callbackQueryHandle(Thread): self._rssMeta.config.display_entry_link = not self._rssMeta.config.display_entry_link elif self._inlineKeyBoardCommand == InlineKeyBoardCallBack.SendImgAsFile: self._rssMeta.config.send_img_as_file = not self._rssMeta.config.send_img_as_file + elif self._inlineKeyBoardCommand == InlineKeyBoardCallBack.SendUgoiraWithOriginPixFmt: + self._rssMeta.config.send_ugoira_with_origin_pix_fmt = not self._rssMeta.config.send_ugoira_with_origin_pix_fmt di = {'chat_id': self._rssMeta.chatId, 'message_id': self._rssMeta.messageId} di['text'] = getMediaInfo( @@ -1478,7 +1578,7 @@ class callbackQueryHandle(Thread): self._main._request("editMessageText", "post", json=di) self.answer() return - elif self._inlineKeyBoardForRSSListCommand in [InlineKeyBoardForRSSList.DisableWebPagePreview, InlineKeyBoardForRSSList.ShowRSSTitle, InlineKeyBoardForRSSList.ShowContentTitle, InlineKeyBoardForRSSList.ShowContent, InlineKeyBoardForRSSList.SendMedia, InlineKeyBoardForRSSList.DisplayEntryLink, InlineKeyBoardForRSSList.SendImgAsFile]: + elif self._inlineKeyBoardForRSSListCommand in [InlineKeyBoardForRSSList.DisableWebPagePreview, InlineKeyBoardForRSSList.ShowRSSTitle, InlineKeyBoardForRSSList.ShowContentTitle, InlineKeyBoardForRSSList.ShowContent, InlineKeyBoardForRSSList.SendMedia, InlineKeyBoardForRSSList.DisplayEntryLink, InlineKeyBoardForRSSList.SendImgAsFile, InlineKeyBoardForRSSList.SendUgoiraWithOriginPixFmt]: di = {'chat_id': self._data['message']['chat']['id'], 'message_id': self._data['message']['message_id']} rssList = self._main._db.getRSSListByChatId(chatId) @@ -1505,6 +1605,8 @@ class callbackQueryHandle(Thread): config.display_entry_link = not config.display_entry_link elif self._inlineKeyBoardForRSSListCommand == InlineKeyBoardForRSSList.SendImgAsFile: config.send_img_as_file = not config.send_img_as_file + elif self._inlineKeyBoardForRSSListCommand == InlineKeyBoardForRSSList.SendUgoiraWithOriginPixFmt: + config.send_ugoira_with_origin_pix_fmt = not config.send_ugoira_with_origin_pix_fmt updated = self._main._db.updateChatConfig(chatEntry) if updated: self.answer('修改设置成功') diff --git a/rssbotlib.py b/rssbotlib.py index 4734d9b..ee92cd1 100644 --- a/rssbotlib.py +++ b/rssbotlib.py @@ -16,10 +16,12 @@ from enum import unique, Enum from traceback import print_exc try: - from _rssbotlib import version, VideoInfo + from _rssbotlib import version, VideoInfo, convert_ugoira_to_mp4, AVDict have_rssbotlib = True except ImportError: have_rssbotlib = False +if have_rssbotlib: + from fileEntry import FileEntry, remove @unique @@ -35,7 +37,7 @@ if have_rssbotlib: from rssbot import main self._main: main = m self._version = version() - if self._version is None or self._version != [1, 0, 0, 0]: + if self._version is None or self._version != [1, 0, 0, 1]: raise ValueError('RSSBotLib Version unknown or not supported.') def addVideoInfo(self, url: str, data: dict, loc: str = None) -> AddVideoInfoResult: @@ -72,6 +74,27 @@ if have_rssbotlib: print_exc() return AddVideoInfoResult.ERROR + def convert_ugoira_to_mp4(self, f: FileEntry, frames, force_yuv420p: bool): + try: + na = '_yuv420p' if force_yuv420p else '_origin' + if f.getSubFile(na, 'mp4') is not None: + return True + dst = f.getSubPath(na, 'mp4') + opt = AVDict() + if force_yuv420p: + opt['force_yuv420p'] = '' + if not convert_ugoira_to_mp4(f._abspath, dst, frames, opts=opt): + return False + f.addSubFile(na, 'mp4') + return True + except Exception: + print_exc() + try: + remove(dst) + except Exception: + print_exc() + return False + def loadRSSBotLib(m): if have_rssbotlib: diff --git a/rsslist.py b/rsslist.py index 0b4f611..de882ae 100644 --- a/rsslist.py +++ b/rsslist.py @@ -19,6 +19,7 @@ from enum import Enum, unique from math import ceil, floor from textc import textc, timeToStr from readset import settings +from rssbotlib import have_rssbotlib @unique @@ -45,6 +46,7 @@ class InlineKeyBoardForRSSList(Enum): SendImgAsFile = 19 GlobalSettingsPage = 20 SendOriginFileName = 21 + SendUgoiraWithOriginPixFmt = 22 def getTextContentForRSSInList(rssEntry: RSSEntry, s: settings) -> str: @@ -76,6 +78,8 @@ def getTextContentForRSSInList(rssEntry: RSSEntry, s: settings) -> str: text.addtotext(f"发送媒体：{config.send_media}") text += f"单独一行显示链接：{config.display_entry_link}" text += f"发送图片为文件：{config.send_img_as_file}" + if have_rssbotlib: + text += f'发送原始像素格式的Pixiv动图：{config.send_ugoira_with_origin_pix_fmt}' text += f"RSS全局设置：" text += f"发送时使用原文件名：{config.send_origin_file_name}" return text.tostr() @@ -204,6 +208,9 @@ def getInlineKeyBoardForRSSSettingsInList(chatId: int, rssEntry: RSSEntry, index i += 1 temp = '禁用发送图片为文件' if config.send_img_as_file else '启用发送图片为文件' d[i].append({'text': temp, 'callback_data': f'1,{chatId},{InlineKeyBoardForRSSList.SendImgAsFile.value},{index},{rssEntry.id}'}) + if have_rssbotlib: + temp = f"{'禁用' if config.send_ugoira_with_origin_pix_fmt else '启用'}发送原始像素格式的Pixiv动图" + d[i].append({'text': temp, 'callback_data': f'1,{chatId},{InlineKeyBoardForRSSList.SendUgoiraWithOriginPixFmt.value},{index},{rssEntry.id}'}) d.append([]) i = i + 1 d[i].append( diff --git a/rssparser.py b/rssparser.py index 8432a00..3f58e27 100644 --- a/rssparser.py +++ b/rssparser.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import List from xml.dom import minidom defusedxmlSupported = True try: @@ -27,6 +28,7 @@ import sys import requests from traceback import format_exc from urllib.parse import urljoin +from json import loads as loadjson class HTMLContent: @@ -50,13 +52,14 @@ class HTMLSimpleParser(HTMLParser): def __init__(self, baseUrl: str = None): self.data = '' self.tagName = [] - self.tagContent = [] + self.tagContent: List[HTMLContent] = [] self.tagAttrs = [] self.imgList = [] self.videoList = [] self.baseUrl = '' if baseUrl is not None: self.baseUrl = baseUrl + self.ugoiraList = [] HTMLParser.__init__(self) def handle_startendtag(self, tag, attrs): @@ -65,6 +68,9 @@ class HTMLSimpleParser(HTMLParser): self.data = self.data + '\n' else: self.tagContent[-1].add('\n') + else: + self.handle_starttag(tag, attrs) + self.handle_endtag(tag) def handle_starttag(self, tag, attrs): if tag == 'br': @@ -89,6 +95,29 @@ class HTMLSimpleParser(HTMLParser): if 'src' in p: self.videoList.append(p) return + elif tag == 'ugoira': + p = {} + for key, value in attrs: + if key == 'src': + p['src'] = urljoin(self.baseUrl, value) + elif key == 'poster': + p['poster'] = urljoin(self.baseUrl, value) + elif key == 'frames': + try: + frames = loadjson(value) + if not isinstance(frames, list): + raise ValueError(f"Invaild frames: {frames}") + for i in frames: + if not isinstance(i['file'], str): + raise ValueError(f"Invalid file: {i['file']}") + if not isinstance(i['delay'], (int, float)): + raise ValueError(f"Invalid delay: {i['delay']}") + p['frames'] = frames + except Exception: + print(format_exc()) + if 'src' in p and 'poster' in p and 'frames' in p: + self.ugoiraList.append(p) + return self.tagName.append(tag) self.tagContent.append(HTMLContent()) self.tagAttrs.append('') @@ -110,7 +139,7 @@ class HTMLSimpleParser(HTMLParser): elif len(self.tagName) > 1: self.tagContent[-2].add( f"<{tag}{self.tagAttrs[-1]}>{self.tagContent[-1].export()}") - elif tag not in ['img', 'video', 'br']: + elif tag not in ['img', 'video', 'br', 'ugoira']: if len(self.tagName) == 1: self.data = f"{self.data}{self.tagContent[-1].export()}" elif len(self.tagName) > 1: @@ -233,6 +262,7 @@ class RSSParser: del m['content:encoded'] m['imgList'] = p.imgList m['videoList'] = p.videoList + m['ugoiraList'] = p.ugoiraList else: m[i.nodeName] = '' for k in i.childNodes: @@ -274,6 +304,7 @@ class RSSParser: if i.nodeName in ['content', 'summary']: m['imgList'] = p.imgList m['videoList'] = p.videoList + m['ugoiraList'] = p.ugoiraList m['description'] = m[i.nodeName] del m[i.nodeName] elif i.nodeValue is None and len(i.childNodes) == 0: @@ -304,6 +335,7 @@ class RSSParser: if i.nodeName in ['content', 'summary']: m['imgList'] = p.imgList m['videoList'] = p.videoList + m['ugoiraList'] = p.ugoiraList m['description'] = m[i.nodeName] del m[i.nodeName] elif typ == 'xhtml': @@ -318,6 +350,7 @@ class RSSParser: if i.nodeName in ['content', 'summary']: m['imgList'] = p.imgList m['videoList'] = p.videoList + m['ugoiraList'] = p.ugoiraLists m['description'] = m[i.nodeName] del m[i.nodeName] elif len(i.childNodes) == 0: