basic support ugoira

This commit is contained in:
2021-11-14 10:21:02 +08:00
parent a973b596fa
commit 655fdcd76c
7 changed files with 247 additions and 12 deletions

View File

@@ -53,6 +53,7 @@ rssbotLib=rssbot.dll
已弃用。
新版本的[RSSBotLib](https://github.com/lifegpc/ffmpeg-study/tree/master/rssbotlib)采用Python Extension Module模式,将动态库放置于Module Path内即可加载。加载后默认启用以下功能:
- 发送视频时附带时长,视频分辨率大小信息(在视频文件大于10MB时非常有用)。
- 支持Pixiv的[动图](https://www.pixiv.help/hc/zh-cn/articles/235584628-动图是什么-)(对HTML标签有要求,详见[wiki](https://github.com/lifegpc/rssbot/wiki/Pixiv动图自定义HTML标签:-ugoira-))。
### databaseLocation
可选参数。数据库位置。默认值为`data.db`
### retryTTL

View File

@@ -26,10 +26,11 @@ class RSSConfig:
self.display_entry_link = False
self.send_img_as_file = False
self.send_origin_file_name = False
self.send_ugoira_with_origin_pix_fmt = False
self.update(d)
def toJson(self):
return dumps({'disable_web_page_preview': self.disable_web_page_preview, 'show_RSS_title': self.show_RSS_title, 'show_Content_title': self.show_Content_title, 'show_content': self.show_content, 'send_media': self.send_media, 'display_entry_link': self.display_entry_link, 'send_img_as_file': self.send_img_as_file}, ensure_ascii=False)
return dumps({'disable_web_page_preview': self.disable_web_page_preview, 'show_RSS_title': self.show_RSS_title, 'show_Content_title': self.show_Content_title, 'show_content': self.show_content, 'send_media': self.send_media, 'display_entry_link': self.display_entry_link, 'send_img_as_file': self.send_img_as_file, 'send_ugoira_with_origin_pix_fmt': self.send_ugoira_with_origin_pix_fmt}, ensure_ascii=False)
def update(self, d: dict):
if d is not None:

View File

@@ -19,7 +19,7 @@ from time import time_ns
from random import randint
from requests import get
from os import remove as removeFile, mkdir, listdir, removedirs
from typing import List
from typing import Dict, List
from threading import Lock
from config import RSSConfig
@@ -39,6 +39,40 @@ def remove(s: str):
remove(s)
class SubFileEntry:
def __init__(self, path: str) -> None:
self._path = path
self._abspath = path
self._fileExist = True if exists(self._path) else False
if self._fileExist:
self._fileSize = getsize(self._path)
self._localURI = f"file://{self._path}" if self._path[0] == '/' else f"file:///{self._path}"
self._f = None
def delete(self):
if not self._fileExist:
return
if self._f is not None and not self._f.closed:
self._f.close()
try:
remove(self._path)
self._fileExist = False
except:
pass
def open(self) -> bool:
if not self._fileExist:
return False
if self._f is not None and not self._f.closed:
self._f.seek(0, 0)
return True
try:
self._f = open(self._path, 'rb')
return True
except:
return False
class FileEntry:
def __init__(self, url: str, m, config: RSSConfig):
if not exists('Temp'):
@@ -98,8 +132,26 @@ class FileEntry:
self._fileSize = getsize(self._abspath)
self._localURI = f"file://{self._abspath}" if self._abspath[0] == '/' else f"file:///{self._abspath}"
self._f = None
self._subFileDict: Dict[str, SubFileEntry] = {}
def addSubFile(self, name: str, ext: str):
if not isinstance(name, str) or len(name) == 0:
raise ValueError('At least 1 char.')
if not isinstance(ext, str) or len(ext) == 0:
ext = 'temp'
na = f"{name}.{ext}"
if na in self._subFileDict:
return False
p = self.getSubPath(name, ext)
if not exists(p):
raise FileNotFoundError(p)
self._subFileDict[na] = SubFileEntry(p)
return True
def delete(self):
for key in self._subFileDict:
self._subFileDict[key].delete()
self._subFileDict = {}
if not self._fileExist:
return
if self._f is not None and not self._f.closed:
@@ -112,6 +164,22 @@ class FileEntry:
except:
pass
def getSubFile(self, name: str, ext: str):
if not isinstance(name, str) or len(name) == 0:
raise ValueError('At least 1 char.')
if not isinstance(ext, str) or len(ext) == 0:
ext = 'temp'
na = f"{name}.{ext}"
if na in self._subFileDict:
return self._subFileDict[na]
def getSubPath(self, name: str, ext: str):
if not isinstance(name, str) or len(name) == 0:
raise ValueError('At least 1 char.')
if not isinstance(ext, str) or len(ext) == 0:
ext = 'temp'
return splitext(self._abspath)[0] + name + '.' + ext
def open(self) -> bool:
if not self._fileExist:
return False

114
rssbot.py
View File

@@ -35,7 +35,7 @@ from usercheck import checkUserPermissionsInChat, UserPermissionsInChatCheckResu
import sys
from fileEntry import FileEntries, remove
from dictdeal import json2data
from rssbotlib import loadRSSBotLib, AddVideoInfoResult
from rssbotlib import loadRSSBotLib, AddVideoInfoResult, have_rssbotlib
from time import sleep, time
from miraiDatabase import MiraiDatabase
from mirai import Mirai
@@ -73,6 +73,8 @@ def getMediaInfo(m: dict, config: RSSConfig = RSSConfig()) -> str:
s = f"{s}\n发送媒体:{config.send_media}"
s = f"{s}\n单独一行显示链接:{config.display_entry_link}"
s += f"\n发送图片为文件:{config.send_img_as_file}"
if have_rssbotlib:
s += f"\n发送原始像素格式的Pixiv动图:{config.send_ugoira_with_origin_pix_fmt}"
s += f"\nRSS全局设置:"
s += f"\n发送时使用原文件名:{config.send_origin_file_name}"
return s
@@ -95,6 +97,7 @@ class InlineKeyBoardCallBack(Enum):
SendImgAsFile = 12
GlobalSettingsPage = 13
SendOriginFileName = 14
SendUgoiraWithOriginPixFmt = 15
def getInlineKeyBoardWhenRSS(hashd: str, m: dict, isOwn: bool) -> dict:
@@ -158,6 +161,9 @@ def getInlineKeyBoardWhenRSS2(hashd: str, config: RSSConfig) -> str:
i += 1
temp = '禁用发送图片为文件' if config.send_img_as_file else '启用发送图片为文件'
d[i].append({'text': temp, 'callback_data': f'0,{hashd},{InlineKeyBoardCallBack.SendImgAsFile.value}'})
if have_rssbotlib:
temp = f"{'禁用' if config.send_ugoira_with_origin_pix_fmt else '启用'}发送原始像素格式的Pixiv动图"
d[i].append({'text': temp, 'callback_data': f'0,{hashd},{InlineKeyBoardCallBack.SendUgoiraWithOriginPixFmt.value}'})
d.append([])
i += 1
d[i].append(
@@ -230,7 +236,7 @@ class main:
if key not in content or content[key] is None:
return 0
return len(content[key])
if not config.send_media or (getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 0):
if not config.send_media or (getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 0 and getListCount(content, 'ugoiraList') == 0):
if config.disable_web_page_preview:
di['disable_web_page_preview'] = True
while len(text) > 0:
@@ -249,7 +255,7 @@ class main:
else:
return False
sleep(5)
elif getListCount(content, 'imgList') == 1 and getListCount(content, 'videoList') == 0:
elif getListCount(content, 'imgList') == 1 and getListCount(content, 'videoList') == 0 and getListCount(content, 'ugoiraList') == 0:
f = True
while len(text) > 0 or f:
if f:
@@ -305,7 +311,7 @@ class main:
else:
return False
sleep(5)
elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 1:
elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 1 and getListCount(content, 'ugoiraList') == 0:
f = True
while len(text) > 0 or f:
if f:
@@ -402,6 +408,98 @@ class main:
else:
return False
sleep(5)
elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 0 and getListCount(content, 'ugoiraList') == 1:
f = True
while len(text) > 0 or f:
if f:
di['caption'] = text.tostr(1024)
else:
di['text'] = text.tostr()
di['parse_mode'] = 'HTML'
for i in range(self._setting.maxRetryCount + 1):
if f:
if self._setting.downloadMediaFile and not self._setting.sendFileURLScheme:
di2 = {}
if not self._setting.downloadMediaFile:
di['photo'] = content['ugoiraList'][0]['poster']
else:
fileEntry = self._tempFileEntries.add(
content['ugoiraList'][0]['poster'], config)
if not fileEntry:
continue
if self._setting.sendFileURLScheme:
di['thumb'] = fileEntry._localURI
else:
fileEntry.open()
di2['thumb'] = (
fileEntry._fullfn, fileEntry._f)
z = self._tempFileEntries.add(content['ugoiraList'][0]['src'], config)
force_yuv420p = not config.send_ugoira_with_origin_pix_fmt
mp4_ok = z.ok and self._rssbotLib is not None and self._rssbotLib.convert_ugoira_to_mp4(z, content['ugoiraList'][0]['frames'], force_yuv420p)
if mp4_ok:
mp4 = z.getSubFile('_yuv420p' if force_yuv420p else '_origin', 'mp4')
# TODO: Generate a better thumb
if self._setting.sendFileURLScheme:
del di['thumb']
di['animation'] = mp4._localURI
else:
del di2['thumb']
mp4.open()
di2['animation'] = (mp4._path, mp4._f)
self._rssbotLib.addVideoInfo(mp4._path, di)
if self._setting.sendFileURLScheme:
re = self._request('sendAnimation', 'post', json=di)
else:
re = self._request('sendAnimation', 'post', json=di, files=di2)
else:
should_use_file = False if fileEntry._fileSize < MAX_PHOTO_SIZE and not config.send_img_as_file else True
if self._setting.sendFileURLScheme:
if not should_use_file:
di['photo'] = di['thumb']
re = self._request('sendPhoto', 'post', json=di)
else:
di['document'] = di['thumb']
re = self._request('sendDocument', 'post', json=di)
else:
if not should_use_file:
di2['photo'] = di2['thumb']
re = self._request('sendPhoto', 'post', json=di, files=di2)
else:
di2['document'] = di2['thumb']
re = self._request('sendDocument', 'post', json=di, files=di2)
else:
re = self._request('sendMessage', 'post', json=di)
if re is not None and 'ok' in re and re['ok']:
di['reply_to_message_id'] = re['result']['message_id']
if f:
if 'photo' in di:
del di['photo']
if 'document' in di:
del di['document']
if 'animation' in di:
del di['video']
if 'thumb' in di:
del di['thumb']
if 'caption' in di:
del di['caption']
if 'duration' in di:
del di['duration']
if 'width' in di:
del di['width']
if 'height' in di:
del di['height']
if config.disable_web_page_preview:
di['disable_web_page_preview'] = True
f = False
break
if i == self._setting.maxRetryCount:
if returnError and re is not None and 'description' in re:
return False, re['description']
elif returnError:
return False, ''
else:
return False
sleep(5)
else:
ind = 0
if self._setting.downloadMediaFile and not self._setting.sendFileURLScheme:
@@ -1277,7 +1375,7 @@ class callbackQueryHandle(Thread):
self._main._request("editMessageText", "post", json=di)
self.answer()
return
elif self._inlineKeyBoardCommand in [InlineKeyBoardCallBack.DisableWebPagePreview, InlineKeyBoardCallBack.ShowRSSTitle, InlineKeyBoardCallBack.ShowContentTitle, InlineKeyBoardCallBack.ShowContent, InlineKeyBoardCallBack.SendMedia, InlineKeyBoardCallBack.DisplayEntryLink, InlineKeyBoardCallBack.SendImgAsFile]:
elif self._inlineKeyBoardCommand in [InlineKeyBoardCallBack.DisableWebPagePreview, InlineKeyBoardCallBack.ShowRSSTitle, InlineKeyBoardCallBack.ShowContentTitle, InlineKeyBoardCallBack.ShowContent, InlineKeyBoardCallBack.SendMedia, InlineKeyBoardCallBack.DisplayEntryLink, InlineKeyBoardCallBack.SendImgAsFile, InlineKeyBoardCallBack.SendUgoiraWithOriginPixFmt]:
if self._inlineKeyBoardCommand == InlineKeyBoardCallBack.DisableWebPagePreview:
self._rssMeta.config.disable_web_page_preview = not self._rssMeta.config.disable_web_page_preview
elif self._inlineKeyBoardCommand == InlineKeyBoardCallBack.ShowRSSTitle:
@@ -1292,6 +1390,8 @@ class callbackQueryHandle(Thread):
self._rssMeta.config.display_entry_link = not self._rssMeta.config.display_entry_link
elif self._inlineKeyBoardCommand == InlineKeyBoardCallBack.SendImgAsFile:
self._rssMeta.config.send_img_as_file = not self._rssMeta.config.send_img_as_file
elif self._inlineKeyBoardCommand == InlineKeyBoardCallBack.SendUgoiraWithOriginPixFmt:
self._rssMeta.config.send_ugoira_with_origin_pix_fmt = not self._rssMeta.config.send_ugoira_with_origin_pix_fmt
di = {'chat_id': self._rssMeta.chatId,
'message_id': self._rssMeta.messageId}
di['text'] = getMediaInfo(
@@ -1478,7 +1578,7 @@ class callbackQueryHandle(Thread):
self._main._request("editMessageText", "post", json=di)
self.answer()
return
elif self._inlineKeyBoardForRSSListCommand in [InlineKeyBoardForRSSList.DisableWebPagePreview, InlineKeyBoardForRSSList.ShowRSSTitle, InlineKeyBoardForRSSList.ShowContentTitle, InlineKeyBoardForRSSList.ShowContent, InlineKeyBoardForRSSList.SendMedia, InlineKeyBoardForRSSList.DisplayEntryLink, InlineKeyBoardForRSSList.SendImgAsFile]:
elif self._inlineKeyBoardForRSSListCommand in [InlineKeyBoardForRSSList.DisableWebPagePreview, InlineKeyBoardForRSSList.ShowRSSTitle, InlineKeyBoardForRSSList.ShowContentTitle, InlineKeyBoardForRSSList.ShowContent, InlineKeyBoardForRSSList.SendMedia, InlineKeyBoardForRSSList.DisplayEntryLink, InlineKeyBoardForRSSList.SendImgAsFile, InlineKeyBoardForRSSList.SendUgoiraWithOriginPixFmt]:
di = {'chat_id': self._data['message']['chat']['id'],
'message_id': self._data['message']['message_id']}
rssList = self._main._db.getRSSListByChatId(chatId)
@@ -1505,6 +1605,8 @@ class callbackQueryHandle(Thread):
config.display_entry_link = not config.display_entry_link
elif self._inlineKeyBoardForRSSListCommand == InlineKeyBoardForRSSList.SendImgAsFile:
config.send_img_as_file = not config.send_img_as_file
elif self._inlineKeyBoardForRSSListCommand == InlineKeyBoardForRSSList.SendUgoiraWithOriginPixFmt:
config.send_ugoira_with_origin_pix_fmt = not config.send_ugoira_with_origin_pix_fmt
updated = self._main._db.updateChatConfig(chatEntry)
if updated:
self.answer('修改设置成功')

View File

@@ -16,10 +16,12 @@
from enum import unique, Enum
from traceback import print_exc
try:
from _rssbotlib import version, VideoInfo
from _rssbotlib import version, VideoInfo, convert_ugoira_to_mp4, AVDict
have_rssbotlib = True
except ImportError:
have_rssbotlib = False
if have_rssbotlib:
from fileEntry import FileEntry, remove
@unique
@@ -35,7 +37,7 @@ if have_rssbotlib:
from rssbot import main
self._main: main = m
self._version = version()
if self._version is None or self._version != [1, 0, 0, 0]:
if self._version is None or self._version != [1, 0, 0, 1]:
raise ValueError('RSSBotLib Version unknown or not supported.')
def addVideoInfo(self, url: str, data: dict, loc: str = None) -> AddVideoInfoResult:
@@ -72,6 +74,27 @@ if have_rssbotlib:
print_exc()
return AddVideoInfoResult.ERROR
def convert_ugoira_to_mp4(self, f: FileEntry, frames, force_yuv420p: bool):
try:
na = '_yuv420p' if force_yuv420p else '_origin'
if f.getSubFile(na, 'mp4') is not None:
return True
dst = f.getSubPath(na, 'mp4')
opt = AVDict()
if force_yuv420p:
opt['force_yuv420p'] = ''
if not convert_ugoira_to_mp4(f._abspath, dst, frames, opts=opt):
return False
f.addSubFile(na, 'mp4')
return True
except Exception:
print_exc()
try:
remove(dst)
except Exception:
print_exc()
return False
def loadRSSBotLib(m):
if have_rssbotlib:

View File

@@ -19,6 +19,7 @@ from enum import Enum, unique
from math import ceil, floor
from textc import textc, timeToStr
from readset import settings
from rssbotlib import have_rssbotlib
@unique
@@ -45,6 +46,7 @@ class InlineKeyBoardForRSSList(Enum):
SendImgAsFile = 19
GlobalSettingsPage = 20
SendOriginFileName = 21
SendUgoiraWithOriginPixFmt = 22
def getTextContentForRSSInList(rssEntry: RSSEntry, s: settings) -> str:
@@ -76,6 +78,8 @@ def getTextContentForRSSInList(rssEntry: RSSEntry, s: settings) -> str:
text.addtotext(f"发送媒体:{config.send_media}")
text += f"单独一行显示链接:{config.display_entry_link}"
text += f"发送图片为文件:{config.send_img_as_file}"
if have_rssbotlib:
text += f'发送原始像素格式的Pixiv动图:{config.send_ugoira_with_origin_pix_fmt}'
text += f"RSS全局设置:"
text += f"发送时使用原文件名:{config.send_origin_file_name}"
return text.tostr()
@@ -204,6 +208,9 @@ def getInlineKeyBoardForRSSSettingsInList(chatId: int, rssEntry: RSSEntry, index
i += 1
temp = '禁用发送图片为文件' if config.send_img_as_file else '启用发送图片为文件'
d[i].append({'text': temp, 'callback_data': f'1,{chatId},{InlineKeyBoardForRSSList.SendImgAsFile.value},{index},{rssEntry.id}'})
if have_rssbotlib:
temp = f"{'禁用' if config.send_ugoira_with_origin_pix_fmt else '启用'}发送原始像素格式的Pixiv动图"
d[i].append({'text': temp, 'callback_data': f'1,{chatId},{InlineKeyBoardForRSSList.SendUgoiraWithOriginPixFmt.value},{index},{rssEntry.id}'})
d.append([])
i = i + 1
d[i].append(

View File

@@ -13,6 +13,7 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from typing import List
from xml.dom import minidom
defusedxmlSupported = True
try:
@@ -27,6 +28,7 @@ import sys
import requests
from traceback import format_exc
from urllib.parse import urljoin
from json import loads as loadjson
class HTMLContent:
@@ -50,13 +52,14 @@ class HTMLSimpleParser(HTMLParser):
def __init__(self, baseUrl: str = None):
self.data = ''
self.tagName = []
self.tagContent = []
self.tagContent: List[HTMLContent] = []
self.tagAttrs = []
self.imgList = []
self.videoList = []
self.baseUrl = ''
if baseUrl is not None:
self.baseUrl = baseUrl
self.ugoiraList = []
HTMLParser.__init__(self)
def handle_startendtag(self, tag, attrs):
@@ -65,6 +68,9 @@ class HTMLSimpleParser(HTMLParser):
self.data = self.data + '\n'
else:
self.tagContent[-1].add('\n')
else:
self.handle_starttag(tag, attrs)
self.handle_endtag(tag)
def handle_starttag(self, tag, attrs):
if tag == 'br':
@@ -89,6 +95,29 @@ class HTMLSimpleParser(HTMLParser):
if 'src' in p:
self.videoList.append(p)
return
elif tag == 'ugoira':
p = {}
for key, value in attrs:
if key == 'src':
p['src'] = urljoin(self.baseUrl, value)
elif key == 'poster':
p['poster'] = urljoin(self.baseUrl, value)
elif key == 'frames':
try:
frames = loadjson(value)
if not isinstance(frames, list):
raise ValueError(f"Invaild frames: {frames}")
for i in frames:
if not isinstance(i['file'], str):
raise ValueError(f"Invalid file: {i['file']}")
if not isinstance(i['delay'], (int, float)):
raise ValueError(f"Invalid delay: {i['delay']}")
p['frames'] = frames
except Exception:
print(format_exc())
if 'src' in p and 'poster' in p and 'frames' in p:
self.ugoiraList.append(p)
return
self.tagName.append(tag)
self.tagContent.append(HTMLContent())
self.tagAttrs.append('')
@@ -110,7 +139,7 @@ class HTMLSimpleParser(HTMLParser):
elif len(self.tagName) > 1:
self.tagContent[-2].add(
f"<{tag}{self.tagAttrs[-1]}>{self.tagContent[-1].export()}</{tag}>")
elif tag not in ['img', 'video', 'br']:
elif tag not in ['img', 'video', 'br', 'ugoira']:
if len(self.tagName) == 1:
self.data = f"{self.data}{self.tagContent[-1].export()}"
elif len(self.tagName) > 1:
@@ -233,6 +262,7 @@ class RSSParser:
del m['content:encoded']
m['imgList'] = p.imgList
m['videoList'] = p.videoList
m['ugoiraList'] = p.ugoiraList
else:
m[i.nodeName] = ''
for k in i.childNodes:
@@ -274,6 +304,7 @@ class RSSParser:
if i.nodeName in ['content', 'summary']:
m['imgList'] = p.imgList
m['videoList'] = p.videoList
m['ugoiraList'] = p.ugoiraList
m['description'] = m[i.nodeName]
del m[i.nodeName]
elif i.nodeValue is None and len(i.childNodes) == 0:
@@ -304,6 +335,7 @@ class RSSParser:
if i.nodeName in ['content', 'summary']:
m['imgList'] = p.imgList
m['videoList'] = p.videoList
m['ugoiraList'] = p.ugoiraList
m['description'] = m[i.nodeName]
del m[i.nodeName]
elif typ == 'xhtml':
@@ -318,6 +350,7 @@ class RSSParser:
if i.nodeName in ['content', 'summary']:
m['imgList'] = p.imgList
m['videoList'] = p.videoList
m['ugoiraList'] = p.ugoiraLists
m['description'] = m[i.nodeName]
del m[i.nodeName]
elif len(i.childNodes) == 0: