add support to download media file and then send

.gitignore

@@ -3,3 +3,4 @@ __pycache__/
 *.txt
 !requirements.txt
 *.json
+Temp/

dictdeal.py

@@ -0,0 +1,26 @@
+# (C) 2021 lifegpc
+# This file is part of rssbot.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from json import dumps
+
+
+def json2data(json: dict) -> dict:
+    r = {}
+    for key in json.keys():
+        if isinstance(json[key], (dict, list, tuple)):
+            r[key] = dumps(json[key], ensure_ascii=False)
+        else:
+            r[key] = json[key]
+    return r

fileEntry.py

@@ -0,0 +1,136 @@
+# (C) 2021 lifegpc
+# This file is part of rssbot.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from urllib.parse import urlsplit
+from os.path import abspath, splitext, getsize, exists, isdir, isfile
+from time import time_ns
+from random import randint
+from requests import get
+from os import remove as removeFile, mkdir, listdir, removedirs
+from typing import List
+
+
+def remove(s: str):
+    try:
+        if not exists(s):
+            return
+        if isfile(s):
+            removeFile(s)
+        elif isdir(s):
+            p = s if s[-1] in ['/', '\\'] else f"{s}/"
+            for v in listdir(s):
+                remove(f"{p}{v}")
+            removedirs(s)
+    except:
+        remove(s)
+
+
+class FileEntry:
+    def __init__(self, url: str):
+        if not exists('Temp'):
+            mkdir('Temp')
+        if not isdir('Temp'):
+            remove('Temp')
+            mkdir('Temp')
+        self._url = url
+        self._ext = splitext(urlsplit(url).path)[1]
+        if self._ext == '':
+            self._ext = 'temp'
+        self._fn = f"{time_ns()}{randint(0, 9999)}"
+        self._fullfn = f"{self._fn}.{self._ext}"
+        self._abspath = abspath(f'Temp/{self._fn}.{self._ext}')
+        try:
+            self._r = get(url, stream=True)
+            if self._r.ok:
+                with open(self._abspath, 'wb') as f:
+                    for chunk in self._r.iter_content(1024):
+                        if chunk:
+                            f.write(chunk)
+        except:
+            self.ok = False
+        self.ok = self._r.ok
+        self._fileSize = getsize(self._abspath)
+        self._fileExist = True if exists(self._abspath) else False
+        self._localURI = f"file://{self._abspath}" if self._abspath[0] == '/' else f"file:///{self._abspath}"
+        self._f = None
+
+    def delete(self):
+        if not self._fileExist:
+            return
+        if self._f is not None and not self._f.closed:
+            self._f.close()
+        try:
+            remove(self._abspath)
+            self._fileExist = False
+        except:
+            pass
+
+    def open(self) -> bool:
+        if not self._fileExist:
+            return False
+        if self._f is not None and not self._f.closed:
+            self._f.seek(0, 0)
+            return True
+        try:
+            self._f = open(self._abspath, 'rb')
+            return True
+        except:
+            return False
+
+
+class FileEntries:
+    def __init__(self):
+        self.__list = []
+
+    def add(self, url: str) -> FileEntry:
+        if self.has(url):
+            return self.get(url)
+        fileEntry = FileEntry(url)
+        if fileEntry.ok and fileEntry._fileExist:
+            self.__list.append(fileEntry)
+            return fileEntry
+        return None
+
+    def clear(self):
+        for v in self.__list:
+            fileEntry: FileEntry = v
+            fileEntry.delete()
+        i = 0
+        while i < len(self.__list):
+            fileEntry = self.__list[i]
+            if not fileEntry._fileExist:
+                self.__list.remove(fileEntry)
+                i = i - 1
+            i = i + 1
+
+    def get(self, url: str) -> FileEntry:
+        for v in self.__list:
+            fileEntry: FileEntry = v
+            if fileEntry._url == url:
+                return fileEntry
+        return None
+
+    def getList(self) -> List[FileEntry]:
+        r = []
+        for v in self.__list:
+            r.append(v)
+        return r
+
+    def has(self, url: str):
+        for v in self.__list:
+            fileEntry: FileEntry = v
+            if fileEntry._url == url:
+                return True
+        return False

readset.py

@@ -40,6 +40,8 @@ class settings:
         self._maxRetryCount = int(d['maxRetryCount']) if 'maxRetryCount' in d and d['maxRetryCount'].isnumeric(
         ) and int(d['maxRetryCount']) >= 0 else 3
         self._telegramBotApiServer = d['telegramBotApiServer'] if 'telegramBotApiServer' in d else 'https://api.telegram.org'
+        self._downloadMediaFile = bool(int(d['downloadMediaFile'])) if 'downloadMediaFile' in d and d['downloadMediaFile'].isnumeric() else False
+        self._sendFileURLScheme = bool(int(d['sendFileURLScheme'])) if 'sendFileURLScheme' in d and d['sendFileURLScheme'].isnumeric() else False
 
 
 class commandline:

rssbot.py

@@ -24,7 +24,7 @@ from typing import List
 from rssparser import RSSParser
 from html import escape
 from hashl import md5WithBase64
-from enum import Enum
+from enum import Enum, unique
 from rsstempdict import rssMetaInfo, rssMetaList
 from random import randrange
 from textc import textc, removeEmptyLine, decodeURI
@@ -33,6 +33,8 @@ from rsschecker import RSSCheckerThread
 from rsslist import getInlineKeyBoardForRSSList, InlineKeyBoardForRSSList, getInlineKeyBoardForRSSInList, getTextContentForRSSInList, getInlineKeyBoardForRSSUnsubscribeInList, getTextContentForRSSUnsubscribeInList, getInlineKeyBoardForRSSSettingsInList
 from usercheck import checkUserPermissionsInChat, UserPermissionsInChatCheckResult
 import sys
+from fileEntry import FileEntries, remove
+from dictdeal import json2data
 
 
 def getMediaInfo(m: dict, config: RSSConfig = RSSConfig()) -> str:
@@ -62,6 +64,7 @@ def getMediaInfo(m: dict, config: RSSConfig = RSSConfig()) -> str:
     return s
 
 
+@unique
 class InlineKeyBoardCallBack(Enum):
     Subscribe = 0
     SendPriview = 1
@@ -138,6 +141,9 @@ class main:
 
     def _request(self, methodName: str, HTTPMethod: str = 'get', data: dict = None, json: dict = None, files: dict = None, returnType: str = 'json', telegramBotApiServer: str = None):
         try:
+            if json is not None and files is not None:
+                data = json2data(json)
+                json = None
             r = self._r.request(
                 HTTPMethod, f'{self._telegramBotApiServer if telegramBotApiServer is None else telegramBotApiServer}/bot{self._setting._token}/{methodName}', data=data, json=json, files=files)
             if r.status_code != 200:
@@ -183,24 +189,85 @@ class main:
         elif getListCount(content, 'imgList') == 1 and getListCount(content, 'videoList') == 0:
             di['caption'] = text.tostr()
             di['parse_mode'] = 'HTML'
-            di['photo'] = content['imgList'][0]
+            if not self._setting._downloadMediaFile:
-            re = self._request('sendPhoto', 'post', json=di)
+                di['photo'] = content['imgList'][0]
+                re = self._request('sendPhoto', 'post', json=di)
+            else:
+                fileEntry = self._tempFileEntries.add(content['imgList'][0])
+                if not fileEntry.ok:
+                    return None
+                if self._setting._sendFileURLScheme:
+                    di['photo'] = fileEntry._localURI
+                    re = self._request('sendPhoto', 'post', json=di)
+                else:
+                    fileEntry.open()
+                    re = self._request('sendPhoto', 'post', json=di, files={
+                                       'photo': (fileEntry._fullfn, fileEntry._f)})
         elif getListCount(content, 'imgList') == 0 and getListCount(content, 'videoList') == 1:
             di['caption'] = text.tostr()
             di['parse_mode'] = 'HTML'
-            di['video'] = content['videoList'][0]['src']
+            if self._setting._downloadMediaFile and not self._setting._sendFileURLScheme:
+                di2 = {}
+            if not self._setting._downloadMediaFile:
+                di['video'] = content['videoList'][0]['src']
+            else:
+                fileEntry = self._tempFileEntries.add(
+                    content['videoList'][0]['src'])
+                if not fileEntry.ok:
+                    return None
+                if self._setting._sendFileURLScheme:
+                    di['video'] = fileEntry._localURI
+                else:
+                    fileEntry.open()
+                    di2['video'] = (fileEntry._fullfn, fileEntry._f)
             if 'poster' in content['videoList'][0] and content['videoList'][0]['poster'] is not None and content['videoList'][0]['poster'] != '':
-                di['thumb'] = content['videoList'][0]['poster']
+                if not self._setting._downloadMediaFile:
+                    di['thumb'] = content['videoList'][0]['poster']
+                else:
+                    fileEntry = self._tempFileEntries.add(
+                        content['videoList'][0]['poster'])
+                    if not fileEntry.ok:
+                        return None
+                    if self._setting._sendFileURLScheme:
+                        di['thumb'] = fileEntry._localURI
+                    else:
+                        fileEntry.open()
+                        di2['thumb'] = (fileEntry._fullfn, fileEntry._f)
             di['supports_streaming'] = True
-            re = self._request('sendVideo', 'post', json=di)
+            if not self._setting._downloadMediaFile or self._setting._sendFileURLScheme:
+                re = self._request('sendVideo', 'post', json=di)
+            else:
+                re = self._request('sendVideo', 'post', json=di, files=di2)
         else:
             ind = 0
+            if self._setting._downloadMediaFile and not self._setting._sendFileURLScheme:
+                ind2 = 0
+                di3 = {}
             di['media'] = []
             for i in content['imgList']:
                 if ind % 9 == 0 and ind != 0:
-                    re = self._request('sendMediaGroup', 'post', json=di)
+                    if not self._setting._downloadMediaFile or self._setting._sendFileURLScheme:
-                    di['media'] = []
+                        re = self._request('sendMediaGroup', 'post', json=di)
-                di2 = {'type': 'photo', 'media': i}
+                        di['media'] = []
+                    else:
+                        re = self._request(
+                            'sendMediaGroup', 'post', json=di, files=di3)
+                        di['media'] = []
+                        di3 = {}
+                di2 = {'type': 'photo'}
+                if not self._setting._downloadMediaFile:
+                    di2['media'] = i
+                else:
+                    fileEntry = self._tempFileEntries.add(i)
+                    if not fileEntry.ok:
+                        return None
+                    if self._setting._sendFileURLScheme:
+                        di2['media'] = fileEntry._localURI
+                    else:
+                        fileEntry.open()
+                        di2['media'] = f'attach://file{ind2}'
+                        di3[f'file{ind2}'] = (fileEntry._fullfn, fileEntry._f)
+                        ind2 = ind2 + 1
                 if ind == 0:
                     di2['caption'] = text.tostr()
                     di2['parse_mode'] = 'HTML'
@@ -208,18 +275,53 @@ class main:
                 ind = ind + 1
             for i in content['videoList']:
                 if ind % 9 == 0 and ind != 0:
-                    re = self._request('sendMediaGroup', 'post', json=di)
+                    if not self._setting._downloadMediaFile or self._setting._sendFileURLScheme:
-                    di['media'] = []
+                        re = self._request('sendMediaGroup', 'post', json=di)
-                di2 = {'type': 'video',
+                        di['media'] = []
-                       'media': i['src'], 'supports_streaming': True}
+                    else:
+                        re = self._request(
+                            'sendMediaGroup', 'post', json=di, files=di3)
+                        di['media'] = []
+                        di3 = {}
+                di2 = {'type': 'video', 'supports_streaming': True}
+                if not self._setting._downloadMediaFile:
+                    di2['media'] = i['src']
+                else:
+                    fileEntry = self._tempFileEntries.add(i['src'])
+                    if not fileEntry.ok:
+                        return None
+                    if self._setting._sendFileURLScheme:
+                        di2['media'] = fileEntry._localURI
+                    else:
+                        fileEntry.open()
+                        di2['media'] = f'attach://file{ind2}'
+                        di3[f'file{ind2}'] = (fileEntry._fullfn, fileEntry._f)
+                        ind2 = ind2 + 1
                 if 'poster' in i and i['poster'] is not None and i['poster'] != '':
-                    di2['thumb'] = i['poster']
+                    if not self._setting._downloadMediaFile:
+                        di2['thumb'] = i['poster']
+                    else:
+                        fileEntry = self._tempFileEntries.add(i['poster'])
+                        if not fileEntry.ok:
+                            return None
+                        if self._setting._sendFileURLScheme:
+                            di2['thumb'] = fileEntry._localURI
+                        else:
+                            fileEntry.open()
+                            di2['thumb'] = f'attach://file{ind2}'
+                            di3[f'file{ind2}'] = (
+                                fileEntry._fullfn, fileEntry._f)
+                            ind2 = ind2 + 1
                 if ind == 0:
                     di2['caption'] = text.tostr()
                     di2['parse_mode'] = 'HTML'
                 di['media'].append(di2)
                 ind = ind + 1
-            re = self._request('sendMediaGroup', 'post', json=di)
+            if not self._setting._downloadMediaFile or self._setting._sendFileURLScheme:
+                re = self._request('sendMediaGroup', 'post', json=di)
+            else:
+                re = self._request('sendMediaGroup', 'post',
+                                   json=di, files=di3)
         if re is not None and 'ok' in re and re['ok']:
             return True
         return False
@@ -259,6 +361,8 @@ class main:
         if self._telegramBotApiServer != 'https://api.telegram.org':
             self._request("logOut", "post",
                           telegramBotApiServer="https://api.telegram.org")
+        remove('Temp')
+        self._tempFileEntries = FileEntries()
         self._me = self._request('getMe')
         self._rssMetaList = rssMetaList()
         print(self._me)

rsschecker.py

@@ -64,6 +64,7 @@ class RSSCheckerThread(Thread):
         if self._main._commandLine._rebuildHashlist and self._main._commandLine._exitAfterRebuild:
             _exit(0)
         self._main._commandLine._rebuildHashlist = False
+        self._main._tempFileEntries.clear()
 
     def __init__(self, m):
         Thread.__init__(self)