Project Rebase

- Changelog:
 Improved Performance
 Fixed Stucked Messages
 Seperated Download Progress Status From Listener
 Disabled Upload Progress (Temporarily)

Author: JaskaranSM

Dockerfile

@@ -1,10 +1,10 @@
-FROM python:3.8-slim-buster
+FROM ubuntu:18.04
 
 WORKDIR /usr/src/app
 RUN chmod 777 /usr/src/app
 COPY . .
-RUN pip install --no-cache-dir -r requirements.txt
 RUN apt-get update
-RUN apt-get install -y aria2
+RUN apt-get install -y aria2 megatools ffmpeg python3 python3-pip
+RUN pip3 install --no-cache-dir -r requirements.txt
 
-CMD ["bash","start.sh"]
+CMD ["bash","start.sh"]

aria.sh

@@ -1,5 +1,5 @@
 export MAX_DOWNLOAD_SPEED=0
-export MAX_CONCURRENT_DOWNLOADS=2
+export MAX_CONCURRENT_DOWNLOADS=3
 aria2c --enable-rpc --rpc-listen-all=false --rpc-listen-port 6800 \
   --max-connection-per-server=10 --rpc-max-request-size=1024M \
   --seed-time=0.01 --min-split-size=10M --follow-torrent=mem --split=10 \

bot/__init__.py

@@ -1,9 +1,10 @@
 import logging
 import aria2p
 import threading
-from telegram.ext import Updater
+import telegram.ext as tg
+from telegram import Bot
 import os
-from dotenv import load_dotenv
+import configparser
 import time
 
 botStartTime = time.time()
@@ -15,11 +16,13 @@
                     handlers=[logging.FileHandler('log.txt'), logging.StreamHandler()],
                     level=logging.INFO)
 
-load_dotenv('config.env')
+config = configparser.ConfigParser()
+config.read('bot/config.ini')
 
+Interval = []
 
 def getConfig(name: str):
-    return os.environ[name]
+    return config['DEFAULT'][name]
 
 
 LOGGER = logging.getLogger(__name__)
@@ -39,6 +42,7 @@ def getConfig(name: str):
     )
 )
 
+
 DOWNLOAD_DIR = None
 BOT_TOKEN = None
 
@@ -71,6 +75,8 @@ def getConfig(name: str):
     LOGGER.error("One or more env variables missing! Exiting now")
     exit(1)
 
-bot = Bot(BOT_TOKEN)
-updater = Updater(token=BOT_TOKEN, use_context=True)
+bot  = Bot(BOT_TOKEN)
+updater = tg.Updater(token=BOT_TOKEN)
 dispatcher = updater.dispatcher
+
+

bot/__main__.py

@@ -8,10 +8,10 @@
 import shutil
 from .helper.telegram_helper.filters import CustomFilters
 from bot.helper.telegram_helper.bot_commands import BotCommands
-from .modules import authorize, list, cancel_mirror, mirror_status, mirror
+from .modules import authorize, list, cancel_mirror, mirror_status, mirror,eval, shell,lydia
 
 @run_async
-def stats(update, context):
+def stats(bot,update):
     currentTime = get_readable_time((time.time() - botStartTime))
     total, used, free = shutil.disk_usage('.')
     total = get_readable_file_size(total)
@@ -21,31 +21,31 @@ def stats(update, context):
             f'Total disk space: {total}\n' \
                         f'Used: {used}\n' \
                         f'Free: {free}'
-    sendMessage(stats, context, update)
+    sendMessage(stats, bot, update)
 
 
 
 @run_async
-def start(update, context):
+def start(bot,update):
     sendMessage("This is a bot which can mirror all your links to Google drive!\n"
-                "Type /help to get a list of available commands", context, update)
+                "Type /help to get a list of available commands", bot, update)
 
 
 @run_async
-def ping(update, context):
+def ping(bot,update):
     start_time = int(round(time.time() * 1000))
-    reply = sendMessage("Starting Ping", context, update)
+    reply = sendMessage("Starting Ping", bot, update)
     end_time = int(round(time.time()*1000))
-    editMessage(f'{end_time - start_time} ms', context, reply)
+    editMessage(f'{end_time - start_time} ms',reply)
 
 
 @run_async
-def log(update, context):
-    sendLogFile(context, update)
+def log(bot,update):
+    sendLogFile(bot, update)
 
 
 @run_async
-def bot_help(update, context):
+def bot_help(bot,update):
     help_string = f'''
 /{BotCommands.HelpCommand}: To get this message
 
@@ -66,7 +66,7 @@ def bot_help(update, context):
 /{BotCommands.LogCommand}: Get a log file of the bot. Handy for getting crash reports
 
 '''
-    sendMessage(help_string, context, update)
+    sendMessage(help_string, bot, update)
 
 
 def main():

bot/helper/ext_utils/bot_utils.py

@@ -1,6 +1,9 @@
 from bot import download_dict, download_dict_lock
 import logging
 import re
+import threading
+import time
+import math
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,14 +25,30 @@ class MirrorStatus:
 
 SIZE_UNITS = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
 
+class setInterval :
+    def __init__(self,interval,action) :
+        self.interval=interval
+        self.action=action
+        self.stopEvent=threading.Event()
+        thread=threading.Thread(target=self.__setInterval)
+        thread.start()
+
+    def __setInterval(self) :
+        nextTime=time.time()+self.interval
+        while not self.stopEvent.wait(nextTime-time.time()) :
+            nextTime+=self.interval
+            self.action()
+
+    def cancel(self) :
+        self.stopEvent.set()
 
 def get_readable_file_size(size_in_bytes) -> str:
     index = 0
     while size_in_bytes >= 1024:
         size_in_bytes /= 1024
         index += 1
     try:
-        return f'{round(size_in_bytes, 2)} {SIZE_UNITS[index]}'
+        return f'{round(size_in_bytes, 2)}{SIZE_UNITS[index]}'
     except IndexError:
         return 'File too large'
 
@@ -80,20 +99,21 @@ def get_download_str():
             result += (status.progress() + status.speed() + status.status())
         return result
 
-
 def get_readable_message():
     with download_dict_lock:
-        progress_list = list(download_dict.values())
-    msg = ''
-    for download in progress_list:
-        msg += f'<b>Name:</b> {download.name()}\n' \
-               f'<b>Status:</b> {download.status()}\n' \
-               f'<code>{get_progress_bar_string(download)}</code> {download.progress()} of {download.size()}\n' \
-               f'<b>Speed:</b> {download.speed()}\n' \
-               f'<b>ETA:</b> {download.eta()}\n\n'
-#    LOGGER.info(msg)
-    return msg
-
+        msg = ""
+        for download in list(download_dict.values()):
+            msg += f"<i>{download.name()}</i> - "
+            if download.status() == MirrorStatus.STATUS_UPLOADING:
+                msg += "Uploading\n\n"
+            elif download.status() == MirrorStatus.STATUS_ARCHIVING:
+                msg += "Archiving\n\n"
+            elif download.status() == MirrorStatus.STATUS_WAITING:
+                msg += "Queued\n\n"
+            else:
+                msg += f"<code>{get_progress_bar_string(download)} {download.progress()}</code> of {get_readable_file_size(download.download().total_length)}" \
+                       f" at {get_readable_file_size(download.download().download_speed)}ps, ETA: {download.eta()}\n\n"
+        return msg
 
 def get_readable_time(seconds: int) -> str:
     result = ''

bot/helper/mirror_utils/aria2_download.py

@@ -1,90 +1,71 @@
-from bot import DOWNLOAD_STATUS_UPDATE_INTERVAL, aria2
+from bot import DOWNLOAD_STATUS_UPDATE_INTERVAL, aria2, Interval
 from bot.helper.ext_utils.bot_utils import *
 from .download_helper import DownloadHelper
 from .download_status import DownloadStatus
+from bot.helper.telegram_helper.message_utils import *
+from bot.helper.mirror_utils.gdriveTools import GoogleDriveHelper
 import threading
 from aria2p import API, ClientException
 import schedule
 import time
 
-
 class AriaDownloadHelper(DownloadHelper):
 
-    def __init__(self, listener):
-        super(AriaDownloadHelper, self).__init__(listener)
-        self.__is_torrent = False
+    def __init__(self,listener):
         self.gid = None
-        self.__scheduler = schedule.every(DOWNLOAD_STATUS_UPDATE_INTERVAL).seconds.do(self.__onDownloadProgress())
-
-    def __updater(self):
-        while True:
-            with self._resource_lock:
-                if self.__scheduler is None:
-                    break
-            schedule.run_pending()
-            time.sleep(1)
+        self._listener = listener
+        self._resource_lock = threading.Lock()
 
     def __onDownloadStarted(self, api, gid):
         with self._resource_lock:
             if self.gid == gid:
                 download = api.get_download(gid)
                 self.name = download.name
-                self.size = download.length
-                self._listener.onDownloadStarted()
-                self.should_update = True
-
-    def __onDownloadProgress(self):
-        with self._resource_lock:
-            download = aria2.get_download(self.gid)
-            self.progress = download.progress
-            self.progress_string = download.progress_string
-            self.eta_string = download.eta_string
-            self.eta = download.eta
+                update_all_messages()
 
     def __onDownloadComplete(self, api: API, gid):
         with self._resource_lock:
             if self.gid == gid:
-                if self.__is_torrent:
-                    self.__is_torrent = False
+                if api.get_download(gid).followed_by_ids:
                     self.gid = api.get_download(gid).followed_by_ids[0]
+                    download_dict[self._listener.uid] = DownloadStatus(self.gid,self._listener)                 
+                    update_all_messages()
                     LOGGER.info(f'Changed gid from {gid} to {self.gid}')
                 else:
-                    self._listener.onDownloadComplete()
-                    self.__scheduler = None
+                    self._listener.onDownloadComplete() 
 
     def __onDownloadPause(self, api, gid):
         if self.gid == gid:
+            LOGGER.info("Called onDownloadPause")
             self._listener.onDownloadError('Download stopped by user!')
 
     def __onDownloadStopped(self, api, gid):
         if self.gid == gid:
-            self._listener.onDownloadError()
+            LOGGER.info("Called on_download_stop")
 
     def __onDownloadError(self, api, gid):
         with self._resource_lock:
             if self.gid == gid:
                 download = api.get_download(gid)
                 error = download.error_message
+                LOGGER.info(f"Download Error: {error}")
                 self._listener.onDownloadError(error)
 
     def add_download(self, link: str, path):
         if is_magnet(link):
             download = aria2.add_magnet(link, {'dir': path})
-            self.__is_torrent = True
         else:
             download = aria2.add_uris([link], {'dir': path})
-            if download.name.endswith('.torrent'):
-                self.__is_torrent = True
         self.gid = download.gid
+        with download_dict_lock:
+            download_dict[self._listener.uid] = DownloadStatus(self.gid,self._listener)
+        if download.error_message:
+            self._listener.onDownloadError(download.error_message)
+            return 
+        LOGGER.info(f"Started: {self.gid} DIR:{download.dir} ")
         aria2.listen_to_notifications(threaded=True, on_download_start=self.__onDownloadStarted,
-                                      on_download_error=self.__onDownloadError,
-                                      on_download_complete=self.__onDownloadComplete)
-        threading.Thread(target=self.__updater).start()
+                                    on_download_error=self.__onDownloadError,
+                                    on_download_pause=self.__onDownloadPause,
+                                    on_download_stop=self.__onDownloadStopped,
+                                    on_download_complete=self.__onDownloadComplete)
 
-    def cancel_download(self):
-        # Returns None if successfully cancelled, else error string
-        download = aria2.get_download(self.gid)
-        try:
-            download.pause(force=True)
-        except ClientException:
-            return 'Unable to cancel download! Internal error.'

bot/helper/mirror_utils/download_helper.py

@@ -3,21 +3,20 @@
 
 
 class MethodNotImplementedError(NotImplementedError):
-    def __int__(self):
+    def __init__(self):
         super(self, 'Not implemented method')
 
 
 class DownloadHelper:
-    def __int__(self, listener):
+    def __init__(self):
         self.name = ''  # Name of the download; empty string if no download has been started
         self.size = 0.0  # Size of the download
         self.downloaded_bytes = 0.0  # Bytes downloaded
         self.speed = 0.0  # Download speed in bytes per second
         self.progress = 0.0
         self.progress_string = '0.00%'
         self.eta = 0  # Estimated time of download complete
-        self.eta_string = '0s'
-        self._listener = listener  # A listener class which have event callbacks
+        self.eta_string = '0s' # A listener class which have event callbacks
         self._resource_lock = threading.Lock()
 
     def add_download(self, link: str, path):

bot/helper/mirror_utils/download_status.py

@@ -8,13 +8,16 @@ def get_download(gid):
 
 class DownloadStatus:
 
-    def __init__(self, gid, message_id):
+    def __init__(self, gid, listener):
         self.upload_name = None
         self.is_archiving = False
         self.__gid = gid
         self.__download = get_download(gid)
-        self.__uid = message_id
+        self.__uid = listener.uid
+        self._listener = listener
         self.upload_helper = None
+        self.last = None
+        self.is_waiting = False
 
     def __update(self):
         self.__download = get_download(self.__gid)
@@ -54,6 +57,7 @@ def speed(self):
     def name(self):
         if self.upload_name is not None:
             return self.upload_name
+        self.__update()
         return self.__download.name
 
     def path(self):

bot/helper/mirror_utils/gdriveTools.py

@@ -140,9 +140,9 @@ def upload(self, file_name: str):
         self.__listener.onUploadStarted(_list, index)
         file_dir = f"{DOWNLOAD_DIR}{self.__listener.message.message_id}"
         file_path = f"{file_dir}/{file_name}"
-        LOGGER.info("Uploading File: " + file_name)
+        LOGGER.info("Uploading File: " + file_path)
         self.start_time = time.time()
-        threading.Thread(target=self._on_upload_progress).start()
+        #threading.Thread(target=self._on_upload_progress).start()
         if os.path.isfile(file_path):
             try:
                 mime_type = get_mime_type(file_path)

bot/helper/mirror_utils/listeners.py

@@ -1,6 +1,6 @@
 class MirrorListeners:
     def __init__(self, context, update):
-        self.context = context
+        self.bot = context
         self.update = update
         self.message = update.message
         self.uid = self.message.message_id