diff --git a/bot/__init__.py b/bot/__init__.py index caf50c0..8b26f2d 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -4,6 +4,7 @@ import threading import time import random import string +import subprocess import aria2p import qbittorrentapi as qba @@ -72,7 +73,7 @@ def get_client() -> qba.TorrentsAPIMixIn: qb_client = qba.Client(host="localhost", port=8090, username="admin", password="adminadmin") try: qb_client.auth_log_in() - qb_client.application.set_preferences({"disk_cache":64, "incomplete_files_ext":True, "max_connec":10000, "max_connec_per_torrent":1000, "async_io_threads":32, "preallocate_all":True, "upnp":True, "dl_limit":-1, "up_limit":-1, "dht":True, "pex":True, "lsd":True, "encryption":0, "queueing_enabled":True, "max_active_downloads":15, "max_active_torrents":50, "dont_count_slow_torrents":True, "bittorrent_protocol":0, "recheck_completed_torrents":True, "enable_multi_connections_from_same_ip":True, "slow_torrent_dl_rate_threshold":100,"slow_torrent_inactive_timer":600}) + qb_client.application.set_preferences({"disk_cache":64, "incomplete_files_ext":True, "max_connec":10000, "max_connec_per_torrent":1000, "async_io_threads":32, "preallocate_all":True, "upnp":True, "dl_limit":-1, "up_limit":-1, "dht":True, "pex":True, "lsd":True, "encryption":0, "queueing_enabled":True, "max_active_downloads":15, "max_active_torrents":50, "dont_count_slow_torrents":True, "bittorrent_protocol":0, "recheck_completed_torrents":True, "auto_delete_mode":True, "enable_multi_connections_from_same_ip":True, "slow_torrent_dl_rate_threshold":100,"slow_torrent_inactive_timer":600}) return qb_client except qba.LoginFailed as e: LOGGER.error(str(e)) @@ -351,6 +352,31 @@ except KeyError: logging.warning('SERVER_PORT not provided!') SERVER_PORT = None +try: + TOKEN_PICKLE_URL = getConfig('TOKEN_PICKLE_URL') + if len(TOKEN_PICKLE_URL) == 0: + TOKEN_PICKLE_URL = None + else: + out = subprocess.run(["wget", "-q", "-O", "token.pickle", TOKEN_PICKLE_URL]) + if out.returncode != 0: + logging.error(out) +except KeyError: + TOKEN_PICKLE_URL = None + +try: + ACCOUNTS_ZIP_URL = getConfig('ACCOUNTS_ZIP_URL') + if len(ACCOUNTS_ZIP_URL) == 0: + ACCOUNTS_ZIP_URL = None + else: + out = subprocess.run(["wget", "-q", "-O", "accounts.zip", ACCOUNTS_ZIP_URL]) + if out.returncode != 0: + logging.error(out) + raise KeyError + subprocess.run(["unzip", "-q", "-o", "accounts.zip"]) + os.remove("accounts.zip") +except KeyError: + ACCOUNTS_ZIP_URL = None + updater = tg.Updater(token=BOT_TOKEN) bot = updater.bot dispatcher = updater.dispatcher diff --git a/bot/helper/ext_utils/bot_utils.py b/bot/helper/ext_utils/bot_utils.py index 8f7206a..5440dbd 100644 --- a/bot/helper/ext_utils/bot_utils.py +++ b/bot/helper/ext_utils/bot_utils.py @@ -181,6 +181,20 @@ def flip(update, context): message_utils.update_all_messages() +def check_limit(size, limit, tar_unzip_limit=None, is_tar_ext=False): + LOGGER.info(f"Checking File/Folder Size...") + if is_tar_ext and tar_unzip_limit is not None: + limit = tar_unzip_limit + if limit is not None: + limit = limit.split(' ', maxsplit=1) + limitint = int(limit[0]) + if 'G' in limit[1] or 'g' in limit[1]: + if size > limitint * 1024**3: + return True + elif 'T' in limit[1] or 't' in limit[1]: + if size > limitint * 1024**4: + return True + def get_readable_time(seconds: int) -> str: result = '' (days, remainder) = divmod(seconds, 86400) diff --git a/bot/helper/mirror_utils/download_utils/aria2_download.py b/bot/helper/mirror_utils/download_utils/aria2_download.py index f9eeab1..6f0df38 100644 --- a/bot/helper/mirror_utils/download_utils/aria2_download.py +++ b/bot/helper/mirror_utils/download_utils/aria2_download.py @@ -36,29 +36,18 @@ class AriaDownloadHelper(DownloadHelper): sendMarkup("Here are the search results:", dl.getListener().bot, dl.getListener().update, button) return if (TORRENT_DIRECT_LIMIT is not None or TAR_UNZIP_LIMIT is not None) and dl is not None: - limit = None - if TAR_UNZIP_LIMIT is not None and (dl.getListener().isTar or dl.getListener().extract): - LOGGER.info(f"Checking File/Folder Size...") - limit = TAR_UNZIP_LIMIT + size = aria2.get_download(gid).total_length + if dl.getListener().isTar or dl.getListener().extract: + is_tar_ext = True mssg = f'Tar/Unzip limit is {TAR_UNZIP_LIMIT}' - elif TORRENT_DIRECT_LIMIT is not None and limit is None: - LOGGER.info(f"Checking File/Folder Size...") - limit = TORRENT_DIRECT_LIMIT + else: + is_tar_ext = False mssg = f'Torrent/Direct limit is {TORRENT_DIRECT_LIMIT}' - if limit is not None: - size = aria2.get_download(gid).total_length - limit = limit.split(' ', maxsplit=1) - limitint = int(limit[0]) - if 'G' in limit[1] or 'g' in limit[1]: - if size > limitint * 1024**3: - dl.getListener().onDownloadError(f'{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}') - aria2.remove([download], force=True) - return - elif 'T' in limit[1] or 't' in limit[1]: - if size > limitint * 1024**4: - dl.getListener().onDownloadError(f'{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}') - aria2.remove([download], force=True) - return + result = check_limit(size, TORRENT_DIRECT_LIMIT, TAR_UNZIP_LIMIT, is_tar_ext) + if result: + dl.getListener().onDownloadError(f'{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}') + aria2.remove([download], force=True) + return update_all_messages() def __onDownloadComplete(self, api: API, gid): diff --git a/bot/helper/mirror_utils/download_utils/mega_downloader.py b/bot/helper/mirror_utils/download_utils/mega_downloader.py index 8f4778a..0d0b474 100644 --- a/bot/helper/mirror_utils/download_utils/mega_downloader.py +++ b/bot/helper/mirror_utils/download_utils/mega_downloader.py @@ -3,7 +3,7 @@ import threading from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError) from bot.helper.telegram_helper.message_utils import * import os -from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type, get_readable_file_size +from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type, get_readable_file_size, check_limit from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper from bot import MEGA_LIMIT, STOP_DUPLICATE, TAR_UNZIP_LIMIT @@ -180,27 +180,18 @@ class MegaDownloadHelper: executor.continue_event.set() return if MEGA_LIMIT is not None or TAR_UNZIP_LIMIT is not None: - limit = None - LOGGER.info(f'Checking File/Folder Size') - if TAR_UNZIP_LIMIT is not None and (listener.isTar or listener.extract): - limit = TAR_UNZIP_LIMIT + size = api.getSize(node) + if listener.isTar or listener.extract: + is_tar_ext = True msg3 = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(api.getSize(node))}.' - elif MEGA_LIMIT is not None and limit is None: - limit = MEGA_LIMIT + else: + is_tar_ext = False msg3 = f'Failed, Mega limit is {MEGA_LIMIT}.\nYour File/Folder size is {get_readable_file_size(api.getSize(node))}.' - if limit is not None: - limit = limit.split(' ', maxsplit=1) - limitint = int(limit[0]) - if 'G' in limit[1] or 'g' in limit[1]: - if api.getSize(node) > limitint * 1024**3: - sendMessage(msg3, listener.bot, listener.update) - executor.continue_event.set() - return - elif 'T' in limit[1] or 't' in limit[1]: - if api.getSize(node) > limitint * 1024**4: - sendMessage(msg3, listener.bot, listener.update) - executor.continue_event.set() - return + result = check_limit(size, MEGA_LIMIT, TAR_UNZIP_LIMIT, is_tar_ext) + if result: + sendMessage(msg3, listener.bot, listener.update) + executor.continue_event.set() + return with download_dict_lock: download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener) os.makedirs(path) diff --git a/bot/helper/mirror_utils/download_utils/qbit_downloader.py b/bot/helper/mirror_utils/download_utils/qbit_downloader.py index 98d2bb2..17ef702 100644 --- a/bot/helper/mirror_utils/download_utils/qbit_downloader.py +++ b/bot/helper/mirror_utils/download_utils/qbit_downloader.py @@ -9,15 +9,16 @@ import time import logging import qbittorrentapi as qba +from fnmatch import fnmatch from urllib.parse import urlparse, parse_qs from torrentool.api import Torrent from telegram import InlineKeyboardMarkup from telegram.ext import CallbackQueryHandler -from bot import download_dict, download_dict_lock, BASE_URL, dispatcher, get_client +from bot import download_dict, download_dict_lock, BASE_URL, dispatcher, get_client, TORRENT_DIRECT_LIMIT, TAR_UNZIP_LIMIT from bot.helper.mirror_utils.status_utils.qbit_download_status import QbDownloadStatus from bot.helper.telegram_helper.message_utils import * -from bot.helper.ext_utils.bot_utils import setInterval, new_thread, MirrorStatus, getDownloadByGid +from bot.helper.ext_utils.bot_utils import setInterval, new_thread, MirrorStatus, getDownloadByGid, get_readable_file_size, check_limit from bot.helper.telegram_helper import button_build LOGGER = logging.getLogger(__name__) @@ -29,15 +30,18 @@ class qbittorrent: def __init__(self): self.update_interval = 2 self.meta_time = time.time() + self.stalled_time = time.time() + self.checked = False @new_thread def add_torrent(self, link, dire, listener, qbitsel): self.client = get_client() self.listener = listener + self.dire = dire + self.qbitsel = qbitsel is_file = False count = 0 pincode = "" - markup = None try: if os.path.exists(link): is_file = True @@ -54,7 +58,6 @@ class qbittorrent: else: op = self.client.torrents_add(link, save_path=dire) if op.lower() == "ok.": - LOGGER.info(f"QbitDownload started: {self.ext_hash}") tor_info = self.client.torrents_info(torrent_hashes=self.ext_hash) if len(tor_info) == 0: while True: @@ -72,6 +75,7 @@ class qbittorrent: download_dict[listener.uid] = QbDownloadStatus(gid, listener, self.ext_hash, self.client) self.updater = setInterval(self.update_interval, self.update) tor_info = tor_info[0] + LOGGER.info(f"QbitDownload started: {tor_info.name}") if BASE_URL is not None and qbitsel: if not is_file and (tor_info.state == "checkingResumeData" or tor_info.state == "metaDL"): meta = sendMessage("Downloading Metadata...Please wait then you can select files or mirror torrent file if it have low seeders", listener.bot, listener.update) @@ -84,8 +88,9 @@ class qbittorrent: if tor_info.state == "metaDL" or tor_info.state == "checkingResumeData": time.sleep(1) else: - break + break deleteMessage(listener.bot, meta) + self.client.torrents_pause(torrent_hashes=self.ext_hash) for n in str(self.ext_hash): if n.isdigit(): pincode += str(n) @@ -102,7 +107,6 @@ class qbittorrent: QBBUTTONS = InlineKeyboardMarkup(buttons.build_menu(2)) msg = "Your download paused. Choose files then press Done Selecting button to start downloading." markup = sendMarkup(msg, listener.bot, listener.update, QBBUTTONS) - self.client.torrents_pause(torrent_hashes=self.ext_hash) with download_dict_lock: download = download_dict[listener.uid] download.markup = markup @@ -114,7 +118,7 @@ class qbittorrent: except Exception as e: LOGGER.error(str(e)) sendMessage(str(e), listener.bot, listener.update) - self.client.torrents_delete(torrent_hashes=self.ext_hash) + self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True) def update(self): @@ -125,18 +129,49 @@ class qbittorrent: else: tor_info = tor_info[0] if tor_info.state == "metaDL": - if time.time() - self.meta_time > 600: - self.client.torrents_delete(torrent_hashes=self.ext_hash) + self.stalled_time = time.time() + if time.time() - self.meta_time >= 600: self.listener.onDownloadError("Dead Torrent!") + self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True) + self.updater.cancel() + return + elif tor_info.state == "downloading": + self.stalled_time = time.time() + if (TORRENT_DIRECT_LIMIT is not None or TAR_UNZIP_LIMIT is not None) and not self.checked: + if self.listener.isTar or self.listener.extract: + is_tar_ext = True + mssg = f'Tar/Unzip limit is {TAR_UNZIP_LIMIT}' + else: + is_tar_ext = False + mssg = f'Torrent/Direct limit is {TORRENT_DIRECT_LIMIT}' + size = tor_info.size + result = check_limit(size, TORRENT_DIRECT_LIMIT, TAR_UNZIP_LIMIT, is_tar_ext) + self.checked = True + if result: + self.listener.onDownloadError(f"{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}") + self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True) + self.updater.cancel() + return + elif tor_info.state == "stalledDL": + if time.time() - self.stalled_time >= 900: + self.listener.onDownloadError("Dead Torrent!") + self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True) self.updater.cancel() return elif tor_info.state == "error": - self.client.torrents_delete(torrent_hashes=self.ext_hash) self.listener.onDownloadError("Error. IDK why, report in support group") + self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True) self.updater.cancel() return elif tor_info.state == "uploading" or tor_info.state.lower().endswith("up"): self.client.torrents_pause(torrent_hashes=self.ext_hash) + if self.qbitsel: + for dirpath, subdir, files in os.walk(f"{self.dire}", topdown=False): + for file in files: + if fnmatch(file, "*.!qB"): + os.remove(os.path.join(dirpath, file)) + if not os.listdir(dirpath): + os.rmdir(dirpath) self.listener.onDownloadComplete() self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True) self.updater.cancel() @@ -164,7 +199,6 @@ def get_confirm(update, context): query.delete_message() - def get_hash_magnet(mgt): if mgt.startswith('magnet:'): _, _, _, _, query, _ = urlparse(mgt) diff --git a/bot/helper/mirror_utils/status_utils/qbit_download_status.py b/bot/helper/mirror_utils/status_utils/qbit_download_status.py index af43847..eabec4f 100644 --- a/bot/helper/mirror_utils/status_utils/qbit_download_status.py +++ b/bot/helper/mirror_utils/status_utils/qbit_download_status.py @@ -32,7 +32,7 @@ class QbDownloadStatus(Status): Gets total size of the mirror file/folder :return: total size of mirror """ - return self.torrent_info().total_size + return self.torrent_info().size def processed_bytes(self): return self.torrent_info().downloaded @@ -47,7 +47,7 @@ class QbDownloadStatus(Status): return f"{DOWNLOAD_DIR}{self.__uid}" def size(self): - return get_readable_file_size(self.torrent_info().total_size) + return get_readable_file_size(self.torrent_info().size) def eta(self): return get_readable_time(self.torrent_info().eta) @@ -79,4 +79,4 @@ class QbDownloadStatus(Status): def cancel_download(self): LOGGER.info(f"Cancelling Download: {self.name()}") self.listener.onDownloadError('Download stopped by user!') - self.client.torrents_delete(torrent_hashes=self.__hash) + self.client.torrents_delete(torrent_hashes=self.__hash, delete_files=True) diff --git a/bot/helper/mirror_utils/upload_utils/gdriveTools.py b/bot/helper/mirror_utils/upload_utils/gdriveTools.py index 8f2e75b..61190c2 100644 --- a/bot/helper/mirror_utils/upload_utils/gdriveTools.py +++ b/bot/helper/mirror_utils/upload_utils/gdriveTools.py @@ -510,9 +510,8 @@ class GoogleDriveHelper: mime_type = get_mime_type(current_file_name) file_name = current_file_name.split("/")[-1] # current_file_name will have the full path - if not file_name.endswith(".!qB"): - self.upload_file(current_file_name, file_name, mime_type, parent_id) - self.total_files += 1 + self.upload_file(current_file_name, file_name, mime_type, parent_id) + self.total_files += 1 new_id = parent_id if self.is_cancelled: break diff --git a/bot/modules/clone.py b/bot/modules/clone.py index 9ef506f..dd31288 100644 --- a/bot/modules/clone.py +++ b/bot/modules/clone.py @@ -5,7 +5,7 @@ from bot.helper.telegram_helper.filters import CustomFilters from bot.helper.telegram_helper.bot_commands import BotCommands from bot.helper.mirror_utils.status_utils.clone_status import CloneStatus from bot import dispatcher, LOGGER, CLONE_LIMIT, STOP_DUPLICATE, download_dict, download_dict_lock, Interval -from bot.helper.ext_utils.bot_utils import get_readable_file_size +from bot.helper.ext_utils.bot_utils import get_readable_file_size, check_limit import random import string @@ -15,7 +15,7 @@ def cloneNode(update, context): if len(args) > 1: link = args[1] gd = gdriveTools.GoogleDriveHelper() - res, clonesize, name, files = gd.clonehelper(link) + res, size, name, files = gd.clonehelper(link) if res != "": sendMessage(res, context.bot, update) return @@ -27,19 +27,11 @@ def cloneNode(update, context): sendMarkup(msg3, context.bot, update, button) return if CLONE_LIMIT is not None: - LOGGER.info(f"Checking File/Folder Size...") - limit = CLONE_LIMIT - limit = limit.split(' ', maxsplit=1) - limitint = int(limit[0]) - msg2 = f'Failed, Clone limit is {CLONE_LIMIT}.\nYour File/Folder size is {get_readable_file_size(clonesize)}.' - if 'G' in limit[1] or 'g' in limit[1]: - if clonesize > limitint * 1024**3: - sendMessage(msg2, context.bot, update) - return - elif 'T' in limit[1] or 't' in limit[1]: - if clonesize > limitint * 1024**4: - sendMessage(msg2, context.bot, update) - return + result = check_limit(size, CLONE_LIMIT) + if result: + msg2 = f'Failed, Clone limit is {CLONE_LIMIT}.\nYour File/Folder size is {get_readable_file_size(clonesize)}.' + sendMessage(msg2, context.bot, update) + return if files < 15: msg = sendMessage(f"Cloning: {link}", context.bot, update) result, button = gd.clone(link) diff --git a/bot/modules/mirror.py b/bot/modules/mirror.py index fe409ef..af98a86 100644 --- a/bot/modules/mirror.py +++ b/bot/modules/mirror.py @@ -90,7 +90,7 @@ class MirrorListener(listeners.MirrorListeners): else: archive_result = subprocess.run(["extract", m_path]) if archive_result.returncode == 0: - threading.Thread(target=os.remove, args=(m_path,)).start() + threading.Thread(target=os.remove, args=(m_path)).start() LOGGER.info(f"Deleting archive: {m_path}") else: LOGGER.warning('Unable to extract archive! Uploading anyway') @@ -280,7 +280,7 @@ def _mirror(bot, update, isTar=False, extract=False): listener = MirrorListener(bot, update, pswd, isTar, extract) tg_downloader = TelegramDownloadHelper(listener) ms = update.message - tg_downloader.add_download(ms, f'{DOWNLOAD_DIR}{listener.uid}/', name) + tg_downloader.add_download(ms, f'{DOWNLOAD_DIR}{listener.uid}', name) return else: if qbit: @@ -315,19 +315,11 @@ def _mirror(bot, update, isTar=False, extract=False): sendMessage(res, bot, update) return if TAR_UNZIP_LIMIT is not None: - LOGGER.info(f'Checking File/Folder Size') - limit = TAR_UNZIP_LIMIT - limit = limit.split(' ', maxsplit=1) - limitint = int(limit[0]) - msg = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(size)}.' - if 'G' in limit[1] or 'g' in limit[1]: - if size > limitint * 1024**3: - sendMessage(msg, listener.bot, listener.update) - return - elif 'T' in limit[1] or 't' in limit[1]: - if size > limitint * 1024**4: - sendMessage(msg, listener.bot, listener.update) - return + result = check_limit(size, TAR_UNZIP_LIMIT) + if result: + msg = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(size)}.' + sendMessage(msg, listener.bot, listener.update) + return LOGGER.info(f"Download Name : {name}") drive = gdriveTools.GoogleDriveHelper(name, listener) gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=12)) @@ -345,14 +337,14 @@ def _mirror(bot, update, isTar=False, extract=False): sendMessage("Mega links are blocked!", bot, update) else: mega_dl = MegaDownloadHelper() - mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener) + mega_dl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}', listener) elif qbit and (bot_utils.is_magnet(link) or os.path.exists(link)): qbit = qbittorrent() - qbit.add_torrent(link, f'{DOWNLOAD_DIR}{listener.uid}/', listener, qbitsel) + qbit.add_torrent(link, f'{DOWNLOAD_DIR}{listener.uid}', listener, qbitsel) else: - ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener, name) + ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}', listener, name) sendStatusMessage(update, bot) diff --git a/start.sh b/start.sh index db2ce37..3d46f03 100755 --- a/start.sh +++ b/start.sh @@ -1,11 +1 @@ -if [[ -n $TOKEN_PICKLE_URL ]]; then - wget -q $TOKEN_PICKLE_URL -O /usr/src/app/token.pickle -fi - -if [[ -n $ACCOUNTS_ZIP_URL ]]; then - wget -q $ACCOUNTS_ZIP_URL -O /usr/src/app/accounts.zip - unzip accounts.zip -d /usr/src/app/accounts - rm accounts.zip -fi - gunicorn wserver:start_server --bind 0.0.0.0:$PORT --worker-class aiohttp.GunicornWebWorker & qbittorrent-nox -d --webui-port=8090 & python3 alive.py & ./aria.sh; python3 -m bot