-fix upload and downlaod size after selecting with qbit
-fix upload empty folders after selecting wit qbit
-add size limit for qbit -add timeout for dead torrents 900sec by default
-remove duplicated codes for limiting size

Signed-off-by: anas <e.anastayyar@gmail.com>
This commit is contained in:
anas 2021-08-15 18:11:00 +03:00
parent 4308cd463c
commit e29487b613
10 changed files with 129 additions and 102 deletions

View File

@ -4,6 +4,7 @@ import threading
import time
import random
import string
import subprocess
import aria2p
import qbittorrentapi as qba
@ -72,7 +73,7 @@ def get_client() -> qba.TorrentsAPIMixIn:
qb_client = qba.Client(host="localhost", port=8090, username="admin", password="adminadmin")
try:
qb_client.auth_log_in()
qb_client.application.set_preferences({"disk_cache":64, "incomplete_files_ext":True, "max_connec":10000, "max_connec_per_torrent":1000, "async_io_threads":32, "preallocate_all":True, "upnp":True, "dl_limit":-1, "up_limit":-1, "dht":True, "pex":True, "lsd":True, "encryption":0, "queueing_enabled":True, "max_active_downloads":15, "max_active_torrents":50, "dont_count_slow_torrents":True, "bittorrent_protocol":0, "recheck_completed_torrents":True, "enable_multi_connections_from_same_ip":True, "slow_torrent_dl_rate_threshold":100,"slow_torrent_inactive_timer":600})
qb_client.application.set_preferences({"disk_cache":64, "incomplete_files_ext":True, "max_connec":10000, "max_connec_per_torrent":1000, "async_io_threads":32, "preallocate_all":True, "upnp":True, "dl_limit":-1, "up_limit":-1, "dht":True, "pex":True, "lsd":True, "encryption":0, "queueing_enabled":True, "max_active_downloads":15, "max_active_torrents":50, "dont_count_slow_torrents":True, "bittorrent_protocol":0, "recheck_completed_torrents":True, "auto_delete_mode":True, "enable_multi_connections_from_same_ip":True, "slow_torrent_dl_rate_threshold":100,"slow_torrent_inactive_timer":600})
return qb_client
except qba.LoginFailed as e:
LOGGER.error(str(e))
@ -351,6 +352,31 @@ except KeyError:
logging.warning('SERVER_PORT not provided!')
SERVER_PORT = None
try:
TOKEN_PICKLE_URL = getConfig('TOKEN_PICKLE_URL')
if len(TOKEN_PICKLE_URL) == 0:
TOKEN_PICKLE_URL = None
else:
out = subprocess.run(["wget", "-q", "-O", "token.pickle", TOKEN_PICKLE_URL])
if out.returncode != 0:
logging.error(out)
except KeyError:
TOKEN_PICKLE_URL = None
try:
ACCOUNTS_ZIP_URL = getConfig('ACCOUNTS_ZIP_URL')
if len(ACCOUNTS_ZIP_URL) == 0:
ACCOUNTS_ZIP_URL = None
else:
out = subprocess.run(["wget", "-q", "-O", "accounts.zip", ACCOUNTS_ZIP_URL])
if out.returncode != 0:
logging.error(out)
raise KeyError
subprocess.run(["unzip", "-q", "-o", "accounts.zip"])
os.remove("accounts.zip")
except KeyError:
ACCOUNTS_ZIP_URL = None
updater = tg.Updater(token=BOT_TOKEN)
bot = updater.bot
dispatcher = updater.dispatcher

View File

@ -181,6 +181,20 @@ def flip(update, context):
message_utils.update_all_messages()
def check_limit(size, limit, tar_unzip_limit=None, is_tar_ext=False):
LOGGER.info(f"Checking File/Folder Size...")
if is_tar_ext and tar_unzip_limit is not None:
limit = tar_unzip_limit
if limit is not None:
limit = limit.split(' ', maxsplit=1)
limitint = int(limit[0])
if 'G' in limit[1] or 'g' in limit[1]:
if size > limitint * 1024**3:
return True
elif 'T' in limit[1] or 't' in limit[1]:
if size > limitint * 1024**4:
return True
def get_readable_time(seconds: int) -> str:
result = ''
(days, remainder) = divmod(seconds, 86400)

View File

@ -36,29 +36,18 @@ class AriaDownloadHelper(DownloadHelper):
sendMarkup("Here are the search results:", dl.getListener().bot, dl.getListener().update, button)
return
if (TORRENT_DIRECT_LIMIT is not None or TAR_UNZIP_LIMIT is not None) and dl is not None:
limit = None
if TAR_UNZIP_LIMIT is not None and (dl.getListener().isTar or dl.getListener().extract):
LOGGER.info(f"Checking File/Folder Size...")
limit = TAR_UNZIP_LIMIT
size = aria2.get_download(gid).total_length
if dl.getListener().isTar or dl.getListener().extract:
is_tar_ext = True
mssg = f'Tar/Unzip limit is {TAR_UNZIP_LIMIT}'
elif TORRENT_DIRECT_LIMIT is not None and limit is None:
LOGGER.info(f"Checking File/Folder Size...")
limit = TORRENT_DIRECT_LIMIT
else:
is_tar_ext = False
mssg = f'Torrent/Direct limit is {TORRENT_DIRECT_LIMIT}'
if limit is not None:
size = aria2.get_download(gid).total_length
limit = limit.split(' ', maxsplit=1)
limitint = int(limit[0])
if 'G' in limit[1] or 'g' in limit[1]:
if size > limitint * 1024**3:
dl.getListener().onDownloadError(f'{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}')
aria2.remove([download], force=True)
return
elif 'T' in limit[1] or 't' in limit[1]:
if size > limitint * 1024**4:
dl.getListener().onDownloadError(f'{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}')
aria2.remove([download], force=True)
return
result = check_limit(size, TORRENT_DIRECT_LIMIT, TAR_UNZIP_LIMIT, is_tar_ext)
if result:
dl.getListener().onDownloadError(f'{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}')
aria2.remove([download], force=True)
return
update_all_messages()
def __onDownloadComplete(self, api: API, gid):

View File

@ -3,7 +3,7 @@ import threading
from mega import (MegaApi, MegaListener, MegaRequest, MegaTransfer, MegaError)
from bot.helper.telegram_helper.message_utils import *
import os
from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type, get_readable_file_size
from bot.helper.ext_utils.bot_utils import new_thread, get_mega_link_type, get_readable_file_size, check_limit
from bot.helper.mirror_utils.status_utils.mega_download_status import MegaDownloadStatus
from bot.helper.mirror_utils.upload_utils.gdriveTools import GoogleDriveHelper
from bot import MEGA_LIMIT, STOP_DUPLICATE, TAR_UNZIP_LIMIT
@ -180,27 +180,18 @@ class MegaDownloadHelper:
executor.continue_event.set()
return
if MEGA_LIMIT is not None or TAR_UNZIP_LIMIT is not None:
limit = None
LOGGER.info(f'Checking File/Folder Size')
if TAR_UNZIP_LIMIT is not None and (listener.isTar or listener.extract):
limit = TAR_UNZIP_LIMIT
size = api.getSize(node)
if listener.isTar or listener.extract:
is_tar_ext = True
msg3 = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(api.getSize(node))}.'
elif MEGA_LIMIT is not None and limit is None:
limit = MEGA_LIMIT
else:
is_tar_ext = False
msg3 = f'Failed, Mega limit is {MEGA_LIMIT}.\nYour File/Folder size is {get_readable_file_size(api.getSize(node))}.'
if limit is not None:
limit = limit.split(' ', maxsplit=1)
limitint = int(limit[0])
if 'G' in limit[1] or 'g' in limit[1]:
if api.getSize(node) > limitint * 1024**3:
sendMessage(msg3, listener.bot, listener.update)
executor.continue_event.set()
return
elif 'T' in limit[1] or 't' in limit[1]:
if api.getSize(node) > limitint * 1024**4:
sendMessage(msg3, listener.bot, listener.update)
executor.continue_event.set()
return
result = check_limit(size, MEGA_LIMIT, TAR_UNZIP_LIMIT, is_tar_ext)
if result:
sendMessage(msg3, listener.bot, listener.update)
executor.continue_event.set()
return
with download_dict_lock:
download_dict[listener.uid] = MegaDownloadStatus(mega_listener, listener)
os.makedirs(path)

View File

@ -9,15 +9,16 @@ import time
import logging
import qbittorrentapi as qba
from fnmatch import fnmatch
from urllib.parse import urlparse, parse_qs
from torrentool.api import Torrent
from telegram import InlineKeyboardMarkup
from telegram.ext import CallbackQueryHandler
from bot import download_dict, download_dict_lock, BASE_URL, dispatcher, get_client
from bot import download_dict, download_dict_lock, BASE_URL, dispatcher, get_client, TORRENT_DIRECT_LIMIT, TAR_UNZIP_LIMIT
from bot.helper.mirror_utils.status_utils.qbit_download_status import QbDownloadStatus
from bot.helper.telegram_helper.message_utils import *
from bot.helper.ext_utils.bot_utils import setInterval, new_thread, MirrorStatus, getDownloadByGid
from bot.helper.ext_utils.bot_utils import setInterval, new_thread, MirrorStatus, getDownloadByGid, get_readable_file_size, check_limit
from bot.helper.telegram_helper import button_build
LOGGER = logging.getLogger(__name__)
@ -29,15 +30,18 @@ class qbittorrent:
def __init__(self):
self.update_interval = 2
self.meta_time = time.time()
self.stalled_time = time.time()
self.checked = False
@new_thread
def add_torrent(self, link, dire, listener, qbitsel):
self.client = get_client()
self.listener = listener
self.dire = dire
self.qbitsel = qbitsel
is_file = False
count = 0
pincode = ""
markup = None
try:
if os.path.exists(link):
is_file = True
@ -54,7 +58,6 @@ class qbittorrent:
else:
op = self.client.torrents_add(link, save_path=dire)
if op.lower() == "ok.":
LOGGER.info(f"QbitDownload started: {self.ext_hash}")
tor_info = self.client.torrents_info(torrent_hashes=self.ext_hash)
if len(tor_info) == 0:
while True:
@ -72,6 +75,7 @@ class qbittorrent:
download_dict[listener.uid] = QbDownloadStatus(gid, listener, self.ext_hash, self.client)
self.updater = setInterval(self.update_interval, self.update)
tor_info = tor_info[0]
LOGGER.info(f"QbitDownload started: {tor_info.name}")
if BASE_URL is not None and qbitsel:
if not is_file and (tor_info.state == "checkingResumeData" or tor_info.state == "metaDL"):
meta = sendMessage("Downloading Metadata...Please wait then you can select files or mirror torrent file if it have low seeders", listener.bot, listener.update)
@ -84,8 +88,9 @@ class qbittorrent:
if tor_info.state == "metaDL" or tor_info.state == "checkingResumeData":
time.sleep(1)
else:
break
break
deleteMessage(listener.bot, meta)
self.client.torrents_pause(torrent_hashes=self.ext_hash)
for n in str(self.ext_hash):
if n.isdigit():
pincode += str(n)
@ -102,7 +107,6 @@ class qbittorrent:
QBBUTTONS = InlineKeyboardMarkup(buttons.build_menu(2))
msg = "Your download paused. Choose files then press Done Selecting button to start downloading."
markup = sendMarkup(msg, listener.bot, listener.update, QBBUTTONS)
self.client.torrents_pause(torrent_hashes=self.ext_hash)
with download_dict_lock:
download = download_dict[listener.uid]
download.markup = markup
@ -114,7 +118,7 @@ class qbittorrent:
except Exception as e:
LOGGER.error(str(e))
sendMessage(str(e), listener.bot, listener.update)
self.client.torrents_delete(torrent_hashes=self.ext_hash)
self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True)
def update(self):
@ -125,18 +129,49 @@ class qbittorrent:
else:
tor_info = tor_info[0]
if tor_info.state == "metaDL":
if time.time() - self.meta_time > 600:
self.client.torrents_delete(torrent_hashes=self.ext_hash)
self.stalled_time = time.time()
if time.time() - self.meta_time >= 600:
self.listener.onDownloadError("Dead Torrent!")
self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True)
self.updater.cancel()
return
elif tor_info.state == "downloading":
self.stalled_time = time.time()
if (TORRENT_DIRECT_LIMIT is not None or TAR_UNZIP_LIMIT is not None) and not self.checked:
if self.listener.isTar or self.listener.extract:
is_tar_ext = True
mssg = f'Tar/Unzip limit is {TAR_UNZIP_LIMIT}'
else:
is_tar_ext = False
mssg = f'Torrent/Direct limit is {TORRENT_DIRECT_LIMIT}'
size = tor_info.size
result = check_limit(size, TORRENT_DIRECT_LIMIT, TAR_UNZIP_LIMIT, is_tar_ext)
self.checked = True
if result:
self.listener.onDownloadError(f"{mssg}.\nYour File/Folder size is {get_readable_file_size(size)}")
self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True)
self.updater.cancel()
return
elif tor_info.state == "stalledDL":
if time.time() - self.stalled_time >= 900:
self.listener.onDownloadError("Dead Torrent!")
self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True)
self.updater.cancel()
return
elif tor_info.state == "error":
self.client.torrents_delete(torrent_hashes=self.ext_hash)
self.listener.onDownloadError("Error. IDK why, report in support group")
self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True)
self.updater.cancel()
return
elif tor_info.state == "uploading" or tor_info.state.lower().endswith("up"):
self.client.torrents_pause(torrent_hashes=self.ext_hash)
if self.qbitsel:
for dirpath, subdir, files in os.walk(f"{self.dire}", topdown=False):
for file in files:
if fnmatch(file, "*.!qB"):
os.remove(os.path.join(dirpath, file))
if not os.listdir(dirpath):
os.rmdir(dirpath)
self.listener.onDownloadComplete()
self.client.torrents_delete(torrent_hashes=self.ext_hash, delete_files=True)
self.updater.cancel()
@ -164,7 +199,6 @@ def get_confirm(update, context):
query.delete_message()
def get_hash_magnet(mgt):
if mgt.startswith('magnet:'):
_, _, _, _, query, _ = urlparse(mgt)

View File

@ -32,7 +32,7 @@ class QbDownloadStatus(Status):
Gets total size of the mirror file/folder
:return: total size of mirror
"""
return self.torrent_info().total_size
return self.torrent_info().size
def processed_bytes(self):
return self.torrent_info().downloaded
@ -47,7 +47,7 @@ class QbDownloadStatus(Status):
return f"{DOWNLOAD_DIR}{self.__uid}"
def size(self):
return get_readable_file_size(self.torrent_info().total_size)
return get_readable_file_size(self.torrent_info().size)
def eta(self):
return get_readable_time(self.torrent_info().eta)
@ -79,4 +79,4 @@ class QbDownloadStatus(Status):
def cancel_download(self):
LOGGER.info(f"Cancelling Download: {self.name()}")
self.listener.onDownloadError('Download stopped by user!')
self.client.torrents_delete(torrent_hashes=self.__hash)
self.client.torrents_delete(torrent_hashes=self.__hash, delete_files=True)

View File

@ -510,9 +510,8 @@ class GoogleDriveHelper:
mime_type = get_mime_type(current_file_name)
file_name = current_file_name.split("/")[-1]
# current_file_name will have the full path
if not file_name.endswith(".!qB"):
self.upload_file(current_file_name, file_name, mime_type, parent_id)
self.total_files += 1
self.upload_file(current_file_name, file_name, mime_type, parent_id)
self.total_files += 1
new_id = parent_id
if self.is_cancelled:
break

View File

@ -5,7 +5,7 @@ from bot.helper.telegram_helper.filters import CustomFilters
from bot.helper.telegram_helper.bot_commands import BotCommands
from bot.helper.mirror_utils.status_utils.clone_status import CloneStatus
from bot import dispatcher, LOGGER, CLONE_LIMIT, STOP_DUPLICATE, download_dict, download_dict_lock, Interval
from bot.helper.ext_utils.bot_utils import get_readable_file_size
from bot.helper.ext_utils.bot_utils import get_readable_file_size, check_limit
import random
import string
@ -15,7 +15,7 @@ def cloneNode(update, context):
if len(args) > 1:
link = args[1]
gd = gdriveTools.GoogleDriveHelper()
res, clonesize, name, files = gd.clonehelper(link)
res, size, name, files = gd.clonehelper(link)
if res != "":
sendMessage(res, context.bot, update)
return
@ -27,19 +27,11 @@ def cloneNode(update, context):
sendMarkup(msg3, context.bot, update, button)
return
if CLONE_LIMIT is not None:
LOGGER.info(f"Checking File/Folder Size...")
limit = CLONE_LIMIT
limit = limit.split(' ', maxsplit=1)
limitint = int(limit[0])
msg2 = f'Failed, Clone limit is {CLONE_LIMIT}.\nYour File/Folder size is {get_readable_file_size(clonesize)}.'
if 'G' in limit[1] or 'g' in limit[1]:
if clonesize > limitint * 1024**3:
sendMessage(msg2, context.bot, update)
return
elif 'T' in limit[1] or 't' in limit[1]:
if clonesize > limitint * 1024**4:
sendMessage(msg2, context.bot, update)
return
result = check_limit(size, CLONE_LIMIT)
if result:
msg2 = f'Failed, Clone limit is {CLONE_LIMIT}.\nYour File/Folder size is {get_readable_file_size(clonesize)}.'
sendMessage(msg2, context.bot, update)
return
if files < 15:
msg = sendMessage(f"Cloning: <code>{link}</code>", context.bot, update)
result, button = gd.clone(link)

View File

@ -90,7 +90,7 @@ class MirrorListener(listeners.MirrorListeners):
else:
archive_result = subprocess.run(["extract", m_path])
if archive_result.returncode == 0:
threading.Thread(target=os.remove, args=(m_path,)).start()
threading.Thread(target=os.remove, args=(m_path)).start()
LOGGER.info(f"Deleting archive: {m_path}")
else:
LOGGER.warning('Unable to extract archive! Uploading anyway')
@ -280,7 +280,7 @@ def _mirror(bot, update, isTar=False, extract=False):
listener = MirrorListener(bot, update, pswd, isTar, extract)
tg_downloader = TelegramDownloadHelper(listener)
ms = update.message
tg_downloader.add_download(ms, f'{DOWNLOAD_DIR}{listener.uid}/', name)
tg_downloader.add_download(ms, f'{DOWNLOAD_DIR}{listener.uid}', name)
return
else:
if qbit:
@ -315,19 +315,11 @@ def _mirror(bot, update, isTar=False, extract=False):
sendMessage(res, bot, update)
return
if TAR_UNZIP_LIMIT is not None:
LOGGER.info(f'Checking File/Folder Size')
limit = TAR_UNZIP_LIMIT
limit = limit.split(' ', maxsplit=1)
limitint = int(limit[0])
msg = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(size)}.'
if 'G' in limit[1] or 'g' in limit[1]:
if size > limitint * 1024**3:
sendMessage(msg, listener.bot, listener.update)
return
elif 'T' in limit[1] or 't' in limit[1]:
if size > limitint * 1024**4:
sendMessage(msg, listener.bot, listener.update)
return
result = check_limit(size, TAR_UNZIP_LIMIT)
if result:
msg = f'Failed, Tar/Unzip limit is {TAR_UNZIP_LIMIT}.\nYour File/Folder size is {get_readable_file_size(size)}.'
sendMessage(msg, listener.bot, listener.update)
return
LOGGER.info(f"Download Name : {name}")
drive = gdriveTools.GoogleDriveHelper(name, listener)
gid = ''.join(random.SystemRandom().choices(string.ascii_letters + string.digits, k=12))
@ -345,14 +337,14 @@ def _mirror(bot, update, isTar=False, extract=False):
sendMessage("Mega links are blocked!", bot, update)
else:
mega_dl = MegaDownloadHelper()
mega_dl.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener)
mega_dl.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}', listener)
elif qbit and (bot_utils.is_magnet(link) or os.path.exists(link)):
qbit = qbittorrent()
qbit.add_torrent(link, f'{DOWNLOAD_DIR}{listener.uid}/', listener, qbitsel)
qbit.add_torrent(link, f'{DOWNLOAD_DIR}{listener.uid}', listener, qbitsel)
else:
ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/', listener, name)
ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}{listener.uid}', listener, name)
sendStatusMessage(update, bot)

View File

@ -1,11 +1 @@
if [[ -n $TOKEN_PICKLE_URL ]]; then
wget -q $TOKEN_PICKLE_URL -O /usr/src/app/token.pickle
fi
if [[ -n $ACCOUNTS_ZIP_URL ]]; then
wget -q $ACCOUNTS_ZIP_URL -O /usr/src/app/accounts.zip
unzip accounts.zip -d /usr/src/app/accounts
rm accounts.zip
fi
gunicorn wserver:start_server --bind 0.0.0.0:$PORT --worker-class aiohttp.GunicornWebWorker & qbittorrent-nox -d --webui-port=8090 & python3 alive.py & ./aria.sh; python3 -m bot