add ability to unzip files (#93)

handle unsupported request

use shell utils to extract archives

use shell utils to extract archives

update mirror calls for unzipmirror

fix extract location of files

install deps

handling if archive is not extracted correctly

typo fix

implement Extract Status

handle filenames with spaces

handle special character

Dockerfile

@@ -3,10 +3,12 @@ FROM ubuntu:18.04
 WORKDIR /usr/src/app
 RUN chmod 777 /usr/src/app
 RUN apt-get -qq update
-RUN apt-get -qq install -y aria2 python3 python3-pip \
+RUN apt-get -qq install -y aria2 python3 python3-pip rar unzip \
     locales python3-lxml \
     curl pv jq ffmpeg
 COPY requirements.txt .
+COPY extract /usr/local/bin
+RUN chmod +x /usr/local/bin/extract
 RUN pip3 install --no-cache-dir -r requirements.txt
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8

bot/__main__.py

@@ -65,6 +65,8 @@ def bot_help(update, context):
 
 /{BotCommands.MirrorCommand} [download_url][magnet_link]: Start mirroring the link to google drive
 
+/{BotCommands.UnzipMirrorCommand} [download_url][magnet_link] : starts mirroring and if downloaded file is any archive , extracts it to google drive
+
 /{BotCommands.TarMirrorCommand} [download_url][magnet_link]: start mirroring and upload the archived (.tar) version of the download
 
 /{BotCommands.WatchCommand} [youtube-dl supported link]: Mirror through youtube-dl 

bot/helper/ext_utils/bot_utils.py

@@ -19,6 +19,7 @@ class MirrorStatus:
     STATUS_FAILED = "Failed. Cleaning download"
     STATUS_CANCELLED = "Cancelled"
     STATUS_ARCHIVING = "Archiving"
+    STATUS_EXTRACTING = "Extracting"
 
 
 PROGRESS_MAX_SIZE = 100 // 8
@@ -91,7 +92,7 @@ def get_readable_message():
         for download in list(download_dict.values()):
             msg += f"<i>{download.name()}</i> - "
             msg += download.status()
-            if download.status() != MirrorStatus.STATUS_ARCHIVING:
+            if download.status() != MirrorStatus.STATUS_ARCHIVING and download.status() != MirrorStatus.STATUS_EXTRACTING:
                 msg += f"\n<code>{get_progress_bar_string(download)} {download.progress()}</code> of " \
                        f"{download.size()}" \
                        f" at {download.speed()}, ETA: {download.eta()} "

bot/helper/ext_utils/fs_utils.py

@@ -56,6 +56,31 @@ def tar(org_path):
     return tar_path
 
 
+def get_base_name(orig_path: str):
+    if orig_path.endswith(".tar.bz2"):
+        return orig_path.replace(".tar.bz2", "")
+    elif orig_path.endswith(".tar.gz"):
+        return orig_path.replace(".tar.gz","")
+    elif orig_path.endswith(".bz2"):
+        return orig_path.replace(".bz2","")
+    elif orig_path.endswith(".gz"):
+        return orig_path.replace(".gz","")
+    elif orig_path.endswith(".tar"):
+        return orig_path.replace(".tar","")
+    elif orig_path.endswith(".tbz2"):
+        return orig_path.replace("tbz2","")
+    elif orig_path.endswith(".tgz"):
+        return orig_path.replace(".tgz","")
+    elif orig_path.endswith(".zip"):
+        return orig_path.replace(".zip","")
+    elif orig_path.endswith(".Z"):
+        return orig_path.replace(".Z","")
+    elif orig_path.endswith(".rar"):
+        return orig_path.replace(".rar","")
+    else:
+        return "unsupported"
+
+
 def get_mime_type(file_path):
     mime = magic.Magic(mime=True)
     mime_type = mime.from_file(file_path)

bot/helper/mirror_utils/status_utils/aria_download_status.py

@@ -20,6 +20,7 @@ class AriaDownloadStatus(Status):
         self.message = listener.message
         self.last = None
         self.is_waiting = False
+        self.is_extracting = False
 
     def __update(self):
         self.__download = get_download(self.__gid)

bot/helper/mirror_utils/status_utils/extract_status.py

@@ -0,0 +1,36 @@
+from .status import Status
+from bot.helper.ext_utils.bot_utils import get_readable_file_size, MirrorStatus
+
+
+class ExtractStatus(Status):
+    def __init__(self, name, path, size):
+        self.__name = name
+        self.__path = path
+        self.__size = size
+
+    # The progress of Tar function cannot be tracked. So we just return dummy values.
+    # If this is possible in future,we should implement it
+
+    def progress(self):
+        return '0'
+
+    def speed(self):
+        return '0'
+
+    def name(self):
+        return self.__name
+
+    def path(self):
+        return self.__path
+
+    def size(self):
+        return get_readable_file_size(self.__size)
+
+    def eta(self):
+        return '0s'
+
+    def status(self):
+        return MirrorStatus.STATUS_EXTRACTING
+
+    def processed_bytes(self):
+        return 0

bot/helper/telegram_helper/bot_commands.py

@@ -2,6 +2,7 @@ class _BotCommands:
     def __init__(self):
         self.StartCommand = 'start'
         self.MirrorCommand = 'mirror'
+        self.UnzipMirrorCommand = 'unzipmirror'
         self.TarMirrorCommand = 'tarmirror'
         self.CancelMirror = 'cancel'
         self.CancelAllCommand = 'cancelall'

bot/modules/mirror.py

@@ -10,6 +10,7 @@ from bot.helper.mirror_utils.download_utils.aria2_download import AriaDownloadHe
 from bot.helper.mirror_utils.download_utils.direct_link_generator import direct_link_generator
 from bot.helper.mirror_utils.download_utils.telegram_downloader import TelegramDownloadHelper
 from bot.helper.mirror_utils.status_utils import listeners
+from bot.helper.mirror_utils.status_utils.extract_status import ExtractStatus
 from bot.helper.mirror_utils.status_utils.tar_status import TarStatus
 from bot.helper.mirror_utils.status_utils.upload_status import UploadStatus
 from bot.helper.mirror_utils.upload_utils import gdriveTools
@@ -24,11 +25,11 @@ ariaDlManager.start_listener()
 
 
 class MirrorListener(listeners.MirrorListeners):
-
-    def __init__(self, bot, update, isTar=False, tag=None):
+    def __init__(self, bot, update, isTar=False,tag=None, extract=False):
         super().__init__(bot, update)
         self.isTar = isTar
         self.tag = tag
+        self.extract = extract
 
     def onDownloadStarted(self):
         pass
@@ -62,6 +63,30 @@ class MirrorListener(listeners.MirrorListeners):
                 LOGGER.info('File to archive not found!')
                 self.onUploadError('Internal error occurred!!')
                 return
+        elif self.extract:
+            download.is_extracting = True
+            
+            path = fs_utils.get_base_name(m_path)
+            if path != "unsupported":
+                LOGGER.info(
+                    f"Extracting : {download_dict[self.uid].name()} "
+                )
+                download_dict[self.uid] = ExtractStatus(name, m_path, size)
+                os.system(f"extract '{m_path}'")
+                if not os.path.exists(path):
+                    self.onUploadError("Cannot extract file, check integrity of the file")
+                    return
+                LOGGER.info(
+                    f'got path : {path}'
+                )
+                try:
+                    os.remove(m_path)
+                    LOGGER.info(f"Deleting archive : {m_path}")
+                except Exception as e:
+                    LOGGER.error(str(e))
+            else:
+                LOGGER.info("Not any valid archive, uploading file as it is.")
+                path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}'
         else:
             path = f'{DOWNLOAD_DIR}{self.uid}/{download_dict[self.uid].name()}'
         up_name = pathlib.PurePath(path).name
@@ -145,8 +170,7 @@ class MirrorListener(listeners.MirrorListeners):
         else:
             update_all_messages()
 
-
-def _mirror(bot, update, isTar=False):
+def _mirror(bot, update, isTar=False, extract=False):
     message_args = update.message.text.split(' ')
     try:
         link = message_args[1]
@@ -186,7 +210,7 @@ def _mirror(bot, update, isTar=False):
         link = direct_link_generator(link)
     except DirectDownloadLinkException as e:
         LOGGER.info(f'{link}: {e}')
-    listener = MirrorListener(bot, update, isTar, tag)
+    listener = MirrorListener(bot, update, isTar, tag, extract)
     ariaDlManager.add_download(link, f'{DOWNLOAD_DIR}/{listener.uid}/',listener)
     sendStatusMessage(update, bot)
     if len(Interval) == 0:
@@ -203,9 +227,17 @@ def tar_mirror(update, context):
     _mirror(context.bot, update, True)
 
 
+@run_async
+def unzip_mirror(update, context):
+    _mirror(context.bot,update, extract=True)
+
+
 mirror_handler = CommandHandler(BotCommands.MirrorCommand, mirror,
                                 filters=CustomFilters.authorized_chat | CustomFilters.authorized_user)
 tar_mirror_handler = CommandHandler(BotCommands.TarMirrorCommand, tar_mirror,
                                     filters=CustomFilters.authorized_chat | CustomFilters.authorized_user)
+unzip_mirror_handler = CommandHandler(BotCommands.UnzipMirrorCommand, unzip_mirror,
+                                      filters=CustomFilters.authorized_chat | CustomFilters.authorized_user)
 dispatcher.add_handler(mirror_handler)
 dispatcher.add_handler(tar_mirror_handler)
+dispatcher.add_handler(unzip_mirror_handler)

extract

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ $# -lt 1 ];then
+  echo "Usage: `basename $0` FILES"
+  exit 1
+fi
+
+extract () {
+	arg="$1"
+		cd $(dirname "$arg")
+            case "$arg" in
+                *.tar.bz2) 
+					tar xjf "$arg" --one-top-level ;;
+                *.tar.gz)
+					tar xzf "$arg" --one-top-level ;;
+                *.bz2) bunzip2 "$arg"  ;;
+                *.gz)  gunzip "$arg"       ;;
+                *.tar) tar xf "$arg" --one-top-level ;;
+                *.tbz2) tar xjf "$arg" --one-top-level ;;
+                *.tgz) tar xzf "$arg" --one-top-level ;;
+                *.zip)
+					a_dir=`expr "$arg" : '\(.*\).zip'`
+					unzip "$arg" -d "$a_dir"       ;;
+                *.Z)  uncompress "$arg"   ;;
+                *.rar)   
+					a_dir=`expr "$arg" : '\(.*\).rar'` ;
+					mkdir "$a_dir" ;
+					rar x "$arg"  "$a_dir"      ;;  # 'rar' must to be installed
+                *)          echo "'$arg' cannot be extracted via extract()" ;;
+            esac
+		cd -
+}
+
+extract "$1"