[common] resolve conflicts

src/you_get/common.py

@@ -100,12 +100,10 @@ SITES = {
     'zhanqi'           : 'zhanqi',
 }
 
-import getopt
 import json
 import locale
 import logging
 import os
-import platform
 import re
 import socket
 import sys
@@ -113,6 +111,7 @@ import time
 from urllib import request, parse, error
 from http import cookiejar
 from importlib import import_module
+import argparse
 
 from .version import __version__
 from .util import log, term
@@ -1072,217 +1071,211 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         else:
             download(url, **kwargs)
 
-def script_main(script_name, download, download_playlist, **kwargs):
+def load_cookies(cookiefile):
-    def version():
+    global cookies
-        log.i('version %s, a tiny downloader that scrapes the web.'
+    try:
-              % get_version(kwargs['repo_path']
+        cookies = cookiejar.MozillaCookieJar(a)
-            if 'repo_path' in kwargs else __version__))
+        cookies.load()
+    except Exception:
+        import sqlite3
+        cookies = cookiejar.MozillaCookieJar()
+        con = sqlite3.connect(a)
+        cur = con.cursor()
+        try:
+            cur.execute("""SELECT host, path, isSecure, expiry, name, value
+                        FROM moz_cookies""")
+            for item in cur.fetchall():
+                c = cookiejar.Cookie(
+                    0, item[4], item[5], None, False, item[0],
+                    item[0].startswith('.'), item[0].startswith('.'),
+                    item[1], False, item[2], item[3], item[3]=="", None,
+                    None, {},
+                )
+                cookies.set_cookie(c)
+        except Exception:
+            pass
+        # TODO: Chromium Cookies
+        # SELECT host_key, path, secure, expires_utc, name, encrypted_value
+        # FROM cookies
+        # http://n8henrie.com/2013/11/use-chromes-cookies-for-easier-downloading-with-python-requests/
 
+def set_socks_proxy(proxy):
+    try:
+        import socks
+        socks_proxy_addrs = socks_proxy.split(':')
+        socks.set_default_proxy(socks.SOCKS5,
+                                socks_proxy_addrs[0],
+                                int(socks_proxy_addrs[1]))
+        socket.socket = socks.socksocket
+        def getaddrinfo(*args):
+            return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+        socket.getaddrinfo = getaddrinfo
+    except ImportError:
+        log.w('Error importing PySocks library, socks proxy ignored.'
+            'In order to use use socks proxy, please install PySocks.')
+
+def script_main(download, download_playlist, **kwargs):
     logging.basicConfig(format='[%(levelname)s] %(message)s')
 
-    help = 'Usage: %s [OPTION]... [URL]...\n\n' % script_name
+    def print_version():
-    help += '''Startup options:
+        log.i('version %s, a tiny downloader that scrapes the web.'
-    -V | --version                      Print version and exit.
+                % get_version(kwargs['repo_path']
-    -h | --help                         Print help and exit.
+            if 'repo_path' in kwargs else __version__))
-    \n'''
-    help += '''Dry-run options: (no actual downloading)
-    -i | --info                         Print extracted information.
-    -u | --url                          Print extracted information with URLs.
-         --json                         Print extracted URLs in JSON format.
-    \n'''
-    help += '''Download options:
-    -n | --no-merge                     Do not merge video parts.
-         --no-caption                   Do not download captions.
-                                        (subtitles, lyrics, danmaku, ...)
-    -f | --force                        Force overwriting existed files.
-    -F | --format <STREAM_ID>           Set video format to STREAM_ID.
-    -O | --output-filename <FILE>       Set output filename.
-    -o | --output-dir <PATH>            Set output directory.
-    -p | --player <PLAYER [OPTIONS]>    Stream extracted URL to a PLAYER.
-    -c | --cookies <COOKIES_FILE>       Load cookies.txt or cookies.sqlite.
-    -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
-    -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
-         --no-proxy                     Never use a proxy.
-    -s | --socks-proxy <HOST:PORT>      Use an SOCKS5 proxy for downloading.
-    -t | --timeout <SECONDS>            Set socket timeout.
-    -d | --debug                        Show traceback and other debug info.
-    -I | --input-file                   Read non-playlist urls from file.
-    -P | --password <PASSWORD>          Set video visit password to PASSWORD.
-    -l | --playlist                     Download a playlist.
-    '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:I:P:'
+    parser = argparse.ArgumentParser(
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=', 'input-file=', 'password=']
+        prog='you-get',
-#dead code? download_playlist is a function and always True
+        usage='you-get [OPTION]... URL...',
-#if download_playlist:
+        description='A tiny downloader that scrapes the web',
-    short_opts = 'l' + short_opts
+        add_help=False,
-    opts = ['playlist'] + opts
+    )
+    parser.add_argument('-V', '--version', action='store_true',
+                        help='Print version and exit')
+    parser.add_argument('-h', '--help', action='store_true',
+                        help='Print this help message and exit')
 
-    try:
+    dry_run_grp = parser.add_argument_group('Dry-run options', '(no actual downloading)')
-        opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, opts)
+    dry_run_grp = dry_run_grp.add_mutually_exclusive_group()
-    except getopt.GetoptError as err:
+    dry_run_grp.add_argument('-i', '--info', action='store_true',
-        log.e(err)
+                        help='Print extracted information')
-        log.e("try 'you-get --help' for more options")
+    dry_run_grp.add_argument('-u', '--url', action='store_true',
-        sys.exit(2)
+                        help='Print extracted information with URLs')
+    dry_run_grp.add_argument('--json', action='store_true',
+                        help='Print extracted URLs in JSON format')
+
+    download_grp = parser.add_argument_group('Download options')
+    download_grp.add_argument('-n', '--no-merge', action='store_true', default=False,
+                        help='Do not merge video parts')
+    download_grp.add_argument('--no-caption', action='store_true',
+                        help='Do not download captions (subtitles, lyrics, danmaku, ...)')
+    download_grp.add_argument('-f', '--force', action='store_true', default=False,
+                        help='Force overwriting existing files')
+    download_grp.add_argument('-F', '--format', metavar='STREAM_ID',
+                        help='Set video format to STREAM_ID')
+    download_grp.add_argument('-O', '--output-filename', metavar='FILE',
+                        help='Set output filename')
+    download_grp.add_argument('-o', '--output-dir', metavar='DIR', default='.',
+                        help='Set output directory')
+    download_grp.add_argument('-p', '--player', metavar='PLAYER',
+                        help='Stream extracted URL to a PLAYER')
+    download_grp.add_argument('-c', '--cookies', metavar='COOKIES_FILE',
+                        help='Load cookies.txt or cookies.sqlite')
+    download_grp.add_argument('-t', '--timeout', metavar='SECONDS', type=int, default=600,
+                        help='Set socket timeout')
+    download_grp.add_argument('-d', '--debug', action='store_true',
+                        help='Show traceback and other debug info')
+    download_grp.add_argument('-I', '--input-file', metavar='FILE', type=argparse.FileType('r'),
+                        help='Read non-playlist URLs from FILE')
+    download_grp.add_argument('-P', '--password',
+                        help='Set video visit password to PASSWORD')
+    download_grp.add_argument('-l', '--playlist', action='store_true',
+                        help='Prefer to download a playlist')
+
+    proxy_grp = parser.add_argument_group('Proxy options')
+    proxy_grp = proxy_grp.add_mutually_exclusive_group()
+    proxy_grp.add_argument('-x', '--http-proxy', metavar='HOST:PORT',
+                        help='Use an HTTP proxy for downloading')
+    proxy_grp.add_argument('-y', '--extractor-proxy', metavar='HOST:PORT',
+                        help='Use an HTTP proxy for extracting only')
+    proxy_grp.add_argument('--no-proxy', action='store_true',
+                        help='Never use a proxy')
+    proxy_grp.add_argument('-s', '--socks-proxy', metavar='HOST:PORT',
+                        help='Use an SOCKS5 proxy for downloading')
+
+    download_grp.add_argument('--stream',
+                        help=argparse.SUPPRESS)
+    download_grp.add_argument('--itag',
+                        help=argparse.SUPPRESS)
+
+    parser.add_argument('URL', nargs='*',
+                        help=argparse.SUPPRESS)
+
+    args = parser.parse_args()
+
+    if args.help:
+        print_version()
+        parser.print_help()
+        sys.exit()
+    if args.version:
+        print_version()
+        sys.exit()
+
+    if args.debug:
+        # Set level of root logger to DEBUG
+        logging.getLogger().setLevel(logging.DEBUG)
 
     global force
     global dry_run
     global json_output
     global player
     global extractor_proxy
-    global cookies
     global output_filename
 
-    info_only = False
+    output_filename = args.output_filename
-    playlist = False
+    extractor_proxy = args.extractor_proxy
+
+    info_only = args.info
+    if args.url:
+        dry_run = True
+    if args.json:
+        json_output = True
+        # to fix extractors not use VideoExtractor
+        dry_run = True
+        info_only = False
+
+    if args.cookies:
+        load_cookies(args.cookies)
+
     caption = True
-    merge = True
+    stream_id = args.format or args.stream or args.itag
-    stream_id = None
+    if args.no_caption:
-    lang = None
+        caption = False
-    output_dir = '.'
+    if args.player:
-    proxy = None
+        player = args.player
-    socks_proxy = None
+        caption = False
-    extractor_proxy = None
-    traceback = False
-    timeout = 600
-    urls_from_file = []
-    password = None
 
-    for o, a in opts:
+    if args.no_proxy:
-        if o in ('-V', '--version'):
+        set_http_proxy('')
-            version()
-            sys.exit()
-        elif o in ('-h', '--help'):
-            version()
-            print(help)
-            sys.exit()
-        elif o in ('-f', '--force'):
-            force = True
-        elif o in ('-i', '--info'):
-            info_only = True
-        elif o in ('-u', '--url'):
-            dry_run = True
-        elif o in ('--json', ):
-            json_output = True
-            # to fix extractors not use VideoExtractor
-            dry_run = True
-            info_only = False
-        elif o in ('-c', '--cookies'):
-            try:
-                cookies = cookiejar.MozillaCookieJar(a)
-                cookies.load()
-            except:
-                import sqlite3
-                cookies = cookiejar.MozillaCookieJar()
-                con = sqlite3.connect(a)
-                cur = con.cursor()
-                try:
-                    cur.execute("SELECT host, path, isSecure, expiry, name, value FROM moz_cookies")
-                    for item in cur.fetchall():
-                        c = cookiejar.Cookie(0, item[4], item[5],
-                                             None, False,
-                                             item[0],
-                                             item[0].startswith('.'),
-                                             item[0].startswith('.'),
-                                             item[1], False,
-                                             item[2],
-                                             item[3], item[3]=="",
-                                             None, None, {})
-                        cookies.set_cookie(c)
-                except: pass
-                # TODO: Chromium Cookies
-                # SELECT host_key, path, secure, expires_utc, name, encrypted_value FROM cookies
-                # http://n8henrie.com/2013/11/use-chromes-cookies-for-easier-downloading-with-python-requests/
-
-        elif o in ('-l', '--playlist'):
-            playlist = True
-        elif o in ('--no-caption',):
-            caption = False
-        elif o in ('-n', '--no-merge'):
-            merge = False
-        elif o in ('--no-proxy',):
-            proxy = ''
-        elif o in ('-d', '--debug'):
-            traceback = True
-            # Set level of root logger to DEBUG
-            logging.getLogger().setLevel(logging.DEBUG)
-        elif o in ('-F', '--format', '--stream', '--itag'):
-            stream_id = a
-        elif o in ('-O', '--output-filename'):
-            output_filename = a
-        elif o in ('-o', '--output-dir'):
-            output_dir = a
-        elif o in ('-p', '--player'):
-            player = a
-            caption = False
-        elif o in ('-x', '--http-proxy'):
-            proxy = a
-        elif o in ('-s', '--socks-proxy'):
-            socks_proxy = a
-        elif o in ('-y', '--extractor-proxy'):
-            extractor_proxy = a
-        elif o in ('--lang',):
-            lang = a
-        elif o in ('-t', '--timeout'):
-            timeout = int(a)
-        elif o in ('-P', '--password',):
-            password = a
-        elif o in ('-I', '--input-file'):
-            logging.debug('you are trying to load urls from {}'.format(a))
-            if playlist:
-                log.e("reading playlist from a file is unsupported and won't make your life easier")
-                sys.exit(2)
-            with open(a, 'r') as input_file:
-                for line in input_file:
-                    url = line.strip()
-                    urls_from_file.append(url)
-        else:
-            log.e("try 'you-get --help' for more options")
-            sys.exit(2)
-    if not args and not urls_from_file:
-        print(help)
-        sys.exit()
-    args.extend(urls_from_file)
-
-    if (socks_proxy):
-        try:
-            import socket
-            import socks
-            socks_proxy_addrs = socks_proxy.split(':')
-            socks.set_default_proxy(socks.SOCKS5,
-                                    socks_proxy_addrs[0],
-                                    int(socks_proxy_addrs[1]))
-            socket.socket = socks.socksocket
-            def getaddrinfo(*args):
-                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
-            socket.getaddrinfo = getaddrinfo
-        except ImportError:
-            log.w('Error importing PySocks library, socks proxy ignored.'
-                'In order to use use socks proxy, please install PySocks.')
     else:
-        import socket
+        set_http_proxy(args.http_proxy)
-        set_http_proxy(proxy)
+    if args.socks_proxy:
+        set_socks_proxy(args.socks_proxy)
 
-    socket.setdefaulttimeout(timeout)
+    URLs = []
+    if args.input_file:
+        logging.debug('you are trying to load urls from %s', args.input_file)
+        if args.playlist:
+            log.e("reading playlist from a file is unsupported and won't make your life easier")
+            sys.exit(2)
+        URLs.extend(args.input_file.read().splitlines())
+        args.input_file.close()
+    URLs.extend(args.URL)
+
+    if not URLs:
+        parser.print_help()
+        sys.exit()
+
+    socket.setdefaulttimeout(args.timeout)
 
     try:
+        extra = {}
+        if extractor_proxy:
+            extra['extractor_proxy'] = extractor_proxy
         if stream_id:
-            if not extractor_proxy:
+            extra['stream_id'] = stream_id
-                download_main(download, download_playlist, args, playlist, stream_id=stream_id, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
+        download_main(
-            else:
+            download, download_playlist,
-                download_main(download, download_playlist, args, playlist, stream_id=stream_id, extractor_proxy=extractor_proxy, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
+            URLs, args.playlist,
-        else:
+            stream_id=stream_id, output_dir=args.output_dir, merge=not args.no_merge,
-            if not extractor_proxy:
+            info_only=info_only, json_output=json_output, caption=caption,
-                download_main(download, download_playlist, args, playlist, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
+            **extra
-            else:
+        )
-                download_main(download, download_playlist, args, playlist, extractor_proxy=extractor_proxy, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
     except KeyboardInterrupt:
-        if traceback:
+        if args.debug:
             raise
         else:
             sys.exit(1)
     except UnicodeEncodeError:
-        if traceback:
+        if args.debug:
             raise
         log.e('[error] oops, the current environment does not seem to support Unicode.')
         log.e('please set it to a UTF-8-aware locale first,')
@@ -1292,7 +1285,7 @@ def script_main(script_name, download, download_playlist, **kwargs):
         log.e('    (Linux)      $ LC_CTYPE=en_US.UTF-8')
         sys.exit(1)
     except Exception:
-        if not traceback:
+        if not args.debug:
             log.e('[error] oops, something went wrong.')
             log.e('don\'t panic, c\'est la vie. please try the following steps:')
             log.e('  (1) Rule out any network problem.')
@@ -1303,7 +1296,7 @@ def script_main(script_name, download, download_playlist, **kwargs):
             log.e('  (4) Run the command with \'--debug\' option,')
             log.e('      and report this issue with the full output.')
         else:
-            version()
+            print_version()
             log.i(args)
             raise
         sys.exit(1)
@@ -1366,4 +1359,4 @@ def any_download_playlist(url, **kwargs):
     m.download_playlist(url, **kwargs)
 
 def main(**kwargs):
-    script_main('you-get', any_download, any_download_playlist, **kwargs)
+    script_main(any_download, any_download_playlist, **kwargs)