Merge remote-tracking branch 'upstream/develop' into develop

.gitignore

@@ -5,6 +5,7 @@
 *.py[cod]
 
 _*/
+*_
 
 *.bak
 *.download

Makefile

@@ -14,6 +14,7 @@ clean:
 	zenity --question
 	rm -fr build/ dist/ src/*.egg-info/
 	find . | grep __pycache__ | xargs rm -fr
+	find . | grep .pyc | xargs rm -f
 
 all: build sdist bdist bdist_egg
 

README.md

@@ -65,6 +65,7 @@ __中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%A
 * SongTaste <http://www.songtaste.com>
 * Alive.in.th <http://alive.in.th>
 * VK <http://vk.com>
+* Catfun (喵星球) <http://www.catfun.tv>
 
 ## Dependencies
 
@@ -186,8 +187,6 @@ For a complete list of all available options, see:
         -x | --http-proxy <HOST:PORT>            Use specific HTTP proxy for downloading.
         -y | --extractor-proxy <HOST:PORT>       Use specific HTTP proxy for extracting stream data.
              --no-proxy                          Don't use any proxy. (ignore $http_proxy)
-        -S | --sogou                             Use a Sogou proxy server for downloading.
-             --sogou-proxy <HOST:PORT>           Run a standalone Sogou proxy server.
              --debug                             Show traceback on KeyboardInterrupt.
 
 ## License

README.txt

@@ -68,6 +68,7 @@ Supported Sites (As of Now)
 * SongTaste http://www.songtaste.com
 * Alive.in.th http://alive.in.th
 * VK http://vk.com
+* Catfun (喵星球) http://www.catfun.tv
 
 Dependencies
 ------------
@@ -194,8 +195,6 @@ For a complete list of all available options, see::
         -x | --http-proxy <HOST:PORT>            Use specific HTTP proxy for downloading.
         -y | --extractor-proxy <HOST:PORT>       Use specific HTTP proxy for extracting stream data.
              --no-proxy                          Don't use any proxy. (ignore $http_proxy)
-        -S | --sogou                             Use a Sogou proxy server for downloading.
-             --sogou-proxy <HOST:PORT>           Run a standalone Sogou proxy server.
              --debug                             Show traceback on KeyboardInterrupt.
 
 License

src/you_get/__init__.py

@@ -1,7 +1,18 @@
 #!/usr/bin/env python
+# This file is Python 2 compliant.
 
-from .common import *
+import sys
-from .version import *
 
-from .cli_wrapper import *
+if sys.version_info[0] == 3:
-from .extractor import *
+    #from .extractor import Extractor, VideoExtractor
+    #from .util import log
+
+    from .__main__ import *
+
+    #from .common import *
+    #from .version import *
+    #from .cli_wrapper import *
+    #from .extractor import *
+else:
+    # Don't import anything.
+    pass

src/you_get/__main__.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+import getopt
+import os
+import platform
+import sys
+from .version import script_name, __version__
+from .util import git, log
+
+_options = [
+    'help',
+    'version',
+    'gui',
+    'force',
+    'playlists',
+]
+_short_options = 'hVgfl'
+
+_help = """Usage: {} [OPTION]... [URL]...
+TODO
+""".format(script_name)
+
+def main_dev(**kwargs):
+    """Main entry point.
+    you-get-dev
+    """
+
+    # Get (branch, commit) if running from a git repo.
+    head = git.get_head(kwargs['repo_path'])
+
+    # Get options and arguments.
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], _short_options, _options)
+    except getopt.GetoptError as e:
+        log.wtf("""
+    [Fatal] {}.
+    Try '{} --help' for more options.""".format(e, script_name))
+
+    if not opts and not args:
+        # Display help.
+        print(_help)
+        # Enter GUI mode.
+        #from .gui import gui_main
+        #gui_main()
+    else:
+        conf = {}
+        for opt, arg in opts:
+            if opt in ('-h', '--help'):
+                # Display help.
+                print(_help)
+
+            elif opt in ('-V', '--version'):
+                # Display version.
+                log.println("you-get:", log.BOLD)
+                log.println("    version:  {}".format(__version__))
+                if head is not None:
+                    log.println("    branch:   {}\n    commit:   {}".format(*head))
+                else:
+                    log.println("    branch:   {}\n    commit:   {}".format("(stable)", "(tag v{})".format(__version__)))
+
+                log.println("    platform: {}".format(platform.platform()))
+                log.println("    python:   {}".format(sys.version.split('\n')[0]))
+
+            elif opt in ('-g', '--gui'):
+                # Run using GUI.
+                conf['gui'] = True
+
+            elif opt in ('-f', '--force'):
+                # Force download.
+                conf['force'] = True
+
+            elif opt in ('-l', '--playlist', '--playlists'):
+                # Download playlist whenever possible.
+                conf['playlist'] = True
+
+        if args:
+            if 'gui' in conf and conf['gui']:
+                # Enter GUI mode.
+                from .gui import gui_main
+                gui_main(*args, **conf)
+            else:
+                # Enter console mode.
+                from .console import console_main
+                console_main(*args, **conf)
+
+def main(**kwargs):
+    """Main entry point.
+    you-get (legacy)
+    """
+    from .common import main
+    main()

src/you_get/common.py

@@ -4,21 +4,19 @@ import getopt
 import json
 import locale
 import os
+import platform
 import re
 import sys
 from urllib import request, parse
-import platform
-import threading
 
 from .version import __version__
-from .util import log, sogou_proxy_server, get_filename, unescape_html
+from .util import log
+from .util.strings import get_filename, unescape_html
 
 dry_run = False
 force = False
 player = None
 extractor_proxy = None
-sogou_proxy = None
-sogou_env = None
 cookies_txt = None
 
 fake_headers = {
@@ -752,6 +750,18 @@ def print_info(site_info, title, type, size):
     print("Size:      ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
     print()
 
+def mime_to_container(mime):
+    mapping = {
+        'video/3gpp': '3gp',
+        'video/mp4': 'mp4',
+        'video/webm': 'webm',
+        'video/x-flv': 'flv',
+    }
+    if mime in mapping:
+        return mapping[mime]
+    else:
+        return mime.split('/')[1]
+
 def parse_host(host):
     """Parses host name and port number from a string.
     """
@@ -764,9 +774,6 @@ def parse_host(host):
     port = o.port or 0
     return (hostname, port)
 
-def get_sogou_proxy():
-    return sogou_proxy
-
 def set_proxy(proxy):
     proxy_handler = request.ProxyHandler({
         'http': '%s:%s' % proxy,
@@ -791,6 +798,8 @@ def set_http_proxy(proxy):
     opener = request.build_opener(proxy_support)
     request.install_opener(opener)
 
+
+
 def download_main(download, download_playlist, urls, playlist, **kwargs):
     for url in urls:
         if url.startswith('https://'):
@@ -803,18 +812,8 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         else:
             download(url, **kwargs)
 
-def get_version():
-    try:
-        import subprocess
-        real_dir = os.path.dirname(os.path.realpath(__file__))
-        git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=real_dir, stderr=subprocess.DEVNULL).decode('utf-8').strip()
-        assert git_hash
-        return '%s-%s' % (__version__, git_hash)
-    except:
-        return __version__
-
 def script_main(script_name, download, download_playlist = None):
-    version = 'You-Get %s, a video downloader.' % get_version()
+    version = 'You-Get %s, a video downloader.' % __version__
     help = 'Usage: %s [OPTION]... [URL]...\n' % script_name
     help += '''\nStartup options:
     -V | --version                           Display the version and exit.
@@ -832,13 +831,11 @@ def script_main(script_name, download, download_playlist = None):
     -x | --http-proxy <HOST:PORT>            Use specific HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>       Use specific HTTP proxy for extracting stream data.
          --no-proxy                          Don't use any proxy. (ignore $http_proxy)
-    -S | --sogou                             Use a Sogou proxy server for downloading.
-         --sogou-proxy <HOST:PORT>           Run a standalone Sogou proxy server.
          --debug                             Show traceback on KeyboardInterrupt.
     '''
 
-    short_opts = 'Vhfiuc:nSF:o:p:x:y:'
+    short_opts = 'Vhfiuc:nF:o:p:x:y:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'sogou', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'sogou-proxy=', 'sogou-env=']
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
     if download_playlist:
         short_opts = 'l' + short_opts
         opts = ['playlist'] + opts
@@ -854,8 +851,6 @@ def script_main(script_name, download, download_playlist = None):
     global dry_run
     global player
     global extractor_proxy
-    global sogou_proxy
-    global sogou_env
     global cookies_txt
     cookies_txt = None
 
@@ -863,6 +858,7 @@ def script_main(script_name, download, download_playlist = None):
     playlist = False
     merge = True
     stream_id = None
+    lang = None
     output_dir = '.'
     proxy = None
     extractor_proxy = None
@@ -903,31 +899,14 @@ def script_main(script_name, download, download_playlist = None):
             proxy = a
         elif o in ('-y', '--extractor-proxy'):
             extractor_proxy = a
-        elif o in ('-S', '--sogou'):
+        elif o in ('--lang',):
-            sogou_proxy = ("0.0.0.0", 0)
+            lang = a
-        elif o in ('--sogou-proxy',):
-            sogou_proxy = parse_host(a)
-        elif o in ('--sogou-env',):
-            sogou_env = a
         else:
             log.e("try 'you-get --help' for more options")
             sys.exit(2)
     if not args:
-        if sogou_proxy is not None:
+        print(help)
-            try:
+        sys.exit()
-                if sogou_env is not None:
-                    server = sogou_proxy_server(sogou_proxy, network_env=sogou_env)
-                else:
-                    server = sogou_proxy_server(sogou_proxy)
-                server.serve_forever()
-            except KeyboardInterrupt:
-                if traceback:
-                    raise
-                else:
-                    sys.exit()
-        else:
-            print(help)
-            sys.exit()
 
     set_http_proxy(proxy)
 
@@ -942,173 +921,95 @@ def script_main(script_name, download, download_playlist = None):
         else:
             sys.exit(1)
 
+def url_to_module(url):
+    from .extractors import netease, w56, acfun, baidu, bilibili, blip, catfun, cntv, cbs, coursera, dailymotion, douban, ehow, facebook, freesound, google, sina, ifeng, alive, instagram, iqiyi, joy, jpopsuki, khan, ku6, kugou, kuwo, letv, magisto, miomio, mixcloud, mtv81, nicovideo, pptv, qq, sohu, songtaste, soundcloud, ted, theplatform, tudou, tumblr, vid48, vimeo, vine, vk, xiami, yinyuetai, youku, youtube
 
+    video_host = r1(r'https?://([^/]+)/', url)
+    video_url = r1(r'https?://[^/]+(.*)', url)
+    assert video_host and video_url, 'invalid url: ' + url
 
-def mime_to_container(mime):
+    if video_host.endswith('.com.cn'):
-    mapping = {
+        video_host = video_host[:-3]
-        'video/3gpp': '3gp',
+    domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
-        'video/mp4': 'mp4',
+    assert domain, 'unsupported url: ' + url
-        'video/webm': 'webm',
+
-        'video/x-flv': 'flv',
+    k = r1(r'([^.]+)', domain)
+    downloads = {
+        '163': netease,
+        '56': w56,
+        'acfun': acfun,
+        'baidu': baidu,
+        'bilibili': bilibili,
+        'blip': blip,
+        'catfun': catfun,
+        'cntv': cntv,
+        'cbs': cbs,
+        'coursera': coursera,
+        'dailymotion': dailymotion,
+        'douban': douban,
+        'ehow': ehow,
+        'facebook': facebook,
+        'freesound': freesound,
+        'google': google,
+        'iask': sina,
+        'ifeng': ifeng,
+        'in': alive,
+        'instagram': instagram,
+        'iqiyi': iqiyi,
+        'joy': joy,
+        'jpopsuki': jpopsuki,
+        'kankanews': bilibili,
+        'khanacademy': khan,
+        'ku6': ku6,
+        'kugou': kugou,
+        'kuwo': kuwo,
+        'letv': letv,
+        'magisto': magisto,
+        'miomio': miomio,
+        'mixcloud': mixcloud,
+        'mtv81': mtv81,
+        'nicovideo': nicovideo,
+        'pptv': pptv,
+        'qq': qq,
+        'sina': sina,
+        'smgbb': bilibili,
+        'sohu': sohu,
+        'songtaste': songtaste,
+        'soundcloud': soundcloud,
+        'ted': ted,
+        'theplatform': theplatform,
+        'tudou': tudou,
+        'tumblr': tumblr,
+        'vid48': vid48,
+        'vimeo': vimeo,
+        'vine': vine,
+        'vk': vk,
+        'xiami': xiami,
+        'yinyuetai': yinyuetai,
+        'youku': youku,
+        'youtu': youtube,
+        'youtube': youtube,
     }
-    if mime in mapping:
+    if k in downloads:
-        return mapping[mime]
+        return downloads[k], url
     else:
-        return mime.split('/')[1]
+        import http.client
-
+        conn = http.client.HTTPConnection(video_host)
-
+        conn.request("HEAD", video_url)
-
+        res = conn.getresponse()
-class VideoExtractor():
+        location = res.getheader('location')
-    def __init__(self, *args):
+        if location is None:
-        self.url = None
+            raise NotImplementedError(url)
-        self.title = None
-        self.vid = None
-        self.streams = {}
-        self.streams_sorted = []
-
-        if args:
-            self.url = args[0]
-
-    def download_by_url(self, url, **kwargs):
-        self.url = url
-
-        global extractor_proxy
-        if extractor_proxy:
-            set_proxy(parse_host(extractor_proxy))
-        self.prepare(**kwargs)
-
-        try:
-            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
-        except:
-            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
-
-        self.extract(**kwargs)
-        if extractor_proxy:
-            unset_proxy()
-
-        self.download(**kwargs)
-
-    def download_by_vid(self, vid, **kwargs):
-        self.vid = vid
-
-        global extractor_proxy
-        if extractor_proxy:
-            set_proxy(parse_host(extractor_proxy))
-        self.prepare(**kwargs)
-
-        try:
-            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
-        except:
-            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
-
-        self.extract(**kwargs)
-        if extractor_proxy:
-            unset_proxy()
-
-        self.download(**kwargs)
-
-    def prepare(self, **kwargs):
-        pass
-        #raise NotImplementedError()
-
-    def extract(self, **kwargs):
-        pass
-        #raise NotImplementedError()
-
-    def p_stream(self, stream_id):
-        stream = self.streams[stream_id]
-        if 'itag' in stream:
-            print("    - itag:          \033[7m%s\033[0m" % stream_id)
         else:
-            print("    - id:            \033[7m%s\033[0m" % stream_id)
+            return url_to_module(location)
 
-        if 'container' in stream:
+def any_download(url, **kwargs):
-            print("      container:     %s" % stream['container'])
+    m, url = url_to_module(url)
+    m.download(url, **kwargs)
 
-        if 'video_profile' in stream:
+def any_download_playlist(url, **kwargs):
-            print("      video-profile: %s" % stream['video_profile'])
+    m, url = url_to_module(url)
+    m.download_playlist(url, **kwargs)
 
-        if 'quality' in stream:
+def main():
-            print("      quality:       %s" % stream['quality'])
+    script_main('you-get', any_download, any_download_playlist)
-
-        if 'size' in stream:
-            print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
-
-        if 'itag' in stream:
-            print("    # download-with: \033[4myou-get --itag=%s [URL]\033[0m" % stream_id)
-        else:
-            print("    # download-with: \033[4myou-get --format=%s [URL]\033[0m" % stream_id)
-
-        print()
-
-    def p_i(self, stream_id):
-        stream = self.streams[stream_id]
-        print("    - title:         %s" % self.title)
-        print("       size:         %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
-        print("        url:         %s" % self.url)
-        print()
-
-    def p(self, stream_id=None):
-        print("site:                %s" % self.__class__.name)
-        print("title:               %s" % self.title)
-        if stream_id:
-            # Print the stream
-            print("stream:")
-            self.p_stream(stream_id)
-
-        elif stream_id is None:
-            # Print stream with best quality
-            print("stream:              # Best quality")
-            stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
-            self.p_stream(stream_id)
-
-        elif stream_id == []:
-            # Print all available streams
-            print("streams:             # Available quality and codecs")
-            for stream in self.streams_sorted:
-                self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
-
-    def p_playlist(self, stream_id=None):
-        print("site:                %s" % self.__class__.name)
-        print("playlist:            %s" % self.title)
-        print("videos:")
-
-    def download(self, **kwargs):
-        if 'info_only' in kwargs and kwargs['info_only']:
-            if 'stream_id' in kwargs and kwargs['stream_id']:
-                # Display the stream
-                stream_id = kwargs['stream_id']
-                if 'index' not in kwargs:
-                    self.p(stream_id)
-                else:
-                    self.p_i(stream_id)
-            else:
-                # Display all available streams
-                if 'index' not in kwargs:
-                    self.p([])
-                else:
-                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
-                    self.p_i(stream_id)
-
-        else:
-            if 'stream_id' in kwargs and kwargs['stream_id']:
-                # Download the stream
-                stream_id = kwargs['stream_id']
-            else:
-                # Download stream with the best quality
-                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
-
-            if 'index' not in kwargs:
-                self.p(None)
-            else:
-                self.p_i(stream_id)
-
-            urls = self.streams[stream_id]['src']
-            if not urls:
-                log.e('[Failed] Cannot extract video source.')
-                log.e('This is most likely because the video has not been made available in your country.')
-                log.e('You may try to use a proxy via \'-y\' for extracting stream data.')
-                exit(1)
-            download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge'])
-
-        self.__init__()

src/you_get/extractor.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python
+
+from .common import match1, download_urls
+from .util import log
+
+class Extractor():
+    def __init__(self, *args):
+        self.url = None
+        self.title = None
+        self.vid = None
+        self.streams = {}
+        self.streams_sorted = []
+
+        if args:
+            self.url = args[0]
+
+class VideoExtractor():
+    def __init__(self, *args):
+        self.url = None
+        self.title = None
+        self.vid = None
+        self.streams = {}
+        self.streams_sorted = []
+        self.audiolang = None
+
+        if args:
+            self.url = args[0]
+
+    def download_by_url(self, url, **kwargs):
+        self.url = url
+
+        #global extractor_proxy
+        #if extractor_proxy:
+        #    set_proxy(parse_host(extractor_proxy))
+        self.prepare(**kwargs)
+
+        try:
+            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+        except:
+            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
+
+        self.extract(**kwargs)
+
+        #if extractor_proxy:
+        #    unset_proxy()
+
+        self.download(**kwargs)
+
+    def download_by_vid(self, vid, **kwargs):
+        self.vid = vid
+
+        #global extractor_proxy
+        #if extractor_proxy:
+        #    set_proxy(parse_host(extractor_proxy))
+        self.prepare(**kwargs)
+
+        try:
+            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+        except:
+            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
+
+        self.extract(**kwargs)
+        #if extractor_proxy:
+        #    unset_proxy()
+
+        self.download(**kwargs)
+
+    def prepare(self, **kwargs):
+        pass
+        #raise NotImplementedError()
+
+    def extract(self, **kwargs):
+        pass
+        #raise NotImplementedError()
+
+    def p_stream(self, stream_id):
+        stream = self.streams[stream_id]
+        if 'itag' in stream:
+            print("    - itag:          %s" % log.sprint(stream_id, log.NEGATIVE))
+        else:
+            print("    - format:        %s" % log.sprint(stream_id, log.NEGATIVE))
+
+        if 'container' in stream:
+            print("      container:     %s" % stream['container'])
+
+        if 'video_profile' in stream:
+            print("      video-profile: %s" % stream['video_profile'])
+
+        if 'quality' in stream:
+            print("      quality:       %s" % stream['quality'])
+
+        if 'size' in stream:
+            print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
+
+        if 'itag' in stream:
+            print("    # download-with: %s" % log.sprint("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE))
+        else:
+            print("    # download-with: %s" % log.sprint("you-get --format=%s [URL]" % stream_id, log.UNDERLINE))
+
+        print()
+
+    def p_i(self, stream_id):
+        stream = self.streams[stream_id]
+        print("    - title:         %s" % self.title)
+        print("       size:         %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
+        print("        url:         %s" % self.url)
+        print()
+
+    def p(self, stream_id=None):
+        print("site:                %s" % self.__class__.name)
+        print("title:               %s" % self.title)
+        if stream_id:
+            # Print the stream
+            print("stream:")
+            self.p_stream(stream_id)
+
+        elif stream_id is None:
+            # Print stream with best quality
+            print("stream:              # Best quality")
+            stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+            self.p_stream(stream_id)
+
+        elif stream_id == []:
+            # Print all available streams
+            print("streams:             # Available quality and codecs")
+            for stream in self.streams_sorted:
+                self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
+
+        if self.audiolang:
+            print("audio-languages:")
+            for i in self.audiolang:
+                print("    - lang:          {}".format(i['lang']))
+                print("      download-url:  {}\n".format(i['url']))
+
+    def p_playlist(self, stream_id=None):
+        print("site:                %s" % self.__class__.name)
+        print("playlist:            %s" % self.title)
+        print("videos:")
+
+    def download(self, **kwargs):
+        if 'info_only' in kwargs and kwargs['info_only']:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Display the stream
+                stream_id = kwargs['stream_id']
+                if 'index' not in kwargs:
+                    self.p(stream_id)
+                else:
+                    self.p_i(stream_id)
+            else:
+                # Display all available streams
+                if 'index' not in kwargs:
+                    self.p([])
+                else:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                    self.p_i(stream_id)
+
+        else:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Download the stream
+                stream_id = kwargs['stream_id']
+            else:
+                # Download stream with the best quality
+                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+
+            if 'index' not in kwargs:
+                self.p(None)
+            else:
+                self.p_i(stream_id)
+
+            urls = self.streams[stream_id]['src']
+            if not urls:
+                log.e('[Failed] Cannot extract video source.')
+                log.e('This is most likely because the video has not been made available in your country.')
+                log.e('You may try to use a proxy via \'-y\' for extracting stream data.')
+                exit(1)
+            #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge'])
+            download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
+
+        self.__init__()

src/you_get/extractor/__main__.py

@@ -1,100 +0,0 @@
-#!/usr/bin/env python
-__all__ = ['main', 'any_download', 'any_download_playlist']
-
-from ..extractor import *
-from ..common import *
-
-def url_to_module(url):
-    video_host = r1(r'https?://([^/]+)/', url)
-    video_url = r1(r'https?://[^/]+(.*)', url)
-    assert video_host and video_url, 'invalid url: ' + url
-
-    if video_host.endswith('.com.cn'):
-        video_host = video_host[:-3]
-    domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
-    assert domain, 'unsupported url: ' + url
-
-    k = r1(r'([^.]+)', domain)
-    downloads = {
-        '163': netease,
-        '56': w56,
-        'acfun': acfun,
-        'baidu': baidu,
-        'bilibili': bilibili,
-        'blip': blip,
-        'catfun':catfun,
-        'cntv': cntv,
-        'cbs': cbs,
-        'coursera': coursera,
-        'dailymotion': dailymotion,
-        'douban': douban,
-        'ehow': ehow,
-        'facebook': facebook,
-        'freesound': freesound,
-        'google': google,
-        'iask': sina,
-        'ifeng': ifeng,
-        'in': alive,
-        'instagram': instagram,
-        'iqiyi': iqiyi,
-        'joy': joy,
-        'jpopsuki': jpopsuki,
-        'kankanews': bilibili,
-        'ku6': ku6,
-        'kugou':kugou,
-        'kuwo':kuwo,
-        'letv': letv,
-        'magisto': magisto,
-        'miomio': miomio,
-        'mixcloud': mixcloud,
-        'mtv81':mtv81,
-        'nicovideo': nicovideo,
-        'pptv': pptv,
-        'qq': qq,
-        'sina': sina,
-        'smgbb': bilibili,
-        'sohu': sohu,
-        'songtaste':songtaste,
-        'soundcloud': soundcloud,
-        'ted': ted,
-        'theplatform': theplatform,
-        'tudou': tudou,
-        'tumblr': tumblr,
-        'vid48': vid48,
-        'vimeo': vimeo,
-        'vine': vine,
-        'vk': vk,
-        'xiami': xiami,
-        'yinyuetai': yinyuetai,
-        'youku': youku,
-        'youtu': youtube,
-        'youtube': youtube,
-        'khanacademy': khan,
-        #TODO
-    }
-    if k in downloads:
-        return downloads[k], url
-    else:
-        import http.client
-        conn = http.client.HTTPConnection(video_host)
-        conn.request("HEAD", video_url)
-        res = conn.getresponse()
-        location = res.getheader('location')
-        if location is None:
-            raise NotImplementedError(url)
-        else:
-            return url_to_module(location)
-
-def any_download(url, **kwargs):
-    m, url = url_to_module(url)
-    m.download(url, **kwargs)
-
-def any_download_playlist(url, **kwargs):
-    m, url = url_to_module(url)
-    m.download_playlist(url, **kwargs)
-
-def main():
-    script_main('you-get', any_download, any_download_playlist)
-
-if __name__ == "__main__":
-    main()

src/you_get/extractor/catfun.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['catfun_download']
-from .tudou import tudou_download_by_id
-from .sina import sina_download_by_vid
-
-from ..common import *
-from xml.dom.minidom import *
-
-def parse_item(item):
-    if item["type"]=="youku":
-        page=get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_youku_video_info&youku_id="+item["vid"])
-        dom=parseString(page)
-        ext=dom.getElementsByTagName("format")[0].firstChild.nodeValue;
-        size=0
-        urls=[]
-        for i in dom.getElementsByTagName("durl"):
-            urls.append(i.getElementsByTagName("url")[0].firstChild.nodeValue)
-            size+=int(i.getElementsByTagName("size")[0].firstChild.nodeValue);
-        return urls,ext,size
-        pass
-
-    elif item["type"]=="qq":
-        page=get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_qq_video_info&qq_id="+item["vid"])
-        dom=parseString(page)
-        size=0
-        urls=[]
-        for i in dom.getElementsByTagName("durl"):
-            url=i.getElementsByTagName("url")[0].firstChild.nodeValue
-            urls.append(url) 
-            vtype,ext,_size=url_info(url)
-            size+=_size
-        return urls,ext,size
-        pass
-
-    elif item["type"]=="sina":
-        page=get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_sina_video_info&sina_id=" + item["vid"])
-        try:
-            dom=parseString(page)
-        except:
-            #refresh page encountered
-            page=get_content(match1(page,r'url=(.+?)"'))
-            dom=parseString(page)
-        size=0
-        urls=[]
-        for i in dom.getElementsByTagName("durl"):
-            url=i.getElementsByTagName("url")[0].firstChild.nodeValue
-            urls.append(url)
-            vtype,ext,_size=url_info(url)
-            if not ext:
-                ext=match1(url,r'\.(\w+?)\?')
-            size+=_size
-        #sina's result does not contains content-type 
-        return urls,ext,size
-        pass
-
-
-
-def catfun_download(url, output_dir = '.', merge = True, info_only = False):
-    # html=get_content(url)
-    title=match1(get_content(url),r'<h1 class="title">(.+?)</h1>')
-    vid=match1(url,r"v\d+/cat(\d+)")
-    j=json.loads(get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_video&modelid=11&id={}".format(vid)))
-    for item in j:
-        if item["name"]!="\u672a\u547d\u540d1":
-                t=title+"-"+item["name"]
-        else:
-            t=title
-        if item["type"]=="tudou":
-            tudou_download_by_id(item["vid"], title, output_dir, merge, info_only)
-
-        else:
-            urls,ext,size=parse_item(item)
-            
-            download_urls(urls,t,ext,size,output_dir)
-
-
-site_info = "catfun.com"
-download = catfun_download
-download_playlist = playlist_not_supported('catfun')

src/you_get/extractors/__init__.py

@@ -50,5 +50,3 @@ from .youku import *
 from .youtube import *
 from .ted import *
 from .khan import *
-
-from .__main__ import *

src/you_get/extractors/baidu.py

@@ -4,7 +4,6 @@
 __all__ = ['baidu_download']
 
 from ..common import *
-from .. import common
 
 from urllib import parse
 

src/you_get/extractors/catfun.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+__all__ = ['catfun_download']
+from .tudou import tudou_download_by_id
+from .sina import sina_download_by_vid
+
+from ..common import *
+from xml.dom.minidom import *
+
+def parse_item(item):
+    if item["type"] == "youku":
+        page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_youku_video_info&youku_id=" + item["vid"])
+        dom = parseString(page)
+        ext = dom.getElementsByTagName("format")[0].firstChild.nodeValue;
+        size = 0
+        urls = []
+        for i in dom.getElementsByTagName("durl"):
+            urls.append(i.getElementsByTagName("url")[0].firstChild.nodeValue)
+            size += int(i.getElementsByTagName("size")[0].firstChild.nodeValue);
+        return urls, ext, size
+
+    elif item["type"] == "qq":
+        page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_qq_video_info&qq_id=" + item["vid"])
+        dom = parseString(page)
+        size = 0
+        urls = []
+        for i in dom.getElementsByTagName("durl"):
+            url = i.getElementsByTagName("url")[0].firstChild.nodeValue
+            urls.append(url)
+            vtype, ext, _size = url_info(url)
+            size += _size
+        return urls, ext, size
+
+    elif item["type"] == "sina":
+        page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_sina_video_info&sina_id=" + item["vid"])
+        try:
+            dom = parseString(page)
+        except:
+            #refresh page encountered
+            page = get_content(match1(page, r'url=(.+?)"'))
+            dom = parseString(page)
+        size = 0
+        urls = []
+        for i in dom.getElementsByTagName("durl"):
+            url = i.getElementsByTagName("url")[0].firstChild.nodeValue
+            urls.append(url)
+            vtype, ext, _size = url_info(url)
+            if not ext:
+                ext = match1(url,r'\.(\w+?)\?')
+            size += _size
+        #sina's result does not contains content-type
+        return urls, ext, size
+
+def catfun_download(url, output_dir = '.', merge = True, info_only = False):
+    # html = get_content(url)
+    title = match1(get_content(url), r'<h1 class="title">(.+?)</h1>')
+    vid = match1(url, r"v\d+/cat(\d+)")
+    j = json.loads(get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_video&modelid=11&id={}".format(vid)))
+    for item in j:
+        if item["name"] != "\u672a\u547d\u540d1":
+            t = title + "-" + item["name"]
+        else:
+            t = title
+        if item["type"] == "tudou":
+            tudou_download_by_id(item["vid"], title, output_dir, merge, info_only)
+
+        else:
+            urls, ext, size = parse_item(item)
+
+            print_info(site_info, title, ext, size)
+            if not info_only:
+                download_urls(urls, t, ext, size, output_dir, merge=merge)
+
+site_info = "CatFun.tv"
+download = catfun_download
+download_playlist = playlist_not_supported('catfun')

src/you_get/extractors/sohu.py

@@ -19,14 +19,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
         vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
     assert vid
 
-    # Open Sogou proxy if required
-    if get_sogou_proxy() is not None:
-        server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w'))
-        server_thread = threading.Thread(target=server.serve_forever)
-        server_thread.daemon = True
-        server_thread.start()
-        set_proxy(server.server_address)
-
     if re.match(r'http://tv.sohu.com/', url):
         data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
         for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
@@ -58,11 +50,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
             urls.append(real_url(host, prot, file, new))
         assert data['clipsURL'][0].endswith('.mp4')
 
-    # Close Sogou proxy if required
-    if get_sogou_proxy() is not None:
-        server.shutdown()
-        unset_proxy()
-
     print_info(site_info, title, 'mp4', size)
     if not info_only:
         download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)

src/you_get/extractors/youku.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from ..common import *
+from ..extractor import VideoExtractor
 
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"
@@ -63,6 +64,11 @@ class Youku(VideoExtractor):
 
         self.title = metadata0['title']
 
+        if 'dvd' in metadata0 and 'audiolang' in metadata0['dvd']:
+            self.audiolang = metadata0['dvd']['audiolang']
+            for i in self.audiolang:
+                i['url'] = 'http://v.youku.com/v_show/id_{}'.format(i['vid'])
+
         for stream_type in self.stream_types:
             if stream_type['id'] in metadata0['streamsizes']:
                 stream_id = stream_type['id']

src/you_get/extractors/youtube.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from ..common import *
+from ..extractor import VideoExtractor
 
 class YouTube(VideoExtractor):
     name = "YouTube"

src/you_get/util/__init__.py

@@ -1,6 +0,0 @@
-#!/usr/bin/env python
-
-from .fs import *
-from .log import *
-from .sogou_proxy import *
-from .strings import *

src/you_get/util/git.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import os
+
+def get_head(repo_path):
+    """Get (branch, commit) from HEAD of a git repo."""
+    try:
+        ref = open(os.path.join(repo_path, '.git', 'HEAD'), 'r').read().strip()[5:].split('/')
+        branch = ref[-1]
+        commit = open(os.path.join(repo_path, '.git', *ref), 'r').read().strip()[:7]
+        return branch, commit
+    except:
+        return None

src/you_get/util/log.py

@@ -1,130 +1,97 @@
 #!/usr/bin/env python
+# This file is Python 2 compliant.
 
-from ..version import __name__
+from .. import __name__ as library_name
 
-import os, sys, subprocess
+import os, sys
 
-# Is terminal ANSI/VT100 compatible
+IS_ANSI_TERMINAL = os.getenv('TERM') in (
-if os.getenv('TERM') in (
+    'eterm-color',
-        'xterm',
+    'linux',
-        'vt100',
+    'screen',
-        'linux',
+    'vt100',
-        'eterm-color',
+    'xterm')
-        'screen',
-    ):
-    has_colors = True
-else:
-    try:
-        # Eshell
-        ppid = os.getppid()
-        has_colors = (subprocess.getoutput('ps -p %d -ocomm=' % ppid)
-                      == 'emacs')
-    except:
-        has_colors = False
 
-# ANSI/VT100 escape code
+# ANSI escape code
-# http://en.wikipedia.org/wiki/ANSI_escape_code
+# See <http://en.wikipedia.org/wiki/ANSI_escape_code>
-colors = {
+RESET = 0
-    'none': '',
+BOLD = 1
-    'reset': '\033[0m',
+UNDERLINE = 4
+NEGATIVE = 7
+NO_BOLD = 21
+NO_UNDERLINE = 24
+POSITIVE = 27
+BLACK = 30
+RED = 31
+GREEN = 32
+YELLOW = 33
+BLUE = 34
+MAGENTA = 35
+CYAN = 36
+LIGHT_GRAY = 37
+DEFAULT = 39
+BLACK_BACKGROUND = 40
+RED_BACKGROUND = 41
+GREEN_BACKGROUND = 42
+YELLOW_BACKGROUND = 43
+BLUE_BACKGROUND = 44
+MAGENTA_BACKGROUND = 45
+CYAN_BACKGROUND = 46
+LIGHT_GRAY_BACKGROUND = 47
+DEFAULT_BACKGROUND = 49
+DARK_GRAY = 90                 # xterm
+LIGHT_RED = 91                 # xterm
+LIGHT_GREEN = 92               # xterm
+LIGHT_YELLOW = 93              # xterm
+LIGHT_BLUE = 94                # xterm
+LIGHT_MAGENTA = 95             # xterm
+LIGHT_CYAN = 96                # xterm
+WHITE = 97                     # xterm
+DARK_GRAY_BACKGROUND = 100     # xterm
+LIGHT_RED_BACKGROUND = 101     # xterm
+LIGHT_GREEN_BACKGROUND = 102   # xterm
+LIGHT_YELLOW_BACKGROUND = 103  # xterm
+LIGHT_BLUE_BACKGROUND = 104    # xterm
+LIGHT_MAGENTA_BACKGROUND = 105 # xterm
+LIGHT_CYAN_BACKGROUND = 106    # xterm
+WHITE_BACKGROUND = 107         # xterm
 
-    'black': '\033[30m',
+def sprint(text, *colors):
-    'bold-black': '\033[30;1m',
+    """Format text with color or other effects into ANSI escaped string."""
-    'dark-gray': '\033[90m',
+    return "\33[{}m{content}\33[{}m".format(";".join([str(color) for color in colors]), RESET, content=text) if IS_ANSI_TERMINAL and colors else text
-    'bold-dark-gray': '\033[90;1m',
 
-    'red': '\033[31m',
+def println(text, *colors):
-    'bold-red': '\033[31;1m',
+    """Print text to standard output."""
-    'light-red': '\033[91m',
+    sys.stdout.write(sprint(text, *colors) + "\n")
-    'bold-light-red': '\033[91;1m',
 
-    'green': '\033[32m',
+def print_err(text, *colors):
-    'bold-green': '\033[32;1m',
+    """Print text to standard error."""
-    'light-green': '\033[92m',
+    sys.stderr.write(sprint(text, *colors) + "\n")
-    'bold-light-green': '\033[92;1m',
 
-    'yellow': '\033[33m',
+def print_log(text, *colors):
-    'bold-yellow': '\033[33;1m',
+    """Print a log message to standard error."""
-    'light-yellow': '\033[93m',
+    sys.stderr.write(sprint("{}: {}".format(library_name, text), *colors) + "\n")
-    'bold-light-yellow': '\033[93;1m',
 
-    'blue': '\033[34m',
+def i(message):
-    'bold-blue': '\033[34;1m',
+    """Print a normal log message."""
-    'light-blue': '\033[94m',
+    print_log(message)
-    'bold-light-blue': '\033[94;1m',
 
-    'magenta': '\033[35m',
+def d(message):
-    'bold-magenta': '\033[35;1m',
+    """Print a debug log message."""
-    'light-magenta': '\033[95m',
+    print_log(message, BLUE)
-    'bold-light-magenta': '\033[95;1m',
 
-    'cyan': '\033[36m',
+def w(message):
-    'bold-cyan': '\033[36;1m',
+    """Print a warning log message."""
-    'light-cyan': '\033[96m',
+    print_log(message, YELLOW)
-    'bold-light-cyan': '\033[96;1m',
 
-    'light-gray': '\033[37m',
+def e(message, exit_code=None):
-    'bold-light-gray': '\033[37;1m',
+    """Print an error log message."""
-    'white': '\033[97m',
+    print_log(message, YELLOW, BOLD)
-    'bold-white': '\033[97;1m',
-}
-
-def underlined(text):
-    """Returns an underlined text.
-    """
-    return "\33[4m%s\33[24m" % text if has_colors else text
-
-def println(text, color=None, ostream=sys.stdout):
-    """Prints a text line to stream.
-    """
-    if has_colors and color in colors:
-        ostream.write("{0}{1}{2}\n".format(colors[color], text, colors['reset']))
-    else:
-        ostream.write("{0}\n".format(text))
-
-def printlog(message, color=None, ostream=sys.stderr):
-    """Prints a log message to stream.
-    """
-    if has_colors and color in colors:
-        ostream.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset']))
-    else:
-        ostream.write("{0}: {1}\n".format(__name__, message))
-
-def i(message, ostream=sys.stderr):
-    """Sends an info log message.
-    """
-    printlog(message,
-             None,
-             ostream=ostream)
-
-def d(message, ostream=sys.stderr):
-    """Sends a debug log message.
-    """
-    printlog(message,
-             'blue' if has_colors else None,
-             ostream=ostream)
-
-def w(message, ostream=sys.stderr):
-    """Sends a warning log message.
-    """
-    printlog(message,
-             'yellow' if has_colors else None,
-             ostream=ostream)
-
-def e(message, ostream=sys.stderr, exit_code=None):
-    """Sends an error log message.
-    """
-    printlog(message,
-             'bold-yellow' if has_colors else None,
-             ostream=ostream)
     if exit_code is not None:
         exit(exit_code)
 
-def wtf(message, ostream=sys.stderr, exit_code=-1):
+def wtf(message, exit_code=-1):
-    """What a Terrible Failure.
+    """What a Terrible Failure!"""
-    """
+    print_log(message, RED, BOLD)
-    printlog(message,
-             'bold-red' if has_colors else None,
-             ostream=ostream)
     if exit_code is not None:
         exit(exit_code)

src/you_get/util/sogou_proxy.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-
-# Original code from:
-# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/
-
-from . import log
-
-from http.client import HTTPResponse
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from socketserver import ThreadingMixIn
-from threading import Thread
-import random, socket, struct, sys, time
-
-def sogou_proxy_server(
-    host=("0.0.0.0", 0),
-    network_env='CERNET',
-    ostream=sys.stderr):
-    """Returns a Sogou proxy server object.
-    """
-
-    x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27'
-    proxy_host = 'h0.cnc.bj.ie.sogou.com'
-    proxy_port = 80
-
-    def sogou_hash(t, host):
-        s = (t + host + 'SogouExplorerProxy').encode('ascii')
-        code = len(s)
-        dwords = int(len(s) / 4)
-        rest = len(s) % 4
-        v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s)
-        for vv in v:
-            if type(vv) != bytes:
-                a = (vv & 0xFFFF)
-                b = (vv >> 16)
-                code += a
-                code = code ^ (((code << 5) ^ b) << 0xb)
-                # To avoid overflows
-                code &= 0xffffffff
-                code += code >> 0xb
-        if rest == 3:
-            code += s[len(s) - 2] * 256 + s[len(s) - 3]
-            code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10)
-            code &= 0xffffffff
-            code += code >> 0xb
-        elif rest == 2:
-            code += (s[len(s) - 1]) * 256 + (s[len(s) - 2])
-            code ^= code << 0xb
-            code &= 0xffffffff
-            code += code >> 0x11
-        elif rest == 1:
-            code += s[len(s) - 1]
-            code ^= code << 0xa
-            code &= 0xffffffff
-            code += code >> 0x1
-        code ^= code * 8
-        code &= 0xffffffff
-        code += code >> 5
-        code ^= code << 4
-        code = code & 0xffffffff
-        code += code >> 0x11
-        code ^= code << 0x19
-        code = code & 0xffffffff
-        code += code >> 6
-        code = code & 0xffffffff
-        return hex(code)[2:].rstrip('L').zfill(8)
-
-    class Handler(BaseHTTPRequestHandler):
-        _socket = None
-        def do_proxy(self):
-            try:
-                if self._socket is None:
-                    self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                    self._socket.connect((proxy_host, proxy_port))
-                self._socket.send(self.requestline.encode('ascii') + b'\r\n')
-                log.d(self.requestline, ostream)
-
-                # Add Sogou Verification Tags
-                self.headers['X-Sogou-Auth'] = x_sogou_auth
-                t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
-                self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host'])
-                self.headers['X-Sogou-Timestamp'] = t
-                self._socket.send(str(self.headers).encode('ascii') + b'\r\n')
-
-                # Send POST data
-                if self.command == 'POST':
-                    self._socket.send(self.rfile.read(int(self.headers['Content-Length'])))
-                response = HTTPResponse(self._socket, method=self.command)
-                response.begin()
-
-                # Response
-                status = 'HTTP/1.1 %s %s' % (response.status, response.reason)
-                self.wfile.write(status.encode('ascii') + b'\r\n')
-                h = ''
-                for hh, vv in response.getheaders():
-                    if hh.upper() != 'TRANSFER-ENCODING':
-                        h += hh + ': ' + vv + '\r\n'
-                self.wfile.write(h.encode('ascii') + b'\r\n')
-                while True:
-                    response_data = response.read(8192)
-                    if len(response_data) == 0:
-                        break
-                    self.wfile.write(response_data)
-
-            except socket.error:
-                log.e('Socket error for ' + self.requestline, ostream)
-
-        def do_POST(self):
-            self.do_proxy()
-
-        def do_GET(self):
-            self.do_proxy()
-
-    class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
-        pass
-
-    # Server starts
-    log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream)
-
-    try:
-        server = ThreadingHTTPServer(host, Handler)
-    except Exception as ex:
-        log.wtf("Socket error: %s" % ex, ostream)
-        exit(1)
-    host = server.server_address
-
-    if network_env.upper() == 'CERNET':
-        proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
-    elif network_env.upper() == 'CTCNET':
-        proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3)
-    elif network_env.upper() == 'CNCNET':
-        proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3)
-    elif network_env.upper() == 'DXT':
-        proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10)
-    else:
-        proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
-
-    log.i('Remote host: %s' % log.underlined(proxy_host), ostream)
-    log.i('Proxy server running on %s' %
-        log.underlined("%s:%s" % host), ostream)
-
-    return server

src/you_get/version.py

@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-__all__ = ['__version__', '__date__']
 
-__name__ = 'you-get'
+script_name = 'you-get'
-__version__ = '0.3.30dev-20140716'
+__version__ = '0.3.30dev'
-__date__ = '2014-07-16'

tests/test.py

@@ -1,47 +1,31 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
 import unittest
 
 from you_get import *
-from you_get.extractor.__main__ import url_to_module
+from you_get.extractors import *
-
+from you_get.common import *
-def test_urls(urls):
-    for url in urls:
-        url_to_module(url)[0].download(url, info_only = True)
 
 class YouGetTests(unittest.TestCase):
 
     def test_freesound(self):
-        test_urls([
+        freesound.download("http://www.freesound.org/people/Corsica_S/sounds/184419/", info_only=True)
-            "http://www.freesound.org/people/Corsica_S/sounds/184419/",
-        ])
 
     def test_magisto(self):
-        test_urls([
+        magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA", info_only=True)
-            "http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA",
-        ])
 
     def test_mixcloud(self):
-        test_urls([
+        mixcloud.download("http://www.mixcloud.com/beatbopz/beat-bopz-disco-mix/", info_only=True)
-            "http://www.mixcloud.com/beatbopz/beat-bopz-disco-mix/",
+        mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)
-            "http://www.mixcloud.com/DJVadim/north-america-are-you-ready/",
-        ])
 
     def test_ted(self):
-        test_urls([
+        ted.download("http://www.ted.com/talks/jennifer_lin_improvs_piano_magic.html", info_only=True)
-            "http://www.ted.com/talks/jennifer_lin_improvs_piano_magic.html",
+        ted.download("http://www.ted.com/talks/derek_paravicini_and_adam_ockelford_in_the_key_of_genius.html", info_only=True)
-            "http://www.ted.com/talks/derek_paravicini_and_adam_ockelford_in_the_key_of_genius.html",
-        ])
 
     def test_vimeo(self):
-        test_urls([
+        vimeo.download("http://vimeo.com/56810854", info_only=True)
-            "http://vimeo.com/56810854",
-        ])
 
     def test_youtube(self):
-        test_urls([
+        youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
-            "http://www.youtube.com/watch?v=pzKerr0JIPA",
+        youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)
-            "http://youtu.be/pzKerr0JIPA",
+        youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare", info_only=True)
-            "http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare"
-        ])

tests/test_common.py

@@ -2,7 +2,7 @@
 
 import unittest
 
-from you_get import *
+from you_get.common import *
 
 class TestCommon(unittest.TestCase):
     

tests/test_util.py

@@ -2,7 +2,7 @@
 
 import unittest
 
-from you_get.util import *
+from you_get.util.fs import *
 
 class TestUtil(unittest.TestCase):
     def test_legitimize(self):

you-get

@@ -1,10 +1,18 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
+# This file is Python 2 compliant.
 
 import os, sys
-__path__ = os.path.dirname(os.path.realpath(__file__))
-__srcdir__ = 'src'
-sys.path.insert(1, os.path.join(__path__, __srcdir__))
-from you_get.extractor import main
 
-if __name__ == '__main__':
+_srcdir = 'src/'
-    main()
+_filepath = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(1, os.path.join(_filepath, _srcdir))
+
+if sys.version_info[0] == 3:
+    import you_get
+    if __name__ == '__main__':
+        you_get.main(repo_path=_filepath)
+else:
+    from you_get.util import log
+    log.wtf("""
+    [Fatal] Python 3 is required.
+    If Python 3 is already installed on your machine, try to run this script using 'python3 you-get'.""")

you-get.json

@@ -4,10 +4,10 @@
     "author_email": "mort.yao@gmail.com",
     "url": "http://www.soimort.org/you-get/",
     "license": "MIT",
-    
+
     "description": "A YouTube/Youku/Niconico video downloader written in Python 3.",
     "keywords": "video download youtube youku niconico",
-    
+
     "classifiers": [
         "Development Status :: 2 - Pre-Alpha",
         "Environment :: Console",
@@ -23,14 +23,15 @@
         "Programming Language :: Python :: 3.1",
         "Programming Language :: Python :: 3.2",
         "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
         "Topic :: Internet",
         "Topic :: Internet :: WWW/HTTP",
         "Topic :: Multimedia",
         "Topic :: Multimedia :: Video",
         "Topic :: Utilities"
         ],
-    
+
     "console_scripts": [
-        "you-get = you_get.extractor.__main__:main"
+        "you-get = you_get.__main__:main"
         ]
 }