Merge pull request #1 from soimort/develop

Update as Oct.21

src/you_get/__main__.py

@@ -20,6 +20,7 @@ _help = """Usage: {} [OPTION]... [URL]...
 TODO
 """.format(script_name)
 
+# TBD
 def main_dev(**kwargs):
     """Main entry point.
     you-get-dev
@@ -88,7 +89,7 @@ def main(**kwargs):
     you-get (legacy)
     """
     from .common import main
-    main()
+    main(**kwargs)
 
 if __name__ == '__main__':
     main()

src/you_get/common.py

@@ -91,6 +91,7 @@ from importlib import import_module
 
 from .version import __version__
 from .util import log, term
+from .util.git import get_version
 from .util.strings import get_filename, unescape_html
 from . import json_output as json_output_
 
@@ -981,8 +982,11 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         else:
             download(url, **kwargs)
 
-def script_main(script_name, download, download_playlist = None):
-    version = 'You-Get %s, a video downloader.' % __version__
+def script_main(script_name, download, download_playlist, **kwargs):
+    def version():
+        log.i('version %s' % get_version(kwargs['repo_path']
+            if 'repo_path' in kwargs else __version__))
+
     help = 'Usage: %s [OPTION]... [URL]...\n' % script_name
     help += '''\nStartup options:
     -V | --version                           Display the version and exit.
@@ -1035,10 +1039,10 @@ def script_main(script_name, download, download_playlist = None):
     traceback = False
     for o, a in opts:
         if o in ('-V', '--version'):
-            print(version)
+            version()
             sys.exit()
         elif o in ('-h', '--help'):
-            print(version)
+            version()
             print(help)
             sys.exit()
         elif o in ('-f', '--force'):
@@ -1125,10 +1129,29 @@ def script_main(script_name, download, download_playlist = None):
         else:
             sys.exit(1)
 
+def google_search(url):
+    keywords = r1(r'https?://(.*)', url)
+    url = 'https://www.google.com/search?tbm=vid&q=%s' % parse.quote(keywords)
+    page = get_content(url, headers=fake_headers)
+    videos = re.findall(r'<a href="([^"]+)" onmousedown="[^"]+">([^<]+)<', page)
+    durs = re.findall(r'<span class="vdur _dwc">[^<]+(\d+:\d+)', page)
+    print("Google Videos search:")
+    for v in zip(videos, durs):
+        print("- video:  %s [%s]" % (unescape_html(v[0][1]), v[1]))
+        print("# you-get %s" % log.sprint(v[0][0], log.UNDERLINE))
+        print()
+    print("Best matched result:")
+    return(videos[0][0])
+
 def url_to_module(url):
+    try:
+        video_host = r1(r'https?://([^/]+)/', url)
+        video_url = r1(r'https?://[^/]+(.*)', url)
+        assert video_host and video_url
+    except:
+        url = google_search(url)
         video_host = r1(r'https?://([^/]+)/', url)
         video_url = r1(r'https?://[^/]+(.*)', url)
-    assert video_host and video_url, 'invalid url: ' + url
 
     if video_host.endswith('.com.cn'):
         video_host = video_host[:-3]
@@ -1157,5 +1180,5 @@ def any_download_playlist(url, **kwargs):
     m, url = url_to_module(url)
     m.download_playlist(url, **kwargs)
 
-def main():
-    script_main('you-get', any_download, any_download_playlist)
+def main(**kwargs):
+    script_main('you-get', any_download, any_download_playlist, **kwargs)

src/you_get/extractors/embed.py

@@ -21,7 +21,8 @@ youku_embed_patterns = [ 'youku\.com/v_show/id_([a-zA-Z0-9=]+)',
 """
 http://www.tudou.com/programs/view/html5embed.action?type=0&code=3LS_URGvl54&lcode=&resourceId=0_06_05_99
 """
-tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([[a-zA-Z0-9_]+)\&'
+tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([[a-zA-Z0-9_]+)\&',
+                         'www\.tudou\.com/v/([[a-zA-Z0-9_]+)/[^"]*v\.swf'
                        ]
 
 """

src/you_get/extractors/miomio.py

@@ -4,9 +4,9 @@ __all__ = ['miomio_download']
 
 from ..common import *
 
-from .sina import sina_download_by_xml
 from .tudou import tudou_download_by_id
 from .youku import youku_download_by_vid
+from xml.dom.minidom import parseString
 
 def miomio_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_html(url)
@@ -20,13 +20,35 @@ def miomio_download(url, output_dir = '.', merge = True, info_only = False, **kw
         youku_download_by_vid(id, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif t == 'tudou':
         tudou_download_by_id(id, title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'sina' or t=='video':
+    elif t == 'sina' or t == 'video':
         url = "http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?vid=" + id
-        xml = get_content (url, headers=fake_headers, decoded=True)
-        sina_download_by_xml(xml, title, output_dir=output_dir, merge=merge, info_only=info_only)
+        xml_data = get_content(url, headers=fake_headers, decoded=True)
+        url_list = sina_xml_to_url_list(xml_data)
+        
+        size_full = 0
+        for url in url_list:
+            type_, ext, size = url_info(url)
+            size_full += size
+        
+        print_info(site_info, title, type_, size_full)
+        if not info_only:
+            download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)        
     else:
         raise NotImplementedError(flashvars)
 
+#----------------------------------------------------------------------
+def sina_xml_to_url_list(xml_data):
+    """str->list
+    Convert XML to URL List.
+    From Biligrab.
+    """
+    rawurl = []
+    dom = parseString(xml_data)
+    for node in dom.getElementsByTagName('durl'):
+        url = node.getElementsByTagName('url')[0]
+        rawurl.append(url.childNodes[0].data)
+    return rawurl
+
 site_info = "MioMio.tv"
 download = miomio_download
 download_playlist = playlist_not_supported('miomio')

src/you_get/util/git.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 
 import os
+import subprocess
+from ..version import __version__
 
 def get_head(repo_path):
     """Get (branch, commit) from HEAD of a git repo."""
@@ -11,3 +13,21 @@ def get_head(repo_path):
         return branch, commit
     except:
         return None
+
+def get_version(repo_path):
+    try:
+        version = __version__.split('.')
+        major, minor = version[0], version[1]
+
+        p = subprocess.Popen(['git', 'rev-list', 'HEAD', '--count'],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        raw, err = p.communicate()
+        c_head = int(raw.decode('ascii'))
+        q = subprocess.Popen(['git', 'rev-list', 'master', '--count'],
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        raw, err = q.communicate()
+        c_master = int(raw.decode('ascii'))
+        cc = c_head - c_master
+        return '%s.%s.%s' % (major, minor, cc)
+    except:
+        return __version__

src/you_get/util/log.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # This file is Python 2 compliant.
 
-from .. import __name__ as library_name
+from ..version import script_name
 
 import os, sys
 
@@ -10,7 +10,8 @@ IS_ANSI_TERMINAL = os.getenv('TERM') in (
     'linux',
     'screen',
     'vt100',
-    'xterm')
+    'xterm',
+)
 
 # ANSI escape code
 # See <http://en.wikipedia.org/wiki/ANSI_escape_code>
@@ -70,7 +71,7 @@ def print_err(text, *colors):
 
 def print_log(text, *colors):
     """Print a log message to standard error."""
-    sys.stderr.write(sprint("{}: {}".format(library_name, text), *colors) + "\n")
+    sys.stderr.write(sprint("{}: {}".format(script_name, text), *colors) + "\n")
 
 def i(message):
     """Print a normal log message."""

src/you_get/version.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.3.36'
+__version__ = '0.4.0'

you-get

@@ -1,6 +1,4 @@
 #!/usr/bin/env python
-# This file is Python 2 compliant.
-
 import os, sys
 
 _srcdir = 'src/'
@@ -17,8 +15,7 @@ if sys.version_info[0] == 3:
     import you_get
     if __name__ == '__main__':
         you_get.main(repo_path=_filepath)
-else:
+else: # Python 2
     from you_get.util import log
-    log.wtf("""
-    [Fatal] Python 3 is required.
-    If Python 3 is already installed on your machine, try to run this script using 'python3 you-get'.""")
+    log.e("[fatal] Python 3 is required!")
+    log.wtf("try to run this script using 'python3 you-get'.")