Merge remote-tracking branch 'upstream/master' into upstream

CHANGELOG.txt

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+0.3.6
+-----
+
+*Date: 2013-03-22*
+
+* Add support for:
+    - Vine
+* Fix issue for:
+    - YouTube
+
 0.3.5
 -----
 

README.md

@@ -19,6 +19,7 @@ Fork me on GitHub: <https://github.com/soimort/you-get>
 * Facebook <http://facebook.com>
 * Google+ <http://plus.google.com>
 * Tumblr <http://www.tumblr.com>
+* Vine <http://vine.co>
 * SoundCloud <http://soundcloud.com>
 * Mixcloud <http://www.mixcloud.com>
 * JPopsuki <http://jpopsuki.tv>
@@ -107,7 +108,13 @@ Fork me on GitHub: <https://github.com/soimort/you-get>
 
    Click [here](https://aur.archlinux.org/packages.php\?ID=62576).
 
-### FAQ (For Windows Users)
+### Upgrading:
+
+Using Pip:
+
+    $ pip install --upgrade you-get
+
+### FAQ (For Windows Users):
 
 * Q: I don't know how to install it on Windows.
 
@@ -117,26 +124,6 @@ Fork me on GitHub: <https://github.com/soimort/you-get>
 
 * A: Run `set PYTHONIOENCODING=utf-8`.
 
-## Upgrading
-
-Using Pip:
-
-    $ pip install --upgrade you-get
-
-### Error When Upgrading from Pip
-
-If you see this error:
-
-```
-  File "/usr/lib/python3.3/site-packages/pip-1.2.1-py3.3.egg/pip/backwardcompat.py", line 44, in u
-    return s.decode('utf-8')
-UnicodeDecodeError: 'utf-8' codec can't decode byte 0xae in position 68: invalid start byte
-```
-
-This is an existing bug in Pip 1.2.1. However, this does not affect your upgrading.
-
-In Pip 1.3+, this should be already fixed.
-
 ## Examples (For End-Users)
 
 Display the information of the video without downloading:
@@ -243,6 +230,7 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y
 * Facebook <http://facebook.com>
 * Google+ <http://plus.google.com>
 * Tumblr <http://www.tumblr.com>
+* Vine <http://vine.co>
 * SoundCloud <http://soundcloud.com>
 * Mixcloud <http://www.mixcloud.com>
 * JPopsuki <http://jpopsuki.tv>
@@ -333,7 +321,13 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y
 
    点击[这里](https://aur.archlinux.org/packages.php\?ID=62576)。
 
-### FAQ（针对Windows用户)
+### 升级：
+
+使用Pip：
+
+    $ pip install --upgrade you-get
+
+### FAQ（针对Windows用户)：
 
 * Q：我不知道该如何在Windows下安装。
 
@@ -343,26 +337,6 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y
 
 * A：执行`set PYTHONIOENCODING=utf-8`。
 
-## 升级
-
-使用Pip：
-
-    $ pip install --upgrade you-get
-
-### 从Pip升级时可能的错误
-
-若出现以下错误提示：
-
-```
-  File "/usr/lib/python3.3/site-packages/pip-1.2.1-py3.3.egg/pip/backwardcompat.py", line 44, in u
-    return s.decode('utf-8')
-UnicodeDecodeError: 'utf-8' codec can't decode byte 0xae in position 68: invalid start byte
-```
-
-这被证实是Pip 1.2.1的一个bug。不过，它并不影响到正常的升级。
-
-这在Pip 1.3+中应当已经被修复。
-
 ## 使用方法示例
 
 ### 如何下载视频

README.txt

@@ -22,6 +22,7 @@ Supported Sites (As of Now)
 * Facebook http://facebook.com
 * Google+ http://plus.google.com
 * Tumblr http://www.tumblr.com
+* Vine http://vine.co
 * SoundCloud http://soundcloud.com
 * Mixcloud http://www.mixcloud.com
 * JPopsuki http://jpopsuki.tv

src/you_get/__main__.py

@@ -48,6 +48,7 @@ def url_to_module(url):
         'tumblr': tumblr,
         'vid48': vid48,
         'vimeo': vimeo,
+        'vine': vine,
         'xiami': xiami,
         'yinyuetai': yinyuetai,
         'youku': youku,

src/you_get/downloader/__init__.py

@@ -26,6 +26,7 @@ from .tudou import *
 from .tumblr import *
 from .vid48 import *
 from .vimeo import *
+from .vine import *
 from .w56 import *
 from .xiami import *
 from .yinyuetai import *

src/you_get/downloader/vine.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+__all__ = ['vine_download']
+
+from ..common import *
+
+def vine_download(url, output_dir = '.', merge = True, info_only = False):
+    html = get_html(url)
+    
+    title = r1(r'<meta property="og:title" content="([^"]*)"', html)
+    url = r1(r'<source src="([^"]*)"', html)
+    type, ext, size = url_info(url)
+    
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls([url], title, ext, size, output_dir, merge = merge)
+
+site_info = "Vine.co"
+download = vine_download
+download_playlist = playlist_not_supported('vine')

src/you_get/downloader/youtube.py

@@ -4,13 +4,99 @@ __all__ = ['youtube_download', 'youtube_download_by_id']
 
 from ..common import *
 
-import json
+# YouTube media encoding options, in descending quality order.
+# taken from http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs, 3/22/2013.
+youtube_codecs = [
+    {'itag': 38, 'container': 'MP4', 'video_resolution': '3072p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3.5-5', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
+    {'itag': 46, 'container': 'WebM', 'video_resolution': '1080p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
+    {'itag': 37, 'container': 'MP4', 'video_resolution': '1080p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3-4.3', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
+    {'itag': 102, 'container': '', 'video_resolution': '', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '2', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
+    {'itag': 45, 'container': 'WebM', 'video_resolution': '720p', 'video_encoding': '', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': '', 'audio_bitrate': ''},
+    {'itag': 22, 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
+    {'itag': 84, 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '152'},
+    {'itag': 120, 'container': 'FLV', 'video_resolution': '720p', 'video_encoding': 'AVC', 'video_profile': 'Main@L3.1', 'video_bitrate': '2', 'audio_encoding': 'AAC', 'audio_bitrate': '128'},
+    {'itag': 85, 'container': 'MP4', 'video_resolution': '520p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '152'},
+    {'itag': 44, 'container': 'WebM', 'video_resolution': '480p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '1', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
+    {'itag': 35, 'container': 'FLV', 'video_resolution': '480p', 'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.8-1', 'audio_encoding': 'AAC', 'audio_bitrate': '128'},
+    {'itag': 101, 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
+    {'itag': 100, 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
+    {'itag': 43, 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '0.5', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
+    {'itag': 34, 'container': 'FLV', 'video_resolution': '360p', 'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '128'},
+    {'itag': 82, 'container': 'MP4', 'video_resolution': '360p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'},
+    {'itag': 18, 'container': 'MP4', 'video_resolution': '270p/360p', 'video_encoding': 'H.264', 'video_profile': 'Baseline', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'},
+    {'itag': 6, 'container': 'FLV', 'video_resolution': '270p', 'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.8', 'audio_encoding': 'MP3', 'audio_bitrate': '64'},
+    {'itag': 83, 'container': 'MP4', 'video_resolution': '240p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'},
+    {'itag': 13, 'container': '3GP', 'video_resolution': '', 'video_encoding': 'MPEG-4 Visual', 'video_profile': '', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': ''},
+    {'itag': 5, 'container': 'FLV', 'video_resolution': '240p', 'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.25', 'audio_encoding': 'MP3', 'audio_bitrate': '64'},
+    {'itag': 36, 'container': '3GP', 'video_resolution': '240p', 'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.17', 'audio_encoding': 'AAC', 'audio_bitrate': '38'},
+    {'itag': 17, 'container': '3GP', 'video_resolution': '144p', 'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.05', 'audio_encoding': 'AAC', 'audio_bitrate': '24'},
+]
+
+def parse_video_info(raw_info):
+    """Parser for YouTube's get_video_info data.
+    Returns a dict, where 'url_encoded_fmt_stream_map' maps to a sorted list.
+    """
+    
+    # Percent-encoding reserved characters, used as separators.
+    sepr = {
+        '&': '%26',
+        ',': '%2C',
+        '=': '%3D',
+    }
+    
+    # fmt_level = {'itag': level, ...}
+    # itag of a higher quality maps to a lower level number.
+    # The highest quality has level number 0.
+    fmt_level = dict(
+        zip(
+            [str(codec['itag'])
+                for codec in
+                    youtube_codecs],
+            range(len(youtube_codecs))))
+    
+    # {key1: value1, key2: value2, ...,
+    #   'url_encoded_fmt_stream_map': [{'itag': '38', ...}, ...]
+    # }
+    return dict(
+        [(lambda metadata:
+            ['url_encoded_fmt_stream_map', (
+                lambda stream_map:
+                    sorted(
+                        [dict(
+                            [subitem.split(sepr['='])
+                                for subitem in
+                                    item.split(sepr['&'])])
+                            for item in
+                                stream_map.split(sepr[','])],
+                        key =
+                            lambda stream:
+                                fmt_level[stream['itag']]))
+                (metadata[1])]
+            if metadata[0] == 'url_encoded_fmt_stream_map'
+            else metadata)
+        (item.split('='))
+            for item in
+                raw_info.split('&')])
 
 def youtube_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
-    html = request.urlopen('http://www.youtube.com/watch?v=' + id).read().decode('utf-8')
     
+    raw_info = request.urlopen('http://www.youtube.com/get_video_info?video_id=%s' % id).read().decode('utf-8')
+    
+    video_info = parse_video_info(raw_info)
+    
+    if video_info['status'] == 'ok': # use get_video_info data
+        
+        title = parse.unquote(video_info['title'].replace('+', ' '))
+        
+        signature = video_info['url_encoded_fmt_stream_map'][0]['sig']
+        url = parse.unquote(parse.unquote(video_info['url_encoded_fmt_stream_map'][0]['url'])) + "&signature=%s" % signature
+        
+    else: # parse video page when "embedding disabled by request"
+        
+        import json
+        html = request.urlopen('http://www.youtube.com/watch?v=' + id).read().decode('utf-8')
         html = unescape_html(html)
-    yt_player_config = json.loads(r1(r'yt.playerConfig = ([^\n]+);\n', html))
+        yt_player_config = json.loads(r1(r'ytplayer.config = ([^\n]+);', html))
         title = yt_player_config['args']['title']
         title = unicodize(title)
         title = parse.unquote(title)
@@ -25,10 +111,7 @@ def youtube_download_by_id(id, title = None, output_dir = '.', merge = True, inf
             '85',
             '44', '35',
             '101', '100', '43', '34', '82', '18',
-        '6',
+            '6', '83', '13', '5', '36', '17',
-        '83', '5', '36',
-        '17',
-        '13',
         ]:
             fmt = r1(r'([^,\"]*itag=' + itag + "[^,\"]*)", html)
             if fmt:
@@ -41,7 +124,7 @@ def youtube_download_by_id(id, title = None, output_dir = '.', merge = True, inf
         try:
             url
         except NameError:
-        url = r1(r'crossdomain.xml"\);yt.preload.start\("([^"]+)"\)', html)
+            url = r1(r'ytdns.ping\("([^"]+)"[^;]*;</script>', html)
             url = unicodize(url)
             url = re.sub(r'\\/', '/', url)
             url = re.sub(r'generate_204', 'videoplayback', url)

src/you_get/version.py

@@ -2,5 +2,5 @@
 
 __all__ = ['__version__', '__date__']
 
-__version__ = '0.3.5'
+__version__ = '0.3.6'
-__date__ = '2013-03-15'
+__date__ = '2013-03-22'