From df740c10e2e8d9c8b268d44be2ecc935cd753bdb Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 22 Mar 2013 04:24:01 +0100 Subject: [PATCH 1/5] YouTube: fix #135 --- src/you_get/downloader/youtube.py | 151 +++++++++++++++++++++++------- 1 file changed, 115 insertions(+), 36 deletions(-) diff --git a/src/you_get/downloader/youtube.py b/src/you_get/downloader/youtube.py index f398e466..67f2096c 100644 --- a/src/you_get/downloader/youtube.py +++ b/src/you_get/downloader/youtube.py @@ -4,47 +4,126 @@ __all__ = ['youtube_download', 'youtube_download_by_id'] from ..common import * -import json +# YouTube media encoding options, in descending quality order. +# taken from http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs, 3/22/2013. +youtube_codecs = [ + {'itag': 38, 'container': 'MP4', 'video_resolution': '3072p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3.5-5', 'audio_encoding': 'AAC', 'audio_bitrate': '192'}, + {'itag': 46, 'container': 'WebM', 'video_resolution': '1080p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'}, + {'itag': 37, 'container': 'MP4', 'video_resolution': '1080p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3-4.3', 'audio_encoding': 'AAC', 'audio_bitrate': '192'}, + {'itag': 102, 'container': '', 'video_resolution': '', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '2', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'}, + {'itag': 45, 'container': 'WebM', 'video_resolution': '720p', 'video_encoding': '', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': '', 'audio_bitrate': ''}, + {'itag': 22, 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '192'}, + {'itag': 84, 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '152'}, + {'itag': 120, 'container': 'FLV', 'video_resolution': '720p', 'video_encoding': 'AVC', 'video_profile': 'Main@L3.1', 'video_bitrate': '2', 'audio_encoding': 'AAC', 'audio_bitrate': '128'}, + {'itag': 85, 'container': 'MP4', 'video_resolution': '520p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '152'}, + {'itag': 44, 'container': 'WebM', 'video_resolution': '480p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '1', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'}, + {'itag': 35, 'container': 'FLV', 'video_resolution': '480p', 'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.8-1', 'audio_encoding': 'AAC', 'audio_bitrate': '128'}, + {'itag': 101, 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'}, + {'itag': 100, 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'}, + {'itag': 43, 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '0.5', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'}, + {'itag': 34, 'container': 'FLV', 'video_resolution': '360p', 'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '128'}, + {'itag': 82, 'container': 'MP4', 'video_resolution': '360p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'}, + {'itag': 18, 'container': 'MP4', 'video_resolution': '270p/360p', 'video_encoding': 'H.264', 'video_profile': 'Baseline', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'}, + {'itag': 6, 'container': 'FLV', 'video_resolution': '270p', 'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.8', 'audio_encoding': 'MP3', 'audio_bitrate': '64'}, + {'itag': 83, 'container': 'MP4', 'video_resolution': '240p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'}, + {'itag': 13, 'container': '3GP', 'video_resolution': '', 'video_encoding': 'MPEG-4 Visual', 'video_profile': '', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': ''}, + {'itag': 5, 'container': 'FLV', 'video_resolution': '240p', 'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.25', 'audio_encoding': 'MP3', 'audio_bitrate': '64'}, + {'itag': 36, 'container': '3GP', 'video_resolution': '240p', 'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.17', 'audio_encoding': 'AAC', 'audio_bitrate': '38'}, + {'itag': 17, 'container': '3GP', 'video_resolution': '144p', 'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.05', 'audio_encoding': 'AAC', 'audio_bitrate': '24'}, +] + +def parse_video_info(raw_info): + """Parser for YouTube's get_video_info data. + Returns a map, with 'url_encoded_fmt_stream_map' field being a sorted list. + """ + + # Percent-encoding reserved characters, used as separators. + separator = { + '&': '%26', + '=': '%3D', + } + + # fmt_level = {'itag': level, ...} + # itag of a higher quality maps to a lower level number. + # The highest quality has level number 0. + fmt_level = dict( + zip( + [str(codec['itag']) + for codec in + youtube_codecs], + range(len(youtube_codecs)))) + + return dict( + [(lambda metadata: + ['url_encoded_fmt_stream_map', ( + lambda url_encoded_fmt_stream_map: + sorted( + [dict( + [sub_item.split(separator['=']) + for sub_item in + item.split(separator['&'])]) + for item in + url_encoded_fmt_stream_map.split('%2C')], + key = + lambda stream: + fmt_level[stream['itag']])) + (metadata[1])] + if metadata[0] == 'url_encoded_fmt_stream_map' + else metadata) + (item.split('=')) + for item in + raw_info.split('&')]) def youtube_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False): - html = request.urlopen('http://www.youtube.com/watch?v=' + id).read().decode('utf-8') - html = unescape_html(html) - yt_player_config = json.loads(r1(r'yt.playerConfig = ([^\n]+);\n', html)) - title = yt_player_config['args']['title'] - title = unicodize(title) - title = parse.unquote(title) - title = escape_file_path(title) + raw_info = request.urlopen('http://www.youtube.com/get_video_info?video_id=%s' % id).read().decode('utf-8') - for itag in [ - '38', - '46', '37', - '102', '45', '22', - '84', - '120', - '85', - '44', '35', - '101', '100', '43', '34', '82', '18', - '6', - '83', '5', '36', - '17', - '13', - ]: - fmt = r1(r'([^,\"]*itag=' + itag + "[^,\"]*)", html) - if fmt: - url = r1(r'url=([^\\]+)', fmt) + video_info = parse_video_info(raw_info) + + if video_info['status'] == 'ok': # use get_video_info data + + title = parse.unquote(video_info['title'].replace('+', ' ')) + + signature = video_info['url_encoded_fmt_stream_map'][0]['sig'] + url = parse.unquote(parse.unquote(video_info['url_encoded_fmt_stream_map'][0]['url'])) + "&signature=%s" % signature + + else: # parse video page when "embedding disabled by request" + + import json + html = request.urlopen('http://www.youtube.com/watch?v=' + id).read().decode('utf-8') + html = unescape_html(html) + yt_player_config = json.loads(r1(r'ytplayer.config = ([^\n]+);', html)) + title = yt_player_config['args']['title'] + title = unicodize(title) + title = parse.unquote(title) + title = escape_file_path(title) + + for itag in [ + '38', + '46', '37', + '102', '45', '22', + '84', + '120', + '85', + '44', '35', + '101', '100', '43', '34', '82', '18', + '6', '83', '13', '5', '36', '17', + ]: + fmt = r1(r'([^,\"]*itag=' + itag + "[^,\"]*)", html) + if fmt: + url = r1(r'url=([^\\]+)', fmt) + url = unicodize(url) + url = parse.unquote(url) + sig = r1(r'sig=([^\\]+)', fmt) + url = url + '&signature=' + sig + break + try: + url + except NameError: + url = r1(r'ytdns.ping\("([^"]+)"[^;]*;', html) url = unicodize(url) - url = parse.unquote(url) - sig = r1(r'sig=([^\\]+)', fmt) - url = url + '&signature=' + sig - break - try: - url - except NameError: - url = r1(r'crossdomain.xml"\);yt.preload.start\("([^"]+)"\)', html) - url = unicodize(url) - url = re.sub(r'\\/', '/', url) - url = re.sub(r'generate_204', 'videoplayback', url) + url = re.sub(r'\\/', '/', url) + url = re.sub(r'generate_204', 'videoplayback', url) type, ext, size = url_info(url) From 9027ed92e02a8d797c21bd5bce68f407ce4dea15 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 22 Mar 2013 04:31:28 +0100 Subject: [PATCH 2/5] add support for Vine.co, fix #134 --- src/you_get/__main__.py | 1 + src/you_get/downloader/__init__.py | 1 + src/you_get/downloader/vine.py | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 src/you_get/downloader/vine.py diff --git a/src/you_get/__main__.py b/src/you_get/__main__.py index 90a409b5..5821d190 100644 --- a/src/you_get/__main__.py +++ b/src/you_get/__main__.py @@ -48,6 +48,7 @@ def url_to_module(url): 'tumblr': tumblr, 'vid48': vid48, 'vimeo': vimeo, + 'vine': vine, 'xiami': xiami, 'yinyuetai': yinyuetai, 'youku': youku, diff --git a/src/you_get/downloader/__init__.py b/src/you_get/downloader/__init__.py index e5853a2d..3db355bc 100644 --- a/src/you_get/downloader/__init__.py +++ b/src/you_get/downloader/__init__.py @@ -26,6 +26,7 @@ from .tudou import * from .tumblr import * from .vid48 import * from .vimeo import * +from .vine import * from .w56 import * from .xiami import * from .yinyuetai import * diff --git a/src/you_get/downloader/vine.py b/src/you_get/downloader/vine.py new file mode 100644 index 00000000..5ff629b7 --- /dev/null +++ b/src/you_get/downloader/vine.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +__all__ = ['vine_download'] + +from ..common import * + +def vine_download(url, output_dir = '.', merge = True, info_only = False): + html = get_html(url) + + title = r1(r' Date: Fri, 22 Mar 2013 04:37:28 +0100 Subject: [PATCH 3/5] version 0.3.6 --- CHANGELOG.txt | 10 ++++++++++ README.md | 2 ++ README.txt | 1 + src/you_get/version.py | 4 ++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3bc1adff..7c3c47a6 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,16 @@ Changelog ========= +0.3.6 +----- + +*Date: 2013-03-22* + +* Add support for: + - Vine +* Fix issue for: + - YouTube + 0.3.5 ----- diff --git a/README.md b/README.md index 0daf0038..f7427cc3 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Fork me on GitHub: * Facebook * Google+ * Tumblr +* Vine * SoundCloud * Mixcloud * JPopsuki @@ -243,6 +244,7 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y * Facebook * Google+ * Tumblr +* Vine * SoundCloud * Mixcloud * JPopsuki diff --git a/README.txt b/README.txt index 9b39133e..d32fcdc9 100644 --- a/README.txt +++ b/README.txt @@ -22,6 +22,7 @@ Supported Sites (As of Now) * Facebook http://facebook.com * Google+ http://plus.google.com * Tumblr http://www.tumblr.com +* Vine http://vine.co * SoundCloud http://soundcloud.com * Mixcloud http://www.mixcloud.com * JPopsuki http://jpopsuki.tv diff --git a/src/you_get/version.py b/src/you_get/version.py index 0bef1d10..ad52199f 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -2,5 +2,5 @@ __all__ = ['__version__', '__date__'] -__version__ = '0.3.5' -__date__ = '2013-03-15' +__version__ = '0.3.6' +__date__ = '2013-03-22' From 533633c71698f685ca33c419b521881e88b4b4cd Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 22 Mar 2013 14:19:44 +0100 Subject: [PATCH 4/5] update youtube.py --- src/you_get/downloader/youtube.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/you_get/downloader/youtube.py b/src/you_get/downloader/youtube.py index 67f2096c..6ec39ccf 100644 --- a/src/you_get/downloader/youtube.py +++ b/src/you_get/downloader/youtube.py @@ -34,12 +34,13 @@ youtube_codecs = [ def parse_video_info(raw_info): """Parser for YouTube's get_video_info data. - Returns a map, with 'url_encoded_fmt_stream_map' field being a sorted list. + Returns a dict, where 'url_encoded_fmt_stream_map' maps to a sorted list. """ # Percent-encoding reserved characters, used as separators. - separator = { + sepr = { '&': '%26', + ',': '%2C', '=': '%3D', } @@ -53,17 +54,20 @@ def parse_video_info(raw_info): youtube_codecs], range(len(youtube_codecs)))) + # {key1: value1, key2: value2, ..., + # 'url_encoded_fmt_stream_map': [{'itag': '38', ...}, ...] + # } return dict( [(lambda metadata: ['url_encoded_fmt_stream_map', ( - lambda url_encoded_fmt_stream_map: + lambda stream_map: sorted( [dict( - [sub_item.split(separator['=']) - for sub_item in - item.split(separator['&'])]) + [subitem.split(sepr['=']) + for subitem in + item.split(sepr['&'])]) for item in - url_encoded_fmt_stream_map.split('%2C')], + stream_map.split(sepr[','])], key = lambda stream: fmt_level[stream['itag']])) From 215ce0fdbcaaa5abc0bcd18529fd8cd1b7ec69b5 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 22 Mar 2013 23:03:00 +0100 Subject: [PATCH 5/5] remove all Download-urls on PyPI, fix #19 --- README.md | 56 ++++++++++++++----------------------------------------- 1 file changed, 14 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index f7427cc3..18c67c23 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,13 @@ Fork me on GitHub: Click [here](https://aur.archlinux.org/packages.php\?ID=62576). -### FAQ (For Windows Users) +### Upgrading: + +Using Pip: + + $ pip install --upgrade you-get + +### FAQ (For Windows Users): * Q: I don't know how to install it on Windows. @@ -118,26 +124,6 @@ Fork me on GitHub: * A: Run `set PYTHONIOENCODING=utf-8`. -## Upgrading - -Using Pip: - - $ pip install --upgrade you-get - -### Error When Upgrading from Pip - -If you see this error: - -``` - File "/usr/lib/python3.3/site-packages/pip-1.2.1-py3.3.egg/pip/backwardcompat.py", line 44, in u - return s.decode('utf-8') -UnicodeDecodeError: 'utf-8' codec can't decode byte 0xae in position 68: invalid start byte -``` - -This is an existing bug in Pip 1.2.1. However, this does not affect your upgrading. - -In Pip 1.3+, this should be already fixed. - ## Examples (For End-Users) Display the information of the video without downloading: @@ -335,7 +321,13 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y 点击[这里](https://aur.archlinux.org/packages.php\?ID=62576)。 -### FAQ（针对Windows用户) +### 升级： + +使用Pip： + + $ pip install --upgrade you-get + +### FAQ（针对Windows用户)： * Q：我不知道该如何在Windows下安装。 @@ -345,26 +337,6 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y * A：执行`set PYTHONIOENCODING=utf-8`。 -## 升级 - -使用Pip： - - $ pip install --upgrade you-get - -### 从Pip升级时可能的错误 - -若出现以下错误提示： - -``` - File "/usr/lib/python3.3/site-packages/pip-1.2.1-py3.3.egg/pip/backwardcompat.py", line 44, in u - return s.decode('utf-8') -UnicodeDecodeError: 'utf-8' codec can't decode byte 0xae in position 68: invalid start byte -``` - -这被证实是Pip 1.2.1的一个bug。不过，它并不影响到正常的升级。 - -这在Pip 1.3+中应当已经被修复。 - ## 使用方法示例 ### 如何下载视频