From d1d02fe7ae3e41133981323f951c3fd540b63e09 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 11 Sep 2013 13:45:34 +0200 Subject: [PATCH 01/48] downloader -> extractor --- src/you_get/__init__.py | 3 ++- src/you_get/{downloader => extractor}/__init__.py | 0 src/you_get/{downloader => extractor}/__main__.py | 2 +- src/you_get/{downloader => extractor}/acfun.py | 0 src/you_get/{downloader => extractor}/alive.py | 0 src/you_get/{downloader => extractor}/baidu.py | 0 src/you_get/{downloader => extractor}/bilibili.py | 0 src/you_get/{downloader => extractor}/blip.py | 0 src/you_get/{downloader => extractor}/cntv.py | 0 src/you_get/{downloader => extractor}/coursera.py | 0 src/you_get/{downloader => extractor}/dailymotion.py | 0 src/you_get/{downloader => extractor}/douban.py | 0 src/you_get/{downloader => extractor}/ehow.py | 0 src/you_get/{downloader => extractor}/facebook.py | 0 src/you_get/{downloader => extractor}/fivesing.py | 0 src/you_get/{downloader => extractor}/freesound.py | 0 src/you_get/{downloader => extractor}/google.py | 0 src/you_get/{downloader => extractor}/ifeng.py | 0 src/you_get/{downloader => extractor}/instagram.py | 0 src/you_get/{downloader => extractor}/iqiyi.py | 0 src/you_get/{downloader => extractor}/joy.py | 0 src/you_get/{downloader => extractor}/khan.py | 0 src/you_get/{downloader => extractor}/ku6.py | 0 src/you_get/{downloader => extractor}/miomio.py | 0 src/you_get/{downloader => extractor}/mixcloud.py | 0 src/you_get/{downloader => extractor}/netease.py | 0 src/you_get/{downloader => extractor}/nicovideo.py | 0 src/you_get/{downloader => extractor}/pptv.py | 0 src/you_get/{downloader => extractor}/qq.py | 0 src/you_get/{downloader => extractor}/sina.py | 0 src/you_get/{downloader => extractor}/sohu.py | 0 src/you_get/{downloader => extractor}/songtaste.py | 0 src/you_get/{downloader => extractor}/soundcloud.py | 0 src/you_get/{downloader => extractor}/ted.py | 0 src/you_get/{downloader => extractor}/tudou.py | 0 src/you_get/{downloader => extractor}/tumblr.py | 0 src/you_get/{downloader => extractor}/vid48.py | 0 src/you_get/{downloader => extractor}/vimeo.py | 0 src/you_get/{downloader => extractor}/vine.py | 0 src/you_get/{downloader => extractor}/w56.py | 0 src/you_get/{downloader => extractor}/xiami.py | 0 src/you_get/{downloader => extractor}/yinyuetai.py | 0 src/you_get/{downloader => extractor}/youku.py | 0 src/you_get/{downloader => extractor}/youtube.py | 0 tests/test.py | 2 +- you-get | 2 +- you-get.json | 2 +- 47 files changed, 6 insertions(+), 5 deletions(-) rename src/you_get/{downloader => extractor}/__init__.py (100%) rename src/you_get/{downloader => extractor}/__main__.py (98%) rename src/you_get/{downloader => extractor}/acfun.py (100%) rename src/you_get/{downloader => extractor}/alive.py (100%) rename src/you_get/{downloader => extractor}/baidu.py (100%) rename src/you_get/{downloader => extractor}/bilibili.py (100%) rename src/you_get/{downloader => extractor}/blip.py (100%) rename src/you_get/{downloader => extractor}/cntv.py (100%) rename src/you_get/{downloader => extractor}/coursera.py (100%) rename src/you_get/{downloader => extractor}/dailymotion.py (100%) rename src/you_get/{downloader => extractor}/douban.py (100%) rename src/you_get/{downloader => extractor}/ehow.py (100%) rename src/you_get/{downloader => extractor}/facebook.py (100%) rename src/you_get/{downloader => extractor}/fivesing.py (100%) rename src/you_get/{downloader => extractor}/freesound.py (100%) rename src/you_get/{downloader => extractor}/google.py (100%) rename src/you_get/{downloader => extractor}/ifeng.py (100%) rename src/you_get/{downloader => extractor}/instagram.py (100%) rename src/you_get/{downloader => extractor}/iqiyi.py (100%) rename src/you_get/{downloader => extractor}/joy.py (100%) rename src/you_get/{downloader => extractor}/khan.py (100%) rename src/you_get/{downloader => extractor}/ku6.py (100%) rename src/you_get/{downloader => extractor}/miomio.py (100%) rename src/you_get/{downloader => extractor}/mixcloud.py (100%) rename src/you_get/{downloader => extractor}/netease.py (100%) rename src/you_get/{downloader => extractor}/nicovideo.py (100%) rename src/you_get/{downloader => extractor}/pptv.py (100%) rename src/you_get/{downloader => extractor}/qq.py (100%) rename src/you_get/{downloader => extractor}/sina.py (100%) rename src/you_get/{downloader => extractor}/sohu.py (100%) rename src/you_get/{downloader => extractor}/songtaste.py (100%) rename src/you_get/{downloader => extractor}/soundcloud.py (100%) rename src/you_get/{downloader => extractor}/ted.py (100%) rename src/you_get/{downloader => extractor}/tudou.py (100%) rename src/you_get/{downloader => extractor}/tumblr.py (100%) rename src/you_get/{downloader => extractor}/vid48.py (100%) rename src/you_get/{downloader => extractor}/vimeo.py (100%) rename src/you_get/{downloader => extractor}/vine.py (100%) rename src/you_get/{downloader => extractor}/w56.py (100%) rename src/you_get/{downloader => extractor}/xiami.py (100%) rename src/you_get/{downloader => extractor}/yinyuetai.py (100%) rename src/you_get/{downloader => extractor}/youku.py (100%) rename src/you_get/{downloader => extractor}/youtube.py (100%) diff --git a/src/you_get/__init__.py b/src/you_get/__init__.py index ecca35d2..7cae2415 100644 --- a/src/you_get/__init__.py +++ b/src/you_get/__init__.py @@ -6,4 +6,5 @@ from .version import * # Easy import #from .cli_wrapper.converter import * #from .cli_wrapper.player import * -from .downloader import * +from .cli_wrapper import * +from .extractor import * diff --git a/src/you_get/downloader/__init__.py b/src/you_get/extractor/__init__.py similarity index 100% rename from src/you_get/downloader/__init__.py rename to src/you_get/extractor/__init__.py diff --git a/src/you_get/downloader/__main__.py b/src/you_get/extractor/__main__.py similarity index 98% rename from src/you_get/downloader/__main__.py rename to src/you_get/extractor/__main__.py index ed07f702..d5d88d04 100644 --- a/src/you_get/downloader/__main__.py +++ b/src/you_get/extractor/__main__.py @@ -1,7 +1,7 @@ #!/usr/bin/env python __all__ = ['main', 'any_download', 'any_download_playlist'] -from ..downloader import * +from ..extractor import * from ..common import * def url_to_module(url): diff --git a/src/you_get/downloader/acfun.py b/src/you_get/extractor/acfun.py similarity index 100% rename from src/you_get/downloader/acfun.py rename to src/you_get/extractor/acfun.py diff --git a/src/you_get/downloader/alive.py b/src/you_get/extractor/alive.py similarity index 100% rename from src/you_get/downloader/alive.py rename to src/you_get/extractor/alive.py diff --git a/src/you_get/downloader/baidu.py b/src/you_get/extractor/baidu.py similarity index 100% rename from src/you_get/downloader/baidu.py rename to src/you_get/extractor/baidu.py diff --git a/src/you_get/downloader/bilibili.py b/src/you_get/extractor/bilibili.py similarity index 100% rename from src/you_get/downloader/bilibili.py rename to src/you_get/extractor/bilibili.py diff --git a/src/you_get/downloader/blip.py b/src/you_get/extractor/blip.py similarity index 100% rename from src/you_get/downloader/blip.py rename to src/you_get/extractor/blip.py diff --git a/src/you_get/downloader/cntv.py b/src/you_get/extractor/cntv.py similarity index 100% rename from src/you_get/downloader/cntv.py rename to src/you_get/extractor/cntv.py diff --git a/src/you_get/downloader/coursera.py b/src/you_get/extractor/coursera.py similarity index 100% rename from src/you_get/downloader/coursera.py rename to src/you_get/extractor/coursera.py diff --git a/src/you_get/downloader/dailymotion.py b/src/you_get/extractor/dailymotion.py similarity index 100% rename from src/you_get/downloader/dailymotion.py rename to src/you_get/extractor/dailymotion.py diff --git a/src/you_get/downloader/douban.py b/src/you_get/extractor/douban.py similarity index 100% rename from src/you_get/downloader/douban.py rename to src/you_get/extractor/douban.py diff --git a/src/you_get/downloader/ehow.py b/src/you_get/extractor/ehow.py similarity index 100% rename from src/you_get/downloader/ehow.py rename to src/you_get/extractor/ehow.py diff --git a/src/you_get/downloader/facebook.py b/src/you_get/extractor/facebook.py similarity index 100% rename from src/you_get/downloader/facebook.py rename to src/you_get/extractor/facebook.py diff --git a/src/you_get/downloader/fivesing.py b/src/you_get/extractor/fivesing.py similarity index 100% rename from src/you_get/downloader/fivesing.py rename to src/you_get/extractor/fivesing.py diff --git a/src/you_get/downloader/freesound.py b/src/you_get/extractor/freesound.py similarity index 100% rename from src/you_get/downloader/freesound.py rename to src/you_get/extractor/freesound.py diff --git a/src/you_get/downloader/google.py b/src/you_get/extractor/google.py similarity index 100% rename from src/you_get/downloader/google.py rename to src/you_get/extractor/google.py diff --git a/src/you_get/downloader/ifeng.py b/src/you_get/extractor/ifeng.py similarity index 100% rename from src/you_get/downloader/ifeng.py rename to src/you_get/extractor/ifeng.py diff --git a/src/you_get/downloader/instagram.py b/src/you_get/extractor/instagram.py similarity index 100% rename from src/you_get/downloader/instagram.py rename to src/you_get/extractor/instagram.py diff --git a/src/you_get/downloader/iqiyi.py b/src/you_get/extractor/iqiyi.py similarity index 100% rename from src/you_get/downloader/iqiyi.py rename to src/you_get/extractor/iqiyi.py diff --git a/src/you_get/downloader/joy.py b/src/you_get/extractor/joy.py similarity index 100% rename from src/you_get/downloader/joy.py rename to src/you_get/extractor/joy.py diff --git a/src/you_get/downloader/khan.py b/src/you_get/extractor/khan.py similarity index 100% rename from src/you_get/downloader/khan.py rename to src/you_get/extractor/khan.py diff --git a/src/you_get/downloader/ku6.py b/src/you_get/extractor/ku6.py similarity index 100% rename from src/you_get/downloader/ku6.py rename to src/you_get/extractor/ku6.py diff --git a/src/you_get/downloader/miomio.py b/src/you_get/extractor/miomio.py similarity index 100% rename from src/you_get/downloader/miomio.py rename to src/you_get/extractor/miomio.py diff --git a/src/you_get/downloader/mixcloud.py b/src/you_get/extractor/mixcloud.py similarity index 100% rename from src/you_get/downloader/mixcloud.py rename to src/you_get/extractor/mixcloud.py diff --git a/src/you_get/downloader/netease.py b/src/you_get/extractor/netease.py similarity index 100% rename from src/you_get/downloader/netease.py rename to src/you_get/extractor/netease.py diff --git a/src/you_get/downloader/nicovideo.py b/src/you_get/extractor/nicovideo.py similarity index 100% rename from src/you_get/downloader/nicovideo.py rename to src/you_get/extractor/nicovideo.py diff --git a/src/you_get/downloader/pptv.py b/src/you_get/extractor/pptv.py similarity index 100% rename from src/you_get/downloader/pptv.py rename to src/you_get/extractor/pptv.py diff --git a/src/you_get/downloader/qq.py b/src/you_get/extractor/qq.py similarity index 100% rename from src/you_get/downloader/qq.py rename to src/you_get/extractor/qq.py diff --git a/src/you_get/downloader/sina.py b/src/you_get/extractor/sina.py similarity index 100% rename from src/you_get/downloader/sina.py rename to src/you_get/extractor/sina.py diff --git a/src/you_get/downloader/sohu.py b/src/you_get/extractor/sohu.py similarity index 100% rename from src/you_get/downloader/sohu.py rename to src/you_get/extractor/sohu.py diff --git a/src/you_get/downloader/songtaste.py b/src/you_get/extractor/songtaste.py similarity index 100% rename from src/you_get/downloader/songtaste.py rename to src/you_get/extractor/songtaste.py diff --git a/src/you_get/downloader/soundcloud.py b/src/you_get/extractor/soundcloud.py similarity index 100% rename from src/you_get/downloader/soundcloud.py rename to src/you_get/extractor/soundcloud.py diff --git a/src/you_get/downloader/ted.py b/src/you_get/extractor/ted.py similarity index 100% rename from src/you_get/downloader/ted.py rename to src/you_get/extractor/ted.py diff --git a/src/you_get/downloader/tudou.py b/src/you_get/extractor/tudou.py similarity index 100% rename from src/you_get/downloader/tudou.py rename to src/you_get/extractor/tudou.py diff --git a/src/you_get/downloader/tumblr.py b/src/you_get/extractor/tumblr.py similarity index 100% rename from src/you_get/downloader/tumblr.py rename to src/you_get/extractor/tumblr.py diff --git a/src/you_get/downloader/vid48.py b/src/you_get/extractor/vid48.py similarity index 100% rename from src/you_get/downloader/vid48.py rename to src/you_get/extractor/vid48.py diff --git a/src/you_get/downloader/vimeo.py b/src/you_get/extractor/vimeo.py similarity index 100% rename from src/you_get/downloader/vimeo.py rename to src/you_get/extractor/vimeo.py diff --git a/src/you_get/downloader/vine.py b/src/you_get/extractor/vine.py similarity index 100% rename from src/you_get/downloader/vine.py rename to src/you_get/extractor/vine.py diff --git a/src/you_get/downloader/w56.py b/src/you_get/extractor/w56.py similarity index 100% rename from src/you_get/downloader/w56.py rename to src/you_get/extractor/w56.py diff --git a/src/you_get/downloader/xiami.py b/src/you_get/extractor/xiami.py similarity index 100% rename from src/you_get/downloader/xiami.py rename to src/you_get/extractor/xiami.py diff --git a/src/you_get/downloader/yinyuetai.py b/src/you_get/extractor/yinyuetai.py similarity index 100% rename from src/you_get/downloader/yinyuetai.py rename to src/you_get/extractor/yinyuetai.py diff --git a/src/you_get/downloader/youku.py b/src/you_get/extractor/youku.py similarity index 100% rename from src/you_get/downloader/youku.py rename to src/you_get/extractor/youku.py diff --git a/src/you_get/downloader/youtube.py b/src/you_get/extractor/youtube.py similarity index 100% rename from src/you_get/downloader/youtube.py rename to src/you_get/extractor/youtube.py diff --git a/tests/test.py b/tests/test.py index 641878ef..5d21da2d 100644 --- a/tests/test.py +++ b/tests/test.py @@ -4,7 +4,7 @@ import unittest from you_get import * -from you_get.downloader.__main__ import url_to_module +from you_get.extractor.__main__ import url_to_module def test_urls(urls): for url in urls: diff --git a/you-get b/you-get index 86b44109..8bdc77c7 100755 --- a/you-get +++ b/you-get @@ -4,7 +4,7 @@ import os, sys __path__ = os.path.dirname(os.path.realpath(__file__)) __srcdir__ = 'src' sys.path.insert(1, os.path.join(__path__, __srcdir__)) -from you_get.downloader import main +from you_get.extractor import main if __name__ == '__main__': main() diff --git a/you-get.json b/you-get.json index 92114cff..dc988868 100644 --- a/you-get.json +++ b/you-get.json @@ -31,6 +31,6 @@ ], "console_scripts": [ - "you-get = you_get.downloader.__main__:main" + "you-get = you_get.extractor.__main__:main" ] } From 610365a37ca3a3f46cd6da67fa4a110e317513cb Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 11 Sep 2013 13:50:39 +0200 Subject: [PATCH 02/48] cli_wrapper --- src/you_get/__init__.py | 3 --- src/you_get/cli_wrapper/__init__.py | 0 src/you_get/cli_wrapper/downloader/__init__.py | 0 src/you_get/cli_wrapper/openssl/__init__.py | 0 src/you_get/cli_wrapper/player/__init__.py | 3 +++ src/you_get/cli_wrapper/player/__main__.py | 7 +++++++ src/you_get/cli_wrapper/player/dragonplayer.py | 0 src/you_get/cli_wrapper/player/gnome_mplayer.py | 0 src/you_get/cli_wrapper/player/mplayer.py | 0 src/you_get/cli_wrapper/player/vlc.py | 1 + src/you_get/cli_wrapper/player/wmp.py | 0 src/you_get/cli_wrapper/transcoder/__init__.py | 0 src/you_get/cli_wrapper/transcoder/ffmpeg.py | 0 src/you_get/cli_wrapper/transcoder/libav.py | 0 src/you_get/cli_wrapper/transcoder/mencoder.py | 0 15 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 src/you_get/cli_wrapper/__init__.py create mode 100644 src/you_get/cli_wrapper/downloader/__init__.py create mode 100644 src/you_get/cli_wrapper/openssl/__init__.py create mode 100644 src/you_get/cli_wrapper/player/__init__.py create mode 100644 src/you_get/cli_wrapper/player/__main__.py create mode 100644 src/you_get/cli_wrapper/player/dragonplayer.py create mode 100644 src/you_get/cli_wrapper/player/gnome_mplayer.py create mode 100644 src/you_get/cli_wrapper/player/mplayer.py create mode 100644 src/you_get/cli_wrapper/player/vlc.py create mode 100644 src/you_get/cli_wrapper/player/wmp.py create mode 100644 src/you_get/cli_wrapper/transcoder/__init__.py create mode 100644 src/you_get/cli_wrapper/transcoder/ffmpeg.py create mode 100644 src/you_get/cli_wrapper/transcoder/libav.py create mode 100644 src/you_get/cli_wrapper/transcoder/mencoder.py diff --git a/src/you_get/__init__.py b/src/you_get/__init__.py index 7cae2415..f8ee6011 100644 --- a/src/you_get/__init__.py +++ b/src/you_get/__init__.py @@ -3,8 +3,5 @@ from .common import * from .version import * -# Easy import -#from .cli_wrapper.converter import * -#from .cli_wrapper.player import * from .cli_wrapper import * from .extractor import * diff --git a/src/you_get/cli_wrapper/__init__.py b/src/you_get/cli_wrapper/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/downloader/__init__.py b/src/you_get/cli_wrapper/downloader/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/openssl/__init__.py b/src/you_get/cli_wrapper/openssl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/player/__init__.py b/src/you_get/cli_wrapper/player/__init__.py new file mode 100644 index 00000000..2f7636de --- /dev/null +++ b/src/you_get/cli_wrapper/player/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +from .mplayer import * diff --git a/src/you_get/cli_wrapper/player/__main__.py b/src/you_get/cli_wrapper/player/__main__.py new file mode 100644 index 00000000..8d4958b9 --- /dev/null +++ b/src/you_get/cli_wrapper/player/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python + +def main(): + script_main('you-get', any_download, any_download_playlist) + +if __name__ == "__main__": + main() diff --git a/src/you_get/cli_wrapper/player/dragonplayer.py b/src/you_get/cli_wrapper/player/dragonplayer.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/player/gnome_mplayer.py b/src/you_get/cli_wrapper/player/gnome_mplayer.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/player/mplayer.py b/src/you_get/cli_wrapper/player/mplayer.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/player/vlc.py b/src/you_get/cli_wrapper/player/vlc.py new file mode 100644 index 00000000..4265cc3e --- /dev/null +++ b/src/you_get/cli_wrapper/player/vlc.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/src/you_get/cli_wrapper/player/wmp.py b/src/you_get/cli_wrapper/player/wmp.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/transcoder/__init__.py b/src/you_get/cli_wrapper/transcoder/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/transcoder/ffmpeg.py b/src/you_get/cli_wrapper/transcoder/ffmpeg.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/transcoder/libav.py b/src/you_get/cli_wrapper/transcoder/libav.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/cli_wrapper/transcoder/mencoder.py b/src/you_get/cli_wrapper/transcoder/mencoder.py new file mode 100644 index 00000000..e69de29b From 2714ad86ef78be1abdbc8b3ece9a088fdbda93f9 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Thu, 12 Sep 2013 23:21:15 +0200 Subject: [PATCH 03/48] JPopsuki.tv is back --- src/you_get/extractor/__init__.py | 1 + src/you_get/extractor/__main__.py | 1 + src/you_get/extractor/jpopsuki.py | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 src/you_get/extractor/jpopsuki.py diff --git a/src/you_get/extractor/__init__.py b/src/you_get/extractor/__init__.py index 99e331f4..874824fe 100644 --- a/src/you_get/extractor/__init__.py +++ b/src/you_get/extractor/__init__.py @@ -18,6 +18,7 @@ from .ifeng import * from .instagram import * from .iqiyi import * from .joy import * +from .jpopsuki import * from .ku6 import * from .miomio import * from .mixcloud import * diff --git a/src/you_get/extractor/__main__.py b/src/you_get/extractor/__main__.py index d5d88d04..ed7d5c10 100644 --- a/src/you_get/extractor/__main__.py +++ b/src/you_get/extractor/__main__.py @@ -38,6 +38,7 @@ def url_to_module(url): 'instagram': instagram, 'iqiyi': iqiyi, 'joy': joy, + 'jpopsuki': jpopsuki, 'kankanews': bilibili, 'ku6': ku6, 'miomio': miomio, diff --git a/src/you_get/extractor/jpopsuki.py b/src/you_get/extractor/jpopsuki.py new file mode 100644 index 00000000..cf4ec052 --- /dev/null +++ b/src/you_get/extractor/jpopsuki.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +__all__ = ['jpopsuki_download'] + +from ..common import * + +def jpopsuki_download(url, output_dir='.', merge=True, info_only=False): + html = get_html(url, faker=True) + + title = r1(r' Date: Fri, 13 Sep 2013 00:52:43 +0200 Subject: [PATCH 04/48] JPopsuki.tv is back --- README.md | 2 ++ README.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 52d5d30c..001fd4b4 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Fork me on GitHub: * SoundCloud * Mixcloud * Freesound +* JPopsuki * VID48 * Niconico (ニコニコ動画) * Youku (优酷) @@ -251,6 +252,7 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y * SoundCloud * Mixcloud * Freesound +* JPopsuki * VID48 * NICONICO动画 * 优酷 diff --git a/README.txt b/README.txt index 7bd8a1ee..74cedfd3 100644 --- a/README.txt +++ b/README.txt @@ -32,6 +32,7 @@ Supported Sites (As of Now) * SoundCloud http://soundcloud.com * Mixcloud http://www.mixcloud.com * Freesound http://www.freesound.org +* JPopsuki http://jpopsuki.tv * VID48 http://vid48.com * Niconico (ニコニコ動画) http://www.nicovideo.jp * Youku (优酷) http://www.youku.com From 4d93cc3dc5c0cba5b4e4f95c7dbeccbb4d963de6 Mon Sep 17 00:00:00 2001 From: gongqijian Date: Sat, 21 Sep 2013 00:16:59 +0800 Subject: [PATCH 05/48] Baidu: fix #242 --- src/you_get/extractor/baidu.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/you_get/extractor/baidu.py b/src/you_get/extractor/baidu.py index 79d7053d..10127aeb 100755 --- a/src/you_get/extractor/baidu.py +++ b/src/you_get/extractor/baidu.py @@ -12,16 +12,16 @@ def baidu_get_song_html(sid): return get_html('http://music.baidu.com/song/%s/download?__o=%%2Fsong%%2F%s' % (sid, sid), faker = True) def baidu_get_song_url(html): - return r1(r'downlink="/data/music/file\?link=(.+?)"', html) + return r1(r'href="/data/music/file\?link=(http.+?)"', html) def baidu_get_song_artist(html): - return r1(r'singer_name:"(.+?)"', html) + return r1(r'singer_name\s*:\s*"(.+?)"', html) def baidu_get_song_album(html): - return r1(r'ablum_name:"(.+?)"', html) + return r1(r'album_name\s*:\s*"(.+?)"', html) def baidu_get_song_title(html): - return r1(r'song_title:"(.+?)"', html) + return r1(r'song_title\s*:\s*"(.+?)"', html) def baidu_download_lyric(sid, file_name, output_dir): if common.dry_run: @@ -41,6 +41,8 @@ def baidu_download_song(sid, output_dir = '.', merge = True, info_only = False): title = baidu_get_song_title(html) artist = baidu_get_song_artist(html) album = baidu_get_song_album(html) + assert url + type, ext, size = url_info(url, faker = True) print_info(site_info, title, type, size) if not info_only: From ced8a92873dc7e6156411913a9857118bf1eea97 Mon Sep 17 00:00:00 2001 From: Star Brilliant Date: Sat, 21 Sep 2013 09:57:29 +0800 Subject: [PATCH 06/48] Use decompressobj for zlib decompressing This solves issue #224 and #238 --- src/you_get/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index 8faf907a..0cb9c337 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -146,7 +146,8 @@ def undeflate(data): (the zlib compression is used.) """ import zlib - return zlib.decompress(data, -zlib.MAX_WBITS) + decompressobj = zlib.decompressobj(-zlib.MAX_WBITS) + return decompressobj.decompress(data)+decompressobj.flush() # DEPRECATED in favor of get_content() def get_response(url, faker = False): From 47753422acbb0578c1ec9328618000225c70f64a Mon Sep 17 00:00:00 2001 From: nagev Date: Fri, 20 Sep 2013 05:09:27 +0100 Subject: [PATCH 07/48] Update youtube.py No longer works on VEVO encoded videos such as http://www.youtube.com/watch?v=3O1_3zBUKM8 This more general regular expression fixes it. --- src/you_get/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/extractor/youtube.py b/src/you_get/extractor/youtube.py index 51fbb07f..1b766cc2 100644 --- a/src/you_get/extractor/youtube.py +++ b/src/you_get/extractor/youtube.py @@ -45,7 +45,7 @@ def decipher(js, s): code = re.sub(r'(\w+).split\(""\)', r'list(\1)', code) return code - f1 = match1(js, r'g.sig\|\|(\w+)\(g.s\)') + f1 = match1(js, r'\w+\.sig\|\|(\w+)\(\w+\.\w+\)') f1def = match1(js, r'(function %s\(\w+\)\{[^\{]+\})' % f1) code = tr_js(f1def) f2 = match1(f1def, r'(\w+)\(\w+,\d+\)') From b6868cbc12e1c05e7101244080c0422811e48972 Mon Sep 17 00:00:00 2001 From: Fwolf Date: Tue, 17 Sep 2013 21:42:46 +0800 Subject: [PATCH 08/48] Sohu: Fix vid get error --- src/you_get/extractor/sohu.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/you_get/extractor/sohu.py b/src/you_get/extractor/sohu.py index 4400836a..dcaac7eb 100644 --- a/src/you_get/extractor/sohu.py +++ b/src/you_get/extractor/sohu.py @@ -12,8 +12,11 @@ def real_url(host, prot, file, new): return '%s%s?key=%s' % (start[:-1], new, key) def sohu_download(url, output_dir = '.', merge = True, info_only = False): - vid = r1('vid\s*=\s*"(\d+)"', get_html(url)) - + html = get_html(url) + vid = r1('vid\s*=\s*"(\d+)"', html) + if not vid: + vid = r1('vid\s*:\s*"(\d+)"', html) + if vid: data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid)) for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]: @@ -31,8 +34,9 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): for file, new in zip(data['clipsURL'], data['su']): urls.append(real_url(host, prot, file, new)) assert data['clipsURL'][0].endswith('.mp4') - + else: + # my.tv link doesn't include clip info anymore, below block is useless vid = r1('vid\s*=\s*\'(\d+)\'', get_html(url)) data = json.loads(get_decoded_html('http://my.tv.sohu.com/videinfo.jhtml?m=viewnew&vid=%s' % vid)) host = data['allot'] @@ -45,7 +49,7 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): for file, new in zip(data['clipsURL'], data['su']): urls.append(real_url(host, prot, file, new)) assert data['clipsURL'][0].endswith('.mp4') - + print_info(site_info, title, 'mp4', size) if not info_only: download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge) From aae2a6cfdc916c38349ecfdadb1085575292047d Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Sat, 21 Sep 2013 14:26:01 +0200 Subject: [PATCH 09/48] Baidu: fix download link from outside mainland China --- src/you_get/extractor/baidu.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/you_get/extractor/baidu.py b/src/you_get/extractor/baidu.py index 10127aeb..54b94ec4 100755 --- a/src/you_get/extractor/baidu.py +++ b/src/you_get/extractor/baidu.py @@ -36,13 +36,18 @@ def baidu_download_lyric(sid, file_name, output_dir): x.write(lrc) def baidu_download_song(sid, output_dir = '.', merge = True, info_only = False): - html = baidu_get_song_html(sid) - url = baidu_get_song_url(html) - title = baidu_get_song_title(html) - artist = baidu_get_song_artist(html) - album = baidu_get_song_album(html) - assert url - + try: + html = baidu_get_song_html(sid) + url = baidu_get_song_url(html) + title = baidu_get_song_title(html) + artist = baidu_get_song_artist(html) + album = baidu_get_song_album(html) + assert url + except: + html = get_html("http://music.baidu.com/song/%s" % sid) + url = r1(r'download_url="([^"]+)"', html) + title = r1(r'sname="([^"]+)"', html) + type, ext, size = url_info(url, faker = True) print_info(site_info, title, type, size) if not info_only: From 36be10805dc61903105dc3d1cd6322581092b0b9 Mon Sep 17 00:00:00 2001 From: Star Brilliant Date: Mon, 30 Sep 2013 22:59:28 +0800 Subject: [PATCH 10/48] Bilibili started to use ssl.bilibili.tv instead of secure.bilibili.tv now --- src/you_get/extractor/bilibili.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/extractor/bilibili.py b/src/you_get/extractor/bilibili.py index 8512d362..322b60fc 100644 --- a/src/you_get/extractor/bilibili.py +++ b/src/you_get/extractor/bilibili.py @@ -83,7 +83,7 @@ def bilibili_download(url, output_dir = '.', merge = True, info_only = False): title = unescape_html(title) title = escape_file_path(title) - flashvars = r1_of([r'player_params=\'(cid=\d+)', r'flashvars="([^"]+)"', r'"https://secure.bilibili.tv/secure,(cid=\d+)(?:&aid=\d+)?"'], html) + flashvars = r1_of([r'player_params=\'(cid=\d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.tv/secure,(cid=\d+)(?:&aid=\d+)?"'], html) assert flashvars t, id = flashvars.split('=', 1) id = id.split('&')[0] From f11d819f9e72f07ff7ccabab1adf9605136adf64 Mon Sep 17 00:00:00 2001 From: gongqijian Date: Fri, 4 Oct 2013 21:15:04 +0800 Subject: [PATCH 11/48] Update baidu.py --- src/you_get/extractor/baidu.py | 111 ++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 42 deletions(-) diff --git a/src/you_get/extractor/baidu.py b/src/you_get/extractor/baidu.py index 54b94ec4..b93b0333 100755 --- a/src/you_get/extractor/baidu.py +++ b/src/you_get/extractor/baidu.py @@ -8,52 +8,71 @@ from .. import common from urllib import parse -def baidu_get_song_html(sid): - return get_html('http://music.baidu.com/song/%s/download?__o=%%2Fsong%%2F%s' % (sid, sid), faker = True) +def baidu_get_song_data(sid): + data = json.loads(get_html('http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker = True))['data'] -def baidu_get_song_url(html): - return r1(r'href="/data/music/file\?link=(http.+?)"', html) + if data['xcode'] != '': + # inside china mainland + return data['songList'][0] + else: + # outside china mainland + html = get_html("http://music.baidu.com/song/%s" % sid) -def baidu_get_song_artist(html): - return r1(r'singer_name\s*:\s*"(.+?)"', html) + # baidu pan link + sourceLink = r1(r'"link-src-info">', html) - if href: - lrc = get_html('http://music.baidu.com' + href) - if len(lrc) > 0: - with open(output_dir + "/" + file_name.replace('/', '-') + '.lrc', 'w') as x: - x.write(lrc) +def baidu_get_song_artist(data): + return data['artistName'] + +def baidu_get_song_album(data): + return data['albumName'] + +def baidu_get_song_title(data): + return data['songName'] + +def baidu_get_song_lyric(data): + lrc = data['lrcLink'] + return None if lrc is '' else "http://music.baidu.com%s" % lrc def baidu_download_song(sid, output_dir = '.', merge = True, info_only = False): - try: - html = baidu_get_song_html(sid) - url = baidu_get_song_url(html) - title = baidu_get_song_title(html) - artist = baidu_get_song_artist(html) - album = baidu_get_song_album(html) - assert url - except: - html = get_html("http://music.baidu.com/song/%s" % sid) - url = r1(r'download_url="([^"]+)"', html) - title = r1(r'sname="([^"]+)"', html) - + data = baidu_get_song_data(sid) + url = baidu_get_song_url(data) + title = baidu_get_song_title(data) + artist = baidu_get_song_artist(data) + album = baidu_get_song_album(data) + lrc = baidu_get_song_lyric(data) + + assert url + file_name = "%s - %s - %s" % (title, album, artist) + type, ext, size = url_info(url, faker = True) print_info(site_info, title, type, size) if not info_only: - file_name = "%s - %s - %s" % (title, album, artist) download_urls([url], file_name, ext, size, output_dir, merge = merge, faker = True) - baidu_download_lyric(sid, file_name, output_dir) + + if lrc: + type, ext, size = url_info(lrc, faker = True) + print_info(site_info, title, type, size) + if not info_only: + download_urls([lrc], file_name, ext, size, output_dir, faker = True) def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False): html = get_html('http://music.baidu.com/album/%s' % aid, faker = True) @@ -63,32 +82,40 @@ def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False) ids = json.loads(r1(r'', html).replace('"', '').replace(';', '"'))['ids'] track_nr = 1 for id in ids: - song_html = baidu_get_song_html(id) - song_url = baidu_get_song_url(song_html) - song_title = baidu_get_song_title(song_html) + song_data = baidu_get_song_data(id) + song_url = baidu_get_song_url(song_data) + song_title = baidu_get_song_title(song_data) + song_lrc = baidu_get_song_lyric(song_data) file_name = '%02d.%s' % (track_nr, song_title) + type, ext, size = url_info(song_url, faker = True) print_info(site_info, song_title, type, size) if not info_only: download_urls([song_url], file_name, ext, size, output_dir, merge = merge, faker = True) - baidu_download_lyric(id, file_name, output_dir) + + if song_lrc: + type, ext, size = url_info(song_lrc, faker = True) + print_info(site_info, song_title, type, size) + if not info_only: + download_urls([song_lrc], file_name, ext, size, output_dir, faker = True) + track_nr += 1 def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False): if re.match(r'http://pan.baidu.com', url): html = get_html(url) - + title = r1(r'server_filename="([^"]+)"', html) if len(title.split('.')) > 1: title = ".".join(title.split('.')[:-1]) - + real_url = r1(r'\\"dlink\\":\\"([^"]*)\\"', html).replace('\\\\/', '/') type, ext, size = url_info(real_url, faker = True) - + print_info(site_info, title, ext, size) if not info_only: download_urls([real_url], title, ext, size, output_dir, merge = merge) - + elif re.match(r'http://music.baidu.com/album/\d+', url): id = r1(r'http://music.baidu.com/album/(\d+)', url) baidu_download_album(id, output_dir, merge, info_only) From f412a9e9c57ca3849a575b9504a938e7785151f9 Mon Sep 17 00:00:00 2001 From: "Lee, Donggu" Date: Tue, 8 Oct 2013 14:00:00 +0000 Subject: [PATCH 12/48] fix length check --- src/you_get/extractor/tudou.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/extractor/tudou.py b/src/you_get/extractor/tudou.py index f28da697..31e90d85 100644 --- a/src/you_get/extractor/tudou.py +++ b/src/you_get/extractor/tudou.py @@ -8,7 +8,7 @@ def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid)) vids = [] for k in data: - if len(data[k]) == 1: + if len(data[k]) > 0: vids.append({"k": data[k][0]["k"], "size": data[k][0]["size"]}) temp = max(vids, key=lambda x:x["size"]) From f1ed97c500ef566714640c00369f801e7c01e22d Mon Sep 17 00:00:00 2001 From: "Lee, Donggu" Date: Tue, 8 Oct 2013 14:00:00 +0000 Subject: [PATCH 13/48] flash link can be downloaded --- src/you_get/extractor/sohu.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/you_get/extractor/sohu.py b/src/you_get/extractor/sohu.py index dcaac7eb..c364917f 100644 --- a/src/you_get/extractor/sohu.py +++ b/src/you_get/extractor/sohu.py @@ -36,9 +36,11 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): assert data['clipsURL'][0].endswith('.mp4') else: - # my.tv link doesn't include clip info anymore, below block is useless - vid = r1('vid\s*=\s*\'(\d+)\'', get_html(url)) - data = json.loads(get_decoded_html('http://my.tv.sohu.com/videinfo.jhtml?m=viewnew&vid=%s' % vid)) + if re.match(r'http://share.vrs.sohu.com', url): + vid = r1('id=(\d+)', url) + else: + vid = r1('vid\s*=\s*\'(\d+)\'', get_html(url)) + data = json.loads(get_decoded_html('http://my.tv.sohu.com/play/videonew.do?vid=%s&referer=http://my.tv.sohu.com' % vid)) host = data['allot'] prot = data['prot'] urls = [] From 3966bcf9cb9146ceda9223c99368b1bd3ee28401 Mon Sep 17 00:00:00 2001 From: "Lee, Donggu" Date: Tue, 8 Oct 2013 14:00:00 +0000 Subject: [PATCH 14/48] removed a filename of the first letter is a period on unix system --- src/you_get/common.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index 0cb9c337..b78ae0e3 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -106,11 +106,7 @@ def filenameable(text): 0: None, ord('/'): '-', }) - if platform.system() == 'Darwin': # For Mac OS - text = text.translate({ - ord(':'): '-', - }) - elif platform.system() == 'Windows': # For Windows + if platform.system() == 'Windows': # For Windows text = text.translate({ ord(':'): '-', ord('*'): '-', @@ -124,6 +120,13 @@ def filenameable(text): ord('['): '(', ord(']'): ')', }) + else: + if text.startswith("."): + text = text[1:] + if platform.system() == 'Darwin': # For Mac OS + text = text.translate({ + ord(':'): '-', + }) return text def unescape_html(html): From 9393e770acf04bdb25f9de349a8f8642570dc564 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Tue, 8 Oct 2013 14:35:59 +0200 Subject: [PATCH 15/48] remove test_xiami because it fails CI build too often --- tests/test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test.py b/tests/test.py index 5d21da2d..76b39ac5 100644 --- a/tests/test.py +++ b/tests/test.py @@ -28,11 +28,6 @@ class YouGetTests(unittest.TestCase): "http://vimeo.com/56810854", ]) - def test_xiami(self): - test_urls([ - "http://www.xiami.com/song/1769835121", - ]) - def test_youtube(self): test_urls([ "http://www.youtube.com/watch?v=pzKerr0JIPA", From 557926e5151c5f2f5d75eeb6c612e5915a75e0b3 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Tue, 8 Oct 2013 14:43:03 +0200 Subject: [PATCH 16/48] add version badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 001fd4b4..e8b0b775 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # You-Get -[![Build Status](https://api.travis-ci.org/soimort/you-get.png)](https://travis-ci.org/soimort/you-get) +[![Build Status](https://api.travis-ci.org/soimort/you-get.png)](https://travis-ci.org/soimort/you-get) [![PyPI version](https://badge.fury.io/py/you-get.png)](http://badge.fury.io/py/you-get) [You-Get](https://github.com/soimort/you-get) is a video downloader runs on Python 3. It aims at easing the download of videos on [YouTube](http://www.youtube.com), [Youku](http://www.youku.com)/[Tudou](http://www.tudou.com) (biggest online video providers in China), [ Niconico](http://www.nicovideo.jp), etc., in one script. From e7ce66c5655f8320cf8dfda1266ac505454cdaf3 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 18 Oct 2013 06:16:06 +0200 Subject: [PATCH 17/48] PPTV: fix multiple segments --- src/you_get/extractor/pptv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/extractor/pptv.py b/src/you_get/extractor/pptv.py index 4fd88e5b..b1d45edf 100644 --- a/src/you_get/extractor/pptv.py +++ b/src/you_get/extractor/pptv.py @@ -14,7 +14,7 @@ def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o key = r1(r']+>([^<>]+)', xml) rid = r1(r'rid="([^"]+)"', xml) title = r1(r'nm="([^"]+)"', xml) - pieces = re.findall(']+fs="(\d+)"', xml) numbers, fs = zip(*pieces) urls = ['http://%s/%s/%s?k=%s' % (host, i, rid, key) for i in numbers] total_size = sum(map(int, fs)) From acfd3b4e2a5a85a1c50e60c8d1e363ea2b61698d Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 18 Oct 2013 14:29:19 +0200 Subject: [PATCH 18/48] TED: fix #256 --- src/you_get/extractor/ted.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/you_get/extractor/ted.py b/src/you_get/extractor/ted.py index 167da2a8..23a7054c 100644 --- a/src/you_get/extractor/ted.py +++ b/src/you_get/extractor/ted.py @@ -11,7 +11,7 @@ def ted_download(url, output_dir = '.', merge = True, info_only = False): title = line.replace("", "").replace("", "").replace("\t", "") title = title[:title.find(' | ')] if line.find("no-flash-video-download") > -1: - url = line.replace(' Date: Fri, 18 Oct 2013 14:31:08 +0200 Subject: [PATCH 19/48] add tests for TED --- tests/test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test.py b/tests/test.py index 76b39ac5..452fbcd8 100644 --- a/tests/test.py +++ b/tests/test.py @@ -23,6 +23,12 @@ class YouGetTests(unittest.TestCase): "http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", ]) + def test_ted(self): + test_urls([ + "http://www.ted.com/talks/jennifer_lin_improvs_piano_magic.html", + "http://www.ted.com/talks/derek_paravicini_and_adam_ockelford_in_the_key_of_genius.html", + ]) + def test_vimeo(self): test_urls([ "http://vimeo.com/56810854", From bf5d2dfbcea26922e37108e0e28b77490576b5a8 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 18 Oct 2013 14:49:29 +0200 Subject: [PATCH 20/48] YouTube: fix #252 --- src/you_get/common.py | 5 ++++- src/you_get/extractor/youtube.py | 4 +++- tests/test.py | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index b78ae0e3..ceb07646 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -85,7 +85,10 @@ def parse_query_param(url, param): The value of the parameter. """ - return parse.parse_qs(parse.urlparse(url).query)[param][0] + try: + return parse.parse_qs(parse.urlparse(url).query)[param][0] + except: + return None def unicodize(text): return re.sub(r'\\u([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])', lambda x: chr(int(x.group(0)[2:], 16)), text) diff --git a/src/you_get/extractor/youtube.py b/src/you_get/extractor/youtube.py index 1b766cc2..96f6987b 100644 --- a/src/you_get/extractor/youtube.py +++ b/src/you_get/extractor/youtube.py @@ -107,7 +107,9 @@ def youtube_download(url, output_dir='.', merge=True, info_only=False): """Downloads YouTube videos by URL. """ - id = match1(url, r'youtu.be/([^/]+)') or parse_query_param(url, 'v') + id = match1(url, r'youtu.be/([^/]+)') or \ + parse_query_param(url, 'v') or \ + parse_query_param(parse_query_param(url, 'u'), 'v') assert id youtube_download_by_id(id, title=None, output_dir=output_dir, merge=merge, info_only=info_only) diff --git a/tests/test.py b/tests/test.py index 452fbcd8..9ecf4d68 100644 --- a/tests/test.py +++ b/tests/test.py @@ -38,4 +38,5 @@ class YouGetTests(unittest.TestCase): test_urls([ "http://www.youtube.com/watch?v=pzKerr0JIPA", "http://youtu.be/pzKerr0JIPA", + "http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare" ]) From 8b5ae3a60940472d26c00eadc6eacb58bc6adb73 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 18 Oct 2013 15:03:01 +0200 Subject: [PATCH 21/48] version 0.3.22 --- CHANGELOG.txt | 17 +++++++++++++++++ README.txt | 2 ++ src/you_get/version.py | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 67cbb1fb..fca035b8 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,23 @@ Changelog ========= +0.3.22 +------ + +*Date: 2013-10-18* + +* Fix issues for: + - Baidu + - Bilibili + - JPopsuki TV + - Niconico + - PPTV + - TED + - Tumblr + - YinYueTai + - YouTube + - ... + 0.3.21 ------ diff --git a/README.txt b/README.txt index 74cedfd3..c4b68af8 100644 --- a/README.txt +++ b/README.txt @@ -3,6 +3,8 @@ You-Get .. image:: https://api.travis-ci.org/soimort/you-get.png +.. image:: https://badge.fury.io/py/you-get.png + `You-Get `_ is a video downloader runs on Python 3. It aims at easing the download of videos on `YouTube `_, `Youku `_/`Tudou `_ (biggest online video providers in China), `Niconico `_, etc., in one script. See the project homepage http://www.soimort.org/you-get for further documentation. diff --git a/src/you_get/version.py b/src/you_get/version.py index 43c2747b..bf14f9c8 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -1,5 +1,5 @@ #!/usr/bin/env python __all__ = ['__version__', '__date__'] -__version__ = '0.3.21' -__date__ = '2013-08-17' +__version__ = '0.3.22' +__date__ = '2013-10-18' From bfc85f8a49ee1eec0cf743664c3741d3303a7632 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 18 Oct 2013 23:03:56 +0200 Subject: [PATCH 22/48] Miomio: add Sina as video source --- src/you_get/extractor/miomio.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/you_get/extractor/miomio.py b/src/you_get/extractor/miomio.py index cbdad765..bac1f64c 100644 --- a/src/you_get/extractor/miomio.py +++ b/src/you_get/extractor/miomio.py @@ -4,6 +4,7 @@ __all__ = ['miomio_download'] from ..common import * +from .sina import sina_download_by_vid from .tudou import tudou_download_by_id from .youku import youku_download_by_id @@ -16,9 +17,11 @@ def miomio_download(url, output_dir = '.', merge = True, info_only = False): t = r1(r'type=(\w+)', flashvars) id = r1(r'vid=([^"]+)', flashvars) if t == 'youku': - youku_download_by_id(id, title, output_dir = output_dir, merge = merge, info_only = info_only) + youku_download_by_id(id, title, output_dir=output_dir, merge=merge, info_only=info_only) elif t == 'tudou': - tudou_download_by_id(id, title, output_dir = output_dir, merge = merge, info_only = info_only) + tudou_download_by_id(id, title, output_dir=output_dir, merge=merge, info_only=info_only) + elif t == 'sina': + sina_download_by_vid(id, title, output_dir=output_dir, merge=merge, info_only=info_only) else: raise NotImplementedError(flashvars) From 1b75cfcc473116d787e4f0908e58a020b90d8768 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Fri, 18 Oct 2013 23:38:24 +0200 Subject: [PATCH 23/48] Sina: fix #246 --- src/you_get/extractor/sina.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/extractor/sina.py b/src/you_get/extractor/sina.py index af030a9e..33cc0c7c 100644 --- a/src/you_get/extractor/sina.py +++ b/src/you_get/extractor/sina.py @@ -5,7 +5,7 @@ __all__ = ['sina_download', 'sina_download_by_vid', 'sina_download_by_vkey'] from ..common import * def video_info(id): - xml = get_content('http://v.iask.com/v_play.php?vid=%s' % id, decoded=True) + xml = get_content('http://www.tucao.cc/api/sina.php?vid=%s' % id, decoded=True) urls = re.findall(r'(?:)?', xml) name = match1(xml, r'(?:)?') vstr = match1(xml, r'(?:)?') From d4855581a60021a84a7b63f4666d70185d6d64e7 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 23 Oct 2013 05:32:53 +0200 Subject: [PATCH 24/48] I can't believe you-get didn't download YouTube playlists (fix #35) --- src/you_get/extractor/youtube.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/you_get/extractor/youtube.py b/src/you_get/extractor/youtube.py index 96f6987b..9ff08e07 100644 --- a/src/you_get/extractor/youtube.py +++ b/src/you_get/extractor/youtube.py @@ -103,6 +103,15 @@ def youtube_download_by_id(id, title=None, output_dir='.', merge=True, info_only if not info_only: download_urls([url], title, ext, size, output_dir, merge = merge) +def youtube_list_download_by_id(list_id, title=None, output_dir='.', merge=True, info_only=False): + """Downloads a YouTube video list by its unique id. + """ + + video_page = get_content('http://www.youtube.com/playlist?list=%s' % list_id) + ids = set(re.findall(r' Date: Wed, 23 Oct 2013 06:11:09 +0200 Subject: [PATCH 25/48] support general short URLs, fix #162 --- src/you_get/extractor/__main__.py | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/you_get/extractor/__main__.py b/src/you_get/extractor/__main__.py index ed7d5c10..f8d9ecea 100644 --- a/src/you_get/extractor/__main__.py +++ b/src/you_get/extractor/__main__.py @@ -5,14 +5,13 @@ from ..extractor import * from ..common import * def url_to_module(url): - site = r1(r'http://([^/]+)/', url) - assert site, 'invalid url: ' + url + video_host = r1(r'http://([^/]+)/', url) + video_url = r1(r'http://[^/]+(.*)', url) + assert video_host and video_url, 'invalid url: ' + url - if site.endswith('.com.cn'): - site = site[:-3] - domain = r1(r'(\.[^.]+\.[^.]+)$', site) - if not domain: - domain = site + if video_host.endswith('.com.cn'): + video_host = video_host[:-3] + domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host assert domain, 'unsupported url: ' + url k = r1(r'([^.]+)', domain) @@ -68,14 +67,28 @@ def url_to_module(url): if k in downloads: return downloads[k] else: - raise NotImplementedError(url) + import http.client + conn = http.client.HTTPConnection(video_host) + conn.request("HEAD", video_url) + res = conn.getresponse() + location = res.getheader('location') + if location is None: + raise NotImplementedError(url) + else: + return url_to_module(location), location def any_download(url, output_dir = '.', merge = True, info_only = False): - m = url_to_module(url) + try: + m, url = url_to_module(url) + except: + m = url_to_module(url) m.download(url, output_dir = output_dir, merge = merge, info_only = info_only) def any_download_playlist(url, output_dir = '.', merge = True, info_only = False): - m = url_to_module(url) + try: + m, url = url_to_module(url) + except: + m = url_to_module(url) m.download_playlist(url, output_dir = output_dir, merge = merge, info_only = info_only) def main(): From 5afdf2661ae7c79a58e7b887a4a52c305f466992 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 23 Oct 2013 06:22:43 +0200 Subject: [PATCH 26/48] version 0.3.23 --- CHANGELOG.txt | 10 ++++++++++ src/you_get/version.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index fca035b8..c4d7da6c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,16 @@ Changelog ========= +0.3.23 +------ + +*Date: 2013-10-23* + +* Support YouTube playlists +* Support general short URLs +* Fix issues for: + - Sina + 0.3.22 ------ diff --git a/src/you_get/version.py b/src/you_get/version.py index bf14f9c8..2295fbec 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -1,5 +1,5 @@ #!/usr/bin/env python __all__ = ['__version__', '__date__'] -__version__ = '0.3.22' -__date__ = '2013-10-18' +__version__ = '0.3.23' +__date__ = '2013-10-23' From b5dc362a5290a9889c181325c27ad1dd51b230aa Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 23 Oct 2013 18:32:56 +0200 Subject: [PATCH 27/48] Vimeo: fix #259 --- src/you_get/extractor/vimeo.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/you_get/extractor/vimeo.py b/src/you_get/extractor/vimeo.py index 1a18dfbb..60611f74 100644 --- a/src/you_get/extractor/vimeo.py +++ b/src/you_get/extractor/vimeo.py @@ -5,19 +5,16 @@ __all__ = ['vimeo_download', 'vimeo_download_by_id'] from ..common import * def vimeo_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False): - html = get_html('http://vimeo.com/%s' % id, faker = True) + video_page = get_content('http://player.vimeo.com/video/%s' % id, headers=fake_headers) + title = r1(r'([^<]+)', video_page) + info = dict(re.findall(r'"([^"]+)":\{[^{]+"url":"([^"]+)"', video_page)) + for quality in ['hd', 'sd', 'mobile']: + if quality in info: + url = info[quality] + break + assert url - signature = r1(r'"signature":"([^"]+)"', html) - timestamp = r1(r'"timestamp":([^,]+)', html) - hd = r1(r',"hd":(\d+),', html) - - title = r1(r'"title":"([^"]+)"', html) - title = escape_file_path(title) - - url = 'http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s' % (id, signature, timestamp) - if hd == "1": - url += '&quality=hd' - type, ext, size = url_info(url, faker = True) + type, ext, size = url_info(url, faker=True) print_info(site_info, title, type, size) if not info_only: From fea0b658efd185b853e778598b2c9550b4292d29 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Tue, 29 Oct 2013 22:11:17 +0100 Subject: [PATCH 28/48] add new module: util.log --- src/you_get/common.py | 9 +-- src/you_get/util/fs.py | 0 src/you_get/util/log.py | 121 ++++++++++++++++++++++++++++++++++++++++ src/you_get/version.py | 1 + 4 files changed, 127 insertions(+), 4 deletions(-) create mode 100644 src/you_get/util/fs.py create mode 100644 src/you_get/util/log.py diff --git a/src/you_get/common.py b/src/you_get/common.py index ceb07646..4ad45053 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -10,6 +10,7 @@ from urllib import request, parse import platform from .version import __version__ +from .util import log dry_run = False force = False @@ -772,8 +773,8 @@ def script_main(script_name, download, download_playlist = None): try: opts, args = getopt.getopt(sys.argv[1:], short_opts, opts) except getopt.GetoptError as err: - print(err) - print(help) + log.e(err) + log.e("try 'you-get --help' for more options") sys.exit(2) info_only = False @@ -811,8 +812,8 @@ def script_main(script_name, download, download_playlist = None): elif o in ('-x', '--http-proxy'): proxy = a else: - print(help) - sys.exit(1) + log.e("try 'you-get --help' for more options") + sys.exit(2) if not args: print(help) sys.exit() diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py new file mode 100644 index 00000000..e69de29b diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py new file mode 100644 index 00000000..738aa209 --- /dev/null +++ b/src/you_get/util/log.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python + +from ..version import __name__ + +import os, sys + +# Is terminal ANSI/VT100 compatible +if os.getenv('TERM') in ( + 'xterm', + 'vt100', + 'linux', + 'eterm-color', + ): + has_colors = True +else: + try: + ppid = os.getppid() + has_colors = (os.popen('ps -p %d -ocomm=' % ppid).read().strip() + == 'emacs') + except: + has_colors = False + +# ANSI/VT100 escape code +# http://en.wikipedia.org/wiki/ANSI_escape_code +colors = { + 'none': '', + 'reset': '\033[0m', + + 'black': '\033[30m', + 'bold-black': '\033[30;1m', + 'dark-gray': '\033[90m', + 'bold-dark-gray': '\033[90;1m', + + 'red': '\033[31m', + 'bold-red': '\033[31;1m', + 'light-red': '\033[91m', + 'bold-light-red': '\033[91;1m', + + 'green': '\033[32m', + 'bold-green': '\033[32;1m', + 'light-green': '\033[92m', + 'bold-light-green': '\033[92;1m', + + 'yellow': '\033[33m', + 'bold-yellow': '\033[33;1m', + 'light-yellow': '\033[93m', + 'bold-light-yellow': '\033[93;1m', + + 'blue': '\033[34m', + 'bold-blue': '\033[34;1m', + 'light-blue': '\033[94m', + 'bold-light-blue': '\033[94;1m', + + 'magenta': '\033[35m', + 'bold-magenta': '\033[35;1m', + 'light-magenta': '\033[95m', + 'bold-light-magenta': '\033[95;1m', + + 'cyan': '\033[36m', + 'bold-cyan': '\033[36;1m', + 'light-cyan': '\033[96m', + 'bold-light-cyan': '\033[96;1m', + + 'light-gray': '\033[37m', + 'bold-light-gray': '\033[37;1m', + 'white': '\033[97m', + 'bold-white': '\033[97;1m', +} + +def println(message): + """Prints a log message. + """ + sys.stderr.write("{0}: {1}\n".format(__name__, message)) + +def writeln(color, message): + """Prints a colorful log message. + """ + if color in colors: + sys.stderr.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset'])) + else: + sys.stderr.write("{0}: {1}\n".format(__name__, message)) + +def i(message): + """Sends an info log message. + """ + if has_colors: + writeln('white', message) + else: + println(message) + +def d(message): + """Sends a debug log message. + """ + if has_colors: + writeln('blue', message) + else: + println(message) + +def w(message): + """Sends a warning log message. + """ + if has_colors: + writeln('yellow', message) + else: + println(message) + +def e(message): + """Sends an error log message. + """ + if has_colors: + writeln('light-red', message) + else: + println(message) + +def wtf(message): + """What a Terrible Failure. + """ + if has_colors: + writeln('bold-red', message) + else: + println(message) diff --git a/src/you_get/version.py b/src/you_get/version.py index 2295fbec..7c09b706 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -1,5 +1,6 @@ #!/usr/bin/env python __all__ = ['__version__', '__date__'] +__name__ = 'you-get' __version__ = '0.3.23' __date__ = '2013-10-23' From a1172526a2998beaed43583c70725cfcad217e2b Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Tue, 29 Oct 2013 22:19:28 +0100 Subject: [PATCH 29/48] add __init__.py --- src/you_get/util/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/you_get/util/__init__.py diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py new file mode 100644 index 00000000..e69de29b From ac30ee318a3974a2e787c1b14b16e9e2546152a4 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Tue, 29 Oct 2013 23:49:33 +0100 Subject: [PATCH 30/48] update module: util.log --- src/you_get/util/log.py | 67 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py index 738aa209..6f02c3a1 100644 --- a/src/you_get/util/log.py +++ b/src/you_get/util/log.py @@ -14,6 +14,7 @@ if os.getenv('TERM') in ( has_colors = True else: try: + # Eshell ppid = os.getppid() has_colors = (os.popen('ps -p %d -ocomm=' % ppid).read().strip() == 'emacs') @@ -67,55 +68,53 @@ colors = { 'bold-white': '\033[97;1m', } -def println(message): - """Prints a log message. - """ - sys.stderr.write("{0}: {1}\n".format(__name__, message)) - -def writeln(color, message): - """Prints a colorful log message. +def println(text, color=None, ostream=sys.stdout): + """Prints a text line to stream. """ if color in colors: - sys.stderr.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset'])) + ostream.write("{0}{1}{2}\n".format(colors[color], text, colors['reset'])) else: - sys.stderr.write("{0}: {1}\n".format(__name__, message)) + ostream.write("{0}\n".format(text)) -def i(message): +def printlog(message, color=None, ostream=sys.stderr): + """Prints a log message to stream. + """ + if color in colors: + ostream.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset'])) + else: + ostream.write("{0}: {1}\n".format(__name__, message)) + +def i(message, ostream=sys.stderr): """Sends an info log message. """ - if has_colors: - writeln('white', message) - else: - println(message) + printlog(message, + 'white' if has_colors else None, + ostream=ostream) -def d(message): +def d(message, ostream=sys.stderr): """Sends a debug log message. """ - if has_colors: - writeln('blue', message) - else: - println(message) + printlog(message, + 'blue' if has_colors else None, + ostream=ostream) -def w(message): +def w(message, ostream=sys.stderr): """Sends a warning log message. """ - if has_colors: - writeln('yellow', message) - else: - println(message) + printlog(message, + 'yellow' if has_colors else None, + ostream=ostream) -def e(message): +def e(message, ostream=sys.stderr): """Sends an error log message. """ - if has_colors: - writeln('light-red', message) - else: - println(message) + printlog(message, + 'bold-yellow' if has_colors else None, + ostream=ostream) -def wtf(message): +def wtf(message, ostream=sys.stderr): """What a Terrible Failure. """ - if has_colors: - writeln('bold-red', message) - else: - println(message) + printlog(message, + 'bold-red' if has_colors else None, + ostream=ostream) From 5c27e57d8c1c2211f06a4ea1ca2a4aa3b8b10032 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Tue, 29 Oct 2013 23:53:21 +0100 Subject: [PATCH 31/48] update __init__.py --- src/you_get/util/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py index e69de29b..b097d246 100644 --- a/src/you_get/util/__init__.py +++ b/src/you_get/util/__init__.py @@ -0,0 +1,3 @@ + +from .fs import * +from .log import * From c3f5f6a320e9a922717238128a6a58bb509327c7 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 00:19:08 +0100 Subject: [PATCH 32/48] add new module: util.fs --- src/you_get/common.py | 9 +++++---- src/you_get/util/fs.py | 45 ++++++++++++++++++++++++++++++++++++++++++ tests/test_util.py | 11 +++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 tests/test_util.py diff --git a/src/you_get/common.py b/src/you_get/common.py index 4ad45053..2c99976f 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -10,7 +10,7 @@ from urllib import request, parse import platform from .version import __version__ -from .util import log +from .util import log, legitimize dry_run = False force = False @@ -94,7 +94,7 @@ def parse_query_param(url, param): def unicodize(text): return re.sub(r'\\u([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])', lambda x: chr(int(x.group(0)[2:], 16)), text) -# DEPRECATED in favor of filenameable() +# DEPRECATED in favor of util.legitimize() def escape_file_path(path): path = path.replace('/', '-') path = path.replace('\\', '-') @@ -102,6 +102,7 @@ def escape_file_path(path): path = path.replace('?', '-') return path +# DEPRECATED in favor of util.legitimize() def filenameable(text): """Converts a string to a legal filename through various OSes. """ @@ -509,7 +510,7 @@ def download_urls(urls, title, ext, total_size, output_dir = '.', refer = None, traceback.print_exc(file = sys.stdout) pass - title = filenameable(title) + title = legitimize(title) filename = '%s.%s' % (title, ext) filepath = os.path.join(output_dir, filename) @@ -585,7 +586,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir = '.', refer assert ext in ('ts') - title = filenameable(title) + title = legitimize(title) filename = '%s.%s' % (title, 'ts') filepath = os.path.join(output_dir, filename) diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py index e69de29b..09aa48a9 100644 --- a/src/you_get/util/fs.py +++ b/src/you_get/util/fs.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +import platform + +def legitimize(text, os=platform.system()): + """Converts a string to a valid filename. + """ + + # POSIX systems + text = text.translate({ + 0: None, + ord('/'): '-', + }) + + if os == 'Windows': + # Windows (non-POSIX namespace) + text = text[:255] # Trim to 255 Unicode characters long + text = text.translate({ + # Reserved in Windows VFAT and NTFS + ord(':'): '-', + ord('*'): '-', + ord('?'): '-', + ord('\\'): '-', + ord('|'): '-', + ord('\"'): '\'', + # Reserved in Windows VFAT + ord('+'): '-', + ord('<'): '-', + ord('>'): '-', + ord('['): '(', + ord(']'): ')', + }) + else: + # *nix + if os == 'Darwin': + # Mac OS HFS+ + text = text.translate({ + ord(':'): '-', + }) + + # Remove leading . + if text.startswith("."): + text = text[1:] + + return text diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 00000000..0b7b0231 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +import unittest + +from you_get.util import * + +class TestUtil(unittest.TestCase): + def test_legitimize(self): + self.assertEqual(legitimize("1*2", os="Linux"), "1*2") + self.assertEqual(legitimize("1*2", os="Darwin"), "1*2") + self.assertEqual(legitimize("1*2", os="Windows"), "1-2") From 26b2be3f7c1c57cc01b74c39800e86059bcd4a7f Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 03:37:47 +0100 Subject: [PATCH 33/48] change log.i() to no color --- src/you_get/util/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py index 6f02c3a1..bb3c6b79 100644 --- a/src/you_get/util/log.py +++ b/src/you_get/util/log.py @@ -88,7 +88,7 @@ def i(message, ostream=sys.stderr): """Sends an info log message. """ printlog(message, - 'white' if has_colors else None, + None, ostream=ostream) def d(message, ostream=sys.stderr): From 1027b925383ef46beb60369a1b561ccf981fc1be Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 04:28:09 +0100 Subject: [PATCH 34/48] add log.underlined() --- src/you_get/util/log.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py index bb3c6b79..299152d0 100644 --- a/src/you_get/util/log.py +++ b/src/you_get/util/log.py @@ -68,10 +68,15 @@ colors = { 'bold-white': '\033[97;1m', } +def underlined(text): + """Returns an underlined text. + """ + return "\33[4m%s\33[24m" % text if has_colors else text + def println(text, color=None, ostream=sys.stdout): """Prints a text line to stream. """ - if color in colors: + if has_colors and color in colors: ostream.write("{0}{1}{2}\n".format(colors[color], text, colors['reset'])) else: ostream.write("{0}\n".format(text)) @@ -79,7 +84,7 @@ def println(text, color=None, ostream=sys.stdout): def printlog(message, color=None, ostream=sys.stderr): """Prints a log message to stream. """ - if color in colors: + if has_colors and color in colors: ostream.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset'])) else: ostream.write("{0}: {1}\n".format(__name__, message)) From 8919897ae884f8a26698da3cf3619d20a242f170 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:29:44 +0100 Subject: [PATCH 35/48] add new module: util.sogou_proxy --- src/you_get/common.py | 78 +++++++++++++++--- src/you_get/util/__init__.py | 2 + src/you_get/util/sogou_proxy.py | 141 ++++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 12 deletions(-) create mode 100644 src/you_get/util/sogou_proxy.py diff --git a/src/you_get/common.py b/src/you_get/common.py index 2c99976f..2233d60d 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -8,12 +8,15 @@ import re import sys from urllib import request, parse import platform +import threading from .version import __version__ -from .util import log, legitimize +from .util import log, legitimize, sogou_proxy_server dry_run = False force = False +sogou_proxy = None +sogou_env = None fake_headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -715,6 +718,35 @@ def print_info(site_info, title, type, size): print("Size: ", round(size / 1048576, 2), "MB (" + str(size) + " Bytes)") print() +def parse_host(host): + """Parses host name and port number from a string. + """ + if re.match(r'^(\d+)$', host) is not None: + return ("0.0.0.0", int(host)) + if re.match(r'^(\w+)://', host) is None: + host = "//" + host + o = parse.urlparse(host) + hostname = o.hostname or "0.0.0.0" + port = o.port or 0 + return (hostname, port) + +def get_sogou_proxy(): + return sogou_proxy + +def set_proxy(proxy): + proxy_handler = request.ProxyHandler({ + 'http': '%s:%s' % proxy, + 'https': '%s:%s' % proxy, + }) + opener = request.build_opener(proxy_handler) + request.install_opener(opener) + +def unset_proxy(): + proxy_handler = request.ProxyHandler({}) + opener = request.build_opener(proxy_handler) + request.install_opener(opener) + +# DEPRECATED in favor of set_proxy() and unset_proxy() def set_http_proxy(proxy): if proxy == None: # Use system default setting proxy_support = request.ProxyHandler() @@ -766,7 +798,7 @@ def script_main(script_name, download, download_playlist = None): ''' short_opts = 'Vhfiuno:x:' - opts = ['version', 'help', 'force', 'info', 'url', 'no-merge', 'no-proxy', 'debug', 'output-dir=', 'http-proxy='] + opts = ['version', 'help', 'force', 'info', 'url', 'no-merge', 'no-proxy', 'debug', 'sogou', 'output-dir=', 'http-proxy=', 'sogou-proxy=', 'sogou-env='] if download_playlist: short_opts = 'l' + short_opts opts = ['playlist'] + opts @@ -778,6 +810,11 @@ def script_main(script_name, download, download_playlist = None): log.e("try 'you-get --help' for more options") sys.exit(2) + global force + global dry_run + global sogou_proxy + global sogou_env + info_only = False playlist = False merge = True @@ -793,12 +830,10 @@ def script_main(script_name, download, download_playlist = None): print(help) sys.exit() elif o in ('-f', '--force'): - global force force = True elif o in ('-i', '--info'): info_only = True elif o in ('-u', '--url'): - global dry_run dry_run = True elif o in ('-l', '--playlist'): playlist = True @@ -812,19 +847,38 @@ def script_main(script_name, download, download_playlist = None): output_dir = a elif o in ('-x', '--http-proxy'): proxy = a + elif o in ('--sogou'): + sogou_proxy = ("0.0.0.0", 0) + elif o in ('--sogou-proxy'): + sogou_proxy = parse_host(a) + elif o in ('--sogou-env'): + sogou_env = a else: log.e("try 'you-get --help' for more options") sys.exit(2) if not args: - print(help) - sys.exit() + if sogou_proxy is not None: + try: + if sogou_env is not None: + server = sogou_proxy_server(sogou_proxy, network_env=sogou_env) + else: + server = sogou_proxy_server(sogou_proxy) + server.serve_forever() + except KeyboardInterrupt: + if traceback: + raise + else: + sys.exit() + else: + print(help) + sys.exit() set_http_proxy(proxy) - - if traceback: + + try: download_main(download, download_playlist, args, playlist, output_dir, merge, info_only) - else: - try: - download_main(download, download_playlist, args, playlist, output_dir, merge, info_only) - except KeyboardInterrupt: + except KeyboardInterrupt: + if traceback: + raise + else: sys.exit(1) diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py index b097d246..4c43c5fa 100644 --- a/src/you_get/util/__init__.py +++ b/src/you_get/util/__init__.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python from .fs import * from .log import * +from .sogou_proxy import * diff --git a/src/you_get/util/sogou_proxy.py b/src/you_get/util/sogou_proxy.py new file mode 100644 index 00000000..ffdc0b7a --- /dev/null +++ b/src/you_get/util/sogou_proxy.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +# Original code from: +# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/ + +from . import log + +from http.client import HTTPResponse +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn +from threading import Thread +import random, socket, struct, sys, time + +def sogou_proxy_server( + host=("0.0.0.0", 0), + network_env='CERNET', + ostream=sys.stderr): + """ + """ + + x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27' + proxy_host = 'h0.cnc.bj.ie.sogou.com' + proxy_port = 80 + + def sogou_hash(t, host): + s = (t + host + 'SogouExplorerProxy').encode('ascii') + code = len(s) + dwords = int(len(s) / 4) + rest = len(s) % 4 + v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s) + for vv in v: + if type(vv) != bytes: + a = (vv & 0xFFFF) + b = (vv >> 16) + code += a + code = code ^ (((code << 5) ^ b) << 0xb) + # To avoid overflows + code &= 0xffffffff + code += code >> 0xb + if rest == 3: + code += s[len(s) - 2] * 256 + s[len(s) - 3] + code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10) + code &= 0xffffffff + code += code >> 0xb + elif rest == 2: + code += (s[len(s) - 1]) * 256 + (s[len(s) - 2]) + code ^= code << 0xb + code &= 0xffffffff + code += code >> 0x11 + elif rest == 1: + code += s[len(s) - 1] + code ^= code << 0xa + code &= 0xffffffff + code += code >> 0x1 + code ^= code * 8 + code &= 0xffffffff + code += code >> 5 + code ^= code << 4 + code = code & 0xffffffff + code += code >> 0x11 + code ^= code << 0x19 + code = code & 0xffffffff + code += code >> 6 + code = code & 0xffffffff + return hex(code)[2:].rstrip('L').zfill(8) + + class Handler(BaseHTTPRequestHandler): + _socket = None + def do_proxy(self): + try: + if self._socket is None: + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket.connect((proxy_host, proxy_port)) + self._socket.send(self.requestline.encode('ascii') + b'\r\n') + log.d(self.requestline, ostream) + + # Add Sogou Verification Tags + self.headers['X-Sogou-Auth'] = x_sogou_auth + t = hex(int(time.time()))[2:].rstrip('L').zfill(8) + self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host']) + self.headers['X-Sogou-Timestamp'] = t + self._socket.send(str(self.headers).encode('ascii') + b'\r\n') + + # Send POST data + if self.command == 'POST': + self._socket.send(self.rfile.read(int(self.headers['Content-Length']))) + response = HTTPResponse(self._socket, method=self.command) + response.begin() + + # Response + status = 'HTTP/1.1 %s %s' % (response.status, response.reason) + self.wfile.write(status.encode('ascii') + b'\r\n') + h = '' + for hh, vv in response.getheaders(): + if hh.upper() != 'TRANSFER-ENCODING': + h += hh + ': ' + vv + '\r\n' + self.wfile.write(h.encode('ascii') + b'\r\n') + while True: + response_data = response.read(8192) + if len(response_data) == 0: + break + self.wfile.write(response_data) + + except socket.error: + log.e('Socket error for ' + self.requestline, ostream) + + def do_POST(self): + self.do_proxy() + + def do_GET(self): + self.do_proxy() + + class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + pass + + # Server starts + log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream) + + try: + server = ThreadingHTTPServer(host, Handler) + except Exception as ex: + log.wtf("Socket error: %s" % ex, ostream) + exit(1) + host = server.server_address + + if network_env.upper() == 'CERNET': + proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10) + elif network_env.upper() == 'CTCNET': + proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3) + elif network_env.upper() == 'CNCNET': + proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3) + elif network_env.upper() == 'DXT': + proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10) + else: + proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10) + + log.i('Remote host: %s' % log.underlined(proxy_host), ostream) + log.i('Proxy server running on %s' % + log.underlined("%s:%s" % host), ostream) + + return server From 8e4bc2f9fa239b735c842d9e5dee23bffa783ff9 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:32:06 +0100 Subject: [PATCH 36/48] enable Sogou proxy for Sohu --- src/you_get/extractor/sohu.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/you_get/extractor/sohu.py b/src/you_get/extractor/sohu.py index c364917f..a084f116 100644 --- a/src/you_get/extractor/sohu.py +++ b/src/you_get/extractor/sohu.py @@ -17,6 +17,14 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): if not vid: vid = r1('vid\s*:\s*"(\d+)"', html) + # Open Sogou proxy if required + if get_sogou_proxy() is not None: + server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w')) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + set_proxy(server.server_address) + if vid: data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid)) for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]: @@ -52,6 +60,11 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): urls.append(real_url(host, prot, file, new)) assert data['clipsURL'][0].endswith('.mp4') + # Close Sogou proxy if required + if get_sogou_proxy() is not None: + server.shutdown() + unset_proxy() + print_info(site_info, title, 'mp4', size) if not info_only: download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge) From ee5dd8f2dd6f224e5e8434c67e6ec903643f2774 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:32:29 +0100 Subject: [PATCH 37/48] enable Sogou proxy for Youku --- src/you_get/extractor/youku.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/you_get/extractor/youku.py b/src/you_get/extractor/youku.py index 20c79c4d..4abedc97 100644 --- a/src/you_get/extractor/youku.py +++ b/src/you_get/extractor/youku.py @@ -121,7 +121,21 @@ def file_type_of_url(url): return str(re.search(r'/st/([^/]+)/', url).group(1)) def youku_download_by_id(id, title, output_dir = '.', stream_type = None, merge = True, info_only = False): + # Open Sogou proxy if required + if get_sogou_proxy() is not None: + server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w')) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + set_proxy(server.server_address) + info = get_info(id) + + # Close Sogou proxy if required + if get_sogou_proxy() is not None: + server.shutdown() + unset_proxy() + urls, sizes = zip(*find_video(info, stream_type)) ext = file_type_of_url(urls[0]) total_size = sum(sizes) From a498ebe2bfa3788d22ffb7547e50b05f4dd89b56 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:36:41 +0100 Subject: [PATCH 38/48] TERM=screen (Tmux) compatible with ANSI/VT100 --- src/you_get/util/log.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py index 299152d0..c28fd4e7 100644 --- a/src/you_get/util/log.py +++ b/src/you_get/util/log.py @@ -10,6 +10,7 @@ if os.getenv('TERM') in ( 'vt100', 'linux', 'eterm-color', + 'screen', ): has_colors = True else: From 86c29abf6ee11996565bd4de710d56c254722a40 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:41:29 +0100 Subject: [PATCH 39/48] correct binary prefix --- src/you_get/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index 2233d60d..b96c8992 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -715,7 +715,7 @@ def print_info(site_info, title, type, size): print("Video Site:", site_info) print("Title: ", tr(title)) print("Type: ", type_info) - print("Size: ", round(size / 1048576, 2), "MB (" + str(size) + " Bytes)") + print("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)") print() def parse_host(host): From 4f7dd97e0246c5af76e6986457ce1b492c1527f6 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:46:22 +0100 Subject: [PATCH 40/48] docstring for sogou_proxy_server() --- src/you_get/util/sogou_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/util/sogou_proxy.py b/src/you_get/util/sogou_proxy.py index ffdc0b7a..01ffb572 100644 --- a/src/you_get/util/sogou_proxy.py +++ b/src/you_get/util/sogou_proxy.py @@ -15,7 +15,7 @@ def sogou_proxy_server( host=("0.0.0.0", 0), network_env='CERNET', ostream=sys.stderr): - """ + """Returns a Sogou proxy server object. """ x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27' From 53aec49f852adc70e5d779e5fa517720ff149dba Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 07:54:30 +0100 Subject: [PATCH 41/48] update help message --- src/you_get/common.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index b96c8992..25df1090 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -792,12 +792,14 @@ def script_main(script_name, download, download_playlist = None): -u | --url Display the real URLs of videos without downloading. -n | --no-merge Don't merge video parts. -o | --output-dir Set the output directory for downloaded videos. - -x | --http-proxy Use specific HTTP proxy for downloading. + -x | --http-proxy Use specific HTTP proxy for downloading. --no-proxy Don't use any proxy. (ignore $http_proxy) + -S | --sogou Use a Sogou proxy server for downloading. + --sogou-proxy Run a standalone Sogou proxy server. --debug Show traceback on KeyboardInterrupt. ''' - short_opts = 'Vhfiuno:x:' + short_opts = 'VhfiunSo:x:' opts = ['version', 'help', 'force', 'info', 'url', 'no-merge', 'no-proxy', 'debug', 'sogou', 'output-dir=', 'http-proxy=', 'sogou-proxy=', 'sogou-env='] if download_playlist: short_opts = 'l' + short_opts @@ -847,7 +849,7 @@ def script_main(script_name, download, download_playlist = None): output_dir = a elif o in ('-x', '--http-proxy'): proxy = a - elif o in ('--sogou'): + elif o in ('-S', '--sogou'): sogou_proxy = ("0.0.0.0", 0) elif o in ('--sogou-proxy'): sogou_proxy = parse_host(a) From 117e03cdc89edf4fd03740ad399ee39779a7708e Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:03:16 +0100 Subject: [PATCH 42/48] update README.md: move zh-CN document to wiki --- README.md | 227 +----------------------------------------------------- 1 file changed, 2 insertions(+), 225 deletions(-) diff --git a/README.md b/README.md index e8b0b775..1f70fb4a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ See the project homepage for further documentation. +中文说明：请参见[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%AD%E6%96%87%E8%AF%B4%E6%98%8E)。 + Fork me on GitHub: ## Features @@ -206,228 +208,3 @@ You-Get is licensed under the [MIT license](https://raw.github.com/soimort/you-g ## Contributing Please see [CONTRIBUTING.md](https://github.com/soimort/you-get/blob/master/CONTRIBUTING.md). - - - -*** - - - -# You-Get - 中文说明 - -[You-Get](https://github.com/soimort/you-get)是一个基于Python 3的视频下载工具。之所以写它的主要原因是，我找不到一个现成的下载工具能够同时支持[YouTube](http://www.youtube.com/)和[优酷](http://www.youku.com/)；而且，几乎所有以前的视频下载程序都是基于Python 2的。 - -项目主页： - -GitHub地址： - -## 特点 - -### 说明 - -You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/youku-lixian)用Python 3改写而成，增加了以下功能： - -* 支持YouTube、Vimeo等国外视频网站 -* 支持断点续传 -* 可设置HTTP代理 - -### 支持的站点（截至目前） - -已实现对以下站点的支持，以后会陆续增加（・∀・） - -* YouTube -* Vimeo -* Coursera -* Blip -* Dailymotion -* eHow -* Facebook -* Google+ -* Google Drive -* Khan Academy -* TED -* Tumblr -* Vine -* Instagram -* SoundCloud -* Mixcloud -* Freesound -* JPopsuki -* VID48 -* NICONICO动画 -* 优酷 -* 土豆 -* 音悦台 -* AcFun -* bilibili -* CNTV -* 豆瓣 -* 凤凰视频 -* 爱奇艺 -* 激动网 -* 酷6网 -* MioMio -* 网易视频 -* PPTV -* 腾讯视频 -* 新浪视频 -* 搜狐视频 -* 56网 -* 虾米 -* 5sing -* 百度音乐 -* 百度网盘 -* SongTaste -* Alive.in.th - -## 依赖 - -* [Python 3](http://www.python.org/download/releases/) -* __（可选）__ [FFmpeg](http://ffmpeg.org) - * 用于转换与合并视频文件。 - -## 安装说明 - -（以下命令格式均以Linux shell为例） - -### 1. 通过[Pip](http://www.pip-installer.org/)安装： - - $ pip install you-get - - 检查安装是否成功： - - $ you-get -V - -### 2. 通过[EasyInstall](http://pypi.python.org/pypi/setuptools)安装： - - $ easy_install you-get - - 检查安装是否成功： - - $ you-get -V - -### 3. 从Git安装： - - $ git clone git://github.com/soimort/you-get.git - - 在不安装的情况下直接使用脚本： - - $ cd you-get/ - $ ./you-get -V - - 若要将Python package安装到系统默认路径，执行： - - $ make install - - 检查安装是否成功： - - $ you-get -V - -### 4. 直接下载（从）： - - $ wget -O you-get.zip https://github.com/soimort/you-get/zipball/master - $ unzip you-get.zip - - 在不安装的情况下直接使用脚本： - - $ cd soimort-you-get-*/ - $ ./you-get -V - - 若要将Python package安装到系统默认路径，执行： - - $ make install - - 检查安装是否成功： - - $ you-get -V - -### 5. 从[AUR (Arch User Repository)](http://aur.archlinux.org/)安装： - - 点击[这里](https://aur.archlinux.org/packages.php\?ID=62576)。 - -### 升级： - -使用Pip： - - $ pip install --upgrade you-get - -### FAQ（针对Windows用户)： - -* Q：我不知道该如何在Windows下安装。 - -* A：不需要安装。直接把`you-get`目录放到系统`%PATH%`中。 - -* Q：出现错误提示`UnicodeDecodeError: 'gbk' codec can't decode byte 0xb0 in position 1012: illegal multibyte sequence`。 - -* A：执行`set PYTHONIOENCODING=utf-8`。 - -## 使用方法示例 - -### 如何下载视频 - -显示视频信息，但不进行下载（`-i`或`--info`选项）： - - $ you-get -i http://www.yinyuetai.com/video/463772 - -下载视频： - - $ you-get http://www.yinyuetai.com/video/463772 - -下载多个视频： - - $ you-get http://www.yinyuetai.com/video/463772 http://www.yinyuetai.com/video/471500 - -若当前目录下已有与视频标题同名的文件，下载时会自动跳过。若有同名的`.download`临时文件，程序会从上次中断处开始下载。 -如要强制重新下载该视频，可使用`-f`（`--force`）选项： - - $ you-get -f http://www.yinyuetai.com/video/463772 - -`-l`（`--playlist`）选项用于下载播放列表（只对某些网站适用）： - - $ you-get -l http://www.youku.com/playlist_show/id_5344313.html - -__注：从0.1.3以后的版本起，`-l`选项不再必须。You-Get可以自动识别并处理播放列表的下载。__ - -指定视频文件的下载目录： - - $ you-get -o ~/Downloads http://www.yinyuetai.com/video/463772 - -显示详细帮助： - - $ you-get -h - -### 如何设置代理 - -默认情况下，Python自动使用系统的代理配置。可以通过环境变量`http_proxy`来设置系统的HTTP代理。 - -`-x`（`--http-proxy`）选项用于手动指定You-Get所使用的HTTP代理。例如：GoAgent的代理服务器是`http://127.0.0.1:8087`，则通过该代理下载某YouTube视频的命令是： - - $ you-get -x 127.0.0.1:8087 http://www.youtube.com/watch?v=KbtO_Ayjw0M - -Windows下的自由门等翻墙软件会自动设置系统全局代理，因此无需指定HTTP代理即可下载YouTube视频： - - $ you-get http://www.youtube.com/watch?v=KbtO_Ayjw0M - -如果不希望程序在下载过程中使用任何代理（包括系统的代理配置），可以显式地指定`--no-proxy`选项： - - $ you-get --no-proxy http://v.youku.com/v_show/id_XMjI0ODc1NTc2.html - -### 断点续传 - -下载未完成时被中止（因为`Ctrl+C`终止程序或者网络中断等原因），在目标路径中会有一个扩展名为`.download`的临时文件。 - -下次运行只要在目标路径中找到相应的`.download`临时文件，程序会自动从中断处继续下载。（除非指定了`-f`选项） - -## 使用Python 2？ - -优酷等国内视频网站的下载，请移步：[iambus/youku-lixian](https://github.com/iambus/youku-lixian) - -YouTube等国外视频网站的下载，请移步：[rg3/youtube-dl](https://github.com/rg3/youtube-dl) - -## 许可证 - -You-Get在[MIT License](https://raw.github.com/soimort/you-get/master/LICENSE.txt)下发布。 - -## 如何参与贡献 / 报告issue - -请阅读 [CONTRIBUTING.md](https://github.com/soimort/you-get/blob/master/CONTRIBUTING.md)。 From 127611a66bb61d4a472f055a3d48b18fc41f9bc3 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:06:33 +0100 Subject: [PATCH 43/48] update README.md: move zh-CN document to wiki --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1f70fb4a..a41ea451 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ See the project homepage for further documentation. -中文说明：请参见[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%AD%E6%96%87%E8%AF%B4%E6%98%8E)。 - Fork me on GitHub: +__中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%AD%E6%96%87%E8%AF%B4%E6%98%8E)。 + ## Features ### Supported Sites (As of Now) From addb3b1c33c12d5177c8412d28fe116fa6d9d56a Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:09:15 +0100 Subject: [PATCH 44/48] update README.md --- README.md | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index a41ea451..60071617 100644 --- a/README.md +++ b/README.md @@ -175,31 +175,23 @@ By default, Python will apply the system proxy settings (i.e. environment variab For a complete list of all available options, see: $ you-get --help - -## Examples (For Developers) - -In Python 3 (interactive): - - >>> from you_get.downloader import * - >>> youtube.download("http://www.youtube.com/watch?v=8bQlxQJEzLk", info_only = True) - Video Site: YouTube.com - Title: If you're good at something, never do it for free! - Type: WebM video (video/webm) - Size: 0.13 MB (133176 Bytes) + Usage: you-get [OPTION]... [URL]... - >>> import you_get - >>> you_get.any_download("http://www.youtube.com/watch?v=sGwy8DsUJ4M") - Video Site: YouTube.com - Title: Mort from Madagascar LIKES - Type: WebM video (video/webm) - Size: 1.78 MB (1867072 Bytes) + Startup options: + -V | --version Display the version and exit. + -h | --help Print this help and exit. - Downloading Mort from Madagascar LIKES.webm ... - 100.0% ( 1.8/1.8 MB) [========================================] 1/1 - -## API Reference - -See source code. + Download options (use with URLs): + -f | --force Force overwriting existed files. + -i | --info Display the information of videos without downloading. + -u | --url Display the real URLs of videos without downloading. + -n | --no-merge Don't merge video parts. + -o | --output-dir Set the output directory for downloaded videos. + -x | --http-proxy Use specific HTTP proxy for downloading. + --no-proxy Don't use any proxy. (ignore $http_proxy) + -S | --sogou Use a Sogou proxy server for downloading. + --sogou-proxy Run a standalone Sogou proxy server. + --debug Show traceback on KeyboardInterrupt. ## License From 740cef54d428e4145a5e64dd7339239ae617d59f Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:09:27 +0100 Subject: [PATCH 45/48] update README.txt --- README.txt | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/README.txt b/README.txt index c4b68af8..83c84ba0 100644 --- a/README.txt +++ b/README.txt @@ -166,33 +166,23 @@ Command-Line Options For a complete list of all available options, see:: $ you-get --help - -Examples (For Developers) -------------------------- - -In Python 3 (interactive):: - - >>> from you_get.downloader import * - >>> youtube.download("http://www.youtube.com/watch?v=8bQlxQJEzLk", info_only = True) - Video Site: YouTube.com - Title: If you're good at something, never do it for free! - Type: WebM video (video/webm) - Size: 0.13 MB (133176 Bytes) + Usage: you-get [OPTION]... [URL]... - >>> import you_get - >>> you_get.any_download("http://www.youtube.com/watch?v=sGwy8DsUJ4M") - Video Site: YouTube.com - Title: Mort from Madagascar LIKES - Type: WebM video (video/webm) - Size: 1.78 MB (1867072 Bytes) + Startup options: + -V | --version Display the version and exit. + -h | --help Print this help and exit. - Downloading Mort from Madagascar LIKES.webm ... - 100.0% ( 1.8/1.8 MB) [========================================] 1/1 - -API Reference -------------- - -See source code. + Download options (use with URLs): + -f | --force Force overwriting existed files. + -i | --info Display the information of videos without downloading. + -u | --url Display the real URLs of videos without downloading. + -n | --no-merge Don't merge video parts. + -o | --output-dir Set the output directory for downloaded videos. + -x | --http-proxy Use specific HTTP proxy for downloading. + --no-proxy Don't use any proxy. (ignore $http_proxy) + -S | --sogou Use a Sogou proxy server for downloading. + --sogou-proxy Run a standalone Sogou proxy server. + --debug Show traceback on KeyboardInterrupt. License ------- From 06ba0b58da67f6cd1f089cbd917c250133fc8dcf Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:42:34 +0100 Subject: [PATCH 46/48] update README.md --- README.md | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 60071617..d8a89bae 100644 --- a/README.md +++ b/README.md @@ -67,23 +67,15 @@ __中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%A ## Installation -### 1. Install via [Pip](http://www.pip-installer.org/): +### 1. Install via Pip: - $ pip install you-get + $ [sudo] pip install you-get Check if the installation was successful: $ you-get -V -### 2. Install via [EasyInstall](http://pypi.python.org/pypi/setuptools): - - $ easy_install you-get - - Check if the installation was successful: - - $ you-get -V - -### 3. Install from Git: +### 2. Install from Git: $ git clone git://github.com/soimort/you-get.git @@ -100,7 +92,7 @@ __中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%A $ you-get -V -### 4. Direct download (from ): +### 3. Direct download (from ): $ wget -O you-get.zip https://github.com/soimort/you-get/zipball/master $ unzip you-get.zip @@ -118,27 +110,19 @@ __中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%A $ you-get -V -### 5. Install from [AUR (Arch User Repository)](http://aur.archlinux.org/): +### 4. Install from your distro's repo: - Click [here](https://aur.archlinux.org/packages.php\?ID=62576). +* __AUR (Arch)__: -### Upgrading: +* __Overlay (Gentoo)__: + +## Upgrading Using Pip: - $ pip install --upgrade you-get + $ [sudo] pip install --upgrade you-get -### FAQ (For Windows Users): - -* Q: I don't know how to install it on Windows. - -* A: Then don't do it. Just put your `you-get` folder into system `%PATH%`. - -* Q: I got something like `UnicodeDecodeError: 'gbk' codec can't decode byte 0xb0 in position 1012: illegal multibyte sequence`. - -* A: Run `set PYTHONIOENCODING=utf-8`. - -## Examples (For End-Users) +## Examples Display the information of the video without downloading: From 884ac10b17d60cb3dc2a6d6b0d08bf81d9c2a08e Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:42:39 +0100 Subject: [PATCH 47/48] update README.txt --- README.txt | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/README.txt b/README.txt index 83c84ba0..02a9408e 100644 --- a/README.txt +++ b/README.txt @@ -72,17 +72,9 @@ Dependencies Installation ------------ -#) Install via `Pip `_:: +#) Install via Pip:: - $ pip install you-get - - Check if the installation was successful:: - - $ you-get -V - -#) Install via `EasyInstall `_:: - - $ easy_install you-get + $ [sudo] pip install you-get Check if the installation was successful:: @@ -123,12 +115,21 @@ Installation $ you-get -V -#) Install from `AUR (Arch User Repository) `_: +#) Install from your distro's repo: - Click `here `_. +* `AUR (Arch) `_ -Examples (For End-Users) ------------------------- +* `Overlay (Gentoo) `_ + +Upgrading +--------- + +Using Pip:: + + $ [sudo] pip install --upgrade you-get + +Examples +-------- Display the information of the video without downloading:: From 35c9e3336c3acebaacd40272f6e8819469a874f8 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Wed, 30 Oct 2013 08:49:28 +0100 Subject: [PATCH 48/48] version 0.3.24 --- CHANGELOG.txt | 9 +++++++++ src/you_get/version.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c4d7da6c..d0409f10 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,15 @@ Changelog ========= +0.3.24 +------ + +*Date: 2013-10-30* + +* Experimental: Sogou proxy server +* Fix issues for: + - Vimeo + 0.3.23 ------ diff --git a/src/you_get/version.py b/src/you_get/version.py index 7c09b706..4e983583 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -2,5 +2,5 @@ __all__ = ['__version__', '__date__'] __name__ = 'you-get' -__version__ = '0.3.23' -__date__ = '2013-10-23' +__version__ = '0.3.24' +__date__ = '2013-10-30'