From 13d40fa694de9bd6496c011206cd876b6ad37cc0 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Mon, 21 Jul 2014 02:39:40 +0200 Subject: [PATCH] Major refactoring * Change entry point to you_get.__main__:main * New entry point for development (you-get-dev): you_get.__main__:main_dev * you_get.extractor -> you_get.extractors --- src/you_get/__init__.py | 19 +- src/you_get/__main__.py | 91 ++++++ src/you_get/common.py | 270 +++++++----------- src/you_get/extractor.py | 179 ++++++++++++ src/you_get/extractor/__main__.py | 100 ------- .../{extractor => extractors}/__init__.py | 2 - .../{extractor => extractors}/acfun.py | 0 .../{extractor => extractors}/alive.py | 0 .../{extractor => extractors}/baidu.py | 1 - .../{extractor => extractors}/bilibili.py | 0 src/you_get/{extractor => extractors}/blip.py | 0 .../{extractor => extractors}/catfun.py | 0 src/you_get/{extractor => extractors}/cbs.py | 0 src/you_get/{extractor => extractors}/cntv.py | 0 .../{extractor => extractors}/coursera.py | 0 .../{extractor => extractors}/dailymotion.py | 0 .../{extractor => extractors}/douban.py | 0 src/you_get/{extractor => extractors}/ehow.py | 0 .../{extractor => extractors}/facebook.py | 0 .../{extractor => extractors}/freesound.py | 0 .../{extractor => extractors}/google.py | 0 .../{extractor => extractors}/ifeng.py | 0 .../{extractor => extractors}/instagram.py | 0 .../{extractor => extractors}/iqiyi.py | 0 src/you_get/{extractor => extractors}/joy.py | 0 .../{extractor => extractors}/jpopsuki.py | 0 src/you_get/{extractor => extractors}/khan.py | 0 src/you_get/{extractor => extractors}/ku6.py | 0 .../{extractor => extractors}/kugou.py | 0 src/you_get/{extractor => extractors}/kuwo.py | 0 src/you_get/{extractor => extractors}/letv.py | 0 .../{extractor => extractors}/magisto.py | 0 .../{extractor => extractors}/miomio.py | 0 .../{extractor => extractors}/mixcloud.py | 0 .../{extractor => extractors}/mtv81.py | 0 .../{extractor => extractors}/netease.py | 0 .../{extractor => extractors}/nicovideo.py | 0 src/you_get/{extractor => extractors}/pptv.py | 0 src/you_get/{extractor => extractors}/qq.py | 0 src/you_get/{extractor => extractors}/sina.py | 0 src/you_get/{extractor => extractors}/sohu.py | 0 .../{extractor => extractors}/songtaste.py | 0 .../{extractor => extractors}/soundcloud.py | 0 src/you_get/{extractor => extractors}/ted.py | 0 .../{extractor => extractors}/theplatform.py | 0 .../{extractor => extractors}/tudou.py | 0 .../{extractor => extractors}/tumblr.py | 0 .../{extractor => extractors}/vid48.py | 0 .../{extractor => extractors}/vimeo.py | 0 src/you_get/{extractor => extractors}/vine.py | 0 src/you_get/{extractor => extractors}/vk.py | 0 src/you_get/{extractor => extractors}/w56.py | 0 .../{extractor => extractors}/xiami.py | 0 .../{extractor => extractors}/yinyuetai.py | 0 .../{extractor => extractors}/youku.py | 1 + .../{extractor => extractors}/youtube.py | 1 + src/you_get/util/__init__.py | 5 - src/you_get/version.py | 6 +- you-get | 22 +- you-get-dev | 18 ++ you-get.json | 2 +- 61 files changed, 422 insertions(+), 295 deletions(-) create mode 100644 src/you_get/__main__.py create mode 100644 src/you_get/extractor.py delete mode 100644 src/you_get/extractor/__main__.py rename src/you_get/{extractor => extractors}/__init__.py (97%) rename src/you_get/{extractor => extractors}/acfun.py (100%) rename src/you_get/{extractor => extractors}/alive.py (100%) rename src/you_get/{extractor => extractors}/baidu.py (99%) rename src/you_get/{extractor => extractors}/bilibili.py (100%) rename src/you_get/{extractor => extractors}/blip.py (100%) rename src/you_get/{extractor => extractors}/catfun.py (100%) rename src/you_get/{extractor => extractors}/cbs.py (100%) rename src/you_get/{extractor => extractors}/cntv.py (100%) rename src/you_get/{extractor => extractors}/coursera.py (100%) rename src/you_get/{extractor => extractors}/dailymotion.py (100%) rename src/you_get/{extractor => extractors}/douban.py (100%) rename src/you_get/{extractor => extractors}/ehow.py (100%) rename src/you_get/{extractor => extractors}/facebook.py (100%) rename src/you_get/{extractor => extractors}/freesound.py (100%) rename src/you_get/{extractor => extractors}/google.py (100%) rename src/you_get/{extractor => extractors}/ifeng.py (100%) rename src/you_get/{extractor => extractors}/instagram.py (100%) rename src/you_get/{extractor => extractors}/iqiyi.py (100%) rename src/you_get/{extractor => extractors}/joy.py (100%) rename src/you_get/{extractor => extractors}/jpopsuki.py (100%) rename src/you_get/{extractor => extractors}/khan.py (100%) rename src/you_get/{extractor => extractors}/ku6.py (100%) rename src/you_get/{extractor => extractors}/kugou.py (100%) rename src/you_get/{extractor => extractors}/kuwo.py (100%) rename src/you_get/{extractor => extractors}/letv.py (100%) rename src/you_get/{extractor => extractors}/magisto.py (100%) rename src/you_get/{extractor => extractors}/miomio.py (100%) rename src/you_get/{extractor => extractors}/mixcloud.py (100%) rename src/you_get/{extractor => extractors}/mtv81.py (100%) rename src/you_get/{extractor => extractors}/netease.py (100%) rename src/you_get/{extractor => extractors}/nicovideo.py (100%) rename src/you_get/{extractor => extractors}/pptv.py (100%) rename src/you_get/{extractor => extractors}/qq.py (100%) rename src/you_get/{extractor => extractors}/sina.py (100%) rename src/you_get/{extractor => extractors}/sohu.py (100%) rename src/you_get/{extractor => extractors}/songtaste.py (100%) rename src/you_get/{extractor => extractors}/soundcloud.py (100%) rename src/you_get/{extractor => extractors}/ted.py (100%) rename src/you_get/{extractor => extractors}/theplatform.py (100%) rename src/you_get/{extractor => extractors}/tudou.py (100%) rename src/you_get/{extractor => extractors}/tumblr.py (100%) rename src/you_get/{extractor => extractors}/vid48.py (100%) rename src/you_get/{extractor => extractors}/vimeo.py (100%) rename src/you_get/{extractor => extractors}/vine.py (100%) rename src/you_get/{extractor => extractors}/vk.py (100%) rename src/you_get/{extractor => extractors}/w56.py (100%) rename src/you_get/{extractor => extractors}/xiami.py (100%) rename src/you_get/{extractor => extractors}/yinyuetai.py (100%) rename src/you_get/{extractor => extractors}/youku.py (99%) rename src/you_get/{extractor => extractors}/youtube.py (99%) create mode 100755 you-get-dev diff --git a/src/you_get/__init__.py b/src/you_get/__init__.py index f8ee6011..5da7138f 100644 --- a/src/you_get/__init__.py +++ b/src/you_get/__init__.py @@ -1,7 +1,18 @@ #!/usr/bin/env python +# This file is Python 2 compliant. -from .common import * -from .version import * +import sys -from .cli_wrapper import * -from .extractor import * +if sys.version_info[0] == 3: + #from .extractor import Extractor, VideoExtractor + #from .util import log + + from .__main__ import * + + #from .common import * + #from .version import * + #from .cli_wrapper import * + #from .extractor import * +else: + # Don't import anything. + pass diff --git a/src/you_get/__main__.py b/src/you_get/__main__.py new file mode 100644 index 00000000..027854a7 --- /dev/null +++ b/src/you_get/__main__.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +import getopt +import os +import platform +import sys +from .version import script_name, __version__ +from .util import git, log + +_options = [ + 'help', + 'version', + 'gui', + 'force', + 'playlists', +] +_short_options = 'hVgfl' + +_help = """Usage: {} [OPTION]... [URL]... +TODO +""".format(script_name) + +def main_dev(**kwargs): + """Main entry point. + you-get-dev + """ + + # Get (branch, commit) if running from a git repo. + head = git.get_head(kwargs['repo_path']) + + # Get options and arguments. + try: + opts, args = getopt.getopt(sys.argv[1:], _short_options, _options) + except getopt.GetoptError as e: + log.wtf(""" + [Fatal] {}. + Try '{} --help' for more options.""".format(e, script_name)) + + if not opts and not args: + # Display help. + print(_help) + # Enter GUI mode. + #from .gui import gui_main + #gui_main() + else: + conf = {} + for opt, arg in opts: + if opt in ('-h', '--help'): + # Display help. + print(_help) + + elif opt in ('-V', '--version'): + # Display version. + log.println("you-get:", log.BOLD) + log.println(" version: {}".format(__version__)) + if head is not None: + log.println(" branch: {}\n commit: {}".format(*head)) + else: + log.println(" branch: {}\n commit: {}".format("(stable)", "(tag v{})".format(__version__))) + + log.println(" platform: {}".format(platform.platform())) + log.println(" python: {}".format(sys.version.split('\n')[0])) + + elif opt in ('-g', '--gui'): + # Run using GUI. + conf['gui'] = True + + elif opt in ('-f', '--force'): + # Force download. + conf['force'] = True + + elif opt in ('-l', '--playlist', '--playlists'): + # Download playlist whenever possible. + conf['playlist'] = True + + if args: + if 'gui' in conf and conf['gui']: + # Enter GUI mode. + from .gui import gui_main + gui_main(*args, **conf) + else: + # Enter console mode. + from .console import console_main + console_main(*args, **conf) + +def main(**kwargs): + """Main entry point. + you-get (legacy) + """ + from .common import main + main() diff --git a/src/you_get/common.py b/src/you_get/common.py index 1c20b538..a7035bf5 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -4,11 +4,10 @@ import getopt import json import locale import os +import platform import re import sys from urllib import request, parse -import platform -import threading from .version import __version__ from .util import log @@ -751,6 +750,18 @@ def print_info(site_info, title, type, size): print("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)") print() +def mime_to_container(mime): + mapping = { + 'video/3gpp': '3gp', + 'video/mp4': 'mp4', + 'video/webm': 'webm', + 'video/x-flv': 'flv', + } + if mime in mapping: + return mapping[mime] + else: + return mime.split('/')[1] + def parse_host(host): """Parses host name and port number from a string. """ @@ -787,6 +798,10 @@ def set_http_proxy(proxy): opener = request.build_opener(proxy_support) request.install_opener(opener) + + +from .extractors import * + def download_main(download, download_playlist, urls, playlist, **kwargs): for url in urls: if url.startswith('https://'): @@ -908,180 +923,93 @@ def script_main(script_name, download, download_playlist = None): else: sys.exit(1) +def url_to_module(url): + video_host = r1(r'https?://([^/]+)/', url) + video_url = r1(r'https?://[^/]+(.*)', url) + assert video_host and video_url, 'invalid url: ' + url + if video_host.endswith('.com.cn'): + video_host = video_host[:-3] + domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host + assert domain, 'unsupported url: ' + url -def mime_to_container(mime): - mapping = { - 'video/3gpp': '3gp', - 'video/mp4': 'mp4', - 'video/webm': 'webm', - 'video/x-flv': 'flv', + k = r1(r'([^.]+)', domain) + downloads = { + '163': netease, + '56': w56, + 'acfun': acfun, + 'baidu': baidu, + 'bilibili': bilibili, + 'blip': blip, + 'catfun':catfun, + 'cntv': cntv, + 'cbs': cbs, + 'coursera': coursera, + 'dailymotion': dailymotion, + 'douban': douban, + 'ehow': ehow, + 'facebook': facebook, + 'freesound': freesound, + 'google': google, + 'iask': sina, + 'ifeng': ifeng, + 'in': alive, + 'instagram': instagram, + 'iqiyi': iqiyi, + 'joy': joy, + 'jpopsuki': jpopsuki, + 'kankanews': bilibili, + 'khanacademy': khan, + 'ku6': ku6, + 'kugou':kugou, + 'kuwo':kuwo, + 'letv': letv, + 'magisto': magisto, + 'miomio': miomio, + 'mixcloud': mixcloud, + 'mtv81': mtv81, + 'nicovideo': nicovideo, + 'pptv': pptv, + 'qq': qq, + 'sina': sina, + 'smgbb': bilibili, + 'sohu': sohu, + 'songtaste':songtaste, + 'soundcloud': soundcloud, + 'ted': ted, + 'theplatform': theplatform, + 'tudou': tudou, + 'tumblr': tumblr, + 'vid48': vid48, + 'vimeo': vimeo, + 'vine': vine, + 'vk': vk, + 'xiami': xiami, + 'yinyuetai': yinyuetai, + 'youku': youku, + 'youtu': youtube, + 'youtube': youtube, } - if mime in mapping: - return mapping[mime] + if k in downloads: + return downloads[k], url else: - return mime.split('/')[1] - - - -class VideoExtractor(): - def __init__(self, *args): - self.url = None - self.title = None - self.vid = None - self.streams = {} - self.streams_sorted = [] - self.audiolang = None - - if args: - self.url = args[0] - - def download_by_url(self, url, **kwargs): - self.url = url - - global extractor_proxy - if extractor_proxy: - set_proxy(parse_host(extractor_proxy)) - self.prepare(**kwargs) - - try: - self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams] - except: - self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams] - - self.extract(**kwargs) - if extractor_proxy: - unset_proxy() - - self.download(**kwargs) - - def download_by_vid(self, vid, **kwargs): - self.vid = vid - - global extractor_proxy - if extractor_proxy: - set_proxy(parse_host(extractor_proxy)) - self.prepare(**kwargs) - - try: - self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams] - except: - self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams] - - self.extract(**kwargs) - if extractor_proxy: - unset_proxy() - - self.download(**kwargs) - - def prepare(self, **kwargs): - pass - #raise NotImplementedError() - - def extract(self, **kwargs): - pass - #raise NotImplementedError() - - def p_stream(self, stream_id): - stream = self.streams[stream_id] - if 'itag' in stream: - print(" - itag: \033[7m%s\033[0m" % stream_id) + import http.client + conn = http.client.HTTPConnection(video_host) + conn.request("HEAD", video_url) + res = conn.getresponse() + location = res.getheader('location') + if location is None: + raise NotImplementedError(url) else: - print(" - format: \033[7m%s\033[0m" % stream_id) + return url_to_module(location) - if 'container' in stream: - print(" container: %s" % stream['container']) +def any_download(url, **kwargs): + m, url = url_to_module(url) + m.download(url, **kwargs) - if 'video_profile' in stream: - print(" video-profile: %s" % stream['video_profile']) +def any_download_playlist(url, **kwargs): + m, url = url_to_module(url) + m.download_playlist(url, **kwargs) - if 'quality' in stream: - print(" quality: %s" % stream['quality']) - - if 'size' in stream: - print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) - - if 'itag' in stream: - print(" # download-with: \033[4myou-get --itag=%s [URL]\033[0m" % stream_id) - else: - print(" # download-with: \033[4myou-get --format=%s [URL]\033[0m" % stream_id) - - print() - - def p_i(self, stream_id): - stream = self.streams[stream_id] - print(" - title: %s" % self.title) - print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) - print(" url: %s" % self.url) - print() - - def p(self, stream_id=None): - print("site: %s" % self.__class__.name) - print("title: %s" % self.title) - if stream_id: - # Print the stream - print("stream:") - self.p_stream(stream_id) - - elif stream_id is None: - # Print stream with best quality - print("stream: # Best quality") - stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] - self.p_stream(stream_id) - - elif stream_id == []: - # Print all available streams - print("streams: # Available quality and codecs") - for stream in self.streams_sorted: - self.p_stream(stream['id'] if 'id' in stream else stream['itag']) - - if self.audiolang: - print("audio-languages:") - for i in self.audiolang: - print(" - lang: {}".format(i['lang'])) - print(" download-url: {}\n".format(i['url'])) - - def p_playlist(self, stream_id=None): - print("site: %s" % self.__class__.name) - print("playlist: %s" % self.title) - print("videos:") - - def download(self, **kwargs): - if 'info_only' in kwargs and kwargs['info_only']: - if 'stream_id' in kwargs and kwargs['stream_id']: - # Display the stream - stream_id = kwargs['stream_id'] - if 'index' not in kwargs: - self.p(stream_id) - else: - self.p_i(stream_id) - else: - # Display all available streams - if 'index' not in kwargs: - self.p([]) - else: - stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] - self.p_i(stream_id) - - else: - if 'stream_id' in kwargs and kwargs['stream_id']: - # Download the stream - stream_id = kwargs['stream_id'] - else: - # Download stream with the best quality - stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] - - if 'index' not in kwargs: - self.p(None) - else: - self.p_i(stream_id) - - urls = self.streams[stream_id]['src'] - if not urls: - log.e('[Failed] Cannot extract video source.') - log.e('This is most likely because the video has not been made available in your country.') - log.e('You may try to use a proxy via \'-y\' for extracting stream data.') - exit(1) - download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge']) - - self.__init__() +def main(): + script_main('you-get', any_download, any_download_playlist) diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py new file mode 100644 index 00000000..7c193db3 --- /dev/null +++ b/src/you_get/extractor.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python + +from .common import match1, download_urls +from .util import log + +class Extractor(): + def __init__(self, *args): + self.url = None + self.title = None + self.vid = None + self.streams = {} + self.streams_sorted = [] + + if args: + self.url = args[0] + +class VideoExtractor(): + def __init__(self, *args): + self.url = None + self.title = None + self.vid = None + self.streams = {} + self.streams_sorted = [] + self.audiolang = None + + if args: + self.url = args[0] + + def download_by_url(self, url, **kwargs): + self.url = url + + #global extractor_proxy + #if extractor_proxy: + # set_proxy(parse_host(extractor_proxy)) + self.prepare(**kwargs) + + try: + self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams] + except: + self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams] + + self.extract(**kwargs) + + #if extractor_proxy: + # unset_proxy() + + self.download(**kwargs) + + def download_by_vid(self, vid, **kwargs): + self.vid = vid + + #global extractor_proxy + #if extractor_proxy: + # set_proxy(parse_host(extractor_proxy)) + self.prepare(**kwargs) + + try: + self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams] + except: + self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams] + + self.extract(**kwargs) + #if extractor_proxy: + # unset_proxy() + + self.download(**kwargs) + + def prepare(self, **kwargs): + pass + #raise NotImplementedError() + + def extract(self, **kwargs): + pass + #raise NotImplementedError() + + def p_stream(self, stream_id): + stream = self.streams[stream_id] + if 'itag' in stream: + print(" - itag: \033[7m%s\033[0m" % stream_id) + else: + print(" - format: \033[7m%s\033[0m" % stream_id) + + if 'container' in stream: + print(" container: %s" % stream['container']) + + if 'video_profile' in stream: + print(" video-profile: %s" % stream['video_profile']) + + if 'quality' in stream: + print(" quality: %s" % stream['quality']) + + if 'size' in stream: + print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) + + if 'itag' in stream: + print(" # download-with: \033[4myou-get --itag=%s [URL]\033[0m" % stream_id) + else: + print(" # download-with: \033[4myou-get --format=%s [URL]\033[0m" % stream_id) + + print() + + def p_i(self, stream_id): + stream = self.streams[stream_id] + print(" - title: %s" % self.title) + print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size'])) + print(" url: %s" % self.url) + print() + + def p(self, stream_id=None): + print("site: %s" % self.__class__.name) + print("title: %s" % self.title) + if stream_id: + # Print the stream + print("stream:") + self.p_stream(stream_id) + + elif stream_id is None: + # Print stream with best quality + print("stream: # Best quality") + stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] + self.p_stream(stream_id) + + elif stream_id == []: + # Print all available streams + print("streams: # Available quality and codecs") + for stream in self.streams_sorted: + self.p_stream(stream['id'] if 'id' in stream else stream['itag']) + + if self.audiolang: + print("audio-languages:") + for i in self.audiolang: + print(" - lang: {}".format(i['lang'])) + print(" download-url: {}\n".format(i['url'])) + + def p_playlist(self, stream_id=None): + print("site: %s" % self.__class__.name) + print("playlist: %s" % self.title) + print("videos:") + + def download(self, **kwargs): + if 'info_only' in kwargs and kwargs['info_only']: + if 'stream_id' in kwargs and kwargs['stream_id']: + # Display the stream + stream_id = kwargs['stream_id'] + if 'index' not in kwargs: + self.p(stream_id) + else: + self.p_i(stream_id) + else: + # Display all available streams + if 'index' not in kwargs: + self.p([]) + else: + stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] + self.p_i(stream_id) + + else: + if 'stream_id' in kwargs and kwargs['stream_id']: + # Download the stream + stream_id = kwargs['stream_id'] + else: + # Download stream with the best quality + stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag'] + + if 'index' not in kwargs: + self.p(None) + else: + self.p_i(stream_id) + + urls = self.streams[stream_id]['src'] + if not urls: + log.e('[Failed] Cannot extract video source.') + log.e('This is most likely because the video has not been made available in your country.') + log.e('You may try to use a proxy via \'-y\' for extracting stream data.') + exit(1) + #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge']) + download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size']) + + self.__init__() diff --git a/src/you_get/extractor/__main__.py b/src/you_get/extractor/__main__.py deleted file mode 100644 index 8bfc15cc..00000000 --- a/src/you_get/extractor/__main__.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python -__all__ = ['main', 'any_download', 'any_download_playlist'] - -from ..extractor import * -from ..common import * - -def url_to_module(url): - video_host = r1(r'https?://([^/]+)/', url) - video_url = r1(r'https?://[^/]+(.*)', url) - assert video_host and video_url, 'invalid url: ' + url - - if video_host.endswith('.com.cn'): - video_host = video_host[:-3] - domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host - assert domain, 'unsupported url: ' + url - - k = r1(r'([^.]+)', domain) - downloads = { - '163': netease, - '56': w56, - 'acfun': acfun, - 'baidu': baidu, - 'bilibili': bilibili, - 'blip': blip, - 'catfun':catfun, - 'cntv': cntv, - 'cbs': cbs, - 'coursera': coursera, - 'dailymotion': dailymotion, - 'douban': douban, - 'ehow': ehow, - 'facebook': facebook, - 'freesound': freesound, - 'google': google, - 'iask': sina, - 'ifeng': ifeng, - 'in': alive, - 'instagram': instagram, - 'iqiyi': iqiyi, - 'joy': joy, - 'jpopsuki': jpopsuki, - 'kankanews': bilibili, - 'ku6': ku6, - 'kugou':kugou, - 'kuwo':kuwo, - 'letv': letv, - 'magisto': magisto, - 'miomio': miomio, - 'mixcloud': mixcloud, - 'mtv81':mtv81, - 'nicovideo': nicovideo, - 'pptv': pptv, - 'qq': qq, - 'sina': sina, - 'smgbb': bilibili, - 'sohu': sohu, - 'songtaste':songtaste, - 'soundcloud': soundcloud, - 'ted': ted, - 'theplatform': theplatform, - 'tudou': tudou, - 'tumblr': tumblr, - 'vid48': vid48, - 'vimeo': vimeo, - 'vine': vine, - 'vk': vk, - 'xiami': xiami, - 'yinyuetai': yinyuetai, - 'youku': youku, - 'youtu': youtube, - 'youtube': youtube, - 'khanacademy': khan, - #TODO - } - if k in downloads: - return downloads[k], url - else: - import http.client - conn = http.client.HTTPConnection(video_host) - conn.request("HEAD", video_url) - res = conn.getresponse() - location = res.getheader('location') - if location is None: - raise NotImplementedError(url) - else: - return url_to_module(location) - -def any_download(url, **kwargs): - m, url = url_to_module(url) - m.download(url, **kwargs) - -def any_download_playlist(url, **kwargs): - m, url = url_to_module(url) - m.download_playlist(url, **kwargs) - -def main(): - script_main('you-get', any_download, any_download_playlist) - -if __name__ == "__main__": - main() diff --git a/src/you_get/extractor/__init__.py b/src/you_get/extractors/__init__.py similarity index 97% rename from src/you_get/extractor/__init__.py rename to src/you_get/extractors/__init__.py index 628f4dbb..bcd926ff 100644 --- a/src/you_get/extractor/__init__.py +++ b/src/you_get/extractors/__init__.py @@ -50,5 +50,3 @@ from .youku import * from .youtube import * from .ted import * from .khan import * - -from .__main__ import * diff --git a/src/you_get/extractor/acfun.py b/src/you_get/extractors/acfun.py similarity index 100% rename from src/you_get/extractor/acfun.py rename to src/you_get/extractors/acfun.py diff --git a/src/you_get/extractor/alive.py b/src/you_get/extractors/alive.py similarity index 100% rename from src/you_get/extractor/alive.py rename to src/you_get/extractors/alive.py diff --git a/src/you_get/extractor/baidu.py b/src/you_get/extractors/baidu.py similarity index 99% rename from src/you_get/extractor/baidu.py rename to src/you_get/extractors/baidu.py index c671fa74..fdf58f28 100755 --- a/src/you_get/extractor/baidu.py +++ b/src/you_get/extractors/baidu.py @@ -4,7 +4,6 @@ __all__ = ['baidu_download'] from ..common import * -from .. import common from urllib import parse diff --git a/src/you_get/extractor/bilibili.py b/src/you_get/extractors/bilibili.py similarity index 100% rename from src/you_get/extractor/bilibili.py rename to src/you_get/extractors/bilibili.py diff --git a/src/you_get/extractor/blip.py b/src/you_get/extractors/blip.py similarity index 100% rename from src/you_get/extractor/blip.py rename to src/you_get/extractors/blip.py diff --git a/src/you_get/extractor/catfun.py b/src/you_get/extractors/catfun.py similarity index 100% rename from src/you_get/extractor/catfun.py rename to src/you_get/extractors/catfun.py diff --git a/src/you_get/extractor/cbs.py b/src/you_get/extractors/cbs.py similarity index 100% rename from src/you_get/extractor/cbs.py rename to src/you_get/extractors/cbs.py diff --git a/src/you_get/extractor/cntv.py b/src/you_get/extractors/cntv.py similarity index 100% rename from src/you_get/extractor/cntv.py rename to src/you_get/extractors/cntv.py diff --git a/src/you_get/extractor/coursera.py b/src/you_get/extractors/coursera.py similarity index 100% rename from src/you_get/extractor/coursera.py rename to src/you_get/extractors/coursera.py diff --git a/src/you_get/extractor/dailymotion.py b/src/you_get/extractors/dailymotion.py similarity index 100% rename from src/you_get/extractor/dailymotion.py rename to src/you_get/extractors/dailymotion.py diff --git a/src/you_get/extractor/douban.py b/src/you_get/extractors/douban.py similarity index 100% rename from src/you_get/extractor/douban.py rename to src/you_get/extractors/douban.py diff --git a/src/you_get/extractor/ehow.py b/src/you_get/extractors/ehow.py similarity index 100% rename from src/you_get/extractor/ehow.py rename to src/you_get/extractors/ehow.py diff --git a/src/you_get/extractor/facebook.py b/src/you_get/extractors/facebook.py similarity index 100% rename from src/you_get/extractor/facebook.py rename to src/you_get/extractors/facebook.py diff --git a/src/you_get/extractor/freesound.py b/src/you_get/extractors/freesound.py similarity index 100% rename from src/you_get/extractor/freesound.py rename to src/you_get/extractors/freesound.py diff --git a/src/you_get/extractor/google.py b/src/you_get/extractors/google.py similarity index 100% rename from src/you_get/extractor/google.py rename to src/you_get/extractors/google.py diff --git a/src/you_get/extractor/ifeng.py b/src/you_get/extractors/ifeng.py similarity index 100% rename from src/you_get/extractor/ifeng.py rename to src/you_get/extractors/ifeng.py diff --git a/src/you_get/extractor/instagram.py b/src/you_get/extractors/instagram.py similarity index 100% rename from src/you_get/extractor/instagram.py rename to src/you_get/extractors/instagram.py diff --git a/src/you_get/extractor/iqiyi.py b/src/you_get/extractors/iqiyi.py similarity index 100% rename from src/you_get/extractor/iqiyi.py rename to src/you_get/extractors/iqiyi.py diff --git a/src/you_get/extractor/joy.py b/src/you_get/extractors/joy.py similarity index 100% rename from src/you_get/extractor/joy.py rename to src/you_get/extractors/joy.py diff --git a/src/you_get/extractor/jpopsuki.py b/src/you_get/extractors/jpopsuki.py similarity index 100% rename from src/you_get/extractor/jpopsuki.py rename to src/you_get/extractors/jpopsuki.py diff --git a/src/you_get/extractor/khan.py b/src/you_get/extractors/khan.py similarity index 100% rename from src/you_get/extractor/khan.py rename to src/you_get/extractors/khan.py diff --git a/src/you_get/extractor/ku6.py b/src/you_get/extractors/ku6.py similarity index 100% rename from src/you_get/extractor/ku6.py rename to src/you_get/extractors/ku6.py diff --git a/src/you_get/extractor/kugou.py b/src/you_get/extractors/kugou.py similarity index 100% rename from src/you_get/extractor/kugou.py rename to src/you_get/extractors/kugou.py diff --git a/src/you_get/extractor/kuwo.py b/src/you_get/extractors/kuwo.py similarity index 100% rename from src/you_get/extractor/kuwo.py rename to src/you_get/extractors/kuwo.py diff --git a/src/you_get/extractor/letv.py b/src/you_get/extractors/letv.py similarity index 100% rename from src/you_get/extractor/letv.py rename to src/you_get/extractors/letv.py diff --git a/src/you_get/extractor/magisto.py b/src/you_get/extractors/magisto.py similarity index 100% rename from src/you_get/extractor/magisto.py rename to src/you_get/extractors/magisto.py diff --git a/src/you_get/extractor/miomio.py b/src/you_get/extractors/miomio.py similarity index 100% rename from src/you_get/extractor/miomio.py rename to src/you_get/extractors/miomio.py diff --git a/src/you_get/extractor/mixcloud.py b/src/you_get/extractors/mixcloud.py similarity index 100% rename from src/you_get/extractor/mixcloud.py rename to src/you_get/extractors/mixcloud.py diff --git a/src/you_get/extractor/mtv81.py b/src/you_get/extractors/mtv81.py similarity index 100% rename from src/you_get/extractor/mtv81.py rename to src/you_get/extractors/mtv81.py diff --git a/src/you_get/extractor/netease.py b/src/you_get/extractors/netease.py similarity index 100% rename from src/you_get/extractor/netease.py rename to src/you_get/extractors/netease.py diff --git a/src/you_get/extractor/nicovideo.py b/src/you_get/extractors/nicovideo.py similarity index 100% rename from src/you_get/extractor/nicovideo.py rename to src/you_get/extractors/nicovideo.py diff --git a/src/you_get/extractor/pptv.py b/src/you_get/extractors/pptv.py similarity index 100% rename from src/you_get/extractor/pptv.py rename to src/you_get/extractors/pptv.py diff --git a/src/you_get/extractor/qq.py b/src/you_get/extractors/qq.py similarity index 100% rename from src/you_get/extractor/qq.py rename to src/you_get/extractors/qq.py diff --git a/src/you_get/extractor/sina.py b/src/you_get/extractors/sina.py similarity index 100% rename from src/you_get/extractor/sina.py rename to src/you_get/extractors/sina.py diff --git a/src/you_get/extractor/sohu.py b/src/you_get/extractors/sohu.py similarity index 100% rename from src/you_get/extractor/sohu.py rename to src/you_get/extractors/sohu.py diff --git a/src/you_get/extractor/songtaste.py b/src/you_get/extractors/songtaste.py similarity index 100% rename from src/you_get/extractor/songtaste.py rename to src/you_get/extractors/songtaste.py diff --git a/src/you_get/extractor/soundcloud.py b/src/you_get/extractors/soundcloud.py similarity index 100% rename from src/you_get/extractor/soundcloud.py rename to src/you_get/extractors/soundcloud.py diff --git a/src/you_get/extractor/ted.py b/src/you_get/extractors/ted.py similarity index 100% rename from src/you_get/extractor/ted.py rename to src/you_get/extractors/ted.py diff --git a/src/you_get/extractor/theplatform.py b/src/you_get/extractors/theplatform.py similarity index 100% rename from src/you_get/extractor/theplatform.py rename to src/you_get/extractors/theplatform.py diff --git a/src/you_get/extractor/tudou.py b/src/you_get/extractors/tudou.py similarity index 100% rename from src/you_get/extractor/tudou.py rename to src/you_get/extractors/tudou.py diff --git a/src/you_get/extractor/tumblr.py b/src/you_get/extractors/tumblr.py similarity index 100% rename from src/you_get/extractor/tumblr.py rename to src/you_get/extractors/tumblr.py diff --git a/src/you_get/extractor/vid48.py b/src/you_get/extractors/vid48.py similarity index 100% rename from src/you_get/extractor/vid48.py rename to src/you_get/extractors/vid48.py diff --git a/src/you_get/extractor/vimeo.py b/src/you_get/extractors/vimeo.py similarity index 100% rename from src/you_get/extractor/vimeo.py rename to src/you_get/extractors/vimeo.py diff --git a/src/you_get/extractor/vine.py b/src/you_get/extractors/vine.py similarity index 100% rename from src/you_get/extractor/vine.py rename to src/you_get/extractors/vine.py diff --git a/src/you_get/extractor/vk.py b/src/you_get/extractors/vk.py similarity index 100% rename from src/you_get/extractor/vk.py rename to src/you_get/extractors/vk.py diff --git a/src/you_get/extractor/w56.py b/src/you_get/extractors/w56.py similarity index 100% rename from src/you_get/extractor/w56.py rename to src/you_get/extractors/w56.py diff --git a/src/you_get/extractor/xiami.py b/src/you_get/extractors/xiami.py similarity index 100% rename from src/you_get/extractor/xiami.py rename to src/you_get/extractors/xiami.py diff --git a/src/you_get/extractor/yinyuetai.py b/src/you_get/extractors/yinyuetai.py similarity index 100% rename from src/you_get/extractor/yinyuetai.py rename to src/you_get/extractors/yinyuetai.py diff --git a/src/you_get/extractor/youku.py b/src/you_get/extractors/youku.py similarity index 99% rename from src/you_get/extractor/youku.py rename to src/you_get/extractors/youku.py index 2cb7a5ce..8b4304e4 100644 --- a/src/you_get/extractor/youku.py +++ b/src/you_get/extractors/youku.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from ..common import * +from ..extractor import VideoExtractor class Youku(VideoExtractor): name = "优酷 (Youku)" diff --git a/src/you_get/extractor/youtube.py b/src/you_get/extractors/youtube.py similarity index 99% rename from src/you_get/extractor/youtube.py rename to src/you_get/extractors/youtube.py index d419488f..ddd63cf0 100644 --- a/src/you_get/extractor/youtube.py +++ b/src/you_get/extractors/youtube.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from ..common import * +from ..extractor import VideoExtractor class YouTube(VideoExtractor): name = "YouTube" diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py index 5345f0ac..e69de29b 100644 --- a/src/you_get/util/__init__.py +++ b/src/you_get/util/__init__.py @@ -1,5 +0,0 @@ -#!/usr/bin/env python - -from .fs import * -from .log import * -from .strings import * diff --git a/src/you_get/version.py b/src/you_get/version.py index c3d6c953..f0e15558 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -__all__ = ['__version__', '__date__'] -__name__ = 'you-get' -__version__ = '0.3.30dev-20140716' -__date__ = '2014-07-16' +script_name = 'you-get' +__version__ = '0.3.30dev' diff --git a/you-get b/you-get index 8bdc77c7..e576af99 100755 --- a/you-get +++ b/you-get @@ -1,10 +1,18 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python +# This file is Python 2 compliant. import os, sys -__path__ = os.path.dirname(os.path.realpath(__file__)) -__srcdir__ = 'src' -sys.path.insert(1, os.path.join(__path__, __srcdir__)) -from you_get.extractor import main -if __name__ == '__main__': - main() +_srcdir = 'src/' +_filepath = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(1, os.path.join(_filepath, _srcdir)) + +if sys.version_info[0] == 3: + import you_get + if __name__ == '__main__': + you_get.main(repo_path=_filepath) +else: + from you_get.util import log + log.wtf(""" + [Fatal] Python 3 is required. + If Python 3 is already installed on your machine, try to run this script using 'python3 you-get'.""") diff --git a/you-get-dev b/you-get-dev new file mode 100755 index 00000000..a63493b4 --- /dev/null +++ b/you-get-dev @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# This file is Python 2 compliant. + +import os, sys + +_srcdir = 'src/' +_filepath = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(1, os.path.join(_filepath, _srcdir)) + +if sys.version_info[0] == 3: + import you_get + if __name__ == '__main__': + you_get.main_dev(repo_path=_filepath) +else: + from you_get.util import log + log.wtf(""" + [Fatal] Python 3 is required. + If Python 3 is already installed on your machine, try to run this script using 'python3 you-get'.""") diff --git a/you-get.json b/you-get.json index d5c1190a..f331b834 100644 --- a/you-get.json +++ b/you-get.json @@ -32,6 +32,6 @@ ], "console_scripts": [ - "you-get = you_get.extractor.__main__:main" + "you-get = you_get.__main__:main" ] }