mirror of
https://github.com/soimort/you-get.git
synced 2025-01-24 05:55:02 +03:00
Merge remote-tracking branch 'upstream/develop' into develop
This commit is contained in:
jackyzy823
commit
6b4c7447cf
1
.gitignore
vendored
1
.gitignore
vendored
@ -5,6 +5,7 @@
|
||||
*.py[cod]
|
||||
|
||||
_*/
|
||||
*_
|
||||
|
||||
*.bak
|
||||
*.download
|
||||
|
||||
1
Makefile
1
Makefile
@ -14,6 +14,7 @@ clean:
|
||||
zenity --question
|
||||
rm -fr build/ dist/ src/*.egg-info/
|
||||
find . | grep __pycache__ | xargs rm -fr
|
||||
find . | grep .pyc | xargs rm -f
|
||||
|
||||
all: build sdist bdist bdist_egg
|
||||
|
||||
|
||||
@ -65,6 +65,7 @@ __中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%A
|
||||
* SongTaste <http://www.songtaste.com>
|
||||
* Alive.in.th <http://alive.in.th>
|
||||
* VK <http://vk.com>
|
||||
* Catfun (喵星球) <http://www.catfun.tv>
|
||||
|
||||
## Dependencies
|
||||
|
||||
@ -186,8 +187,6 @@ For a complete list of all available options, see:
|
||||
-x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
|
||||
-y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
|
||||
--no-proxy Don't use any proxy. (ignore $http_proxy)
|
||||
-S | --sogou Use a Sogou proxy server for downloading.
|
||||
--sogou-proxy <HOST:PORT> Run a standalone Sogou proxy server.
|
||||
--debug Show traceback on KeyboardInterrupt.
|
||||
|
||||
## License
|
||||
|
||||
@ -68,6 +68,7 @@ Supported Sites (As of Now)
|
||||
* SongTaste http://www.songtaste.com
|
||||
* Alive.in.th http://alive.in.th
|
||||
* VK http://vk.com
|
||||
* Catfun (喵星球) http://www.catfun.tv
|
||||
|
||||
Dependencies
|
||||
------------
|
||||
@ -194,8 +195,6 @@ For a complete list of all available options, see::
|
||||
-x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
|
||||
-y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
|
||||
--no-proxy Don't use any proxy. (ignore $http_proxy)
|
||||
-S | --sogou Use a Sogou proxy server for downloading.
|
||||
--sogou-proxy <HOST:PORT> Run a standalone Sogou proxy server.
|
||||
--debug Show traceback on KeyboardInterrupt.
|
||||
|
||||
License
|
||||
|
||||
@ -1,7 +1,18 @@
|
||||
#!/usr/bin/env python
|
||||
# This file is Python 2 compliant.
|
||||
|
||||
from .common import *
|
||||
from .version import *
|
||||
import sys
|
||||
|
||||
from .cli_wrapper import *
|
||||
from .extractor import *
|
||||
if sys.version_info[0] == 3:
|
||||
#from .extractor import Extractor, VideoExtractor
|
||||
#from .util import log
|
||||
|
||||
from .__main__ import *
|
||||
|
||||
#from .common import *
|
||||
#from .version import *
|
||||
#from .cli_wrapper import *
|
||||
#from .extractor import *
|
||||
else:
|
||||
# Don't import anything.
|
||||
pass
|
||||
|
||||
91
src/you_get/__main__.py
Normal file
91
src/you_get/__main__.py
Normal file
@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import getopt
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
from .version import script_name, __version__
|
||||
from .util import git, log
|
||||
|
||||
_options = [
|
||||
'help',
|
||||
'version',
|
||||
'gui',
|
||||
'force',
|
||||
'playlists',
|
||||
]
|
||||
_short_options = 'hVgfl'
|
||||
|
||||
_help = """Usage: {} [OPTION]... [URL]...
|
||||
TODO
|
||||
""".format(script_name)
|
||||
|
||||
def main_dev(**kwargs):
|
||||
"""Main entry point.
|
||||
you-get-dev
|
||||
"""
|
||||
|
||||
# Get (branch, commit) if running from a git repo.
|
||||
head = git.get_head(kwargs['repo_path'])
|
||||
|
||||
# Get options and arguments.
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], _short_options, _options)
|
||||
except getopt.GetoptError as e:
|
||||
log.wtf("""
|
||||
[Fatal] {}.
|
||||
Try '{} --help' for more options.""".format(e, script_name))
|
||||
|
||||
if not opts and not args:
|
||||
# Display help.
|
||||
print(_help)
|
||||
# Enter GUI mode.
|
||||
#from .gui import gui_main
|
||||
#gui_main()
|
||||
else:
|
||||
conf = {}
|
||||
for opt, arg in opts:
|
||||
if opt in ('-h', '--help'):
|
||||
# Display help.
|
||||
print(_help)
|
||||
|
||||
elif opt in ('-V', '--version'):
|
||||
# Display version.
|
||||
log.println("you-get:", log.BOLD)
|
||||
log.println(" version: {}".format(__version__))
|
||||
if head is not None:
|
||||
log.println(" branch: {}\n commit: {}".format(*head))
|
||||
else:
|
||||
log.println(" branch: {}\n commit: {}".format("(stable)", "(tag v{})".format(__version__)))
|
||||
|
||||
log.println(" platform: {}".format(platform.platform()))
|
||||
log.println(" python: {}".format(sys.version.split('\n')[0]))
|
||||
|
||||
elif opt in ('-g', '--gui'):
|
||||
# Run using GUI.
|
||||
conf['gui'] = True
|
||||
|
||||
elif opt in ('-f', '--force'):
|
||||
# Force download.
|
||||
conf['force'] = True
|
||||
|
||||
elif opt in ('-l', '--playlist', '--playlists'):
|
||||
# Download playlist whenever possible.
|
||||
conf['playlist'] = True
|
||||
|
||||
if args:
|
||||
if 'gui' in conf and conf['gui']:
|
||||
# Enter GUI mode.
|
||||
from .gui import gui_main
|
||||
gui_main(*args, **conf)
|
||||
else:
|
||||
# Enter console mode.
|
||||
from .console import console_main
|
||||
console_main(*args, **conf)
|
||||
|
||||
def main(**kwargs):
|
||||
"""Main entry point.
|
||||
you-get (legacy)
|
||||
"""
|
||||
from .common import main
|
||||
main()
|
||||
@ -4,21 +4,19 @@ import getopt
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import sys
|
||||
from urllib import request, parse
|
||||
import platform
|
||||
import threading
|
||||
|
||||
from .version import __version__
|
||||
from .util import log, sogou_proxy_server, get_filename, unescape_html
|
||||
from .util import log
|
||||
from .util.strings import get_filename, unescape_html
|
||||
|
||||
dry_run = False
|
||||
force = False
|
||||
player = None
|
||||
extractor_proxy = None
|
||||
sogou_proxy = None
|
||||
sogou_env = None
|
||||
cookies_txt = None
|
||||
|
||||
fake_headers = {
|
||||
@ -752,6 +750,18 @@ def print_info(site_info, title, type, size):
|
||||
print("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
|
||||
print()
|
||||
|
||||
def mime_to_container(mime):
|
||||
mapping = {
|
||||
'video/3gpp': '3gp',
|
||||
'video/mp4': 'mp4',
|
||||
'video/webm': 'webm',
|
||||
'video/x-flv': 'flv',
|
||||
}
|
||||
if mime in mapping:
|
||||
return mapping[mime]
|
||||
else:
|
||||
return mime.split('/')[1]
|
||||
|
||||
def parse_host(host):
|
||||
"""Parses host name and port number from a string.
|
||||
"""
|
||||
@ -764,9 +774,6 @@ def parse_host(host):
|
||||
port = o.port or 0
|
||||
return (hostname, port)
|
||||
|
||||
def get_sogou_proxy():
|
||||
return sogou_proxy
|
||||
|
||||
def set_proxy(proxy):
|
||||
proxy_handler = request.ProxyHandler({
|
||||
'http': '%s:%s' % proxy,
|
||||
@ -791,6 +798,8 @@ def set_http_proxy(proxy):
|
||||
opener = request.build_opener(proxy_support)
|
||||
request.install_opener(opener)
|
||||
|
||||
|
||||
|
||||
def download_main(download, download_playlist, urls, playlist, **kwargs):
|
||||
for url in urls:
|
||||
if url.startswith('https://'):
|
||||
@ -803,18 +812,8 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
|
||||
else:
|
||||
download(url, **kwargs)
|
||||
|
||||
def get_version():
|
||||
try:
|
||||
import subprocess
|
||||
real_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=real_dir, stderr=subprocess.DEVNULL).decode('utf-8').strip()
|
||||
assert git_hash
|
||||
return '%s-%s' % (__version__, git_hash)
|
||||
except:
|
||||
return __version__
|
||||
|
||||
def script_main(script_name, download, download_playlist = None):
|
||||
version = 'You-Get %s, a video downloader.' % get_version()
|
||||
version = 'You-Get %s, a video downloader.' % __version__
|
||||
help = 'Usage: %s [OPTION]... [URL]...\n' % script_name
|
||||
help += '''\nStartup options:
|
||||
-V | --version Display the version and exit.
|
||||
@ -832,13 +831,11 @@ def script_main(script_name, download, download_playlist = None):
|
||||
-x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
|
||||
-y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
|
||||
--no-proxy Don't use any proxy. (ignore $http_proxy)
|
||||
-S | --sogou Use a Sogou proxy server for downloading.
|
||||
--sogou-proxy <HOST:PORT> Run a standalone Sogou proxy server.
|
||||
--debug Show traceback on KeyboardInterrupt.
|
||||
'''
|
||||
|
||||
short_opts = 'Vhfiuc:nSF:o:p:x:y:'
|
||||
opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'sogou', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'sogou-proxy=', 'sogou-env=']
|
||||
short_opts = 'Vhfiuc:nF:o:p:x:y:'
|
||||
opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
|
||||
if download_playlist:
|
||||
short_opts = 'l' + short_opts
|
||||
opts = ['playlist'] + opts
|
||||
@ -854,8 +851,6 @@ def script_main(script_name, download, download_playlist = None):
|
||||
global dry_run
|
||||
global player
|
||||
global extractor_proxy
|
||||
global sogou_proxy
|
||||
global sogou_env
|
||||
global cookies_txt
|
||||
cookies_txt = None
|
||||
|
||||
@ -863,6 +858,7 @@ def script_main(script_name, download, download_playlist = None):
|
||||
playlist = False
|
||||
merge = True
|
||||
stream_id = None
|
||||
lang = None
|
||||
output_dir = '.'
|
||||
proxy = None
|
||||
extractor_proxy = None
|
||||
@ -903,31 +899,14 @@ def script_main(script_name, download, download_playlist = None):
|
||||
proxy = a
|
||||
elif o in ('-y', '--extractor-proxy'):
|
||||
extractor_proxy = a
|
||||
elif o in ('-S', '--sogou'):
|
||||
sogou_proxy = ("0.0.0.0", 0)
|
||||
elif o in ('--sogou-proxy',):
|
||||
sogou_proxy = parse_host(a)
|
||||
elif o in ('--sogou-env',):
|
||||
sogou_env = a
|
||||
elif o in ('--lang',):
|
||||
lang = a
|
||||
else:
|
||||
log.e("try 'you-get --help' for more options")
|
||||
sys.exit(2)
|
||||
if not args:
|
||||
if sogou_proxy is not None:
|
||||
try:
|
||||
if sogou_env is not None:
|
||||
server = sogou_proxy_server(sogou_proxy, network_env=sogou_env)
|
||||
else:
|
||||
server = sogou_proxy_server(sogou_proxy)
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
if traceback:
|
||||
raise
|
||||
else:
|
||||
sys.exit()
|
||||
else:
|
||||
print(help)
|
||||
sys.exit()
|
||||
print(help)
|
||||
sys.exit()
|
||||
|
||||
set_http_proxy(proxy)
|
||||
|
||||
@ -942,173 +921,95 @@ def script_main(script_name, download, download_playlist = None):
|
||||
else:
|
||||
sys.exit(1)
|
||||
|
||||
def url_to_module(url):
|
||||
from .extractors import netease, w56, acfun, baidu, bilibili, blip, catfun, cntv, cbs, coursera, dailymotion, douban, ehow, facebook, freesound, google, sina, ifeng, alive, instagram, iqiyi, joy, jpopsuki, khan, ku6, kugou, kuwo, letv, magisto, miomio, mixcloud, mtv81, nicovideo, pptv, qq, sohu, songtaste, soundcloud, ted, theplatform, tudou, tumblr, vid48, vimeo, vine, vk, xiami, yinyuetai, youku, youtube
|
||||
|
||||
video_host = r1(r'https?://([^/]+)/', url)
|
||||
video_url = r1(r'https?://[^/]+(.*)', url)
|
||||
assert video_host and video_url, 'invalid url: ' + url
|
||||
|
||||
def mime_to_container(mime):
|
||||
mapping = {
|
||||
'video/3gpp': '3gp',
|
||||
'video/mp4': 'mp4',
|
||||
'video/webm': 'webm',
|
||||
'video/x-flv': 'flv',
|
||||
if video_host.endswith('.com.cn'):
|
||||
video_host = video_host[:-3]
|
||||
domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
|
||||
assert domain, 'unsupported url: ' + url
|
||||
|
||||
k = r1(r'([^.]+)', domain)
|
||||
downloads = {
|
||||
'163': netease,
|
||||
'56': w56,
|
||||
'acfun': acfun,
|
||||
'baidu': baidu,
|
||||
'bilibili': bilibili,
|
||||
'blip': blip,
|
||||
'catfun': catfun,
|
||||
'cntv': cntv,
|
||||
'cbs': cbs,
|
||||
'coursera': coursera,
|
||||
'dailymotion': dailymotion,
|
||||
'douban': douban,
|
||||
'ehow': ehow,
|
||||
'facebook': facebook,
|
||||
'freesound': freesound,
|
||||
'google': google,
|
||||
'iask': sina,
|
||||
'ifeng': ifeng,
|
||||
'in': alive,
|
||||
'instagram': instagram,
|
||||
'iqiyi': iqiyi,
|
||||
'joy': joy,
|
||||
'jpopsuki': jpopsuki,
|
||||
'kankanews': bilibili,
|
||||
'khanacademy': khan,
|
||||
'ku6': ku6,
|
||||
'kugou': kugou,
|
||||
'kuwo': kuwo,
|
||||
'letv': letv,
|
||||
'magisto': magisto,
|
||||
'miomio': miomio,
|
||||
'mixcloud': mixcloud,
|
||||
'mtv81': mtv81,
|
||||
'nicovideo': nicovideo,
|
||||
'pptv': pptv,
|
||||
'qq': qq,
|
||||
'sina': sina,
|
||||
'smgbb': bilibili,
|
||||
'sohu': sohu,
|
||||
'songtaste': songtaste,
|
||||
'soundcloud': soundcloud,
|
||||
'ted': ted,
|
||||
'theplatform': theplatform,
|
||||
'tudou': tudou,
|
||||
'tumblr': tumblr,
|
||||
'vid48': vid48,
|
||||
'vimeo': vimeo,
|
||||
'vine': vine,
|
||||
'vk': vk,
|
||||
'xiami': xiami,
|
||||
'yinyuetai': yinyuetai,
|
||||
'youku': youku,
|
||||
'youtu': youtube,
|
||||
'youtube': youtube,
|
||||
}
|
||||
if mime in mapping:
|
||||
return mapping[mime]
|
||||
if k in downloads:
|
||||
return downloads[k], url
|
||||
else:
|
||||
return mime.split('/')[1]
|
||||
|
||||
|
||||
|
||||
class VideoExtractor():
|
||||
def __init__(self, *args):
|
||||
self.url = None
|
||||
self.title = None
|
||||
self.vid = None
|
||||
self.streams = {}
|
||||
self.streams_sorted = []
|
||||
|
||||
if args:
|
||||
self.url = args[0]
|
||||
|
||||
def download_by_url(self, url, **kwargs):
|
||||
self.url = url
|
||||
|
||||
global extractor_proxy
|
||||
if extractor_proxy:
|
||||
set_proxy(parse_host(extractor_proxy))
|
||||
self.prepare(**kwargs)
|
||||
|
||||
try:
|
||||
self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
|
||||
except:
|
||||
self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
|
||||
|
||||
self.extract(**kwargs)
|
||||
if extractor_proxy:
|
||||
unset_proxy()
|
||||
|
||||
self.download(**kwargs)
|
||||
|
||||
def download_by_vid(self, vid, **kwargs):
|
||||
self.vid = vid
|
||||
|
||||
global extractor_proxy
|
||||
if extractor_proxy:
|
||||
set_proxy(parse_host(extractor_proxy))
|
||||
self.prepare(**kwargs)
|
||||
|
||||
try:
|
||||
self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
|
||||
except:
|
||||
self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
|
||||
|
||||
self.extract(**kwargs)
|
||||
if extractor_proxy:
|
||||
unset_proxy()
|
||||
|
||||
self.download(**kwargs)
|
||||
|
||||
def prepare(self, **kwargs):
|
||||
pass
|
||||
#raise NotImplementedError()
|
||||
|
||||
def extract(self, **kwargs):
|
||||
pass
|
||||
#raise NotImplementedError()
|
||||
|
||||
def p_stream(self, stream_id):
|
||||
stream = self.streams[stream_id]
|
||||
if 'itag' in stream:
|
||||
print(" - itag: \033[7m%s\033[0m" % stream_id)
|
||||
import http.client
|
||||
conn = http.client.HTTPConnection(video_host)
|
||||
conn.request("HEAD", video_url)
|
||||
res = conn.getresponse()
|
||||
location = res.getheader('location')
|
||||
if location is None:
|
||||
raise NotImplementedError(url)
|
||||
else:
|
||||
print(" - id: \033[7m%s\033[0m" % stream_id)
|
||||
return url_to_module(location)
|
||||
|
||||
if 'container' in stream:
|
||||
print(" container: %s" % stream['container'])
|
||||
def any_download(url, **kwargs):
|
||||
m, url = url_to_module(url)
|
||||
m.download(url, **kwargs)
|
||||
|
||||
if 'video_profile' in stream:
|
||||
print(" video-profile: %s" % stream['video_profile'])
|
||||
def any_download_playlist(url, **kwargs):
|
||||
m, url = url_to_module(url)
|
||||
m.download_playlist(url, **kwargs)
|
||||
|
||||
if 'quality' in stream:
|
||||
print(" quality: %s" % stream['quality'])
|
||||
|
||||
if 'size' in stream:
|
||||
print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
|
||||
|
||||
if 'itag' in stream:
|
||||
print(" # download-with: \033[4myou-get --itag=%s [URL]\033[0m" % stream_id)
|
||||
else:
|
||||
print(" # download-with: \033[4myou-get --format=%s [URL]\033[0m" % stream_id)
|
||||
|
||||
print()
|
||||
|
||||
def p_i(self, stream_id):
|
||||
stream = self.streams[stream_id]
|
||||
print(" - title: %s" % self.title)
|
||||
print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
|
||||
print(" url: %s" % self.url)
|
||||
print()
|
||||
|
||||
def p(self, stream_id=None):
|
||||
print("site: %s" % self.__class__.name)
|
||||
print("title: %s" % self.title)
|
||||
if stream_id:
|
||||
# Print the stream
|
||||
print("stream:")
|
||||
self.p_stream(stream_id)
|
||||
|
||||
elif stream_id is None:
|
||||
# Print stream with best quality
|
||||
print("stream: # Best quality")
|
||||
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
|
||||
self.p_stream(stream_id)
|
||||
|
||||
elif stream_id == []:
|
||||
# Print all available streams
|
||||
print("streams: # Available quality and codecs")
|
||||
for stream in self.streams_sorted:
|
||||
self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
|
||||
|
||||
def p_playlist(self, stream_id=None):
|
||||
print("site: %s" % self.__class__.name)
|
||||
print("playlist: %s" % self.title)
|
||||
print("videos:")
|
||||
|
||||
def download(self, **kwargs):
|
||||
if 'info_only' in kwargs and kwargs['info_only']:
|
||||
if 'stream_id' in kwargs and kwargs['stream_id']:
|
||||
# Display the stream
|
||||
stream_id = kwargs['stream_id']
|
||||
if 'index' not in kwargs:
|
||||
self.p(stream_id)
|
||||
else:
|
||||
self.p_i(stream_id)
|
||||
else:
|
||||
# Display all available streams
|
||||
if 'index' not in kwargs:
|
||||
self.p([])
|
||||
else:
|
||||
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
|
||||
self.p_i(stream_id)
|
||||
|
||||
else:
|
||||
if 'stream_id' in kwargs and kwargs['stream_id']:
|
||||
# Download the stream
|
||||
stream_id = kwargs['stream_id']
|
||||
else:
|
||||
# Download stream with the best quality
|
||||
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
|
||||
|
||||
if 'index' not in kwargs:
|
||||
self.p(None)
|
||||
else:
|
||||
self.p_i(stream_id)
|
||||
|
||||
urls = self.streams[stream_id]['src']
|
||||
if not urls:
|
||||
log.e('[Failed] Cannot extract video source.')
|
||||
log.e('This is most likely because the video has not been made available in your country.')
|
||||
log.e('You may try to use a proxy via \'-y\' for extracting stream data.')
|
||||
exit(1)
|
||||
download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge'])
|
||||
|
||||
self.__init__()
|
||||
def main():
|
||||
script_main('you-get', any_download, any_download_playlist)
|
||||
|
||||
179
src/you_get/extractor.py
Normal file
179
src/you_get/extractor.py
Normal file
@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from .common import match1, download_urls
|
||||
from .util import log
|
||||
|
||||
class Extractor():
|
||||
def __init__(self, *args):
|
||||
self.url = None
|
||||
self.title = None
|
||||
self.vid = None
|
||||
self.streams = {}
|
||||
self.streams_sorted = []
|
||||
|
||||
if args:
|
||||
self.url = args[0]
|
||||
|
||||
class VideoExtractor():
|
||||
def __init__(self, *args):
|
||||
self.url = None
|
||||
self.title = None
|
||||
self.vid = None
|
||||
self.streams = {}
|
||||
self.streams_sorted = []
|
||||
self.audiolang = None
|
||||
|
||||
if args:
|
||||
self.url = args[0]
|
||||
|
||||
def download_by_url(self, url, **kwargs):
|
||||
self.url = url
|
||||
|
||||
#global extractor_proxy
|
||||
#if extractor_proxy:
|
||||
# set_proxy(parse_host(extractor_proxy))
|
||||
self.prepare(**kwargs)
|
||||
|
||||
try:
|
||||
self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
|
||||
except:
|
||||
self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
|
||||
|
||||
self.extract(**kwargs)
|
||||
|
||||
#if extractor_proxy:
|
||||
# unset_proxy()
|
||||
|
||||
self.download(**kwargs)
|
||||
|
||||
def download_by_vid(self, vid, **kwargs):
|
||||
self.vid = vid
|
||||
|
||||
#global extractor_proxy
|
||||
#if extractor_proxy:
|
||||
# set_proxy(parse_host(extractor_proxy))
|
||||
self.prepare(**kwargs)
|
||||
|
||||
try:
|
||||
self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
|
||||
except:
|
||||
self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
|
||||
|
||||
self.extract(**kwargs)
|
||||
#if extractor_proxy:
|
||||
# unset_proxy()
|
||||
|
||||
self.download(**kwargs)
|
||||
|
||||
def prepare(self, **kwargs):
|
||||
pass
|
||||
#raise NotImplementedError()
|
||||
|
||||
def extract(self, **kwargs):
|
||||
pass
|
||||
#raise NotImplementedError()
|
||||
|
||||
def p_stream(self, stream_id):
|
||||
stream = self.streams[stream_id]
|
||||
if 'itag' in stream:
|
||||
print(" - itag: %s" % log.sprint(stream_id, log.NEGATIVE))
|
||||
else:
|
||||
print(" - format: %s" % log.sprint(stream_id, log.NEGATIVE))
|
||||
|
||||
if 'container' in stream:
|
||||
print(" container: %s" % stream['container'])
|
||||
|
||||
if 'video_profile' in stream:
|
||||
print(" video-profile: %s" % stream['video_profile'])
|
||||
|
||||
if 'quality' in stream:
|
||||
print(" quality: %s" % stream['quality'])
|
||||
|
||||
if 'size' in stream:
|
||||
print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
|
||||
|
||||
if 'itag' in stream:
|
||||
print(" # download-with: %s" % log.sprint("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE))
|
||||
else:
|
||||
print(" # download-with: %s" % log.sprint("you-get --format=%s [URL]" % stream_id, log.UNDERLINE))
|
||||
|
||||
print()
|
||||
|
||||
def p_i(self, stream_id):
|
||||
stream = self.streams[stream_id]
|
||||
print(" - title: %s" % self.title)
|
||||
print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
|
||||
print(" url: %s" % self.url)
|
||||
print()
|
||||
|
||||
def p(self, stream_id=None):
|
||||
print("site: %s" % self.__class__.name)
|
||||
print("title: %s" % self.title)
|
||||
if stream_id:
|
||||
# Print the stream
|
||||
print("stream:")
|
||||
self.p_stream(stream_id)
|
||||
|
||||
elif stream_id is None:
|
||||
# Print stream with best quality
|
||||
print("stream: # Best quality")
|
||||
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
|
||||
self.p_stream(stream_id)
|
||||
|
||||
elif stream_id == []:
|
||||
# Print all available streams
|
||||
print("streams: # Available quality and codecs")
|
||||
for stream in self.streams_sorted:
|
||||
self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
|
||||
|
||||
if self.audiolang:
|
||||
print("audio-languages:")
|
||||
for i in self.audiolang:
|
||||
print(" - lang: {}".format(i['lang']))
|
||||
print(" download-url: {}\n".format(i['url']))
|
||||
|
||||
def p_playlist(self, stream_id=None):
|
||||
print("site: %s" % self.__class__.name)
|
||||
print("playlist: %s" % self.title)
|
||||
print("videos:")
|
||||
|
||||
def download(self, **kwargs):
|
||||
if 'info_only' in kwargs and kwargs['info_only']:
|
||||
if 'stream_id' in kwargs and kwargs['stream_id']:
|
||||
# Display the stream
|
||||
stream_id = kwargs['stream_id']
|
||||
if 'index' not in kwargs:
|
||||
self.p(stream_id)
|
||||
else:
|
||||
self.p_i(stream_id)
|
||||
else:
|
||||
# Display all available streams
|
||||
if 'index' not in kwargs:
|
||||
self.p([])
|
||||
else:
|
||||
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
|
||||
self.p_i(stream_id)
|
||||
|
||||
else:
|
||||
if 'stream_id' in kwargs and kwargs['stream_id']:
|
||||
# Download the stream
|
||||
stream_id = kwargs['stream_id']
|
||||
else:
|
||||
# Download stream with the best quality
|
||||
stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
|
||||
|
||||
if 'index' not in kwargs:
|
||||
self.p(None)
|
||||
else:
|
||||
self.p_i(stream_id)
|
||||
|
||||
urls = self.streams[stream_id]['src']
|
||||
if not urls:
|
||||
log.e('[Failed] Cannot extract video source.')
|
||||
log.e('This is most likely because the video has not been made available in your country.')
|
||||
log.e('You may try to use a proxy via \'-y\' for extracting stream data.')
|
||||
exit(1)
|
||||
#download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge'])
|
||||
download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
|
||||
|
||||
self.__init__()
|
||||
@ -1,100 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
__all__ = ['main', 'any_download', 'any_download_playlist']
|
||||
|
||||
from ..extractor import *
|
||||
from ..common import *
|
||||
|
||||
def url_to_module(url):
|
||||
video_host = r1(r'https?://([^/]+)/', url)
|
||||
video_url = r1(r'https?://[^/]+(.*)', url)
|
||||
assert video_host and video_url, 'invalid url: ' + url
|
||||
|
||||
if video_host.endswith('.com.cn'):
|
||||
video_host = video_host[:-3]
|
||||
domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
|
||||
assert domain, 'unsupported url: ' + url
|
||||
|
||||
k = r1(r'([^.]+)', domain)
|
||||
downloads = {
|
||||
'163': netease,
|
||||
'56': w56,
|
||||
'acfun': acfun,
|
||||
'baidu': baidu,
|
||||
'bilibili': bilibili,
|
||||
'blip': blip,
|
||||
'catfun':catfun,
|
||||
'cntv': cntv,
|
||||
'cbs': cbs,
|
||||
'coursera': coursera,
|
||||
'dailymotion': dailymotion,
|
||||
'douban': douban,
|
||||
'ehow': ehow,
|
||||
'facebook': facebook,
|
||||
'freesound': freesound,
|
||||
'google': google,
|
||||
'iask': sina,
|
||||
'ifeng': ifeng,
|
||||
'in': alive,
|
||||
'instagram': instagram,
|
||||
'iqiyi': iqiyi,
|
||||
'joy': joy,
|
||||
'jpopsuki': jpopsuki,
|
||||
'kankanews': bilibili,
|
||||
'ku6': ku6,
|
||||
'kugou':kugou,
|
||||
'kuwo':kuwo,
|
||||
'letv': letv,
|
||||
'magisto': magisto,
|
||||
'miomio': miomio,
|
||||
'mixcloud': mixcloud,
|
||||
'mtv81':mtv81,
|
||||
'nicovideo': nicovideo,
|
||||
'pptv': pptv,
|
||||
'qq': qq,
|
||||
'sina': sina,
|
||||
'smgbb': bilibili,
|
||||
'sohu': sohu,
|
||||
'songtaste':songtaste,
|
||||
'soundcloud': soundcloud,
|
||||
'ted': ted,
|
||||
'theplatform': theplatform,
|
||||
'tudou': tudou,
|
||||
'tumblr': tumblr,
|
||||
'vid48': vid48,
|
||||
'vimeo': vimeo,
|
||||
'vine': vine,
|
||||
'vk': vk,
|
||||
'xiami': xiami,
|
||||
'yinyuetai': yinyuetai,
|
||||
'youku': youku,
|
||||
'youtu': youtube,
|
||||
'youtube': youtube,
|
||||
'khanacademy': khan,
|
||||
#TODO
|
||||
}
|
||||
if k in downloads:
|
||||
return downloads[k], url
|
||||
else:
|
||||
import http.client
|
||||
conn = http.client.HTTPConnection(video_host)
|
||||
conn.request("HEAD", video_url)
|
||||
res = conn.getresponse()
|
||||
location = res.getheader('location')
|
||||
if location is None:
|
||||
raise NotImplementedError(url)
|
||||
else:
|
||||
return url_to_module(location)
|
||||
|
||||
def any_download(url, **kwargs):
|
||||
m, url = url_to_module(url)
|
||||
m.download(url, **kwargs)
|
||||
|
||||
def any_download_playlist(url, **kwargs):
|
||||
m, url = url_to_module(url)
|
||||
m.download_playlist(url, **kwargs)
|
||||
|
||||
def main():
|
||||
script_main('you-get', any_download, any_download_playlist)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__all__ = ['catfun_download']
|
||||
from .tudou import tudou_download_by_id
|
||||
from .sina import sina_download_by_vid
|
||||
|
||||
from ..common import *
|
||||
from xml.dom.minidom import *
|
||||
|
||||
def parse_item(item):
|
||||
if item["type"]=="youku":
|
||||
page=get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_youku_video_info&youku_id="+item["vid"])
|
||||
dom=parseString(page)
|
||||
ext=dom.getElementsByTagName("format")[0].firstChild.nodeValue;
|
||||
size=0
|
||||
urls=[]
|
||||
for i in dom.getElementsByTagName("durl"):
|
||||
urls.append(i.getElementsByTagName("url")[0].firstChild.nodeValue)
|
||||
size+=int(i.getElementsByTagName("size")[0].firstChild.nodeValue);
|
||||
return urls,ext,size
|
||||
pass
|
||||
|
||||
elif item["type"]=="qq":
|
||||
page=get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_qq_video_info&qq_id="+item["vid"])
|
||||
dom=parseString(page)
|
||||
size=0
|
||||
urls=[]
|
||||
for i in dom.getElementsByTagName("durl"):
|
||||
url=i.getElementsByTagName("url")[0].firstChild.nodeValue
|
||||
urls.append(url)
|
||||
vtype,ext,_size=url_info(url)
|
||||
size+=_size
|
||||
return urls,ext,size
|
||||
pass
|
||||
|
||||
elif item["type"]=="sina":
|
||||
page=get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_sina_video_info&sina_id=" + item["vid"])
|
||||
try:
|
||||
dom=parseString(page)
|
||||
except:
|
||||
#refresh page encountered
|
||||
page=get_content(match1(page,r'url=(.+?)"'))
|
||||
dom=parseString(page)
|
||||
size=0
|
||||
urls=[]
|
||||
for i in dom.getElementsByTagName("durl"):
|
||||
url=i.getElementsByTagName("url")[0].firstChild.nodeValue
|
||||
urls.append(url)
|
||||
vtype,ext,_size=url_info(url)
|
||||
if not ext:
|
||||
ext=match1(url,r'\.(\w+?)\?')
|
||||
size+=_size
|
||||
#sina's result does not contains content-type
|
||||
return urls,ext,size
|
||||
pass
|
||||
|
||||
|
||||
|
||||
def catfun_download(url, output_dir = '.', merge = True, info_only = False):
|
||||
# html=get_content(url)
|
||||
title=match1(get_content(url),r'<h1 class="title">(.+?)</h1>')
|
||||
vid=match1(url,r"v\d+/cat(\d+)")
|
||||
j=json.loads(get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_video&modelid=11&id={}".format(vid)))
|
||||
for item in j:
|
||||
if item["name"]!="\u672a\u547d\u540d1":
|
||||
t=title+"-"+item["name"]
|
||||
else:
|
||||
t=title
|
||||
if item["type"]=="tudou":
|
||||
tudou_download_by_id(item["vid"], title, output_dir, merge, info_only)
|
||||
|
||||
else:
|
||||
urls,ext,size=parse_item(item)
|
||||
|
||||
download_urls(urls,t,ext,size,output_dir)
|
||||
|
||||
|
||||
site_info = "catfun.com"
|
||||
download = catfun_download
|
||||
download_playlist = playlist_not_supported('catfun')
|
||||
@ -50,5 +50,3 @@ from .youku import *
|
||||
from .youtube import *
|
||||
from .ted import *
|
||||
from .khan import *
|
||||
|
||||
from .__main__ import *
|
||||
@ -4,7 +4,6 @@
|
||||
__all__ = ['baidu_download']
|
||||
|
||||
from ..common import *
|
||||
from .. import common
|
||||
|
||||
from urllib import parse
|
||||
|
||||
76
src/you_get/extractors/catfun.py
Normal file
76
src/you_get/extractors/catfun.py
Normal file
@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__all__ = ['catfun_download']
|
||||
from .tudou import tudou_download_by_id
|
||||
from .sina import sina_download_by_vid
|
||||
|
||||
from ..common import *
|
||||
from xml.dom.minidom import *
|
||||
|
||||
def parse_item(item):
|
||||
if item["type"] == "youku":
|
||||
page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_youku_video_info&youku_id=" + item["vid"])
|
||||
dom = parseString(page)
|
||||
ext = dom.getElementsByTagName("format")[0].firstChild.nodeValue;
|
||||
size = 0
|
||||
urls = []
|
||||
for i in dom.getElementsByTagName("durl"):
|
||||
urls.append(i.getElementsByTagName("url")[0].firstChild.nodeValue)
|
||||
size += int(i.getElementsByTagName("size")[0].firstChild.nodeValue);
|
||||
return urls, ext, size
|
||||
|
||||
elif item["type"] == "qq":
|
||||
page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_qq_video_info&qq_id=" + item["vid"])
|
||||
dom = parseString(page)
|
||||
size = 0
|
||||
urls = []
|
||||
for i in dom.getElementsByTagName("durl"):
|
||||
url = i.getElementsByTagName("url")[0].firstChild.nodeValue
|
||||
urls.append(url)
|
||||
vtype, ext, _size = url_info(url)
|
||||
size += _size
|
||||
return urls, ext, size
|
||||
|
||||
elif item["type"] == "sina":
|
||||
page = get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_sina_video_info&sina_id=" + item["vid"])
|
||||
try:
|
||||
dom = parseString(page)
|
||||
except:
|
||||
#refresh page encountered
|
||||
page = get_content(match1(page, r'url=(.+?)"'))
|
||||
dom = parseString(page)
|
||||
size = 0
|
||||
urls = []
|
||||
for i in dom.getElementsByTagName("durl"):
|
||||
url = i.getElementsByTagName("url")[0].firstChild.nodeValue
|
||||
urls.append(url)
|
||||
vtype, ext, _size = url_info(url)
|
||||
if not ext:
|
||||
ext = match1(url,r'\.(\w+?)\?')
|
||||
size += _size
|
||||
#sina's result does not contains content-type
|
||||
return urls, ext, size
|
||||
|
||||
def catfun_download(url, output_dir = '.', merge = True, info_only = False):
|
||||
# html = get_content(url)
|
||||
title = match1(get_content(url), r'<h1 class="title">(.+?)</h1>')
|
||||
vid = match1(url, r"v\d+/cat(\d+)")
|
||||
j = json.loads(get_content("http://www.catfun.tv/index.php?m=catfun&c=catfun_video&a=get_video&modelid=11&id={}".format(vid)))
|
||||
for item in j:
|
||||
if item["name"] != "\u672a\u547d\u540d1":
|
||||
t = title + "-" + item["name"]
|
||||
else:
|
||||
t = title
|
||||
if item["type"] == "tudou":
|
||||
tudou_download_by_id(item["vid"], title, output_dir, merge, info_only)
|
||||
|
||||
else:
|
||||
urls, ext, size = parse_item(item)
|
||||
|
||||
print_info(site_info, title, ext, size)
|
||||
if not info_only:
|
||||
download_urls(urls, t, ext, size, output_dir, merge=merge)
|
||||
|
||||
site_info = "CatFun.tv"
|
||||
download = catfun_download
|
||||
download_playlist = playlist_not_supported('catfun')
|
||||
@ -19,14 +19,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
|
||||
vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
|
||||
assert vid
|
||||
|
||||
# Open Sogou proxy if required
|
||||
if get_sogou_proxy() is not None:
|
||||
server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w'))
|
||||
server_thread = threading.Thread(target=server.serve_forever)
|
||||
server_thread.daemon = True
|
||||
server_thread.start()
|
||||
set_proxy(server.server_address)
|
||||
|
||||
if re.match(r'http://tv.sohu.com/', url):
|
||||
data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
|
||||
for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
|
||||
@ -58,11 +50,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
|
||||
urls.append(real_url(host, prot, file, new))
|
||||
assert data['clipsURL'][0].endswith('.mp4')
|
||||
|
||||
# Close Sogou proxy if required
|
||||
if get_sogou_proxy() is not None:
|
||||
server.shutdown()
|
||||
unset_proxy()
|
||||
|
||||
print_info(site_info, title, 'mp4', size)
|
||||
if not info_only:
|
||||
download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)
|
||||
@ -2,6 +2,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from ..common import *
|
||||
from ..extractor import VideoExtractor
|
||||
|
||||
class Youku(VideoExtractor):
|
||||
name = "优酷 (Youku)"
|
||||
@ -63,6 +64,11 @@ class Youku(VideoExtractor):
|
||||
|
||||
self.title = metadata0['title']
|
||||
|
||||
if 'dvd' in metadata0 and 'audiolang' in metadata0['dvd']:
|
||||
self.audiolang = metadata0['dvd']['audiolang']
|
||||
for i in self.audiolang:
|
||||
i['url'] = 'http://v.youku.com/v_show/id_{}'.format(i['vid'])
|
||||
|
||||
for stream_type in self.stream_types:
|
||||
if stream_type['id'] in metadata0['streamsizes']:
|
||||
stream_id = stream_type['id']
|
||||
@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from ..common import *
|
||||
from ..extractor import VideoExtractor
|
||||
|
||||
class YouTube(VideoExtractor):
|
||||
name = "YouTube"
|
||||
@ -1,6 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from .fs import *
|
||||
from .log import *
|
||||
from .sogou_proxy import *
|
||||
from .strings import *
|
||||
13
src/you_get/util/git.py
Normal file
13
src/you_get/util/git.py
Normal file
@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os
|
||||
|
||||
def get_head(repo_path):
|
||||
"""Get (branch, commit) from HEAD of a git repo."""
|
||||
try:
|
||||
ref = open(os.path.join(repo_path, '.git', 'HEAD'), 'r').read().strip()[5:].split('/')
|
||||
branch = ref[-1]
|
||||
commit = open(os.path.join(repo_path, '.git', *ref), 'r').read().strip()[:7]
|
||||
return branch, commit
|
||||
except:
|
||||
return None
|
||||
@ -1,130 +1,97 @@
|
||||
#!/usr/bin/env python
|
||||
# This file is Python 2 compliant.
|
||||
|
||||
from ..version import __name__
|
||||
from .. import __name__ as library_name
|
||||
|
||||
import os, sys, subprocess
|
||||
import os, sys
|
||||
|
||||
# Is terminal ANSI/VT100 compatible
|
||||
if os.getenv('TERM') in (
|
||||
'xterm',
|
||||
'vt100',
|
||||
'linux',
|
||||
'eterm-color',
|
||||
'screen',
|
||||
):
|
||||
has_colors = True
|
||||
else:
|
||||
try:
|
||||
# Eshell
|
||||
ppid = os.getppid()
|
||||
has_colors = (subprocess.getoutput('ps -p %d -ocomm=' % ppid)
|
||||
== 'emacs')
|
||||
except:
|
||||
has_colors = False
|
||||
IS_ANSI_TERMINAL = os.getenv('TERM') in (
|
||||
'eterm-color',
|
||||
'linux',
|
||||
'screen',
|
||||
'vt100',
|
||||
'xterm')
|
||||
|
||||
# ANSI/VT100 escape code
|
||||
# http://en.wikipedia.org/wiki/ANSI_escape_code
|
||||
colors = {
|
||||
'none': '',
|
||||
'reset': '\033[0m',
|
||||
# ANSI escape code
|
||||
# See <http://en.wikipedia.org/wiki/ANSI_escape_code>
|
||||
RESET = 0
|
||||
BOLD = 1
|
||||
UNDERLINE = 4
|
||||
NEGATIVE = 7
|
||||
NO_BOLD = 21
|
||||
NO_UNDERLINE = 24
|
||||
POSITIVE = 27
|
||||
BLACK = 30
|
||||
RED = 31
|
||||
GREEN = 32
|
||||
YELLOW = 33
|
||||
BLUE = 34
|
||||
MAGENTA = 35
|
||||
CYAN = 36
|
||||
LIGHT_GRAY = 37
|
||||
DEFAULT = 39
|
||||
BLACK_BACKGROUND = 40
|
||||
RED_BACKGROUND = 41
|
||||
GREEN_BACKGROUND = 42
|
||||
YELLOW_BACKGROUND = 43
|
||||
BLUE_BACKGROUND = 44
|
||||
MAGENTA_BACKGROUND = 45
|
||||
CYAN_BACKGROUND = 46
|
||||
LIGHT_GRAY_BACKGROUND = 47
|
||||
DEFAULT_BACKGROUND = 49
|
||||
DARK_GRAY = 90 # xterm
|
||||
LIGHT_RED = 91 # xterm
|
||||
LIGHT_GREEN = 92 # xterm
|
||||
LIGHT_YELLOW = 93 # xterm
|
||||
LIGHT_BLUE = 94 # xterm
|
||||
LIGHT_MAGENTA = 95 # xterm
|
||||
LIGHT_CYAN = 96 # xterm
|
||||
WHITE = 97 # xterm
|
||||
DARK_GRAY_BACKGROUND = 100 # xterm
|
||||
LIGHT_RED_BACKGROUND = 101 # xterm
|
||||
LIGHT_GREEN_BACKGROUND = 102 # xterm
|
||||
LIGHT_YELLOW_BACKGROUND = 103 # xterm
|
||||
LIGHT_BLUE_BACKGROUND = 104 # xterm
|
||||
LIGHT_MAGENTA_BACKGROUND = 105 # xterm
|
||||
LIGHT_CYAN_BACKGROUND = 106 # xterm
|
||||
WHITE_BACKGROUND = 107 # xterm
|
||||
|
||||
'black': '\033[30m',
|
||||
'bold-black': '\033[30;1m',
|
||||
'dark-gray': '\033[90m',
|
||||
'bold-dark-gray': '\033[90;1m',
|
||||
def sprint(text, *colors):
|
||||
"""Format text with color or other effects into ANSI escaped string."""
|
||||
return "\33[{}m{content}\33[{}m".format(";".join([str(color) for color in colors]), RESET, content=text) if IS_ANSI_TERMINAL and colors else text
|
||||
|
||||
'red': '\033[31m',
|
||||
'bold-red': '\033[31;1m',
|
||||
'light-red': '\033[91m',
|
||||
'bold-light-red': '\033[91;1m',
|
||||
def println(text, *colors):
|
||||
"""Print text to standard output."""
|
||||
sys.stdout.write(sprint(text, *colors) + "\n")
|
||||
|
||||
'green': '\033[32m',
|
||||
'bold-green': '\033[32;1m',
|
||||
'light-green': '\033[92m',
|
||||
'bold-light-green': '\033[92;1m',
|
||||
def print_err(text, *colors):
|
||||
"""Print text to standard error."""
|
||||
sys.stderr.write(sprint(text, *colors) + "\n")
|
||||
|
||||
'yellow': '\033[33m',
|
||||
'bold-yellow': '\033[33;1m',
|
||||
'light-yellow': '\033[93m',
|
||||
'bold-light-yellow': '\033[93;1m',
|
||||
def print_log(text, *colors):
|
||||
"""Print a log message to standard error."""
|
||||
sys.stderr.write(sprint("{}: {}".format(library_name, text), *colors) + "\n")
|
||||
|
||||
'blue': '\033[34m',
|
||||
'bold-blue': '\033[34;1m',
|
||||
'light-blue': '\033[94m',
|
||||
'bold-light-blue': '\033[94;1m',
|
||||
def i(message):
|
||||
"""Print a normal log message."""
|
||||
print_log(message)
|
||||
|
||||
'magenta': '\033[35m',
|
||||
'bold-magenta': '\033[35;1m',
|
||||
'light-magenta': '\033[95m',
|
||||
'bold-light-magenta': '\033[95;1m',
|
||||
def d(message):
|
||||
"""Print a debug log message."""
|
||||
print_log(message, BLUE)
|
||||
|
||||
'cyan': '\033[36m',
|
||||
'bold-cyan': '\033[36;1m',
|
||||
'light-cyan': '\033[96m',
|
||||
'bold-light-cyan': '\033[96;1m',
|
||||
def w(message):
|
||||
"""Print a warning log message."""
|
||||
print_log(message, YELLOW)
|
||||
|
||||
'light-gray': '\033[37m',
|
||||
'bold-light-gray': '\033[37;1m',
|
||||
'white': '\033[97m',
|
||||
'bold-white': '\033[97;1m',
|
||||
}
|
||||
|
||||
def underlined(text):
|
||||
"""Returns an underlined text.
|
||||
"""
|
||||
return "\33[4m%s\33[24m" % text if has_colors else text
|
||||
|
||||
def println(text, color=None, ostream=sys.stdout):
|
||||
"""Prints a text line to stream.
|
||||
"""
|
||||
if has_colors and color in colors:
|
||||
ostream.write("{0}{1}{2}\n".format(colors[color], text, colors['reset']))
|
||||
else:
|
||||
ostream.write("{0}\n".format(text))
|
||||
|
||||
def printlog(message, color=None, ostream=sys.stderr):
|
||||
"""Prints a log message to stream.
|
||||
"""
|
||||
if has_colors and color in colors:
|
||||
ostream.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset']))
|
||||
else:
|
||||
ostream.write("{0}: {1}\n".format(__name__, message))
|
||||
|
||||
def i(message, ostream=sys.stderr):
|
||||
"""Sends an info log message.
|
||||
"""
|
||||
printlog(message,
|
||||
None,
|
||||
ostream=ostream)
|
||||
|
||||
def d(message, ostream=sys.stderr):
|
||||
"""Sends a debug log message.
|
||||
"""
|
||||
printlog(message,
|
||||
'blue' if has_colors else None,
|
||||
ostream=ostream)
|
||||
|
||||
def w(message, ostream=sys.stderr):
|
||||
"""Sends a warning log message.
|
||||
"""
|
||||
printlog(message,
|
||||
'yellow' if has_colors else None,
|
||||
ostream=ostream)
|
||||
|
||||
def e(message, ostream=sys.stderr, exit_code=None):
|
||||
"""Sends an error log message.
|
||||
"""
|
||||
printlog(message,
|
||||
'bold-yellow' if has_colors else None,
|
||||
ostream=ostream)
|
||||
def e(message, exit_code=None):
|
||||
"""Print an error log message."""
|
||||
print_log(message, YELLOW, BOLD)
|
||||
if exit_code is not None:
|
||||
exit(exit_code)
|
||||
|
||||
def wtf(message, ostream=sys.stderr, exit_code=-1):
|
||||
"""What a Terrible Failure.
|
||||
"""
|
||||
printlog(message,
|
||||
'bold-red' if has_colors else None,
|
||||
ostream=ostream)
|
||||
def wtf(message, exit_code=-1):
|
||||
"""What a Terrible Failure!"""
|
||||
print_log(message, RED, BOLD)
|
||||
if exit_code is not None:
|
||||
exit(exit_code)
|
||||
|
||||
@ -1,141 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Original code from:
|
||||
# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/
|
||||
|
||||
from . import log
|
||||
|
||||
from http.client import HTTPResponse
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from socketserver import ThreadingMixIn
|
||||
from threading import Thread
|
||||
import random, socket, struct, sys, time
|
||||
|
||||
def sogou_proxy_server(
|
||||
host=("0.0.0.0", 0),
|
||||
network_env='CERNET',
|
||||
ostream=sys.stderr):
|
||||
"""Returns a Sogou proxy server object.
|
||||
"""
|
||||
|
||||
x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27'
|
||||
proxy_host = 'h0.cnc.bj.ie.sogou.com'
|
||||
proxy_port = 80
|
||||
|
||||
def sogou_hash(t, host):
|
||||
s = (t + host + 'SogouExplorerProxy').encode('ascii')
|
||||
code = len(s)
|
||||
dwords = int(len(s) / 4)
|
||||
rest = len(s) % 4
|
||||
v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s)
|
||||
for vv in v:
|
||||
if type(vv) != bytes:
|
||||
a = (vv & 0xFFFF)
|
||||
b = (vv >> 16)
|
||||
code += a
|
||||
code = code ^ (((code << 5) ^ b) << 0xb)
|
||||
# To avoid overflows
|
||||
code &= 0xffffffff
|
||||
code += code >> 0xb
|
||||
if rest == 3:
|
||||
code += s[len(s) - 2] * 256 + s[len(s) - 3]
|
||||
code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10)
|
||||
code &= 0xffffffff
|
||||
code += code >> 0xb
|
||||
elif rest == 2:
|
||||
code += (s[len(s) - 1]) * 256 + (s[len(s) - 2])
|
||||
code ^= code << 0xb
|
||||
code &= 0xffffffff
|
||||
code += code >> 0x11
|
||||
elif rest == 1:
|
||||
code += s[len(s) - 1]
|
||||
code ^= code << 0xa
|
||||
code &= 0xffffffff
|
||||
code += code >> 0x1
|
||||
code ^= code * 8
|
||||
code &= 0xffffffff
|
||||
code += code >> 5
|
||||
code ^= code << 4
|
||||
code = code & 0xffffffff
|
||||
code += code >> 0x11
|
||||
code ^= code << 0x19
|
||||
code = code & 0xffffffff
|
||||
code += code >> 6
|
||||
code = code & 0xffffffff
|
||||
return hex(code)[2:].rstrip('L').zfill(8)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
_socket = None
|
||||
def do_proxy(self):
|
||||
try:
|
||||
if self._socket is None:
|
||||
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self._socket.connect((proxy_host, proxy_port))
|
||||
self._socket.send(self.requestline.encode('ascii') + b'\r\n')
|
||||
log.d(self.requestline, ostream)
|
||||
|
||||
# Add Sogou Verification Tags
|
||||
self.headers['X-Sogou-Auth'] = x_sogou_auth
|
||||
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
|
||||
self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host'])
|
||||
self.headers['X-Sogou-Timestamp'] = t
|
||||
self._socket.send(str(self.headers).encode('ascii') + b'\r\n')
|
||||
|
||||
# Send POST data
|
||||
if self.command == 'POST':
|
||||
self._socket.send(self.rfile.read(int(self.headers['Content-Length'])))
|
||||
response = HTTPResponse(self._socket, method=self.command)
|
||||
response.begin()
|
||||
|
||||
# Response
|
||||
status = 'HTTP/1.1 %s %s' % (response.status, response.reason)
|
||||
self.wfile.write(status.encode('ascii') + b'\r\n')
|
||||
h = ''
|
||||
for hh, vv in response.getheaders():
|
||||
if hh.upper() != 'TRANSFER-ENCODING':
|
||||
h += hh + ': ' + vv + '\r\n'
|
||||
self.wfile.write(h.encode('ascii') + b'\r\n')
|
||||
while True:
|
||||
response_data = response.read(8192)
|
||||
if len(response_data) == 0:
|
||||
break
|
||||
self.wfile.write(response_data)
|
||||
|
||||
except socket.error:
|
||||
log.e('Socket error for ' + self.requestline, ostream)
|
||||
|
||||
def do_POST(self):
|
||||
self.do_proxy()
|
||||
|
||||
def do_GET(self):
|
||||
self.do_proxy()
|
||||
|
||||
class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
|
||||
pass
|
||||
|
||||
# Server starts
|
||||
log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream)
|
||||
|
||||
try:
|
||||
server = ThreadingHTTPServer(host, Handler)
|
||||
except Exception as ex:
|
||||
log.wtf("Socket error: %s" % ex, ostream)
|
||||
exit(1)
|
||||
host = server.server_address
|
||||
|
||||
if network_env.upper() == 'CERNET':
|
||||
proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
|
||||
elif network_env.upper() == 'CTCNET':
|
||||
proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3)
|
||||
elif network_env.upper() == 'CNCNET':
|
||||
proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3)
|
||||
elif network_env.upper() == 'DXT':
|
||||
proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10)
|
||||
else:
|
||||
proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
|
||||
|
||||
log.i('Remote host: %s' % log.underlined(proxy_host), ostream)
|
||||
log.i('Proxy server running on %s' %
|
||||
log.underlined("%s:%s" % host), ostream)
|
||||
|
||||
return server
|
||||
@ -1,6 +1,4 @@
|
||||
#!/usr/bin/env python
|
||||
__all__ = ['__version__', '__date__']
|
||||
|
||||
__name__ = 'you-get'
|
||||
__version__ = '0.3.30dev-20140716'
|
||||
__date__ = '2014-07-16'
|
||||
script_name = 'you-get'
|
||||
__version__ = '0.3.30dev'
|
||||
|
||||
@ -1,47 +1,31 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import unittest
|
||||
|
||||
from you_get import *
|
||||
from you_get.extractor.__main__ import url_to_module
|
||||
|
||||
def test_urls(urls):
|
||||
for url in urls:
|
||||
url_to_module(url)[0].download(url, info_only = True)
|
||||
from you_get.extractors import *
|
||||
from you_get.common import *
|
||||
|
||||
class YouGetTests(unittest.TestCase):
|
||||
|
||||
def test_freesound(self):
|
||||
test_urls([
|
||||
"http://www.freesound.org/people/Corsica_S/sounds/184419/",
|
||||
])
|
||||
freesound.download("http://www.freesound.org/people/Corsica_S/sounds/184419/", info_only=True)
|
||||
|
||||
def test_magisto(self):
|
||||
test_urls([
|
||||
"http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA",
|
||||
])
|
||||
magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA", info_only=True)
|
||||
|
||||
def test_mixcloud(self):
|
||||
test_urls([
|
||||
"http://www.mixcloud.com/beatbopz/beat-bopz-disco-mix/",
|
||||
"http://www.mixcloud.com/DJVadim/north-america-are-you-ready/",
|
||||
])
|
||||
mixcloud.download("http://www.mixcloud.com/beatbopz/beat-bopz-disco-mix/", info_only=True)
|
||||
mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)
|
||||
|
||||
def test_ted(self):
|
||||
test_urls([
|
||||
"http://www.ted.com/talks/jennifer_lin_improvs_piano_magic.html",
|
||||
"http://www.ted.com/talks/derek_paravicini_and_adam_ockelford_in_the_key_of_genius.html",
|
||||
])
|
||||
ted.download("http://www.ted.com/talks/jennifer_lin_improvs_piano_magic.html", info_only=True)
|
||||
ted.download("http://www.ted.com/talks/derek_paravicini_and_adam_ockelford_in_the_key_of_genius.html", info_only=True)
|
||||
|
||||
def test_vimeo(self):
|
||||
test_urls([
|
||||
"http://vimeo.com/56810854",
|
||||
])
|
||||
vimeo.download("http://vimeo.com/56810854", info_only=True)
|
||||
|
||||
def test_youtube(self):
|
||||
test_urls([
|
||||
"http://www.youtube.com/watch?v=pzKerr0JIPA",
|
||||
"http://youtu.be/pzKerr0JIPA",
|
||||
"http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare"
|
||||
])
|
||||
youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
|
||||
youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)
|
||||
youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare", info_only=True)
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
import unittest
|
||||
|
||||
from you_get import *
|
||||
from you_get.common import *
|
||||
|
||||
class TestCommon(unittest.TestCase):
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
import unittest
|
||||
|
||||
from you_get.util import *
|
||||
from you_get.util.fs import *
|
||||
|
||||
class TestUtil(unittest.TestCase):
|
||||
def test_legitimize(self):
|
||||
|
||||
22
you-get
22
you-get
@ -1,10 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env python
|
||||
# This file is Python 2 compliant.
|
||||
|
||||
import os, sys
|
||||
__path__ = os.path.dirname(os.path.realpath(__file__))
|
||||
__srcdir__ = 'src'
|
||||
sys.path.insert(1, os.path.join(__path__, __srcdir__))
|
||||
from you_get.extractor import main
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
_srcdir = 'src/'
|
||||
_filepath = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.insert(1, os.path.join(_filepath, _srcdir))
|
||||
|
||||
if sys.version_info[0] == 3:
|
||||
import you_get
|
||||
if __name__ == '__main__':
|
||||
you_get.main(repo_path=_filepath)
|
||||
else:
|
||||
from you_get.util import log
|
||||
log.wtf("""
|
||||
[Fatal] Python 3 is required.
|
||||
If Python 3 is already installed on your machine, try to run this script using 'python3 you-get'.""")
|
||||
|
||||
@ -4,10 +4,10 @@
|
||||
"author_email": "mort.yao@gmail.com",
|
||||
"url": "http://www.soimort.org/you-get/",
|
||||
"license": "MIT",
|
||||
|
||||
|
||||
"description": "A YouTube/Youku/Niconico video downloader written in Python 3.",
|
||||
"keywords": "video download youtube youku niconico",
|
||||
|
||||
|
||||
"classifiers": [
|
||||
"Development Status :: 2 - Pre-Alpha",
|
||||
"Environment :: Console",
|
||||
@ -23,14 +23,15 @@
|
||||
"Programming Language :: Python :: 3.1",
|
||||
"Programming Language :: Python :: 3.2",
|
||||
"Programming Language :: Python :: 3.3",
|
||||
"Programming Language :: Python :: 3.4",
|
||||
"Topic :: Internet",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Multimedia",
|
||||
"Topic :: Multimedia :: Video",
|
||||
"Topic :: Utilities"
|
||||
],
|
||||
|
||||
|
||||
"console_scripts": [
|
||||
"you-get = you_get.extractor.__main__:main"
|
||||
"you-get = you_get.__main__:main"
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user