\s*
([^<]{1,9999})
'
- title_patt_backup = r'
([^<]{1,9999})'
+ title_patt = r'
\s*
([^<]{1,9999})
\s*'
roomid = re.findall(room_id_patt,html)[0]
- title = r1_of([title_patt,title_patt_backup], html)
- title = unescape_html(title)
+ title = unescape_html(re.findall(title_patt,html)[0])
conf = get_html("http://www.douyutv.com/api/client/room/"+roomid)
metadata = json.loads(conf)
diff --git a/src/you_get/extractors/ehow.py b/src/you_get/extractors/ehow.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/freesound.py b/src/you_get/extractors/freesound.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/joy.py b/src/you_get/extractors/joy.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/jpopsuki.py b/src/you_get/extractors/jpopsuki.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/kugou.py b/src/you_get/extractors/kugou.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/kuwo.py b/src/you_get/extractors/kuwo.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/letv.py b/src/you_get/extractors/letv.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/magisto.py b/src/you_get/extractors/magisto.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/miomio.py b/src/you_get/extractors/miomio.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/mixcloud.py b/src/you_get/extractors/mixcloud.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/mtv81.py b/src/you_get/extractors/mtv81.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/nicovideo.py b/src/you_get/extractors/nicovideo.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
old mode 100644
new mode 100755
index 5a7f8472..573c844f
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -2,8 +2,8 @@
__all__ = ['qq_download']
-from ..common import *
-import uuid
+from you_get.common import *
+import uuid, urllib
#QQMUSIC
#SINGLE
#1. http://y.qq.com/#type=song&mid=000A9lMb0iEqwN
@@ -17,43 +17,49 @@ import uuid
#can download as video through qq_download_by_id
#1. http://y.qq.com/y/static/mv/mv_play.html?vid=i0014ufczcw
-def qq_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False):
- xml = get_html('http://www.acfun.tv/getinfo?vids=%s' % id)
- from xml.dom.minidom import parseString
- doc = parseString(xml)
- doc_root = doc.getElementsByTagName('root')[0]
- doc_vl = doc_root.getElementsByTagName('vl')[0]
- doc_vi = doc_vl.getElementsByTagName('vi')[0]
- fn = doc_vi.getElementsByTagName('fn')[0].firstChild.data
- # fclip = doc_vi.getElementsByTagName('fclip')[0].firstChild.data
- # fc=doc_vi.getElementsByTagName('fc')[0].firstChild.data
- fvkey = doc_vi.getElementsByTagName('fvkey')[0].firstChild.data
- doc_ul = doc_vi.getElementsByTagName('ul')
+def qq_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, urls_only=False):
+ data = {'vids': id, 'otype': 'json'}
+ url = urllib.request.Request('http://vv.video.qq.com/getinfo', urllib.parse.urlencode(data).encode('utf-8'))
+ f = urllib.request.urlopen(url)
+ json_str = f.read()
+ data = json.loads(json_str[13:-1].decode('utf-8'))
+ format_id = 10202
+ file_id = 1
+ for format_info in data['fl']['fi']:
+ if format_info['sl'] > 0:
+ format_id = format_info['id']
+ file_id = format_info['sl']
+ break
+ file_name = data['vl']['vi'][0]['fn']
+ split_pos = file_name.rfind('.')
+ file_name = file_name[:split_pos] + '.%d' % file_id + file_name[split_pos:]
+ video_urls = [ui['url'] for ui in data['vl']['vi'][0]['ul']['ui']]
- url = doc_ul[0].getElementsByTagName('url')[1].firstChild.data
+ data = {'format': format_id, 'otype': 'json', 'vid': id, 'filename': file_name}
+ url = urllib.request.Request('http://vv.video.qq.com/getkey', urllib.parse.urlencode(data).encode('utf-8'))
+ f = urllib.request.urlopen(url)
+ json_str = f.read()
+ data = json.loads(json_str[13:-1].decode('utf-8'))
+ video_key = data['key']
- # print(i.firstChild.data)
- urls=[]
- ext=fn[-3:]
- size=0
- for i in doc.getElementsByTagName("cs"):
- size+=int(i.firstChild.data)
+ urls = []
+ size = 0
+ ext = ''
+ for url in video_urls:
+ try:
+ url = "%s%s?vkey=%s" % (url, file_name, video_key)
+ _, ext, size = url_info(url)
+ urls = [url]
+ break
+ except:
+ print(url)
- # size=sum(map(int,doc.getElementsByTagName("cs")))
- locid=str(uuid.uuid4())
- for i in doc.getElementsByTagName("ci"):
- urls.append(url+fn[:-4] + "." + i.getElementsByTagName("idx")[0].firstChild.data + fn[-4:] + '?vkey=' + fvkey+ '&sdtfrom=v1000&type='+ fn[-3:0] +'&locid=' + locid + "&&level=1&platform=11&br=133&fmt=hd&sp=0")
+ if urls_only:
+ return urls, size, ext, {}
- # if int(fclip) > 0:
- # fn = fn[:-4] + "." + fclip + fn[-4:]
- # url = url + fn + '?vkey=' + fvkey
-
- # _, ext, size = url_info(url)
-
- print_info(site_info, title, ext, size)
if not info_only:
- download_urls(urls, title, ext, size, output_dir=output_dir, merge=merge)
+ download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)
def qq_download(url, output_dir = '.', merge = True, info_only = False):
if re.match(r'http://v.qq.com/([^\?]+)\?vid', url):
@@ -97,3 +103,9 @@ def qq_download(url, output_dir = '.', merge = True, info_only = False):
site_info = "QQ.com"
download = qq_download
download_playlist = playlist_not_supported('qq')
+
+
+if __name__ == '__main__':
+ #print(qq_download('http://v.qq.com/cover/c/crfns95chw1snp2/t0012q2nz5m.html', urls_only = True))
+ # print(get_videoId('http://v.qq.com/cover/k/kuegopa6s70qeu1.html?vid=t0013jyqbo7'))
+ print(qq_download_by_id('u001428c4av', urls_only=True))
\ No newline at end of file
diff --git a/src/you_get/extractors/sina.py b/src/you_get/extractors/sina.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/songtaste.py b/src/you_get/extractors/songtaste.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/ted.py b/src/you_get/extractors/ted.py
old mode 100644
new mode 100755
index 0c2d2c83..eb99843d
--- a/src/you_get/extractors/ted.py
+++ b/src/you_get/extractors/ted.py
@@ -3,7 +3,43 @@
__all__ = ['ted_download']
from ..common import *
-import json
+import json, os, inspect, logging, time
+from pprint import pprint
+#sys.path += [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))]
+currentDir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
+parentDir = os.path.dirname(os.path.dirname(currentDir))
+se_parentDir = os.path.dirname(parentDir)
+sys.path.append(parentDir)
+sys.path.append(se_parentDir)
+#print currentDir
+#print parentDir
+#print se_parentDir
+# print sys.path
+
+from you_get.common import *
+
+TED_D_DINFO = False
+TED_D_DSUB = False
+TED_D_DFINFO = False
+
+TED_TALKS_URL_PAT = "http://www.ted.com/talks/%s"
+FOUND = False
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+log_handle = logging.StreamHandler(sys.stdout)
+log_handle.setFormatter(logging.Formatter('%(asctime)-15s [%(levelname)s] %(message)s'))
+logger.addHandler(log_handle)
+
+fake_headers_here = {
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+ #'Accept-Charset': 'UTF-8,*;q=0.5',
+ 'Accept-Encoding': 'gzip,deflate,sdch',
+ 'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
+ 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36',
+ 'Connection': 'keep-alive',
+ 'Cookie':'',
+ }
def ted_download(url, output_dir='.', merge=True, info_only=False):
html = get_html(url)
@@ -19,6 +55,298 @@ def ted_download(url, output_dir='.', merge=True, info_only=False):
download_urls([url], title, ext, size, output_dir, merge=merge)
break
+# For ted_download_by_id
+re_url = re.compile('"nativeDownloads":.*"high":"(.+)\?.+},"sub')
+re_slug = re.compile('"slug":"(.*?)"')
+#re_vid = re.compile('http://.+\/(.*\.mp4)')
+re_name = re.compile('"external":.*?,"name":"(.*?)","title":')
+# Inner video ID
+re_in_id = re.compile('http://www.ted.com/talks/(.*?)')
+
+def ted_download_by_id(id, title, output_dir = '.', stream_type = None, merge = True, info_only = False, urls_only = False):
+ # ret: urls,size,ext,headers = callMap[videoType](videoId,"title", urls_only = True)
+
+ try:
+ url = TED_TALKS_URL_PAT % id
+ vpage = get_html(url)
+ except:
+ logger.info("###ted_download_by_id: TED id home can not be accessed")
+ return [url], 0, 'mp4', {}
+
+ logger.info("###ted_download_by_id")
+ logger.info("page url is" + url)
+
+ #print "page content is"
+ # print vpage
+
+ v_url = re.findall(re_url, vpage)[0]
+ v_title = re.findall(re_name, vpage)[0]
+ size = urls_size([v_url], True, None)
+ #size is not used
+ # size = -1
+ urls = [v_url]
+
+ logger.info("###ted_download_by_id")
+ #logger.info("name + v_url + size \n" )
+ #print "%r, %r, %r" % (v_title, v_url, size)
+ logger.info("name: " + str(v_title) + " url:" + str(v_url) + " size: " + str(size))
+
+ # print "ret is",(urls, size, 'mp4', 'fake_headers')
+
+ return urls, size, 'mp4', {}
+
+
+def ted_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False, urls_only = False):
+
+ logger.info("###ted_download")
+ logger.info("page url is " + url)
+
+ vpage = get_html(url)
+ v_title = re.findall(re_name, vpage)[0]
+ v_url = re.findall(re_url, vpage)[0]
+
+ type, ext, size = url_info(v_url)
+ print_info(site_info, v_title, type, size)
+
+ logger.info("v_title is " + str(v_title) + "type is " + str(type) + "size is " + str(size) )
+
+ if not info_only:
+ download_urls([v_url], v_title, ext, size, output_dir, merge = merge)
+
+
+def get_videoId(url):
+ v_in_id = re.findall(re_in_id, url)[0]
+ return v_in_id
+
+
+def srt_time(tst):
+ """Format Time from TED Subtitles format to SRT time Format."""
+ secs, mins, hours = ((tst / 1000) % 60), (tst / 60000), (tst / 3600000)
+ right_srt_time = ("{0:02d}:{1:02d}:{2:02d},{3:3.0f}".
+ format(int(hours), int(mins), int(secs),
+ divmod(secs, 1)[1] * 1000))
+ return right_srt_time
+
+
+def srt_sec_time(tst):
+ """Format Time from TED Subtitles format to SRT time Format."""
+ secs = tst / 1000
+ return secs
+
+
+# regex expressions to search into the webpage
+re_dm_intro = re.compile('"introDuration":(\d+\.?\d+),')
+re_dm_id = re.compile('"id":(\d+),')
+re_dm_url = re.compile('"nativeDownloads":.*"high":"(.+)\?.+},"sub')
+re_dm_vid = re.compile('http://.+\/(.*\.mp4)')
+
+def ted_get_danmu(video_id):
+ """
+ Get Danmu for the unique video_id
+ """
+ logger.info("###ted_get_danmu")
+
+ url = TED_TALKS_URL_PAT % video_id
+ logger.info("page url is " + url)
+
+ try:
+ vpage = get_html(url)
+ except:
+ logger.info("###ted_get_danmu:request faild, ret null danmu list")
+ return []
+
+ ret_list = []
+
+ tt_intro = ((float(re_dm_intro.findall(vpage)[0]) + 1) * 1000)
+ tt_id = int(re_dm_id.findall(vpage)[0])
+ tt_url = re_dm_url.findall(vpage)[0]
+ tt_v_fname = re_dm_vid.findall(tt_url)[0]
+
+ #logger.info("###tt_intro is " + str(tt_intro))
+ subs = get_subs(tt_id, tt_intro, tt_v_fname)
+
+ # we only process english caption currrently
+ # 0(eng) 0(item list)
+ eng_sub = subs[0][0]
+
+ for i in eng_sub:
+ r_item = {}
+ p_item = parse_item(i)
+
+ r_item['text'] = p_item["content"]
+ r_item['color'] = p_item["font_color"]
+ r_item['fontSize'] = p_item["font_size"]
+ r_item['direct'] = p_item["mode"]
+ r_item['startTime'] = p_item["time"]
+ r_item['uuid'] = p_item["uuid"]
+ r_item['publishTime'] = p_item["pub_time"]
+
+ ret_list.append(r_item)
+ #logger.info("###parsed sub item")
+ #pprint(r_item)
+
+ logger.info("###ted_get_danmu:parsed sub item list info:" + " len: " + str(len(ret_list)))
+
+ if TED_D_DINFO:
+ logger.info("###ted_get_danmu:last two items" + " ret_list len " + str(len(ret_list) ) )
+ if len(ret_list) > 0:
+ pprint(ret_list[-1])
+ if len(ret_list) > 1:
+ pprint(ret_list[-2])
+ pass
+
+ if TED_D_DFINFO:
+ logger.info("###ted_get_danmu:full ret list" )
+ logger.info(str(ret_list))
+
+ return ret_list
+
+
+def parse_item(item):
+ """
+ Return a tuple for a+ danmu element
+ """
+ s_time = float(item["start"])
+
+ # Mode is the direct opt
+ # mode 1~3: scrolling
+ # mode 4: bottom
+ # mode 5: top
+ # mode 6: reverse?
+ # mode 7: position
+ # mode 8: advanced
+ mode = 4
+ assert 1 <= mode <= 8
+
+ # pool 0: normal
+ # pool 1: srt
+ # pool 2: special?
+ #pool = int(pool)
+ pool = 0
+ assert 0 <= pool <= 2
+
+ font_size = 25
+ font_color = 16777215
+ pub_time = str(int(time.time() * 1000000 ))[-10:]
+
+ return {"time":s_time, "font_color":font_color, "mode":mode, "font_size":font_size,
+ "uuid":"s_defuuid_z9", "pub_time":pub_time, "content":item["content"]}
+
+
+def get_subs(tt_id, tt_intro, tt_video_fname):
+ """
+ Get the sutitles, currently for english
+ """
+
+ subs = ["{0}.{1}.srt".format(tt_video_fname[:-4], lang) for lang in ('eng', 'chi')]
+ ret_subs = []
+
+ for sub in subs:
+ #logger.info("###get_subs:pls input to continue s sub getting:")
+ #raw_input()
+
+ subtitle = get_single_sub(tt_id, tt_intro, sub)
+ if subtitle:
+ ret_subs.append(subtitle)
+ #logger.info("###get_subs:Subtitle '{0}' downloaded.".format(sub) )
+
+ if TED_D_DSUB:
+ # raw_input()
+ logger.info("\n")
+ for idx, sub in enumerate(subs):
+
+ with open(sub, 'w') as srt_file:
+ for item in ret_subs[idx][0]:
+ srt_file.write(str(item))
+
+ srt_file.write("\n#############\n")
+ srt_file.write("\nSRT formated data\n")
+ srt_file.write(ret_subs[idx][1])
+
+ logger.info("###get_subs:Debug:Subtitle '{0}' downloaded.".format(sub))
+
+ return ret_subs
+
+
+def get_single_sub(tt_id, tt_intro, sub):
+ """
+ Get TED Subtitle in JSON format & convert it to SRT Subtitle.
+ """
+
+ srt_content = ''
+ srt_items = []
+ tt_url = 'http://www.ted.com/talks'
+ sub_url = '{0}/subtitles/id/{1}/lang/{2}'.format(tt_url, tt_id, sub[-7:-4])
+
+ # Get JSON sub
+ json_file = request.urlopen(sub_url).readlines()
+ logger.info("###get_single_sub: sub url is " + sub_url)
+
+ if json_file:
+ try:
+ json_object = json.loads(json_file[0].decode('utf-8'))
+ logger.info("###get_single_sub: json load orig data")
+ #logger.info(json_object)
+ if 'captions' in json_object:
+ caption_idx = 1
+ if not json_object['captions']:
+ logger.info("Subtitle '{0}' not available.".format(sub))
+ for caption in json_object['captions']:
+ start = tt_intro + caption['startTime']
+ end = start + caption['duration']
+ idx_line = '{0}'.format(caption_idx)
+ time_line = '{0} --> {1}'.format(srt_time(start),
+ srt_time(end))
+ text_line = '{0}'.format(caption['content'].encode("utf-8"))
+
+ # Append the srt items and content parellelly
+ srt_items.append({"index":caption_idx, "start":srt_sec_time(start),
+ "duration":srt_sec_time(caption['duration']), "content":text_line})
+ srt_content += '\n'.join([idx_line, time_line, text_line, '\n'])
+ caption_idx += 1
+
+ elif 'status' in json_object:
+ logger.info("This is an error message returned by TED:{0}{0} - "
+ "{1}{0}{0}Probably because the subtitle '{2}' is not "
+ "available.{0}".format(os.linesep, json_object['status']['message'], sub))
+
+ except ValueError:
+ logger.info("Subtitle '{0}' it's a malformed json file.".format(sub))
+
+ return (srt_items, srt_content)
+
+
+def options():
+ """Defines the command line arguments and options for the script."""
+
+ desc = "Downloads the subtitles and the video (optional) for a TED Talk."
+ usage = "Beautifull TED"
+ parser = optparse.OptionParser(usage=usage, version="%prog " + __version__,
+ description=desc)
+
+ parser.add_option("-s", "--only_subs", action='store_true',
+ dest="no_video",
+ help="download only the subs, not the video ",
+ default=False)
+ return parser
+
+
+def check_exec_posix(prog):
+ """
+ Check if the program is installed in a *NIX platform.
+ """
+ return True
+
+
+def main():
+ """main section"""
+ pass
+
+
+# module info
+get_Danmu = ted_get_danmu
+
+
site_info = "TED.com"
download = ted_download
download_playlist = playlist_not_supported('ted')
diff --git a/src/you_get/extractors/theplatform.py b/src/you_get/extractors/theplatform.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/tucao.py b/src/you_get/extractors/tucao.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
old mode 100644
new mode 100755
index a9f78a6d..95cf96fd
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -7,7 +7,7 @@ from xml.dom.minidom import parseString
def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only = False):
data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid))
- temp = max([data[i] for i in data if 'size' in data[i][0]], key=lambda x:x[0]["size"])
+ temp = max([data[i] for i in data], key=lambda x:x[0]["size"])
vids, size = [t["k"] for t in temp], sum([t["size"] for t in temp])
urls = [[n.firstChild.nodeValue.strip()
for n in
diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
old mode 100644
new mode 100755
index 079de707..8a2e2ed1
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -7,23 +7,15 @@ from ..common import *
import re
def tumblr_download(url, output_dir = '.', merge = True, info_only = False):
- html = parse.unquote(get_html(url)).replace('\/', '/')
- feed = r1(r'
', html)
-
- if feed == 'audio':
- real_url = r1(r'source src=\\x22([^\\]+)\\', html)
- if not real_url:
- real_url = r1(r'audio_file=([^&]+)&', html) + '?plead=please-dont-download-this-or-our-lawyers-wont-let-us-host-audio'
- elif feed == 'video':
- iframe_url = r1(r'
', html) or
r1(r'
', html) or
r1(r'
([^<\n]*)', html)).replace('\n', '')
+ real_url = r1(r'source src=\\x22([^\\]+)\\', html)
+ if not real_url:
+ real_url = r1(r'audio_file=([^&]+)&', html) + '?plead=please-dont-download-this-or-our-lawyers-wont-let-us-host-audio'
type, ext, size = url_info(real_url)
diff --git a/src/you_get/extractors/vid48.py b/src/you_get/extractors/vid48.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/videobam.py b/src/you_get/extractors/videobam.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/vine.py b/src/you_get/extractors/vine.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/w56.py b/src/you_get/extractors/w56.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/yinyuetai.py b/src/you_get/extractors/yinyuetai.py
old mode 100644
new mode 100755
diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
old mode 100644
new mode 100755
index c9d98bfd..09b06a61
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -55,14 +55,14 @@ class Youku(VideoExtractor):
def get_vid_from_url(url):
"""Extracts video ID from URL.
"""
- return match1(url, r'youku\.com/v_show/id_([a-zA-Z0-9=]+)') or \
- match1(url, r'player\.youku\.com/player\.php/sid/([a-zA-Z0-9=]+)/v\.swf') or \
- match1(url, r'loader\.swf\?VideoIDS=([a-zA-Z0-9=]+)')
+ return match1(url, r'youku\.com/v_show/id_([\w=]+)') or \
+ match1(url, r'player\.youku\.com/player\.php/sid/([\w=]+)/v\.swf') or \
+ match1(url, r'loader\.swf\?VideoIDS=([\w=]+)')
def get_playlist_id_from_url(url):
"""Extracts playlist ID from URL.
"""
- return match1(url, r'youku\.com/playlist_show/id_([a-zA-Z0-9=]+)')
+ return match1(url, r'youku\.com/playlist_show/id_([\w=]+)')
def download_playlist_by_url(self, url, **kwargs):
self.url = url
diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
old mode 100644
new mode 100755
diff --git a/src/you_get/processor/__init__.py b/src/you_get/processor/__init__.py
old mode 100644
new mode 100755
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
old mode 100644
new mode 100755
diff --git a/src/you_get/processor/rtmpdump.py b/src/you_get/processor/rtmpdump.py
old mode 100644
new mode 100755
diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py
old mode 100644
new mode 100755
diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py
old mode 100644
new mode 100755
diff --git a/src/you_get/util/git.py b/src/you_get/util/git.py
old mode 100644
new mode 100755
diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py
old mode 100644
new mode 100755
diff --git a/src/you_get/util/strings.py b/src/you_get/util/strings.py
old mode 100644
new mode 100755
diff --git a/src/you_get/version.py b/src/you_get/version.py
old mode 100644
new mode 100755
diff --git a/src_bak/you_get/__init__.py b/src_bak/you_get/__init__.py
new file mode 100644
index 00000000..5da7138f
--- /dev/null
+++ b/src_bak/you_get/__init__.py
@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# This file is Python 2 compliant.
+
+import sys
+
+if sys.version_info[0] == 3:
+ #from .extractor import Extractor, VideoExtractor
+ #from .util import log
+
+ from .__main__ import *
+
+ #from .common import *
+ #from .version import *
+ #from .cli_wrapper import *
+ #from .extractor import *
+else:
+ # Don't import anything.
+ pass
diff --git a/src_bak/you_get/__main__.py b/src_bak/you_get/__main__.py
new file mode 100644
index 00000000..027854a7
--- /dev/null
+++ b/src_bak/you_get/__main__.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+import getopt
+import os
+import platform
+import sys
+from .version import script_name, __version__
+from .util import git, log
+
+_options = [
+ 'help',
+ 'version',
+ 'gui',
+ 'force',
+ 'playlists',
+]
+_short_options = 'hVgfl'
+
+_help = """Usage: {} [OPTION]... [URL]...
+TODO
+""".format(script_name)
+
+def main_dev(**kwargs):
+ """Main entry point.
+ you-get-dev
+ """
+
+ # Get (branch, commit) if running from a git repo.
+ head = git.get_head(kwargs['repo_path'])
+
+ # Get options and arguments.
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], _short_options, _options)
+ except getopt.GetoptError as e:
+ log.wtf("""
+ [Fatal] {}.
+ Try '{} --help' for more options.""".format(e, script_name))
+
+ if not opts and not args:
+ # Display help.
+ print(_help)
+ # Enter GUI mode.
+ #from .gui import gui_main
+ #gui_main()
+ else:
+ conf = {}
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ # Display help.
+ print(_help)
+
+ elif opt in ('-V', '--version'):
+ # Display version.
+ log.println("you-get:", log.BOLD)
+ log.println(" version: {}".format(__version__))
+ if head is not None:
+ log.println(" branch: {}\n commit: {}".format(*head))
+ else:
+ log.println(" branch: {}\n commit: {}".format("(stable)", "(tag v{})".format(__version__)))
+
+ log.println(" platform: {}".format(platform.platform()))
+ log.println(" python: {}".format(sys.version.split('\n')[0]))
+
+ elif opt in ('-g', '--gui'):
+ # Run using GUI.
+ conf['gui'] = True
+
+ elif opt in ('-f', '--force'):
+ # Force download.
+ conf['force'] = True
+
+ elif opt in ('-l', '--playlist', '--playlists'):
+ # Download playlist whenever possible.
+ conf['playlist'] = True
+
+ if args:
+ if 'gui' in conf and conf['gui']:
+ # Enter GUI mode.
+ from .gui import gui_main
+ gui_main(*args, **conf)
+ else:
+ # Enter console mode.
+ from .console import console_main
+ console_main(*args, **conf)
+
+def main(**kwargs):
+ """Main entry point.
+ you-get (legacy)
+ """
+ from .common import main
+ main()
diff --git a/src_bak/you_get/cli_wrapper/__init__.py b/src_bak/you_get/cli_wrapper/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/downloader/__init__.py b/src_bak/you_get/cli_wrapper/downloader/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/openssl/__init__.py b/src_bak/you_get/cli_wrapper/openssl/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/player/__init__.py b/src_bak/you_get/cli_wrapper/player/__init__.py
new file mode 100644
index 00000000..2f7636de
--- /dev/null
+++ b/src_bak/you_get/cli_wrapper/player/__init__.py
@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+
+from .mplayer import *
diff --git a/src_bak/you_get/cli_wrapper/player/__main__.py b/src_bak/you_get/cli_wrapper/player/__main__.py
new file mode 100644
index 00000000..8d4958b9
--- /dev/null
+++ b/src_bak/you_get/cli_wrapper/player/__main__.py
@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+def main():
+ script_main('you-get', any_download, any_download_playlist)
+
+if __name__ == "__main__":
+ main()
diff --git a/src_bak/you_get/cli_wrapper/player/dragonplayer.py b/src_bak/you_get/cli_wrapper/player/dragonplayer.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/player/gnome_mplayer.py b/src_bak/you_get/cli_wrapper/player/gnome_mplayer.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/player/mplayer.py b/src_bak/you_get/cli_wrapper/player/mplayer.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/player/vlc.py b/src_bak/you_get/cli_wrapper/player/vlc.py
new file mode 100644
index 00000000..4265cc3e
--- /dev/null
+++ b/src_bak/you_get/cli_wrapper/player/vlc.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/src_bak/you_get/cli_wrapper/player/wmp.py b/src_bak/you_get/cli_wrapper/player/wmp.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/transcoder/__init__.py b/src_bak/you_get/cli_wrapper/transcoder/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/transcoder/ffmpeg.py b/src_bak/you_get/cli_wrapper/transcoder/ffmpeg.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/transcoder/libav.py b/src_bak/you_get/cli_wrapper/transcoder/libav.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/cli_wrapper/transcoder/mencoder.py b/src_bak/you_get/cli_wrapper/transcoder/mencoder.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src_bak/you_get/common.py b/src_bak/you_get/common.py
new file mode 100644
index 00000000..00f06254
--- /dev/null
+++ b/src_bak/you_get/common.py
@@ -0,0 +1,997 @@
+#!/usr/bin/env python
+
+import getopt
+import json
+import locale
+import os
+import platform
+import re
+import sys
+from urllib import request, parse
+
+from .version import __version__
+from .util import log
+from .util.strings import get_filename, unescape_html
+
+dry_run = False
+force = False
+player = None
+extractor_proxy = None
+cookies_txt = None
+
+fake_headers = {
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Charset': 'UTF-8,*;q=0.5',
+ 'Accept-Encoding': 'gzip,deflate,sdch',
+ 'Accept-Language': 'en-US,en;q=0.8',
+ 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) Gecko/20100101 Firefox/13.0'
+}
+
+if sys.stdout.isatty():
+ default_encoding = sys.stdout.encoding.lower()
+else:
+ default_encoding = locale.getpreferredencoding().lower()
+
+def tr(s):
+ if default_encoding == 'utf-8':
+ return s
+ else:
+ return str(s.encode('utf-8'))[2:-1]
+
+# DEPRECATED in favor of match1()
+def r1(pattern, text):
+ m = re.search(pattern, text)
+ if m:
+ return m.group(1)
+
+# DEPRECATED in favor of match1()
+def r1_of(patterns, text):
+ for p in patterns:
+ x = r1(p, text)
+ if x:
+ return x
+
+def match1(text, *patterns):
+ """Scans through a string for substrings matched some patterns (first-subgroups only).
+
+ Args:
+ text: A string to be scanned.
+ patterns: Arbitrary number of regex patterns.
+
+ Returns:
+ When only one pattern is given, returns a string (None if no match found).
+ When more than one pattern are given, returns a list of strings ([] if no match found).
+ """
+
+ if len(patterns) == 1:
+ pattern = patterns[0]
+ match = re.search(pattern, text)
+ if match:
+ return match.group(1)
+ else:
+ return None
+ else:
+ ret = []
+ for pattern in patterns:
+ match = re.search(pattern, text)
+ if match:
+ ret.append(match.group(1))
+ return ret
+
+def launch_player(player, urls):
+ import subprocess
+ import shlex
+ subprocess.call(shlex.split(player) + list(urls))
+
+def parse_query_param(url, param):
+ """Parses the query string of a URL and returns the value of a parameter.
+
+ Args:
+ url: A URL.
+ param: A string representing the name of the parameter.
+
+ Returns:
+ The value of the parameter.
+ """
+
+ try:
+ return parse.parse_qs(parse.urlparse(url).query)[param][0]
+ except:
+ return None
+
+def unicodize(text):
+ return re.sub(r'\\u([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])', lambda x: chr(int(x.group(0)[2:], 16)), text)
+
+# DEPRECATED in favor of util.legitimize()
+def escape_file_path(path):
+ path = path.replace('/', '-')
+ path = path.replace('\\', '-')
+ path = path.replace('*', '-')
+ path = path.replace('?', '-')
+ return path
+
+def ungzip(data):
+ """Decompresses data for Content-Encoding: gzip.
+ """
+ from io import BytesIO
+ import gzip
+ buffer = BytesIO(data)
+ f = gzip.GzipFile(fileobj=buffer)
+ return f.read()
+
+def undeflate(data):
+ """Decompresses data for Content-Encoding: deflate.
+ (the zlib compression is used.)
+ """
+ import zlib
+ decompressobj = zlib.decompressobj(-zlib.MAX_WBITS)
+ return decompressobj.decompress(data)+decompressobj.flush()
+
+# DEPRECATED in favor of get_content()
+def get_response(url, faker = False):
+ if faker:
+ response = request.urlopen(request.Request(url, headers = fake_headers), None)
+ else:
+ response = request.urlopen(url)
+
+ data = response.read()
+ if response.info().get('Content-Encoding') == 'gzip':
+ data = ungzip(data)
+ elif response.info().get('Content-Encoding') == 'deflate':
+ data = undeflate(data)
+ response.data = data
+ return response
+
+# DEPRECATED in favor of get_content()
+def get_html(url, encoding = None, faker = False):
+ content = get_response(url, faker).data
+ return str(content, 'utf-8', 'ignore')
+
+# DEPRECATED in favor of get_content()
+def get_decoded_html(url, faker = False):
+ response = get_response(url, faker)
+ data = response.data
+ charset = r1(r'charset=([\w-]+)', response.headers['content-type'])
+ if charset:
+ return data.decode(charset, 'ignore')
+ else:
+ return data
+
+def get_content(url, headers={}, decoded=True):
+ """Gets the content of a URL via sending a HTTP GET request.
+
+ Args:
+ url: A URL.
+ headers: Request headers used by the client.
+ decoded: Whether decode the response body using UTF-8 or the charset specified in Content-Type.
+
+ Returns:
+ The content as a string.
+ """
+
+ req = request.Request(url, headers=headers)
+ if cookies_txt:
+ cookies_txt.add_cookie_header(req)
+ req.headers.update(req.unredirected_hdrs)
+ response = request.urlopen(req)
+ data = response.read()
+
+ # Handle HTTP compression for gzip and deflate (zlib)
+ content_encoding = response.getheader('Content-Encoding')
+ if content_encoding == 'gzip':
+ data = ungzip(data)
+ elif content_encoding == 'deflate':
+ data = undeflate(data)
+
+ # Decode the response body
+ if decoded:
+ charset = match1(response.getheader('Content-Type'), r'charset=([\w-]+)')
+ if charset is not None:
+ data = data.decode(charset)
+ else:
+ data = data.decode('utf-8')
+
+ return data
+
+def url_size(url, faker = False):
+ if faker:
+ response = request.urlopen(request.Request(url, headers = fake_headers), None)
+ else:
+ response = request.urlopen(url)
+
+ size = response.headers['content-length']
+ return int(size) if size!=None else float('inf')
+
+# TO BE DEPRECATED
+# urls_size() does not have a faker
+# also it takes too long time
+def urls_size(urls):
+ return sum(map(url_size, urls))
+
+def url_info(url, faker = False):
+ if faker:
+ response = request.urlopen(request.Request(url, headers = fake_headers), None)
+ else:
+ response = request.urlopen(request.Request(url))
+
+ headers = response.headers
+
+ type = headers['content-type']
+ mapping = {
+ 'video/3gpp': '3gp',
+ 'video/f4v': 'flv',
+ 'video/mp4': 'mp4',
+ 'video/MP2T': 'ts',
+ 'video/quicktime': 'mov',
+ 'video/webm': 'webm',
+ 'video/x-flv': 'flv',
+ 'video/x-ms-asf': 'asf',
+ 'audio/mp4': 'mp4',
+ 'audio/mpeg': 'mp3'
+ }
+ if type in mapping:
+ ext = mapping[type]
+ else:
+ type = None
+ if headers['content-disposition']:
+ try:
+ filename = parse.unquote(r1(r'filename="?([^"]+)"?', headers['content-disposition']))
+ if len(filename.split('.')) > 1:
+ ext = filename.split('.')[-1]
+ else:
+ ext = None
+ except:
+ ext = None
+ else:
+ ext = None
+
+ if headers['transfer-encoding'] != 'chunked':
+ size = headers['content-length'] and int(headers['content-length'])
+ else:
+ size = None
+
+ return type, ext, size
+
+def url_locations(urls, faker = False):
+ locations = []
+ for url in urls:
+ if faker:
+ response = request.urlopen(request.Request(url, headers = fake_headers), None)
+ else:
+ response = request.urlopen(request.Request(url))
+
+ locations.append(response.url)
+ return locations
+
+def url_save(url, filepath, bar, refer = None, is_part = False, faker = False):
+ file_size = url_size(url, faker = faker)
+
+ if os.path.exists(filepath):
+ if not force and file_size == os.path.getsize(filepath):
+ if not is_part:
+ if bar:
+ bar.done()
+ print('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
+ else:
+ if bar:
+ bar.update_received(file_size)
+ return
+ else:
+ if not is_part:
+ if bar:
+ bar.done()
+ print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
+ elif not os.path.exists(os.path.dirname(filepath)):
+ os.mkdir(os.path.dirname(filepath))
+
+ temp_filepath = filepath + '.download' if file_size!=float('inf') else filepath
+ received = 0
+ if not force:
+ open_mode = 'ab'
+
+ if os.path.exists(temp_filepath):
+ received += os.path.getsize(temp_filepath)
+ if bar:
+ bar.update_received(os.path.getsize(temp_filepath))
+ else:
+ open_mode = 'wb'
+
+ if received < file_size:
+ if faker:
+ headers = fake_headers
+ else:
+ headers = {}
+ if received:
+ headers['Range'] = 'bytes=' + str(received) + '-'
+ if refer:
+ headers['Referer'] = refer
+
+ response = request.urlopen(request.Request(url, headers = headers), None)
+ try:
+ range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0])
+ end_length = end = int(response.headers['content-range'][6:].split('/')[1])
+ range_length = end_length - range_start
+ except:
+ content_length = response.headers['content-length']
+ range_length = int(content_length) if content_length!=None else float('inf')
+
+ if file_size != received + range_length:
+ received = 0
+ if bar:
+ bar.received = 0
+ open_mode = 'wb'
+
+ with open(temp_filepath, open_mode) as output:
+ while True:
+ buffer = response.read(1024 * 256)
+ if not buffer:
+ if received == file_size: # Download finished
+ break
+ else: # Unexpected termination. Retry request
+ headers['Range'] = 'bytes=' + str(received) + '-'
+ response = request.urlopen(request.Request(url, headers = headers), None)
+ output.write(buffer)
+ received += len(buffer)
+ if bar:
+ bar.update_received(len(buffer))
+
+ assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (received, os.path.getsize(temp_filepath), temp_filepath)
+
+ if os.access(filepath, os.W_OK):
+ os.remove(filepath) # on Windows rename could fail if destination filepath exists
+ os.rename(temp_filepath, filepath)
+
+def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker = False):
+ if os.path.exists(filepath):
+ if not force:
+ if not is_part:
+ if bar:
+ bar.done()
+ print('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
+ else:
+ if bar:
+ bar.update_received(os.path.getsize(filepath))
+ return
+ else:
+ if not is_part:
+ if bar:
+ bar.done()
+ print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
+ elif not os.path.exists(os.path.dirname(filepath)):
+ os.mkdir(os.path.dirname(filepath))
+
+ temp_filepath = filepath + '.download'
+ received = 0
+ if not force:
+ open_mode = 'ab'
+
+ if os.path.exists(temp_filepath):
+ received += os.path.getsize(temp_filepath)
+ if bar:
+ bar.update_received(os.path.getsize(temp_filepath))
+ else:
+ open_mode = 'wb'
+
+ if faker:
+ headers = fake_headers
+ else:
+ headers = {}
+ if received:
+ headers['Range'] = 'bytes=' + str(received) + '-'
+ if refer:
+ headers['Referer'] = refer
+
+ response = request.urlopen(request.Request(url, headers = headers), None)
+
+ with open(temp_filepath, open_mode) as output:
+ while True:
+ buffer = response.read(1024 * 256)
+ if not buffer:
+ break
+ output.write(buffer)
+ received += len(buffer)
+ if bar:
+ bar.update_received(len(buffer))
+
+ assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (received, os.path.getsize(temp_filepath))
+
+ if os.access(filepath, os.W_OK):
+ os.remove(filepath) # on Windows rename could fail if destination filepath exists
+ os.rename(temp_filepath, filepath)
+
+class SimpleProgressBar:
+ def __init__(self, total_size, total_pieces = 1):
+ self.displayed = False
+ self.total_size = total_size
+ self.total_pieces = total_pieces
+ self.current_piece = 1
+ self.received = 0
+
+ def update(self):
+ self.displayed = True
+ bar_size = 40
+ percent = round(self.received * 100 / self.total_size, 1)
+ if percent > 100:
+ percent = 100
+ dots = bar_size * int(percent) // 100
+ plus = int(percent) - dots // bar_size * 100
+ if plus > 0.8:
+ plus = '='
+ elif plus > 0.4:
+ plus = '>'
+ else:
+ plus = ''
+ bar = '=' * dots + plus
+ bar = '{0:>5}% ({1:>5}/{2:<5}MB) [{3:<40}] {4}/{5}'.format(percent, round(self.received / 1048576, 1), round(self.total_size / 1048576, 1), bar, self.current_piece, self.total_pieces)
+ sys.stdout.write('\r' + bar)
+ sys.stdout.flush()
+
+ def update_received(self, n):
+ self.received += n
+ self.update()
+
+ def update_piece(self, n):
+ self.current_piece = n
+
+ def done(self):
+ if self.displayed:
+ print()
+ self.displayed = False
+
+class PiecesProgressBar:
+ def __init__(self, total_size, total_pieces = 1):
+ self.displayed = False
+ self.total_size = total_size
+ self.total_pieces = total_pieces
+ self.current_piece = 1
+ self.received = 0
+
+ def update(self):
+ self.displayed = True
+ bar = '{0:>5}%[{1:<40}] {2}/{3}'.format('?', '?' * 40, self.current_piece, self.total_pieces)
+ sys.stdout.write('\r' + bar)
+ sys.stdout.flush()
+
+ def update_received(self, n):
+ self.received += n
+ self.update()
+
+ def update_piece(self, n):
+ self.current_piece = n
+
+ def done(self):
+ if self.displayed:
+ print()
+ self.displayed = False
+
+class DummyProgressBar:
+ def __init__(self, *args):
+ pass
+ def update_received(self, n):
+ pass
+ def update_piece(self, n):
+ pass
+ def done(self):
+ pass
+
+def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False):
+ assert urls
+ if dry_run:
+ print('Real URLs:\n%s\n' % urls)
+ return
+
+ if player:
+ launch_player(player, urls)
+ return
+
+ if not total_size:
+ try:
+ total_size = urls_size(urls)
+ except:
+ import traceback
+ import sys
+ traceback.print_exc(file = sys.stdout)
+ pass
+
+ title = tr(get_filename(title))
+
+ filename = '%s.%s' % (title, ext)
+ filepath = os.path.join(output_dir, filename)
+ if total_size:
+ if not force and os.path.exists(filepath) and os.path.getsize(filepath) >= total_size * 0.9:
+ print('Skipping %s: file already exists' % filepath)
+ print()
+ return
+ bar = SimpleProgressBar(total_size, len(urls))
+ else:
+ bar = PiecesProgressBar(total_size, len(urls))
+
+ if len(urls) == 1:
+ url = urls[0]
+ print('Downloading %s ...' % tr(filename))
+ url_save(url, filepath, bar, refer = refer, faker = faker)
+ bar.done()
+ else:
+ parts = []
+ print('Downloading %s.%s ...' % (tr(title), ext))
+ for i, url in enumerate(urls):
+ filename = '%s[%02d].%s' % (title, i, ext)
+ filepath = os.path.join(output_dir, filename)
+ parts.append(filepath)
+ #print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
+ bar.update_piece(i + 1)
+ url_save(url, filepath, bar, refer = refer, is_part = True, faker = faker)
+ bar.done()
+
+ if not merge:
+ print()
+ return
+ if ext in ['flv', 'f4v']:
+ try:
+ from .processor.ffmpeg import has_ffmpeg_installed
+ if has_ffmpeg_installed():
+ from .processor.ffmpeg import ffmpeg_concat_flv_to_mp4
+ ffmpeg_concat_flv_to_mp4(parts, os.path.join(output_dir, title + '.mp4'))
+ else:
+ from .processor.join_flv import concat_flv
+ concat_flv(parts, os.path.join(output_dir, title + '.flv'))
+ except:
+ raise
+ else:
+ for part in parts:
+ os.remove(part)
+
+ elif ext == 'mp4':
+ try:
+ from .processor.ffmpeg import has_ffmpeg_installed
+ if has_ffmpeg_installed():
+ from .processor.ffmpeg import ffmpeg_concat_mp4_to_mp4
+ ffmpeg_concat_mp4_to_mp4(parts, os.path.join(output_dir, title + '.mp4'))
+ else:
+ from .processor.join_mp4 import concat_mp4
+ concat_mp4(parts, os.path.join(output_dir, title + '.mp4'))
+ except:
+ raise
+ else:
+ for part in parts:
+ os.remove(part)
+
+ else:
+ print("Can't merge %s files" % ext)
+
+ print()
+
+def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False):
+ assert urls
+ if dry_run:
+ print('Real URLs:\n%s\n' % urls)
+ return
+
+ if player:
+ launch_player(player, urls)
+ return
+
+ assert ext in ('ts')
+
+ title = tr(get_filename(title))
+
+ filename = '%s.%s' % (title, 'ts')
+ filepath = os.path.join(output_dir, filename)
+ if total_size:
+ if not force and os.path.exists(filepath[:-3] + '.mkv'):
+ print('Skipping %s: file already exists' % filepath[:-3] + '.mkv')
+ print()
+ return
+ bar = SimpleProgressBar(total_size, len(urls))
+ else:
+ bar = PiecesProgressBar(total_size, len(urls))
+
+ if len(urls) == 1:
+ parts = []
+ url = urls[0]
+ print('Downloading %s ...' % tr(filename))
+ filepath = os.path.join(output_dir, filename)
+ parts.append(filepath)
+ url_save_chunked(url, filepath, bar, refer = refer, faker = faker)
+ bar.done()
+
+ if not merge:
+ print()
+ return
+ if ext == 'ts':
+ from .processor.ffmpeg import has_ffmpeg_installed
+ if has_ffmpeg_installed():
+ from .processor.ffmpeg import ffmpeg_convert_ts_to_mkv
+ if ffmpeg_convert_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')):
+ for part in parts:
+ os.remove(part)
+ else:
+ os.remove(os.path.join(output_dir, title + '.mkv'))
+ else:
+ print('No ffmpeg is found. Conversion aborted.')
+ else:
+ print("Can't convert %s files" % ext)
+ else:
+ parts = []
+ print('Downloading %s.%s ...' % (tr(title), ext))
+ for i, url in enumerate(urls):
+ filename = '%s[%02d].%s' % (title, i, ext)
+ filepath = os.path.join(output_dir, filename)
+ parts.append(filepath)
+ #print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
+ bar.update_piece(i + 1)
+ url_save_chunked(url, filepath, bar, refer = refer, is_part = True, faker = faker)
+ bar.done()
+
+ if not merge:
+ print()
+ return
+ if ext == 'ts':
+ from .processor.ffmpeg import has_ffmpeg_installed
+ if has_ffmpeg_installed():
+ from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv
+ if ffmpeg_concat_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')):
+ for part in parts:
+ os.remove(part)
+ else:
+ os.remove(os.path.join(output_dir, title + '.mkv'))
+ else:
+ print('No ffmpeg is found. Merging aborted.')
+ else:
+ print("Can't merge %s files" % ext)
+
+ print()
+
+def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
+ assert url
+ if dry_run:
+ print('Real URL:\n%s\n' % [url])
+ if params.get("-y",False): #None or unset ->False
+ print('Real Playpath:\n%s\n' % [params.get("-y")])
+ return
+
+ if player:
+ from .processor.rtmpdump import play_rtmpdump_stream
+ play_rtmpdump_stream(player, url, params)
+ return
+
+ from .processor.rtmpdump import has_rtmpdump_installed, download_rtmpdump_stream
+ assert has_rtmpdump_installed(), "RTMPDump not installed."
+ download_rtmpdump_stream(url, title, ext,params, output_dir)
+
+def playlist_not_supported(name):
+ def f(*args, **kwargs):
+ raise NotImplementedError('Playlist is not supported for ' + name)
+ return f
+
+def print_info(site_info, title, type, size):
+ if type:
+ type = type.lower()
+ if type in ['3gp']:
+ type = 'video/3gpp'
+ elif type in ['asf', 'wmv']:
+ type = 'video/x-ms-asf'
+ elif type in ['flv', 'f4v']:
+ type = 'video/x-flv'
+ elif type in ['mkv']:
+ type = 'video/x-matroska'
+ elif type in ['mp3']:
+ type = 'audio/mpeg'
+ elif type in ['mp4']:
+ type = 'video/mp4'
+ elif type in ['mov']:
+ type = 'video/quicktime'
+ elif type in ['ts']:
+ type = 'video/MP2T'
+ elif type in ['webm']:
+ type = 'video/webm'
+
+ if type in ['video/3gpp']:
+ type_info = "3GPP multimedia file (%s)" % type
+ elif type in ['video/x-flv', 'video/f4v']:
+ type_info = "Flash video (%s)" % type
+ elif type in ['video/mp4', 'video/x-m4v']:
+ type_info = "MPEG-4 video (%s)" % type
+ elif type in ['video/MP2T']:
+ type_info = "MPEG-2 transport stream (%s)" % type
+ elif type in ['video/webm']:
+ type_info = "WebM video (%s)" % type
+ #elif type in ['video/ogg']:
+ # type_info = "Ogg video (%s)" % type
+ elif type in ['video/quicktime']:
+ type_info = "QuickTime video (%s)" % type
+ elif type in ['video/x-matroska']:
+ type_info = "Matroska video (%s)" % type
+ #elif type in ['video/x-ms-wmv']:
+ # type_info = "Windows Media video (%s)" % type
+ elif type in ['video/x-ms-asf']:
+ type_info = "Advanced Systems Format (%s)" % type
+ #elif type in ['video/mpeg']:
+ # type_info = "MPEG video (%s)" % type
+ elif type in ['audio/mp4']:
+ type_info = "MPEG-4 audio (%s)" % type
+ elif type in ['audio/mpeg']:
+ type_info = "MP3 (%s)" % type
+ else:
+ type_info = "Unknown type (%s)" % type
+
+ print("Video Site:", site_info)
+ print("Title: ", unescape_html(tr(title)))
+ print("Type: ", type_info)
+ print("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
+ print()
+
+def mime_to_container(mime):
+ mapping = {
+ 'video/3gpp': '3gp',
+ 'video/mp4': 'mp4',
+ 'video/webm': 'webm',
+ 'video/x-flv': 'flv',
+ }
+ if mime in mapping:
+ return mapping[mime]
+ else:
+ return mime.split('/')[1]
+
+def parse_host(host):
+ """Parses host name and port number from a string.
+ """
+ if re.match(r'^(\d+)$', host) is not None:
+ return ("0.0.0.0", int(host))
+ if re.match(r'^(\w+)://', host) is None:
+ host = "//" + host
+ o = parse.urlparse(host)
+ hostname = o.hostname or "0.0.0.0"
+ port = o.port or 0
+ return (hostname, port)
+
+def set_proxy(proxy):
+ proxy_handler = request.ProxyHandler({
+ 'http': '%s:%s' % proxy,
+ 'https': '%s:%s' % proxy,
+ })
+ opener = request.build_opener(proxy_handler)
+ request.install_opener(opener)
+
+def unset_proxy():
+ proxy_handler = request.ProxyHandler({})
+ opener = request.build_opener(proxy_handler)
+ request.install_opener(opener)
+
+# DEPRECATED in favor of set_proxy() and unset_proxy()
+def set_http_proxy(proxy):
+ if proxy == None: # Use system default setting
+ proxy_support = request.ProxyHandler()
+ elif proxy == '': # Don't use any proxy
+ proxy_support = request.ProxyHandler({})
+ else: # Use proxy
+ proxy_support = request.ProxyHandler({'http': '%s' % proxy, 'https': '%s' % proxy})
+ opener = request.build_opener(proxy_support)
+ request.install_opener(opener)
+
+
+
+def download_main(download, download_playlist, urls, playlist, **kwargs):
+ for url in urls:
+ if url.startswith('https://'):
+ url = url[8:]
+ if not url.startswith('http://'):
+ url = 'http://' + url
+
+ if playlist:
+ download_playlist(url, **kwargs)
+ else:
+ download(url, **kwargs)
+
+def script_main(script_name, download, download_playlist = None):
+ version = 'You-Get %s, a video downloader.' % __version__
+ help = 'Usage: %s [OPTION]... [URL]...\n' % script_name
+ help += '''\nStartup options:
+ -V | --version Display the version and exit.
+ -h | --help Print this help and exit.
+ '''
+ help += '''\nDownload options (use with URLs):
+ -f | --force Force overwriting existed files.
+ -i | --info Display the information of videos without downloading.
+ -u | --url Display the real URLs of videos without downloading.
+ -c | --cookies Load NetScape's cookies.txt file.
+ -n | --no-merge Don't merge video parts.
+ -F | --format <STREAM_ID> Video format code.
+ -o | --output-dir <PATH> Set the output directory for downloaded videos.
+ -p | --player <PLAYER [options]> Directly play the video with PLAYER like vlc/smplayer.
+ -x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
+ -y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
+ --no-proxy Don't use any proxy. (ignore $http_proxy)
+ --debug Show traceback on KeyboardInterrupt.
+ '''
+
+ short_opts = 'Vhfiuc:nF:o:p:x:y:'
+ opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
+ if download_playlist:
+ short_opts = 'l' + short_opts
+ opts = ['playlist'] + opts
+
+ try:
+ opts, args = getopt.getopt(sys.argv[1:], short_opts, opts)
+ except getopt.GetoptError as err:
+ log.e(err)
+ log.e("try 'you-get --help' for more options")
+ sys.exit(2)
+
+ global force
+ global dry_run
+ global player
+ global extractor_proxy
+ global cookies_txt
+ cookies_txt = None
+
+ info_only = False
+ playlist = False
+ merge = True
+ stream_id = None
+ lang = None
+ output_dir = '.'
+ proxy = None
+ extractor_proxy = None
+ traceback = False
+ for o, a in opts:
+ if o in ('-V', '--version'):
+ print(version)
+ sys.exit()
+ elif o in ('-h', '--help'):
+ print(version)
+ print(help)
+ sys.exit()
+ elif o in ('-f', '--force'):
+ force = True
+ elif o in ('-i', '--info'):
+ info_only = True
+ elif o in ('-u', '--url'):
+ dry_run = True
+ elif o in ('-c', '--cookies'):
+ from http import cookiejar
+ cookies_txt = cookiejar.MozillaCookieJar(a)
+ cookies_txt.load()
+ elif o in ('-l', '--playlist'):
+ playlist = True
+ elif o in ('-n', '--no-merge'):
+ merge = False
+ elif o in ('--no-proxy',):
+ proxy = ''
+ elif o in ('--debug',):
+ traceback = True
+ elif o in ('-F', '--format', '--stream', '--itag'):
+ stream_id = a
+ elif o in ('-o', '--output-dir'):
+ output_dir = a
+ elif o in ('-p', '--player'):
+ player = a
+ elif o in ('-x', '--http-proxy'):
+ proxy = a
+ elif o in ('-y', '--extractor-proxy'):
+ extractor_proxy = a
+ elif o in ('--lang',):
+ lang = a
+ else:
+ log.e("try 'you-get --help' for more options")
+ sys.exit(2)
+ if not args:
+ print(help)
+ sys.exit()
+
+ set_http_proxy(proxy)
+
+ try:
+ if stream_id:
+ if not extractor_proxy:
+ download_main(download, download_playlist, args, playlist, stream_id=stream_id, output_dir=output_dir, merge=merge, info_only=info_only)
+ else:
+ download_main(download, download_playlist, args, playlist, stream_id=stream_id, extractor_proxy=extractor_proxy, output_dir=output_dir, merge=merge, info_only=info_only)
+ else:
+ if not extractor_proxy:
+ download_main(download, download_playlist, args, playlist, output_dir=output_dir, merge=merge, info_only=info_only)
+ else:
+ download_main(download, download_playlist, args, playlist, extractor_proxy=extractor_proxy, output_dir=output_dir, merge=merge, info_only=info_only)
+ except KeyboardInterrupt:
+ if traceback:
+ raise
+ else:
+ sys.exit(1)
+
+def url_to_module(url):
+ from .extractors import netease, w56, acfun, baidu, baomihua, bilibili, blip, catfun, cntv, cbs, coursera, dailymotion, dongting, douban, douyutv, ehow, facebook, freesound, google, sina, ifeng, alive, instagram, iqiyi, joy, jpopsuki, khan, ku6, kugou, kuwo, letv, magisto, miomio, mixcloud, mtv81, nicovideo, pptv, qq, sohu, songtaste, soundcloud, ted, theplatform, tudou, tucao, tumblr, vid48, videobam, vimeo, vine, vk, xiami, yinyuetai, youku, youtube
+
+ video_host = r1(r'https?://([^/]+)/', url)
+ video_url = r1(r'https?://[^/]+(.*)', url)
+ assert video_host and video_url, 'invalid url: ' + url
+
+ if video_host.endswith('.com.cn'):
+ video_host = video_host[:-3]
+ domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
+ assert domain, 'unsupported url: ' + url
+
+ k = r1(r'([^.]+)', domain)
+ downloads = {
+ '163': netease,
+ '56': w56,
+ 'acfun': acfun,
+ 'baidu': baidu,
+ 'baomihua': baomihua,
+ 'bilibili': bilibili,
+ 'blip': blip,
+ 'catfun': catfun,
+ 'cntv': cntv,
+ 'cbs': cbs,
+ 'coursera': coursera,
+ 'dailymotion': dailymotion,
+ 'dongting': dongting,
+ 'douban': douban,
+ 'douyutv': douyutv,
+ 'ehow': ehow,
+ 'facebook': facebook,
+ 'freesound': freesound,
+ 'google': google,
+ 'iask': sina,
+ 'ifeng': ifeng,
+ 'in': alive,
+ 'instagram': instagram,
+ 'iqiyi': iqiyi,
+ 'joy': joy,
+ 'jpopsuki': jpopsuki,
+ 'kankanews': bilibili,
+ 'khanacademy': khan,
+ 'ku6': ku6,
+ 'kugou': kugou,
+ 'kuwo': kuwo,
+ 'letv': letv,
+ 'magisto': magisto,
+ 'miomio': miomio,
+ 'mixcloud': mixcloud,
+ 'mtv81': mtv81,
+ 'nicovideo': nicovideo,
+ 'pptv': pptv,
+ 'qq': qq,
+ 'sina': sina,
+ 'smgbb': bilibili,
+ 'sohu': sohu,
+ 'songtaste': songtaste,
+ 'soundcloud': soundcloud,
+ 'ted': ted,
+ 'theplatform': theplatform,
+ "tucao":tucao,
+ 'tudou': tudou,
+ 'tumblr': tumblr,
+ 'vid48': vid48,
+ 'videobam': videobam,
+ 'vimeo': vimeo,
+ 'vine': vine,
+ 'vk': vk,
+ 'xiami': xiami,
+ 'yinyuetai': yinyuetai,
+ 'youku': youku,
+ 'youtu': youtube,
+ 'youtube': youtube,
+ }
+ if k in downloads:
+ return downloads[k], url
+ else:
+ import http.client
+ conn = http.client.HTTPConnection(video_host)
+ conn.request("HEAD", video_url)
+ res = conn.getresponse()
+ location = res.getheader('location')
+ if location is None:
+ raise NotImplementedError(url)
+ else:
+ return url_to_module(location)
+
+def any_download(url, **kwargs):
+ m, url = url_to_module(url)
+ m.download(url, **kwargs)
+
+def any_download_playlist(url, **kwargs):
+ m, url = url_to_module(url)
+ m.download_playlist(url, **kwargs)
+
+def main():
+ script_main('you-get', any_download, any_download_playlist)
diff --git a/src_bak/you_get/extractor.py b/src_bak/you_get/extractor.py
new file mode 100644
index 00000000..14fc5b7b
--- /dev/null
+++ b/src_bak/you_get/extractor.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+
+from .common import match1, download_urls, parse_host, set_proxy, unset_proxy
+from .util import log
+
+class Extractor():
+ def __init__(self, *args):
+ self.url = None
+ self.title = None
+ self.vid = None
+ self.streams = {}
+ self.streams_sorted = []
+
+ if args:
+ self.url = args[0]
+
+class VideoExtractor():
+ def __init__(self, *args):
+ self.url = None
+ self.title = None
+ self.vid = None
+ self.streams = {}
+ self.streams_sorted = []
+ self.audiolang = None
+ self.password_protected = False
+
+ if args:
+ self.url = args[0]
+
+ def download_by_url(self, url, **kwargs):
+ self.url = url
+
+ if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+ set_proxy(parse_host(kwargs['extractor_proxy']))
+ self.prepare(**kwargs)
+ if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+ unset_proxy()
+
+ try:
+ self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+ except:
+ self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
+
+ self.extract(**kwargs)
+
+ self.download(**kwargs)
+
+ def download_by_vid(self, vid, **kwargs):
+ self.vid = vid
+
+ if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+ set_proxy(parse_host(kwargs['extractor_proxy']))
+ self.prepare(**kwargs)
+ if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+ unset_proxy()
+
+ try:
+ self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+ except:
+ self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
+
+ self.extract(**kwargs)
+
+ self.download(**kwargs)
+
+ def prepare(self, **kwargs):
+ pass
+ #raise NotImplementedError()
+
+ def extract(self, **kwargs):
+ pass
+ #raise NotImplementedError()
+
+ def p_stream(self, stream_id):
+ stream = self.streams[stream_id]
+ if 'itag' in stream:
+ print(" - itag: %s" % log.sprint(stream_id, log.NEGATIVE))
+ else:
+ print(" - format: %s" % log.sprint(stream_id, log.NEGATIVE))
+
+ if 'container' in stream:
+ print(" container: %s" % stream['container'])
+
+ if 'video_profile' in stream:
+ print(" video-profile: %s" % stream['video_profile'])
+
+ if 'quality' in stream:
+ print(" quality: %s" % stream['quality'])
+
+ if 'size' in stream:
+ print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
+
+ if 'itag' in stream:
+ print(" # download-with: %s" % log.sprint("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE))
+ else:
+ print(" # download-with: %s" % log.sprint("you-get --format=%s [URL]" % stream_id, log.UNDERLINE))
+
+ print()
+
+ def p_i(self, stream_id):
+ stream = self.streams[stream_id]
+ print(" - title: %s" % self.title)
+ print(" size: %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
+ print(" url: %s" % self.url)
+ print()
+
+ def p(self, stream_id=None):
+ print("site: %s" % self.__class__.name)
+ print("title: %s" % self.title)
+ if stream_id:
+ # Print the stream
+ print("stream:")
+ self.p_stream(stream_id)
+
+ elif stream_id is None:
+ # Print stream with best quality
+ print("stream: # Best quality")
+ stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+ self.p_stream(stream_id)
+
+ elif stream_id == []:
+ # Print all available streams
+ print("streams: # Available quality and codecs")
+ for stream in self.streams_sorted:
+ self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
+
+ if self.audiolang:
+ print("audio-languages:")
+ for i in self.audiolang:
+ print(" - lang: {}".format(i['lang']))
+ print(" download-url: {}\n".format(i['url']))
+
+ def p_playlist(self, stream_id=None):
+ print("site: %s" % self.__class__.name)
+ print("playlist: %s" % self.title)
+ print("videos:")
+
+ def download(self, **kwargs):
+ if 'info_only' in kwargs and kwargs['info_only']:
+ if 'stream_id' in kwargs and kwargs['stream_id']:
+ # Display the stream
+ stream_id = kwargs['stream_id']
+ if 'index' not in kwargs:
+ self.p(stream_id)
+ else:
+ self.p_i(stream_id)
+ else:
+ # Display all available streams
+ if 'index' not in kwargs:
+ self.p([])
+ else:
+ stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+ self.p_i(stream_id)
+
+ else:
+ if 'stream_id' in kwargs and kwargs['stream_id']:
+ # Download the stream
+ stream_id = kwargs['stream_id']
+ else:
+ # Download stream with the best quality
+ stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+
+ if 'index' not in kwargs:
+ self.p(stream_id)
+ else:
+ self.p_i(stream_id)
+
+ urls = self.streams[stream_id]['src']
+ if not urls:
+ log.wtf('[Failed] Cannot extract video source.')
+ # For legacy main()
+ download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'], output_dir=kwargs['output_dir'], merge=kwargs['merge'])
+ # For main_dev()
+ #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
+
+ self.__init__()
diff --git a/src_bak/you_get/extractors/__init__.py b/src_bak/you_get/extractors/__init__.py
new file mode 100644
index 00000000..da19036b
--- /dev/null
+++ b/src_bak/you_get/extractors/__init__.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+from .acfun import *
+from .alive import *
+from .baidu import *
+from .bilibili import *
+from .blip import *
+from .catfun import *
+from .cbs import *
+from .cntv import *
+from .coursera import *
+from .dailymotion import *
+from .douban import *
+from .douyutv import *
+from .ehow import *
+from .facebook import *
+from .freesound import *
+from .google import *
+from .ifeng import *
+from .instagram import *
+from .iqiyi import *
+from .joy import *
+from .jpopsuki import *
+from .ku6 import *
+from .kugou import *
+from .kuwo import *
+from .letv import *
+from .magisto import *
+from .miomio import *
+from .mixcloud import *
+from .mtv81 import *
+from .netease import *
+from .nicovideo import *
+from .pptv import *
+from .qq import *
+from .sina import *
+from .sohu import *
+from .songtaste import *
+from .soundcloud import *
+from .theplatform import *
+from .tucao import *
+from .tudou import *
+from .tumblr import *
+from .vid48 import *
+from .videobam import *
+from .vimeo import *
+from .vine import *
+from .vk import *
+from .w56 import *
+from .xiami import *
+from .yinyuetai import *
+from .youku import *
+from .youtube import *
+from .ted import *
+from .khan import *
diff --git a/src_bak/you_get/extractors/acfun.py b/src_bak/you_get/extractors/acfun.py
new file mode 100644
index 00000000..e00c1c52
--- /dev/null
+++ b/src_bak/you_get/extractors/acfun.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+__all__ = ['acfun_download']
+
+from ..common import *
+
+from .letv import letvcloud_download_by_vu
+from .qq import qq_download_by_id
+from .sina import sina_download_by_vid
+from .tudou import tudou_download_by_iid
+from .youku import youku_download_by_vid
+
+import json, re
+
+def get_srt_json(id):
+ # url = 'http://comment.acfun.tv/%s.json' % id
+ url = 'http://static.comment.acfun.mm111.net/%s' %id
+ return get_html(url)
+
+def get_srt_lock_json(id):
+ url = 'http://comment.acfun.tv/%s_lock.json' % id
+ return get_html(url)
+
+def acfun_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
+ info = json.loads(get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
+ sourceType = info['sourceType']
+ sourceId = info['sourceId']
+ # danmakuId = info['danmakuId']
+ if sourceType == 'sina':
+ sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
+ elif sourceType == 'youku':
+ youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+ elif sourceType == 'tudou':
+ tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
+ elif sourceType == 'qq':
+ qq_download_by_id(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
+ elif sourceType == 'letv':
+ letvcloud_download_by_vu(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
+ else:
+ raise NotImplementedError(sourceType)
+
+ if not info_only:
+ title = get_filename(title)
+ try:
+ print('Downloading %s ...\n' % (title + '.cmt.json'))
+ cmt = get_srt_json(vid)
+ with open(os.path.join(output_dir, title + '.cmt.json'), 'w') as x:
+ x.write(cmt)
+ # print('Downloading %s ...\n' % (title + '.cmt_lock.json'))
+ # cmt = get_srt_lock_json(danmakuId)
+ # with open(os.path.join(output_dir, title + '.cmt_lock.json'), 'w') as x:
+ # x.write(cmt)
+ except:
+ pass
+
+def acfun_download(url, output_dir = '.', merge = True, info_only = False):
+ assert re.match(r'http://[^\.]+.acfun.[^\.]+/v/ac(\d+)', url)
+ html = get_html(url)
+
+ title = r1(r'<h1 id="txt-title-view">([^<>]+)<', html)
+ title = unescape_html(title)
+ title = escape_file_path(title)
+ assert title
+
+ videos = re.findall("data-vid=\"(\d+)\".*href=\"[^\"]+\".*title=\"([^\"]+)\"", html)
+ if videos is not None:
+ for video in videos:
+ p_vid = video[0]
+ p_title = title + " - " + video[1]
+ acfun_download_by_vid(p_vid, p_title, output_dir=output_dir, merge=merge, info_only=info_only)
+ else:
+ # Useless - to be removed?
+ id = r1(r"src=\"/newflvplayer/player.*id=(\d+)", html)
+ sina_download_by_vid(id, title, output_dir=output_dir, merge=merge, info_only=info_only)
+
+site_info = "AcFun.tv"
+download = acfun_download
+download_playlist = playlist_not_supported('acfun')
diff --git a/src_bak/you_get/extractors/alive.py b/src_bak/you_get/extractors/alive.py
new file mode 100644
index 00000000..33764c72
--- /dev/null
+++ b/src_bak/you_get/extractors/alive.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+__all__ = ['alive_download']
+
+from ..common import *
+
+def alive_download(url, output_dir = '.', merge = True, info_only = False):
+ html = get_html(url)
+
+ title = r1(r'<meta property="og:title" content="([^"]+)"', html)
+
+ url = r1(r'file: "(http://alive[^"]+)"', html)
+ type, ext, size = url_info(url)
+
+ print_info(site_info, title, type, size)
+ if not info_only:
+ download_urls([url], title, ext, size, output_dir, merge = merge)
+
+site_info = "Alive.in.th"
+download = alive_download
+download_playlist = playlist_not_supported('alive')
diff --git a/src_bak/you_get/extractors/baidu.py b/src_bak/you_get/extractors/baidu.py
new file mode 100755
index 00000000..fdf58f28
--- /dev/null
+++ b/src_bak/you_get/extractors/baidu.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+__all__ = ['baidu_download']
+
+from ..common import *
+
+from urllib import parse
+
+def baidu_get_song_data(sid):
+ data = json.loads(get_html('http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker = True))['data']
+
+ if data['xcode'] != '':
+ # inside china mainland
+ return data['songList'][0]
+ else:
+ # outside china mainland
+ return None
+
+def baidu_get_song_url(data):
+ return data['songLink']
+
+def baidu_get_song_artist(data):
+ return data['artistName']
+
+def baidu_get_song_album(data):
+ return data['albumName']
+
+def baidu_get_song_title(data):
+ return data['songName']
+
+def baidu_get_song_lyric(data):
+ lrc = data['lrcLink']
+ return None if lrc is '' else "http://music.baidu.com%s" % lrc
+
+def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
+ data = baidu_get_song_data(sid)
+ if data is not None:
+ url = baidu_get_song_url(data)
+ title = baidu_get_song_title(data)
+ artist = baidu_get_song_artist(data)
+ album = baidu_get_song_album(data)
+ lrc = baidu_get_song_lyric(data)
+ file_name = "%s - %s - %s" % (title, album, artist)
+ else:
+ html = get_html("http://music.baidu.com/song/%s" % sid)
+ url = r1(r'data_url="([^"]+)"', html)
+ title = r1(r'data_name="([^"]+)"', html)
+ file_name = title
+
+ type, ext, size = url_info(url, faker=True)
+ print_info(site_info, title, type, size)
+ if not info_only:
+ download_urls([url], file_name, ext, size, output_dir, merge=merge, faker=True)
+
+ try:
+ type, ext, size = url_info(lrc, faker=True)
+ print_info(site_info, title, type, size)
+ if not info_only:
+ download_urls([lrc], file_name, ext, size, output_dir, faker=True)
+ except:
+ pass
+
+def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False):
+ html = get_html('http://music.baidu.com/album/%s' % aid, faker = True)
+ album_name = r1(r'<h2 class="album-name">(.+?)<\/h2>', html)
+ artist = r1(r'<span class="author_list" title="(.+?)">', html)
+ output_dir = '%s/%s - %s' % (output_dir, artist, album_name)
+ ids = json.loads(r1(r'<span class="album-add" data-adddata=\'(.+?)\'>', html).replace('"', '').replace(';', '"'))['ids']
+ track_nr = 1
+ for id in ids:
+ song_data = baidu_get_song_data(id)
+ song_url = baidu_get_song_url(song_data)
+ song_title = baidu_get_song_title(song_data)
+ song_lrc = baidu_get_song_lyric(song_data)
+ file_name = '%02d.%s' % (track_nr, song_title)
+
+ type, ext, size = url_info(song_url, faker = True)
+ print_info(site_info, song_title, type, size)
+ if not info_only:
+ download_urls([song_url], file_name, ext, size, output_dir, merge = merge, faker = True)
+
+ if song_lrc:
+ type, ext, size = url_info(song_lrc, faker = True)
+ print_info(site_info, song_title, type, size)
+ if not info_only:
+ download_urls([song_lrc], file_name, ext, size, output_dir, faker = True)
+
+ track_nr += 1
+
+def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False):
+ if re.match(r'http://pan.baidu.com', url):
+ html = get_html(url)
+
+ title = r1(r'server_filename="([^"]+)"', html)
+ if len(title.split('.')) > 1:
+ title = ".".join(title.split('.')[:-1])
+
+ real_url = r1(r'\\"dlink\\":\\"([^"]*)\\"', html).replace('\\\\/', '/')
+ type, ext, size = url_info(real_url, faker = True)
+
+ print_info(site_info, title, ext, size)
+ if not info_only:
+ download_urls([real_url], title, ext, size, output_dir, merge = merge)
+
+ elif re.match(r'http://music.baidu.com/album/\d+', url):
+ id = r1(r'http://music.baidu.com/album/(\d+)', url)
+ baidu_download_album(id, output_dir, merge, info_only)
+
+ elif re.match('http://music.baidu.com/song/\d+', url):
+ id = r1(r'http://music.baidu.com/song/(\d+)', url)
+ baidu_download_song(id, output_dir, merge, info_only)
+
+site_info = "Baidu.com"
+download = baidu_download
+download_playlist = playlist_not_supported("baidu")
diff --git a/src_bak/you_get/extractors/baomihua.py b/src_bak/you_get/extractors/baomihua.py
new file mode 100755
index 00000000..535b746b
--- /dev/null
+++ b/src_bak/you_get/extractors/baomihua.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+__all__ = ['baomihua_download', 'baomihua_download_by_id']
+
+from ..common import *
+
+import urllib
+
+def baomihua_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
+ html = get_html('http://play.baomihua.com/getvideourl.aspx?flvid=%s' % id)
+ host = r1(r'host=([^&]*)', html)
+ assert host
+ type = r1(r'videofiletype=([^&]*)', html)
+ assert type
+ vid = r1(r'&stream_name=([0-9\/]+)&', html)
+ assert vid
+ url = "http://%s/pomoho_video/%s.%s" % (host, vid, type)
+ _, ext, size = url_info(url)
+ print_info(site_info, title, type, size)
+ if not info_only:
+ download_urls([url], title, ext, size, output_dir, merge = merge)
+
+def baomihua_download(url, output_dir = '.', merge = True, info_only = False):
+ html = get_html(url)
+ title = r1(r'<title>(.*)', html)
+ assert title
+ id = r1(r'flvid=(\d+)', html)
+ assert id
+ baomihua_download_by_id(id, title, output_dir = output_dir, merge = merge, info_only = info_only)
+
+site_info = "baomihua.com"
+download = baomihua_download
+download_playlist = playlist_not_supported('baomihua')
diff --git a/src_bak/you_get/extractors/bilibili.py b/src_bak/you_get/extractors/bilibili.py
new file mode 100644
index 00000000..1869f955
--- /dev/null
+++ b/src_bak/you_get/extractors/bilibili.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+
+__all__ = ['bilibili_download']
+
+from ..common import *
+
+from .sina import sina_download_by_vid
+from .tudou import tudou_download_by_id
+from .youku import youku_download_by_vid
+
+import hashlib
+import re
+
+# API key provided by cnbeining
+appkey='85eb6835b0a1034e';
+secretkey = '2ad42749773c441109bdc0191257a664'
+client = {
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Charset': 'UTF-8,*;q=0.5',
+ 'Accept-Encoding': 'gzip,deflate,sdch',
+ 'Accept-Language': 'en-US,en;q=0.8',
+ 'User-Agent': 'Biligrab /0.8 (cnbeining@gmail.com)'
+}
+
+def get_srt_xml(id):
+ url = 'http://comment.bilibili.com/%s.xml' % id
+ return get_html(url)
+
+def parse_srt_p(p):
+ fields = p.split(',')
+ assert len(fields) == 8, fields
+ time, mode, font_size, font_color, pub_time, pool, user_id, history = fields
+ time = float(time)
+
+ mode = int(mode)
+ assert 1 <= mode <= 8
+ # mode 1~3: scrolling
+ # mode 4: bottom
+ # mode 5: top
+ # mode 6: reverse?
+ # mode 7: position
+ # mode 8: advanced
+
+ pool = int(pool)
+ assert 0 <= pool <= 2
+ # pool 0: normal
+ # pool 1: srt
+ # pool 2: special?
+
+ font_size = int(font_size)
+
+ font_color = '#%06x' % int(font_color)
+
+ return pool, mode, font_size, font_color
+
+def parse_srt_xml(xml):
+ d = re.findall(r'
(.*)', xml)
+ for x, y in d:
+ p = parse_srt_p(x)
+ raise NotImplementedError()
+
+def parse_cid_playurl(xml):
+ from xml.dom.minidom import parseString
+ try:
+ doc = parseString(xml.encode('utf-8'))
+ urls = [durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl')]
+ return urls
+ except:
+ return []
+
+def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
+ urls = []
+ for cid in cids:
+ sign_this = hashlib.md5(bytes('appkey=' + appkey + '&cid=' + cid + secretkey, 'utf-8')).hexdigest()
+ url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid + '&sign=' + sign_this
+ urls += [i
+ if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+ else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+ for i in parse_cid_playurl(get_content(url, headers=client))]
+
+ if re.search(r'\.(flv|hlv)\b', urls[0]):
+ type = 'flv'
+ elif re.search(r'/flv/', urls[0]):
+ type = 'flv'
+ elif re.search(r'/mp4/', urls[0]):
+ type = 'mp4'
+ else:
+ type = 'flv'
+
+ size = 0
+ for url in urls:
+ _, _, temp = url_info(url)
+ size += temp
+
+ print_info(site_info, title, type, size)
+ if not info_only:
+ download_urls(urls, title, type, total_size=None, output_dir=output_dir, merge=merge)
+
+def bilibili_download_by_cid(id, title, output_dir='.', merge=True, info_only=False):
+ sign_this = hashlib.md5(bytes('appkey=' + appkey + '&cid=' + id + secretkey, 'utf-8')).hexdigest()
+ url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + id + '&sign=' + sign_this
+ urls = [i
+ if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+ else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+ for i in parse_cid_playurl(get_content(url, headers=client))]
+
+ if re.search(r'\.(flv|hlv)\b', urls[0]):
+ type = 'flv'
+ elif re.search(r'/flv/', urls[0]):
+ type = 'flv'
+ elif re.search(r'/mp4/', urls[0]):
+ type = 'mp4'
+ else:
+ type = 'flv'
+
+ size = 0
+ for url in urls:
+ _, _, temp = url_info(url)
+ size += temp or 0
+
+ print_info(site_info, title, type, size)
+ if not info_only:
+ download_urls(urls, title, type, total_size=None, output_dir=output_dir, merge=merge)
+
+def bilibili_download(url, output_dir='.', merge=True, info_only=False):
+ html = get_html(url)
+
+ title = r1_of([r'
',r'
]*>([^<>]+)'], html)
+ title = unescape_html(title)
+ title = escape_file_path(title)
+
+ flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+ assert flashvars
+ flashvars = flashvars.replace(': ','=')
+ t, id = flashvars.split('=', 1)
+ id = id.split('&')[0]
+ if t == 'cid':
+ # Multi-P
+ cids = [id]
+ p = re.findall('