From 878d1d56c40ddb9783430f0677afd52db46740be Mon Sep 17 00:00:00 2001
From: Timothy Cyrus <tcyrus@users.noreply.github.com>
Date: Fri, 8 Apr 2016 10:31:27 -0400
Subject: [PATCH 001/149] Update README.md

Changed Badges from PNG to SVG
Also Fixed Badge Links
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index abdf39e5..ddfac26d 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # You-Get
 
-[![PyPI version](https://badge.fury.io/py/you-get.png)](http://badge.fury.io/py/you-get)
-[![Build Status](https://api.travis-ci.org/soimort/you-get.png)](https://travis-ci.org/soimort/you-get)
+[![PyPI version](https://img.shields.io/pypi/v/you-get.svg)](https://pypi.python.org/pypi/you-get/)
+[![Build Status](https://travis-ci.org/soimort/you-get.svg)](https://travis-ci.org/soimort/you-get)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 [You-Get](https://you-get.org/) is a tiny command-line utility to download media contents (videos, audios, images) from the Web, in case there is no other handy way to do it.

From 244f77e6b72ddf68afcdcafdec56efeca87521a5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 13 Apr 2016 17:33:34 +0200
Subject: [PATCH 002/149] Antigen: call python3 explicitly

---
 you-get.plugin.zsh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/you-get.plugin.zsh b/you-get.plugin.zsh
index 70f2f484..6282eda5 100644
--- a/you-get.plugin.zsh
+++ b/you-get.plugin.zsh
@@ -1,3 +1,3 @@
 #!/usr/bin/env zsh
-alias you-get="noglob $(dirname $0)/you-get"
-alias you-vlc="noglob $(dirname $0)/you-get --player vlc"
+alias you-get="noglob python3 $(dirname $0)/you-get"
+alias you-vlc="noglob python3 $(dirname $0)/you-get --player vlc"

From 0bd78f5b186957b16ace8659d07da6f873633606 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Fri, 15 Apr 2016 11:25:27 +0800
Subject: [PATCH 003/149] fix v.qq.com 403 code

---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index a0d80664..d6684a3e 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -5,7 +5,7 @@ __all__ = ['qq_download']
 from ..common import *
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    api = "http://h5vv.video.qq.com/getinfo?otype=json&vid=%s" % vid
+    api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
     content = get_html(api)
     output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
     url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']

From 0424cd9f8c54eb5eb036cdff4b07a7bdc64c6b21 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Tue, 19 Apr 2016 17:38:27 +0800
Subject: [PATCH 004/149] support for redirect URLs

---
 src/you_get/extractors/qq.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index d6684a3e..47ca663e 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -18,7 +18,13 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    if 'iframe/player.html' in url:
+    if 'v.qq.com/page' in url:
+        # for URLs like this:
+        # http://v.qq.com/page/k/9/7/k0194pwgw97.html
+        # it will redirect.
+        vid = match1(url, r'\b(\w+).html')
+        title = vid
+    elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is
         title = vid

From 17393a5fc7bbe36ee16085a86fa653b79d46e4db Mon Sep 17 00:00:00 2001
From: yuderbin <591558148@qq.com>
Date: Tue, 19 Apr 2016 18:33:38 +0800
Subject: [PATCH 005/149] add support to kuaibao.qq.com

---
 src/you_get/extractors/qq.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 47ca663e..00a99cd9 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -24,6 +24,10 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         # it will redirect.
         vid = match1(url, r'\b(\w+).html')
         title = vid
+    elif 'kuaibao.qq.com' in url:
+        content = get_html(url)
+        vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
+        title = match1(content, r'title">([^"]+)</p>').strip()
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is

From cc49724b76674c51ebd8b43d3bd2e9c254062f2a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 24 Apr 2016 01:43:37 +0200
Subject: [PATCH 006/149] [youku] fix typos in c054da95

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index b4fc38a7..3135e822 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -171,13 +171,13 @@ class Youku(VideoExtractor):
                     self.password_protected = True
                     self.password = input(log.sprint('Password: ', log.YELLOW))
                     api_url += '&pwd={}'.format(self.password)
-                    api_url12 += '&pwd={}'.format(self.password)
+                    api12_url += '&pwd={}'.format(self.password)
                     meta = json.loads(get_content(
                         api_url,
                         headers={'Referer': 'http://static.youku.com/'}
                     ))
                     meta12 = json.loads(get_content(
-                        api_url12,
+                        api12_url,
                         headers={'Referer': 'http://static.youku.com/'}
                     ))
                     data = meta['data']

From b919fdd8013088daa4f28bdd8e7ed1a4ff8577d5 Mon Sep 17 00:00:00 2001
From: Danieliu <liudanking@gmail.com>
Date: Wed, 27 Apr 2016 17:13:12 +0800
Subject: [PATCH 007/149] Update you-get

fix python version
---
 you-get | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/you-get b/you-get
index fbbdb9f2..85f3f754 100755
--- a/you-get
+++ b/you-get
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os, sys
 
 _srcdir = 'src/'

From 1a018c5261961f86dacb5c0f17648b8daad5b987 Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Wed, 27 Apr 2016 17:08:46 -0400
Subject: [PATCH 008/149] [Dilidili] Fix codec, fix #1085

---
 src/you_get/extractors/dilidili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 615f9861..6bc6ae28 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -40,11 +40,11 @@ def dilidili_download(url, output_dir = '.', merge = False, info_only = False, *
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         
         # player loaded via internal iframe
-        frame_url = re.search(r'<iframe (.+)src="(.+)\" f(.+)</iframe>', html).group(2)
+        frame_url = re.search(r'<iframe src=\"(.+?)\"', html).group(1)
         #print(frame_url)
         
         #https://player.005.tv:60000/?vid=a8760f03fd:a04808d307&v=yun&sign=a68f8110cacd892bc5b094c8e5348432
-        html = get_content(frame_url, headers=headers)
+        html = get_content(frame_url, headers=headers, decoded=False).decode('utf-8')
         
         match = re.search(r'(.+?)var video =(.+?);', html)
         vid = match1(html, r'var vid="(.+)"')

From 58efc40ccb95149027a24098bba8e2db95bf0d9a Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Thu, 28 Apr 2016 01:27:44 -0400
Subject: [PATCH 009/149] [Youku]Add COOP platform

---
 src/you_get/extractors/youku.py | 63 ++++++++++++++++++++++++++++-----
 1 file changed, 54 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 3135e822..fefaf5ee 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -28,7 +28,11 @@ class Youku(VideoExtractor):
     f_code_1 = 'becaf9be'
     f_code_2 = 'bf7e5f01'
 
+    ctype = 12  #differ from 86
+
     def trans_e(a, c):
+        """str, str->str
+        This is an RC4 encryption."""
         f = h = 0
         b = list(range(256))
         result = ''
@@ -49,14 +53,14 @@ class Youku(VideoExtractor):
 
         return result
 
-    def generate_ep(no, streamfileids, sid, token):
+    def generate_ep(self, no, streamfileids, sid, token):
         number = hex(int(str(no), 10))[2:].upper()
         if len(number) == 1:
             number = '0' + number
         fileid = streamfileids[0:8] + number + streamfileids[10:]
         ep = parse.quote(base64.b64encode(
-            ''.join(Youku.trans_e(
-                Youku.f_code_2,
+            ''.join(self.__class__.trans_e(
+                self.f_code_2,  #use the 86 fcode if using 86
                 sid + '_' + fileid + '_' + token)).encode('latin1')),
             safe='~()*!.\''
         )
@@ -150,8 +154,17 @@ class Youku(VideoExtractor):
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
 
-        api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
-        api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
+        #HACK!
+        if 'api_url' in kwargs:
+            api_url = kwargs['api_url']  #85
+            api12_url = kwargs['api12_url']  #86
+            self.ctype = kwargs['ctype']
+            self.title = kwargs['title']
+            
+        else:
+            api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
+            api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
+
         try:
             meta = json.loads(get_content(
                 api_url,
@@ -187,7 +200,8 @@ class Youku(VideoExtractor):
             else:
                 log.wtf('[Failed] Video not found.')
 
-        self.title = data['video']['title']
+        if not self.title:  #86
+            self.title = data['video']['title']
         self.ep = data12['security']['encrypt_string']
         self.ip = data12['security']['ip']
 
@@ -264,7 +278,7 @@ class Youku(VideoExtractor):
             stream_id = self.streams_sorted[0]['id']
 
         e_code = self.__class__.trans_e(
-            self.__class__.f_code_1,
+            self.f_code_1,
             base64.b64decode(bytes(self.ep, 'ascii'))
         )
         sid, token = e_code.split('_')
@@ -279,10 +293,10 @@ class Youku(VideoExtractor):
                     for no in range(0, len(segs)):
                         k = segs[no]['key']
                         if k == -1: break # we hit the paywall; stop here
-                        fileid, ep = self.__class__.generate_ep(no, streamfileid,
+                        fileid, ep = self.__class__.generate_ep(self, no, streamfileid,
                                                                 sid, token)
                         q = parse.urlencode(dict(
-                            ctype = 12,
+                            ctype = self.ctype,
                             ev    = 1,
                             K     = k,
                             ep    = parse.unquote(ep),
@@ -312,9 +326,40 @@ class Youku(VideoExtractor):
         if not kwargs['info_only']:
             self.streams[stream_id]['src'] = ksegs
 
+    def open_download_by_vid(self, client_id, vid, **kwargs):
+        """self, str, str, **kwargs->None
+        Override the original one with VideoExtractor.
+        Most of the credit are to @ERioK, who gave his POC."""
+        self.f_code_1 = '10ehfkbv'  #can be retrived by running r.translate with the keys and the list e
+        self.f_code_2 = 'msjv7h2b'
+        self.url = None
+        self.vid = vid
+        self.name = "优酷开放平台 (Youku COOP)"
+
+        #A little bit of work before self.prepare
+        sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
+        playsign = json.loads(get_content(sign_url))['playsign']
+    
+        api85_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=85&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
+        api86_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=86&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
+        
+        self.prepare(api_url = api85_url, api12_url = api86_url, ctype = 86, **kwargs)
+        if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+            unset_proxy()
+
+        try:
+            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+        except:
+            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
+
+        self.extract(**kwargs)
+
+        self.download(**kwargs)
+
 site = Youku()
 download = site.download_by_url
 download_playlist = site.download_playlist_by_url
 
 youku_download_by_vid = site.download_by_vid
+youku_open_download_by_vid = site.open_download_by_vid
 # Used by: acfun.py bilibili.py miomio.py tudou.py

From 3cc6e7bc661ef4e126a74d6948f154d7b500f4ba Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Thu, 28 Apr 2016 01:29:22 -0400
Subject: [PATCH 010/149] [Acfun]Fix Acfun Youku COOP, fix #1069

---
 src/you_get/extractors/acfun.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/acfun.py

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
old mode 100644
new mode 100755
index e78d3636..983606d4
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -8,7 +8,7 @@ from .le import letvcloud_download_by_vu
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_iid
-from .youku import youku_download_by_vid
+from .youku import youku_download_by_vid, youku_open_download_by_vid
 
 import json, re
 
@@ -32,14 +32,11 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     elif sourceType == 'letv':
         letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'zhuzhan':
-        a = 'http://api.aixifan.com/plays/%s/realSource' % vid
-        s = json.loads(get_content(a, headers={'deviceType': '1'}))
-        urls = s['data']['files'][-1]['url']
-        size = urls_size(urls)
-        print_info(site_info, title, 'mp4', size)
-        if not info_only:
-            download_urls(urls, title, 'mp4', size,
-                          output_dir=output_dir, merge=merge)
+        a = 'http://api.aixifan.com/plays/%s' % vid
+        s = json.loads(get_content(a, headers={'deviceType': '2'}))
+        if s['data']['source'] == "zhuzhan-youku":
+            sourceId = s['data']['sourceId']
+            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
     else:
         raise NotImplementedError(sourceType)
 

From f9c20792fc0ca2b29f0d9b18701abe5ab65b57ef Mon Sep 17 00:00:00 2001
From: David Zhuang <i+david.zhuang-mail.utoronto.ca@cnbeining.com>
Date: Thu, 28 Apr 2016 02:10:11 -0400
Subject: [PATCH 011/149] [Tudou]Fix acfun.tudou.com redirect, fix #695

---
 src/you_get/extractors/tudou.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index d6a7f815..6bbbc12b 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -4,6 +4,7 @@ __all__ = ['tudou_download', 'tudou_download_playlist', 'tudou_download_by_id',
 
 from ..common import *
 from xml.dom.minidom import parseString
+import you_get.extractors.acfun
 
 def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only = False):
     data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid))
@@ -29,6 +30,13 @@ def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only =
     tudou_download_by_iid(iid, title, output_dir = output_dir, merge = merge, info_only = info_only)
 
 def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    if 'acfun.tudou.com' in url:  #wrong way!
+        url = url.replace('acfun.tudou.com', 'www.acfun.tv')
+        you_get.extractors.acfun.acfun_download(url, output_dir, 
+                                               merge, 
+                                               info_only)
+        return  #throw you back
+    
     # Embedded player
     id = r1(r'http://www.tudou.com/v/([^/]+)/', url)
     if id:

From 19bd7a4a48074acfc389c0e653677e64e5e3d8b9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 29 Apr 2016 12:27:05 +0200
Subject: [PATCH 012/149] [youtube] fix #1092

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 2cfd1ec8..0ef390ed 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -56,7 +56,7 @@ class YouTube(VideoExtractor):
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
-        f1def = 'function %s%s' % (re.escape(f1), f1def)
+        f1def = 'function %s%s' % (f1, f1def)
         code = tr_js(f1def)
         f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
         for f2 in f2s:

From b19bc2d5db73166f9efc0181ba1b2ca737e6ac22 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 30 Apr 2016 03:13:38 +0200
Subject: [PATCH 013/149] version 0.4.390

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index d8db720e..0e7b6632 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.365'
+__version__ = '0.4.390'

From 7040db66181d2d6d9cc8f47610a40815decdeb12 Mon Sep 17 00:00:00 2001
From: ryneeverett <ryneeverett@gmail.com>
Date: Mon, 2 May 2016 15:20:57 +0000
Subject: [PATCH 014/149] bandcamp: Sometimes the first key is video_caption

This fixes the exception:

    File "you-get/lib/python3.4/site-packages/you_get/extractors/bandcamp.py", line 9, in bandcamp_download
      trackinfo = json.loads(r1(r'(\[{"video_poster_url".*}\]),', html))
    File "/usr/lib64/python3.4/json/__init__.py", line 312, in loads
      s.__class__.__name__))
---
 src/you_get/extractors/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bandcamp.py b/src/you_get/extractors/bandcamp.py
index de21a590..c24e4f3d 100644
--- a/src/you_get/extractors/bandcamp.py
+++ b/src/you_get/extractors/bandcamp.py
@@ -6,7 +6,7 @@ from ..common import *
 
 def bandcamp_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
-    trackinfo = json.loads(r1(r'(\[{"video_poster_url".*}\]),', html))
+    trackinfo = json.loads(r1(r'(\[{"(video_poster_url|video_caption)".*}\]),', html))
     for track in trackinfo:
         track_num = track['track_num']
         title = '%s. %s' % (track_num, track['title'])

From 08d65c237ca84394816ca594c58a8c84b09907df Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Tue, 3 May 2016 21:07:20 +0800
Subject: [PATCH 015/149] sohu: ignore invalid video format

v1, ignore invalid video format
v2, fix mistake in v1

fix: #1098

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/sohu.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index a3b4e699..9e068921 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -32,9 +32,14 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
             set_proxy(tuple(extractor_proxy.split(":")))
         info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
         for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
-            hqvid = info['data'][qtyp]
+            if 'data' in info:
+                hqvid = info['data'][qtyp]
+            else:
+                hqvid = info[qtyp]
             if hqvid != 0 and hqvid != vid :
                 info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % hqvid))
+                if not 'allot' in info:
+                    continue
                 break
         if extractor_proxy:
             unset_proxy()

From a41a17fdefcca622d23a18dcf70c9d5c19c7680e Mon Sep 17 00:00:00 2001
From: natsu1211 <longyutao1211@gmail.com>
Date: Thu, 5 May 2016 16:35:09 +0900
Subject: [PATCH 016/149] fix url and add support to 189 video source

---
 src/you_get/extractors/tucao.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/tucao.py b/src/you_get/extractors/tucao.py
index 28e296c7..66baa3bf 100644
--- a/src/you_get/extractors/tucao.py
+++ b/src/you_get/extractors/tucao.py
@@ -6,13 +6,14 @@ from ..common import *
 import random
 import time
 from xml.dom import minidom
-
+#possible raw list types
 #1. <li>type=tudou&vid=199687639</li>
 #2. <li>type=tudou&vid=199506910|</li>
 #3. <li>type=video&file=http://xiaoshen140731.qiniudn.com/lovestage04.flv|</li>
 #4 may ? <li>type=video&file=http://xiaoshen140731.qiniudn.com/lovestage04.flv|xx**type=&vid=?</li>
 #5. <li>type=tudou&vid=200003098|07**type=tudou&vid=200000350|08</li>
-
+#6. <li>vid=49454694&type=sina|</li>
+#7. <li>type=189&vid=513031813243909|</li>
 # re_pattern=re.compile(r"(type=(.+?)&(vid|file)=(.*?))[\|<]")
 
 def tucao_single_download(type_link, title, output_dir=".", merge=True, info_only=False):
@@ -22,8 +23,17 @@ def tucao_single_download(type_link, title, output_dir=".", merge=True, info_onl
         print_info(site_info, title, vtype, size)
         if not info_only:
             download_urls([url], title, ext, size, output_dir)
+    #fix for 189 video source, see raw list types 7
+    elif "189" in type_link:
+        vid = match1(type_link, r"vid=(\d+)")
+        assert vid, "vid not exsits"
+        url = "http://api.tucao.tv/api/down/{}".format(vid)
+        vtype, ext, size=url_info(url)
+        print_info(site_info, title, vtype, size)
+        if not info_only:
+            download_urls([url], title, ext, size, output_dir)
     else:
-        u="http://www.tucao.cc/api/playurl.php?{}&key=tucao{:07x}.cc&r={}".format(type_link,random.getrandbits(28),int(time.time()*1000))
+        u="http://www.tucao.tv/api/playurl.php?{}&key=tucao{:07x}.cc&r={}".format(type_link,random.getrandbits(28),int(time.time()*1000))
         xml=minidom.parseString(get_content(u))
         urls=[]
         size=0
@@ -33,12 +43,13 @@ def tucao_single_download(type_link, title, output_dir=".", merge=True, info_onl
             size+=_size
         print_info(site_info, title, vtype, size)
         if not info_only:
-            download_urls(urls, title, ext, size, output_dir) 
+            download_urls(urls, title, ext, size, output_dir)
 
 def tucao_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
     html=get_content(url)
     title=match1(html,r'<h1 class="show_title">(.*?)<\w')
-    raw_list=match1(html,r"<li>(type=.+?)</li>")
+    #fix for raw list that vid goes before type, see raw list types 6
+    raw_list=match1(html,r"<li>\s*(type=.+?|vid=.+?)</li>")
     raw_l=raw_list.split("**")
     if len(raw_l)==1:
         format_link=raw_l[0][:-1] if raw_l[0].endswith("|") else raw_l[0]
@@ -49,6 +60,6 @@ def tucao_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
             tucao_single_download(format_link,title+"-"+sub_title,output_dir,merge,info_only)
 
 
-site_info = "tucao.cc"
+site_info = "tucao.tv"
 download = tucao_download
 download_playlist = playlist_not_supported("tucao")

From 54cafb3d3cc387b2cf2eabe2bc88d76dba173b97 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Fri, 6 May 2016 08:43:12 +0800
Subject: [PATCH 017/149] fix download error like this url
 http://v.qq.com/page/m/a/h/m0195j57pah.html

---
 src/you_get/extractors/qq.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 00a99cd9..66487b8e 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -10,7 +10,12 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
     url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']
     fvkey = output_json['vl']['vi'][0]['fvkey']
-    url = '%s/%s.mp4?vkey=%s' % ( url, vid, fvkey )
+    mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
+    if mp4:
+        mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
+    else:
+        mp4 = output_json['vl']['vi'][0]['fn']
+    url = '%s/%s?vkey=%s' % ( url, mp4, fvkey )
     _, ext, size = url_info(url, faker=True)
 
     print_info(site_info, title, ext, size)

From b9428bd9c73aa61db4bfd4529cd38fa58b743704 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Fri, 6 May 2016 18:41:11 +0800
Subject: [PATCH 018/149] some url like this
 http://kuaibao.qq.com/s/SPO2016050503989500 got the title failed will raise
 exception

---
 src/you_get/extractors/qq.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 66487b8e..ebe1a9ad 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -32,7 +32,8 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     elif 'kuaibao.qq.com' in url:
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
-        title = match1(content, r'title">([^"]+)</p>').strip()
+        title = match1(content, r'title">([^"]+)</p>')
+        title = title.strip() if title else vid
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is

From 6c3828cc32625f3caab6e82bb291a40f2416ff7f Mon Sep 17 00:00:00 2001
From: David Zhuang <i+david.zhuang-mail.utoronto.ca@cnbeining.com>
Date: Fri, 6 May 2016 16:30:08 -0400
Subject: [PATCH 019/149] [MangGuoTV]Add support

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/mgtv.py     | 110 +++++++++++++++++++++++++++++
 3 files changed, 112 insertions(+)
 create mode 100644 src/you_get/extractors/mgtv.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index e20be32b..284b1660 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -47,6 +47,7 @@ SITES = {
     'lizhi'            : 'lizhi',
     'magisto'          : 'magisto',
     'metacafe'         : 'metacafe',
+    'mgtv'             : 'mgtv',
     'miomio'           : 'miomio',
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 5af9cdd3..6c3864fd 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -38,6 +38,7 @@ from .le import *
 from .lizhi import *
 from .magisto import *
 from .metacafe import *
+from .mgtv import *
 from .miaopai import *
 from .miomio import *
 from .mixcloud import *
diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
new file mode 100644
index 00000000..a3b7a06e
--- /dev/null
+++ b/src/you_get/extractors/mgtv.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+from json import loads
+from urllib.parse import urlsplit
+from os.path import dirname
+
+class MGTV(VideoExtractor):
+    name = "芒果 (MGTV)"
+
+    # Last updated: 2015-11-24
+    stream_types = [
+        {'id': 'hd', 'container': 'flv', 'video_profile': '超清'},
+        {'id': 'sd', 'container': 'flv', 'video_profile': '高清'},
+        {'id': 'ld', 'container': 'flv', 'video_profile': '标清'},
+    ]
+    
+    id_dic = {i['video_profile']:(i['id']) for i in stream_types}
+    
+    api_endpoint = 'http://v.api.mgtv.com/player/video?video_id={video_id}'
+
+    @staticmethod
+    def get_vid_from_url(url):
+        """Extracts video ID from URL.
+        """
+        return match1(url, 'http://www.mgtv.com/v/\d/\d+/\w+/(\d+).html')
+    
+    #----------------------------------------------------------------------
+    @staticmethod
+    def get_mgtv_real_url(url):
+        """str->list of str
+        Give you the real URLs."""
+        content = loads(get_content(url))
+        m3u_url = content['info']
+        split = urlsplit(m3u_url)
+        
+        base_url = "{scheme}://{netloc}{path}/".format(scheme = split[0],
+                                                      netloc = split[1],
+                                                      path = dirname(split[2]))
+
+        content = get_content(content['info'])  #get the REAL M3U url, maybe to be changed later?
+        segment_list = []
+        for i in content.split():
+            if not i.startswith('#'):  #not the best way, better we use the m3u8 package
+                segment_list.append(base_url + i)
+        return segment_list
+
+    def download_playlist_by_url(self, url, **kwargs):
+        pass
+
+    def prepare(self, **kwargs):
+        if self.url:
+            self.vid = self.get_vid_from_url(self.url)
+        content = get_content(self.api_endpoint.format(video_id = self.vid))
+        content = loads(content)
+        self.title = content['data']['info']['title']
+        
+        #stream_avalable = [i['name'] for i in content['data']['stream']]
+        stream_available = {}
+        for i in content['data']['stream']:
+            stream_available[i['name']] = i['url']
+
+        for s in self.stream_types:
+            if s['video_profile'] in stream_available.keys():
+                quality_id = self.id_dic[s['video_profile']]
+                url = stream_available[s['video_profile']]
+                segment_list_this = self.get_mgtv_real_url(url)
+                
+                container_this_stream = ''
+                size_this_stream = 0
+                stream_fileid_list = []
+                for i in segment_list_this:
+                    _, container_this_stream, size_this_seg = url_info(i)
+                    size_this_stream += size_this_seg
+                    stream_fileid_list.append(os.path.basename(i).split('.')[0])
+                    
+            #make pieces
+            pieces = []
+            for i in zip(stream_fileid_list, segment_list_this):
+                pieces.append({'fileid': i[0], 'segs': i[1],})
+
+                self.streams[quality_id] = {
+                        'container': 'flv',
+                        'video_profile': s['video_profile'],
+                        'size': size_this_stream,
+                        'pieces': pieces
+                    }
+                
+            if not kwargs['info_only']:
+                self.streams[quality_id]['src'] = segment_list_this
+
+    def extract(self, **kwargs):
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # Extract the stream
+            stream_id = kwargs['stream_id']
+
+            if stream_id not in self.streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # Extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+
+site = MGTV()
+download = site.download_by_url
+download_playlist = site.download_playlist_by_url
\ No newline at end of file

From 23b3a9040b4dad0e74841398aaedbc83ac2106d5 Mon Sep 17 00:00:00 2001
From: ksnzh <ksnzher@gmail.com>
Date: Tue, 10 May 2016 00:23:16 +0800
Subject: [PATCH 020/149] match 'http://www.dilidili.com/watch' and '/watch3'

---
 src/you_get/extractors/dilidili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 6bc6ae28..f3f8e197 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -35,7 +35,7 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch/\w+', url):
+    if re.match(r'http://www.dilidili.com/watch\w+', url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         

From 17533bf6933d3c33be2c238c987275dd1c6da0b4 Mon Sep 17 00:00:00 2001
From: ksnzh <ksnzher@gmail.com>
Date: Tue, 10 May 2016 22:18:34 +0800
Subject: [PATCH 021/149] fix the url 'watch/' and 'watch3/' error

---
 src/you_get/extractors/dilidili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index f3f8e197..8433294b 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -35,7 +35,7 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch\w+', url):
+    if re.match(r'http://www.dilidili.com/watch\S', url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         

From f5c22cdfef99878027f3f9e78ff9ae2701a21a3f Mon Sep 17 00:00:00 2001
From: ksnzh <ksnzher@gmail.com>
Date: Tue, 10 May 2016 23:04:55 +0800
Subject: [PATCH 022/149] fix the url 'watch/' and 'watch3/' error

---
 src/you_get/extractors/dilidili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 8433294b..082f84e1 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -35,7 +35,7 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch\S', url):
+    if re.match(r'http://www.dilidili.com/watch\S+', url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         

From e697287b95e1ddee50047c34acd2f8f9a347e1eb Mon Sep 17 00:00:00 2001
From: iliul <liul.stone@gmail.com>
Date: Wed, 11 May 2016 14:28:50 +0800
Subject: [PATCH 023/149] add support Panda.tv

---
 README.md                          |  1 +
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/panda.py    | 32 ++++++++++++++++++++++++++++++
 4 files changed, 35 insertions(+)
 create mode 100644 src/you_get/extractors/panda.py

diff --git a/README.md b/README.md
index ddfac26d..a4f4fcd9 100644
--- a/README.md
+++ b/README.md
@@ -344,6 +344,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Dilidili | <http://www.dilidili.com/>     |✓| | |
 | 豆瓣     | <http://www.douban.com/>       | | |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
+| Panda<br/>熊猫 | <http://www.panda.tv/>      |✓| | |
 | 凤凰视频 | <http://v.ifeng.com/>          |✓| | |
 | 风行网   | <http://www.fun.tv/>           |✓| | |
 | iQIYI<br/>爱奇艺 | <http://www.iqiyi.com/> |✓| | |
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 284b1660..ab32a62d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -54,6 +54,7 @@ SITES = {
     'musicplayon'      : 'musicplayon',
     '7gogo'            : 'nanagogo',
     'nicovideo'        : 'nicovideo',
+    'panda'            : 'panda',
     'pinterest'        : 'pinterest',
     'pixnet'           : 'pixnet',
     'pptv'             : 'pptv',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 6c3864fd..1bb7a7ab 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -47,6 +47,7 @@ from .musicplayon import *
 from .nanagogo import *
 from .netease import *
 from .nicovideo import *
+from .panda import *
 from .pinterest import *
 from .pixnet import *
 from .pptv import *
diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
new file mode 100644
index 00000000..74064f9f
--- /dev/null
+++ b/src/you_get/extractors/panda.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+__all__ = ['panda_download']
+
+from ..common import *
+import json
+import time
+
+def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    roomid = url[url.rfind('/')+1:]
+    json_request_url = 'http://www.panda.tv/api_room?roomid={}&pub_key=&_={}'.format(roomid, int(time.time()))
+    content = get_html(json_request_url)
+    errno = json.loads(content)['errno']
+    errmsg = json.loads(content)['errmsg']
+    if errno:
+        raise ValueError("Errno : {}, Errmsg : {}".format(errno, errmsg))
+
+    data = json.loads(content)['data']
+    title = data.get('roominfo')['name']
+    room_key = data.get('videoinfo')['room_key']
+    status = data.get('videoinfo')['status']
+    if status is not "2":
+        raise ValueError("The live stream is not online! (status:%s)" % status)
+    real_url = 'http://pl3.live.panda.tv/live_panda/{}.flv'.format(room_key)
+
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+
+site_info = "panda.tv"
+download = panda_download
+download_playlist = playlist_not_supported('panda')

From 03a4d3cc91c91dcc6908b070ae98ac68100c2669 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Thu, 12 May 2016 23:56:34 -0700
Subject: [PATCH 024/149] common.py: Print merged filename after successful
 merge

Instead of just "Done.".
---
 src/you_get/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 284b1660..f35cdf18 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -735,7 +735,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
             if has_ffmpeg_installed():
                 from .processor.ffmpeg import ffmpeg_concat_av
                 ret = ffmpeg_concat_av(parts, output_filepath, ext)
-                print('Done.')
+                print('Merged into %s' % output_filename)
                 if ret == 0:
                     for part in parts: os.remove(part)
 
@@ -748,7 +748,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 else:
                     from .processor.join_flv import concat_flv
                     concat_flv(parts, output_filepath)
-                print('Done.')
+                print('Merged into %s' % output_filename)
             except:
                 raise
             else:
@@ -764,7 +764,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 else:
                     from .processor.join_mp4 import concat_mp4
                     concat_mp4(parts, output_filepath)
-                print('Done.')
+                print('Merged into %s' % output_filename)
             except:
                 raise
             else:
@@ -780,7 +780,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 else:
                     from .processor.join_ts import concat_ts
                     concat_ts(parts, output_filepath)
-                print('Done.')
+                print('Merged into %s' % output_filename)
             except:
                 raise
             else:

From 4f91f908dbb24b19db65659afc6e37b8bcddcc10 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 14 May 2016 23:36:51 +0200
Subject: [PATCH 025/149] [yinyuetai] fix #1102

---
 src/you_get/extractors/yinyuetai.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/yinyuetai.py b/src/you_get/extractors/yinyuetai.py
index dc4a9364..6c39540f 100644
--- a/src/you_get/extractors/yinyuetai.py
+++ b/src/you_get/extractors/yinyuetai.py
@@ -17,7 +17,8 @@ def yinyuetai_download_by_id(vid, title=None, output_dir='.', merge=True, info_o
         download_urls([url], title, ext, size, output_dir, merge = merge)
 
 def yinyuetai_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    id = r1(r'http://\w+.yinyuetai.com/video/(\d+)', url)
+    id = r1(r'http://\w+.yinyuetai.com/video/(\d+)', url) or \
+         r1(r'http://\w+.yinyuetai.com/video/h5/(\d+)', url)
     if not id:
         yinyuetai_download_playlist(url, output_dir=output_dir, merge=merge, info_only=info_only)
         return

From f7c20b236c967e6241b48a430c14a6e97681cefd Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Tue, 17 May 2016 15:28:33 -0400
Subject: [PATCH 026/149] [MGTV]Fix some video cannot be downloaded in full
 length

---
 src/you_get/extractors/mgtv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index a3b7a06e..aeb42490 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -7,6 +7,7 @@ from ..extractor import VideoExtractor
 from json import loads
 from urllib.parse import urlsplit
 from os.path import dirname
+import re
 
 class MGTV(VideoExtractor):
     name = "芒果 (MGTV)"
@@ -67,6 +68,7 @@ class MGTV(VideoExtractor):
             if s['video_profile'] in stream_available.keys():
                 quality_id = self.id_dic[s['video_profile']]
                 url = stream_available[s['video_profile']]
+                url = re.sub( r'(\&arange\=\d+)', '', url)  #Un-Hum
                 segment_list_this = self.get_mgtv_real_url(url)
                 
                 container_this_stream = ''

From 677040a39ef448f8ccf3f4c8056642b6a9a9d585 Mon Sep 17 00:00:00 2001
From: NickeyKim <mmf760@gmail.com>
Date: Thu, 19 May 2016 14:48:45 +0900
Subject: [PATCH 027/149] add Naver support

add Naver support
---
 README.md                          |  1 +
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/naver.py    | 48 ++++++++++++++++++++++++++++++
 4 files changed, 51 insertions(+)
 create mode 100644 src/you_get/extractors/naver.py

diff --git a/README.md b/README.md
index a4f4fcd9..d0a91aa8 100644
--- a/README.md
+++ b/README.md
@@ -373,6 +373,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
 | 央视网   | <http://www.cntv.cn/>          |✓| | |
 | 花瓣     | <http://huaban.com/>           | |✓| |
+| Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6c65bd49..9668df58 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -52,6 +52,7 @@ SITES = {
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
     'musicplayon'      : 'musicplayon',
+    'naver'            : 'naver',
     '7gogo'            : 'nanagogo',
     'nicovideo'        : 'nicovideo',
     'panda'            : 'panda',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 1bb7a7ab..6fdaa340 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -45,6 +45,7 @@ from .mixcloud import *
 from .mtv81 import *
 from .musicplayon import *
 from .nanagogo import *
+from .naver import *
 from .netease import *
 from .nicovideo import *
 from .panda import *
diff --git a/src/you_get/extractors/naver.py b/src/you_get/extractors/naver.py
new file mode 100644
index 00000000..7f7d5548
--- /dev/null
+++ b/src/you_get/extractors/naver.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+__all__ = ['naver_download']
+import urllib.request, urllib.parse
+from ..common import *
+
+def naver_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+
+	assert re.search(r'http://tvcast.naver.com/v/', url), "URL is not supported"
+
+	html = get_html(url)
+	contentid = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',html)
+	videoid = contentid.group(1)
+	inkey = contentid.group(2)
+	assert videoid
+	assert inkey
+	info_key = urllib.parse.urlencode({'vid': videoid, 'inKey': inkey, })
+	down_key = urllib.parse.urlencode({'masterVid': videoid,'protocol': 'p2p','inKey': inkey, })
+	inf_xml = get_html('http://serviceapi.rmcnmv.naver.com/flash/videoInfo.nhn?%s' % info_key )
+
+	from xml.dom.minidom import parseString
+	doc_info = parseString(inf_xml)
+	Subject = doc_info.getElementsByTagName('Subject')[0].firstChild
+	title = Subject.data
+	assert title
+
+	xml = get_html('http://serviceapi.rmcnmv.naver.com/flash/playableEncodingOption.nhn?%s' % down_key )
+	doc = parseString(xml)
+
+	encodingoptions = doc.getElementsByTagName('EncodingOption')
+	old_height = doc.getElementsByTagName('height')[0]
+	real_url= ''
+	#to download the highest resolution one,
+	for node in encodingoptions:
+		new_height = node.getElementsByTagName('height')[0]
+		domain_node = node.getElementsByTagName('Domain')[0]
+		uri_node = node.getElementsByTagName('uri')[0]
+		if int(new_height.firstChild.data) > int (old_height.firstChild.data):
+			real_url= domain_node.firstChild.data+ '/' +uri_node.firstChild.data
+
+	type, ext, size = url_info(real_url)
+	print_info(site_info, title, type, size)
+	if not info_only:
+		download_urls([real_url], title, ext, size, output_dir, merge = merge)
+
+site_info = "tvcast.naver.com"
+download = naver_download
+download_playlist = playlist_not_supported('naver')

From e6af1c6265be5a45f978f3321db20cfe9cb30a05 Mon Sep 17 00:00:00 2001
From: jmargeta <jmargeta@gmail.com>
Date: Thu, 19 May 2016 22:57:48 +0200
Subject: [PATCH 028/149] Support for embedded Vimeo videos

Example URL:
- http://www.miracletutorials.com/howto-embed-vimeo/
---
 src/you_get/extractors/embed.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index fd463c92..a177e663 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -8,6 +8,7 @@ from .netease import netease_download
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_id
+from .vimeo import vimeo_download_by_id
 from .yinyuetai import yinyuetai_download_by_id
 from .youku import youku_download_by_vid
 
@@ -39,6 +40,9 @@ iqiyi_embed_patterns = [ 'player\.video\.qiyi\.com/([^/]+)/[^/]+/[^/]+/[^/]+\.sw
 
 netease_embed_patterns = [ '(http://\w+\.163\.com/movie/[^\'"]+)' ]
 
+vimeo_embed_patters = [ 'player\.vimeo\.com/video/(\d+)' ]
+
+
 def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwargs):
     content = get_content(url, headers=fake_headers)
     found = False
@@ -69,6 +73,11 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         found = True
         netease_download(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
+    urls = matchall(content, vimeo_embed_patters)
+    for url in urls:
+        found = True
+        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+
     if not found:
         raise NotImplementedError(url)
 

From 2cd4656b32c1005edebf3047ba7f1f7470abfbf0 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 20 May 2016 04:28:30 -0400
Subject: [PATCH 029/149] [QiE]Add Support

---
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/qie.py      | 78 ++++++++++++++++++++++++++++++
 src/you_get/extractors/qq.py       |  4 ++
 3 files changed, 83 insertions(+)
 create mode 100644 src/you_get/extractors/qie.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 1bb7a7ab..e2cf656c 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -52,6 +52,7 @@ from .pinterest import *
 from .pixnet import *
 from .pptv import *
 from .qianmo import *
+from .qie import *
 from .qq import *
 from .sina import *
 from .sohu import *
diff --git a/src/you_get/extractors/qie.py b/src/you_get/extractors/qie.py
new file mode 100644
index 00000000..2288106a
--- /dev/null
+++ b/src/you_get/extractors/qie.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+from json import loads
+
+class QiE(VideoExtractor):
+    name = "QiE （企鹅直播）"
+
+    # Last updated: 2015-11-24
+    stream_types = [
+        {'id': 'normal', 'container': 'flv', 'video_profile': '标清'},
+        {'id': 'middle', 'container': 'flv', 'video_profile': '550'},
+        {'id': 'middle2', 'container': 'flv', 'video_profile': '900'},
+    ]
+    
+    id_dic = {i['video_profile']:(i['id']) for i in stream_types}
+    
+    api_endpoint = 'http://www.qie.tv/api/v1/room/{room_id}'
+
+    @staticmethod
+    def get_vid_from_url(url):
+        """Extracts video ID from live.qq.com.
+        """
+        html = get_content(url)
+        return match1(html, r'room_id\":(\d+)')
+
+    def download_playlist_by_url(self, url, **kwargs):
+        pass
+
+    def prepare(self, **kwargs):
+        if self.url:
+            self.vid = self.get_vid_from_url(self.url)
+        
+        content = get_content(self.api_endpoint.format(room_id = self.vid))
+        content = loads(content)
+        self.title = content['data']['room_name']
+        rtmp_url =  content['data']['rtmp_url']
+        #stream_avalable = [i['name'] for i in content['data']['stream']]
+        stream_available = {}
+        stream_available['normal'] = rtmp_url + '/' + content['data']['rtmp_live']
+        if len(content['data']['rtmp_multi_bitrate']) > 0:
+            for k , v in content['data']['rtmp_multi_bitrate'].items():
+                stream_available[k] = rtmp_url + '/' + v
+        
+        for s in self.stream_types:
+            if s['id'] in stream_available.keys():
+                quality_id = s['id']
+                url = stream_available[quality_id]
+                self.streams[quality_id] = {
+                    'container': 'flv',
+                    'video_profile': s['video_profile'],
+                    'size': 0,
+                    'url': url
+                }
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            s['src'] = [s['url']]
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # Extract the stream
+            stream_id = kwargs['stream_id']
+
+            if stream_id not in self.streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # Extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+            s['src'] = [s['url']]
+
+site = QiE()
+download = site.download_by_url
+download_playlist = playlist_not_supported('QiE')
\ No newline at end of file
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index ebe1a9ad..cb4aeebf 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -3,6 +3,7 @@
 __all__ = ['qq_download']
 
 from ..common import *
+from .qie import download as qieDownload
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
@@ -34,6 +35,9 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
         title = title.strip() if title else vid
+    elif 'live.qq.com' in url:
+        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
+        exit()
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is

From b4eb73965ccadd9ba78e9d65d1c05b93c9467979 Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Thu, 7 Apr 2016 16:43:17 -0400
Subject: [PATCH 030/149] Add -bsf:a aac_adtstoasc when merging

As in #813.
---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 src/you_get/processor/ffmpeg.py

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
old mode 100644
new mode 100755
index 320eb642..e7ee35d6
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -169,7 +169,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
 
         params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
         params.append(output + '.txt')
-        params += ['-c', 'copy', output]
+        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', output]
 
         subprocess.check_call(params)
         os.remove(output + '.txt')

From 67d18c766ae832864ad8188e71f981db24983025 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 22 May 2016 17:28:01 +0200
Subject: [PATCH 031/149] processor/ffmpeg.py: 644

---
 src/you_get/processor/ffmpeg.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/you_get/processor/ffmpeg.py

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
old mode 100755
new mode 100644

From d1d62ae3040c98b7c3efb81937db211480ecb859 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 17:28:29 +0200
Subject: [PATCH 032/149] [twitter] fix #1139

---
 src/you_get/extractors/twitter.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 251cb8d5..1794081b 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -5,6 +5,13 @@ __all__ = ['twitter_download']
 from ..common import *
 from .vine import vine_download
 
+def extract_m3u(source):
+    r1 = get_content(source)
+    s1 = re.findall(r'(/ext_tw_video/.*)', r1)
+    r2 = get_content('https://video.twimg.com%s' % s1[-1])
+    s2 = re.findall(r'(/ext_tw_video/.*)', r2)
+    return 'https://video.twimg.com%s' % s2[-1]
+
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
     screen_name = r1(r'data-screen-name="([^"]*)"', html) or \
@@ -63,6 +70,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
             if not item_id: page_title = i['tweet_id']
 
+        source = extract_m3u(source)
         mime, ext, size = url_info(source)
 
         print_info(site_info, page_title, mime, size)

From d2d6fcb95397b969272a53dc9abf852f1181916d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 22:13:12 +0200
Subject: [PATCH 033/149] [twitter] fix #1139 (really)

---
 src/you_get/extractors/twitter.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 1794081b..b0c89189 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -10,7 +10,7 @@ def extract_m3u(source):
     s1 = re.findall(r'(/ext_tw_video/.*)', r1)
     r2 = get_content('https://video.twimg.com%s' % s1[-1])
     s2 = re.findall(r'(/ext_tw_video/.*)', r2)
-    return 'https://video.twimg.com%s' % s2[-1]
+    return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
@@ -70,12 +70,13 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
             if not item_id: page_title = i['tweet_id']
 
-        source = extract_m3u(source)
-        mime, ext, size = url_info(source)
+        urls = extract_m3u(source)
+        size = urls_size(urls)
+        mime, ext = 'video/mp4', 'mp4'
 
         print_info(site_info, page_title, mime, size)
         if not info_only:
-            download_urls([source], page_title, ext, size, output_dir, merge=merge)
+            download_urls(urls, page_title, ext, size, output_dir, merge=merge)
 
 site_info = "Twitter.com"
 download = twitter_download

From 186762b4b51679d1af40139e782e2b722a52c633 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 22:26:36 +0200
Subject: [PATCH 034/149] update README

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index d0a91aa8..60cb125a 100644
--- a/README.md
+++ b/README.md
@@ -360,6 +360,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | PPTV聚力 | <http://www.pptv.com/>         |✓| | |
 | 齐鲁网   | <http://v.iqilu.com/>          |✓| | |
 | QQ<br/>腾讯视频 | <http://v.qq.com/>      |✓| | |
+| 企鹅直播 | <http://live.qq.com/>          |✓| | |
 | 阡陌视频 | <http://qianmo.com/>           |✓| | |
 | THVideo  | <http://thvideo.tv/>           |✓| | |
 | Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
@@ -374,6 +375,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 央视网   | <http://www.cntv.cn/>          |✓| | |
 | 花瓣     | <http://huaban.com/>           | |✓| |
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
+| 芒果TV   | <http://www.mgtv.com/>         |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 

From 5f99e1dfd3ca5d6c83b11e5c060a8c0bfe611d8d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 22:38:19 +0200
Subject: [PATCH 035/149] version 0.4.424

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 0e7b6632..5a9d5581 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.390'
+__version__ = '0.4.424'

From 854386f22fd7c38462e3d89f46c94f5c13ac197b Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Sun, 22 May 2016 23:07:10 -0700
Subject: [PATCH 036/149] Retry request if timeout in get_content

---
 src/you_get/common.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9668df58..dadba69c 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -98,6 +98,7 @@ import logging
 import os
 import platform
 import re
+import socket
 import sys
 import time
 from urllib import request, parse, error
@@ -308,7 +309,14 @@ def get_content(url, headers={}, decoded=True):
     if cookies:
         cookies.add_cookie_header(req)
         req.headers.update(req.unredirected_hdrs)
-    response = request.urlopen(req)
+
+    for i in range(10):
+        try:
+            response = request.urlopen(req, timeout=10)
+            break
+        except socket.timeout:
+            logging.debug('request attempt %s timeout' % str(i + 1))
+
     data = response.read()
 
     # Handle HTTP compression for gzip and deflate (zlib)

From 48640e279f451cb2d365154870ed20e2d8bcc3e4 Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Tue, 24 May 2016 20:58:28 -0700
Subject: [PATCH 037/149] Add -t and --timeout option

To set socket timeout, default is 600 seconds.
---
 src/you_get/common.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index dadba69c..100f3869 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -312,7 +312,7 @@ def get_content(url, headers={}, decoded=True):
 
     for i in range(10):
         try:
-            response = request.urlopen(req, timeout=10)
+            response = request.urlopen(req)
             break
         except socket.timeout:
             logging.debug('request attempt %s timeout' % str(i + 1))
@@ -1071,11 +1071,12 @@ def script_main(script_name, download, download_playlist, **kwargs):
     -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
          --no-proxy                     Never use a proxy.
+    -t | --timeout <SECONDS>            Set socket timeout.
     -d | --debug                        Show traceback and other debug info.
     '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
+    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:t:'
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
     if download_playlist:
         short_opts = 'l' + short_opts
         opts = ['playlist'] + opts
@@ -1105,6 +1106,7 @@ def script_main(script_name, download, download_playlist, **kwargs):
     proxy = None
     extractor_proxy = None
     traceback = False
+    timeout = 600
     for o, a in opts:
         if o in ('-V', '--version'):
             version()
@@ -1178,6 +1180,8 @@ def script_main(script_name, download, download_playlist, **kwargs):
             extractor_proxy = a
         elif o in ('--lang',):
             lang = a
+        elif o in ('-t', '--timeout'):
+            timeout = int(a)
         else:
             log.e("try 'you-get --help' for more options")
             sys.exit(2)
@@ -1187,6 +1191,8 @@ def script_main(script_name, download, download_playlist, **kwargs):
 
     set_http_proxy(proxy)
 
+    socket.setdefaulttimeout(timeout)
+
     try:
         if stream_id:
             if not extractor_proxy:

From c89c977189372c64978b9d8aac0906047c6affde Mon Sep 17 00:00:00 2001
From: osinx zhou <osinx.zhou@gmail.com>
Date: Wed, 25 May 2016 15:35:41 +0800
Subject: [PATCH 038/149]  Support http://baidu.ku6.com #1163

---
 src/you_get/extractors/ku6.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index d9a1ef12..7f28c75b 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -27,13 +27,30 @@ def ku6_download_by_id(id, title = None, output_dir = '.', merge = True, info_on
         download_urls(urls, title, ext, size, output_dir, merge = merge)
 
 def ku6_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    patterns = [r'http://v.ku6.com/special/show_\d+/(.*)\.\.\.html',
-            r'http://v.ku6.com/show/(.*)\.\.\.html',
-            r'http://my.ku6.com/watch\?.*v=(.*)\.\..*']
-    id = r1_of(patterns, url)
+    id = None
+
+    if match1(url, r'http://baidu.ku6.com/watch/(.*)\.html') is not None:
+        id = baidu_ku6(url)
+    else:
+        patterns = [r'http://v.ku6.com/special/show_\d+/(.*)\.\.\.html',
+                r'http://v.ku6.com/show/(.*)\.\.\.html',
+                r'http://my.ku6.com/watch\?.*v=(.*)\.\..*']
+        id = r1_of(patterns, url)
 
     ku6_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
 
+def baidu_ku6(url):
+    id = None
+
+    h1 = get_html(url)
+    isrc = match1(h1, r'<iframe id="innerFrame" src="([^"]*)"')
+
+    if isrc is not None:
+        h2 = get_html(isrc)
+        id = match1(h2, r'http://v.ku6.com/show/(.*)\.\.\.html')
+
+    return id
+
 site_info = "Ku6.com"
 download = ku6_download
 download_playlist = playlist_not_supported('ku6')

From b6a00c3e14114755f4c92dc9eb603e3ff621bf45 Mon Sep 17 00:00:00 2001
From: 4ft35t <4ft35t@gmail.com>
Date: Thu, 26 May 2016 23:52:22 +0800
Subject: [PATCH 039/149] fix path error

---
 you-get | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/you-get b/you-get
index 85f3f754..8529388f 100755
--- a/you-get
+++ b/you-get
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import os, sys
 
-_srcdir = 'src/'
+_srcdir = '%s/src/' % os.path.dirname(os.path.realpath(__file__))
 _filepath = os.path.dirname(sys.argv[0])
 sys.path.insert(1, os.path.join(_filepath, _srcdir))
 

From 8879bca209c4655173134faad2a2dd60ea87ebee Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 27 May 2016 01:48:51 +0200
Subject: [PATCH 040/149] [videomega] fix support

---
 src/you_get/extractors/videomega.py | 68 +++++++++++++----------------
 1 file changed, 31 insertions(+), 37 deletions(-)

diff --git a/src/you_get/extractors/videomega.py b/src/you_get/extractors/videomega.py
index 75e88cd9..a2965e7a 100644
--- a/src/you_get/extractors/videomega.py
+++ b/src/you_get/extractors/videomega.py
@@ -1,47 +1,41 @@
 #!/usr/bin/env python
 
-from ..common import *
-from ..extractor import VideoExtractor
+__all__ = ['videomega_download']
 
+from ..common import *
 import ssl
 
-class Videomega(VideoExtractor):
-    name = "Videomega"
+def videomega_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    # Hot-plug cookie handler
+    ssl_context = request.HTTPSHandler(
+        context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
+    cookie_handler = request.HTTPCookieProcessor()
+    opener = request.build_opener(ssl_context, cookie_handler)
+    opener.addheaders = [('Referer', url),
+                         ('Cookie', 'noadvtday=0')]
+    request.install_opener(opener)
 
-    stream_types = [
-        {'id': 'original'}
-    ]
+    content = get_content(url)
+    m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
+    ref = m.group(1)
+    width, height = m.group(2), m.group(3)
+    php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
+    content = get_content(php_url)
 
-    def prepare(self, **kwargs):
-        # Hot-plug cookie handler
-        ssl_context = request.HTTPSHandler(
-            context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
-        cookie_handler = request.HTTPCookieProcessor()
-        opener = request.build_opener(ssl_context, cookie_handler)
-        opener.addheaders = [('Referer', self.url),
-                             ('Cookie', 'noadvtday=0')]
-        request.install_opener(opener)
+    title = match1(content, r'<title>(.*)</title>')
+    js = match1(content, r'(eval.*)')
+    t = match1(js, r'\$\("\w+"\)\.\w+\("\w+","([^"]+)"\)')
+    t = re.sub(r'(\w)', r'{\1}', t)
+    t = t.translate({87 + i: str(i) for i in range(10, 36)})
+    s = match1(js, r"'([^']+)'\.split").split('|')
+    src = t.format(*s)
 
-        ref = match1(self.url, r'ref=(\w+)')
-        php_url = 'http://videomega.tv/view.php?ref=' + ref
-        content = get_content(php_url)
+    type, ext, size = url_info(src, faker=True)
 
-        self.title = match1(content, r'<title>(.*)</title>')
-        js = match1(content, r'(eval.*)')
-        t = match1(js, r'\$\("\d+"\)\.\d+\("\d+","([^"]+)"\)')
-        t = re.sub(r'(\w)', r'{\1}', t)
-        t = t.translate({87 + i: str(i) for i in range(10, 36)})
-        s = match1(js, r"'([^']+)'\.split").split('|')
-        self.streams['original'] = {
-            'url': t.format(*s)
-        }
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls([src], title, ext, size, output_dir, merge=merge, faker=True)
 
-    def extract(self, **kwargs):
-        for i in self.streams:
-            s = self.streams[i]
-            _, s['container'], s['size'] = url_info(s['url'])
-            s['src'] = [s['url']]
-
-site = Videomega()
-download = site.download_by_url
-download_playlist = site.download_by_url
+site_info = "Videomega.tv"
+download = videomega_download
+download_playlist = playlist_not_supported('videomega')

From 71ec9cad223f4ee2022d2530c2bc5154213907ef Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 27 May 2016 03:12:49 +0200
Subject: [PATCH 041/149] [videomega] support embedded URLs

---
 src/you_get/extractors/videomega.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/videomega.py b/src/you_get/extractors/videomega.py
index a2965e7a..34fb5205 100644
--- a/src/you_get/extractors/videomega.py
+++ b/src/you_get/extractors/videomega.py
@@ -15,11 +15,14 @@ def videomega_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                          ('Cookie', 'noadvtday=0')]
     request.install_opener(opener)
 
-    content = get_content(url)
-    m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
-    ref = m.group(1)
-    width, height = m.group(2), m.group(3)
-    php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
+    if re.search(r'view\.php', url):
+        php_url = url
+    else:
+        content = get_content(url)
+        m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
+        ref = m.group(1)
+        width, height = m.group(2), m.group(3)
+        php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
     content = get_content(php_url)
 
     title = match1(content, r'<title>(.*)</title>')

From 9c67f4025861ac7a4b19562b0255c3e0c3b6f5e1 Mon Sep 17 00:00:00 2001
From: jackyzy823 <jackyzy823@gmail.com>
Date: Fri, 27 May 2016 16:18:56 +0000
Subject: [PATCH 042/149] [Netease] add support for radio & program

---
 src/you_get/extractors/netease.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 1fc7c071..332d48ec 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -61,7 +61,7 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
                 netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
             except: pass
 
-    elif "song" in url:
+    elif "song" in url: 
         j = loads(get_content("http://music.163.com/api/song/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_song_download(j["songs"][0], output_dir=output_dir, info_only=info_only)
         try: # download lyrics
@@ -70,6 +70,15 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
             netease_lyric_download(j["songs"][0], l["lrc"]["lyric"], output_dir=output_dir, info_only=info_only)
         except: pass
 
+    elif "program" in url:
+        j = loads(get_content("http://music.163.com/api/dj/program/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
+        netease_song_download(j["program"]["mainSong"], output_dir=output_dir, info_only=info_only)
+
+    elif "radio" in url:
+        j = loads(get_content("http://music.163.com/api/dj/program/byradio/?radioId=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
+        for i in j['programs']:
+            netease_song_download(i["mainSong"],output_dir=output_dir, info_only=info_only)
+
     elif "mv" in url:
         j = loads(get_content("http://music.163.com/api/mv/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_video_download(j['data'], output_dir=output_dir, info_only=info_only)

From b9431422d8cdd0e74c55bbaf02b79d1f75c16bd8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 May 2016 04:19:25 +0200
Subject: [PATCH 043/149] [twitter] support scribe_playlist_url (vine)

---
 src/you_get/extractors/twitter.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index b0c89189..09956141 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -69,8 +69,14 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             vmap = get_content(vmap_url)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
             if not item_id: page_title = i['tweet_id']
+        elif 'scribe_playlist_url' in i:
+            scribe_playlist_url = i['scribe_playlist_url']
+            return vine_download(scribe_playlist_url, output_dir, merge=merge, info_only=info_only)
 
-        urls = extract_m3u(source)
+        if source.endswith('.mp4'):
+            urls = [source]
+        else:
+            urls = extract_m3u(source)
         size = urls_size(urls)
         mime, ext = 'video/mp4', 'mp4'
 

From fd2a290ffe9193043b025ceb122042cf6623931d Mon Sep 17 00:00:00 2001
From: jackyzy823 <jackyzy823@gmail.com>
Date: Sat, 28 May 2016 09:32:07 +0000
Subject: [PATCH 044/149] [QQ] more robust url extractor

---
 src/you_get/extractors/qq.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index cb4aeebf..783bb2f5 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -4,7 +4,7 @@ __all__ = ['qq_download']
 
 from ..common import *
 from .qie import download as qieDownload
-
+from urllib.parse import urlparse,parse_qs
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
     content = get_html(api)
@@ -24,31 +24,35 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if 'live.qq.com' in url:
+        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
+        return 
+
+    #do redirect
     if 'v.qq.com/page' in url:
         # for URLs like this:
         # http://v.qq.com/page/k/9/7/k0194pwgw97.html
-        # it will redirect.
-        vid = match1(url, r'\b(\w+).html')
-        title = vid
-    elif 'kuaibao.qq.com' in url:
+        content = get_html(url)
+        url = match1(content,r'window\.location\.href="(.*?)"')
+        
+    if 'kuaibao.qq.com' in url:
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
         title = title.strip() if title else vid
-    elif 'live.qq.com' in url:
-        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
-        exit()
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is
         title = vid
     else:
         content = get_html(url)
-        vid = match1(content, r'vid\s*:\s*"\s*([^"]+)"')
-        title = match1(content, r'title\s*:\s*"\s*([^"]+)"')
-        # try to get the right title for URLs like this:
-        # http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        title = matchall(content, [r'title\s*:\s*"\s*([^"]+)"'])[-1]
+        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5 
+        vid = vid[0] if vid else match1(content, r'vid\s*:\s*"\s*([^"]+)"') #general fallback
+        title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
+        title = match1(content, r'title">([^"]+)</p>') if not title else title
+        title = vid if not title else title #general fallback
+
+
 
     qq_download_by_vid(vid, title, output_dir, merge, info_only)
 

From 10d6566552a621bc74e9e3d4e3b695e3797892dd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 May 2016 16:11:49 +0200
Subject: [PATCH 045/149] [bilibili] fix #1176

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 1a6ca325..55293533 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -11,7 +11,7 @@ from .youku import youku_download_by_vid
 import hashlib
 import re
 
-appkey='8e9fc618fbd41e28'
+appkey='f3bb208b3d081dc8'
 
 def get_srt_xml(id):
     url = 'http://comment.bilibili.com/%s.xml' % id

From 3b4a576e6cefa31f0bb25ec91f52814563384502 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Sat, 28 May 2016 18:50:18 -0700
Subject: [PATCH 046/149] Add shell completion definitions for Bash, Fish and
 Zsh

Paths of completion definitions in this repo are inconsequential, and I
don't know a convention for that, so I just modelled it on git.git
https://github.com/git/git/tree/master/contrib/completion. What's
different here is that instead of using a uniform naming scheme for each
file (you-get-completion.<shellname>), I'm using the conventional name
for each shell, which means _you-get for Zsh, for instance. Users of
each shell should be able to pick out the completion definition for
their shell based on the filenames; in the worst case, they can still
tell from the comment at the top of each file.

Note that I'm no expert on Bash or Fish completion, so
you-get-completion.bash and you-get.fish may not be written in the
idiomatic way. (you-get-completion.bash is most certainly not idiomatic,
if there's an idiom at all.)

Closes #1178.
---
 README.md                                  |  4 +++
 contrib/completion/_you-get                | 29 ++++++++++++++++++++
 contrib/completion/you-get-completion.bash | 31 ++++++++++++++++++++++
 contrib/completion/you-get.fish            | 23 ++++++++++++++++
 4 files changed, 87 insertions(+)
 create mode 100644 contrib/completion/_you-get
 create mode 100755 contrib/completion/you-get-completion.bash
 create mode 100644 contrib/completion/you-get.fish

diff --git a/README.md b/README.md
index 60cb125a..af2cdf9d 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,10 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
+### Shell completion
+
+Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](contrib/completion). Please consult your shell's manual for how to take advantage of them.
+
 ## Upgrading
 
 Based on which option you chose to install `you-get`, you may upgrade it via:
diff --git a/contrib/completion/_you-get b/contrib/completion/_you-get
new file mode 100644
index 00000000..696aad89
--- /dev/null
+++ b/contrib/completion/_you-get
@@ -0,0 +1,29 @@
+#compdef you-get
+
+# Zsh completion definition for soimort/you-get.
+
+setopt localoptions noshwordsplit noksharrays
+local -a args
+
+args=(
+    '(- : *)'{-V,--version}'[print version and exit]'
+    '(- : *)'{-h,--help}'[print help and exit]'
+    '(-i --info)'{-i,--info}'[print extracted information]'
+    '(-u --url)'{-u,--url}'[print extracted information with URLs]'
+    '(--json)--json[print extracted URLs in JSON format]'
+    '(-n --no-merge)'{-n,--no-merge}'[do not merge video parts]'
+    '(--no-caption)--no-caption[do not download captions]'
+    '(-f --force)'{-f,--force}'[force overwrite existing files]'
+    '(-F --format)'{-F,--format}'[set video format to the specified stream id]:stream id'
+    '(-O --output-filename)'{-O,--output-filename}'[set output filename]:filename:_files'
+    '(-o --output-dir)'{-o,--output-dir}'[set output directory]:directory:_files -/'
+    '(-p --player)'{-p,--player}'[stream extracted URL to the specified player]:player and options'
+    '(-c --cookies)'{-c,--cookies}'[load cookies.txt or cookies.sqlite]:cookies file:_files'
+    '(-x --http-proxy)'{-x,--http-proxy}'[use the specified HTTP proxy for downloading]:host\:port:'
+    '(-y --extractor-proxy)'{-y,--extractor-proxy}'[use the specified HTTP proxy for extraction only]:host\:port'
+    '(--no-proxy)--no-proxy[do not use a proxy]'
+    '(-t --timeout)'{-t,--timeout}'[set socket timeout]:seconds'
+    '(-d --debug)'{-d,--debug}'[show traceback and other debug info]'
+    '*: :_guard "^-*" url'
+)
+_arguments -S -s $args
diff --git a/contrib/completion/you-get-completion.bash b/contrib/completion/you-get-completion.bash
new file mode 100755
index 00000000..9c6480ec
--- /dev/null
+++ b/contrib/completion/you-get-completion.bash
@@ -0,0 +1,31 @@
+# Bash completion definition for you-get.
+
+_you-get () {
+    COMPREPLY=()
+    local IFS=$' \n'
+    local cur=$2 prev=$3
+    local -a opts_without_arg opts_with_arg
+    opts_without_arg=(
+        -V --version -h --help -i --info -u --url --json -n --no-merge
+        --no-caption -f --force --no-proxy -d --debug
+    )
+    opts_with_arg=(
+        -F --format -O --output-filename -o --output-dir -p --player
+        -c --cookies -x --http-proxy -y --extractor-proxy -t --timeout
+    )
+
+    # Do not complete non option names
+    [[ $cur == -* ]] || return 1
+
+    # Do not complete when the previous arg is an option expecting an argument
+    for opt in "${opts_with_arg[@]}"; do
+        [[ $opt == $prev ]] && return 1
+    done
+
+    # Complete option names
+    COMPREPLY=( $(compgen -W "${opts_without_arg[*]} ${opts_with_arg[*]}" \
+                          -- "$cur") )
+    return 0
+}
+
+complete -F _you-get you-get
diff --git a/contrib/completion/you-get.fish b/contrib/completion/you-get.fish
new file mode 100644
index 00000000..6917c422
--- /dev/null
+++ b/contrib/completion/you-get.fish
@@ -0,0 +1,23 @@
+# Fish completion definition for you-get.
+
+complete -c you-get -s V -l version -d 'print version and exit'
+complete -c you-get -s h -l help -d 'print help and exit'
+complete -c you-get -s i -l info -d 'print extracted information'
+complete -c you-get -s u -l url -d 'print extracted information'
+complete -c you-get -l json -d 'print extracted URLs in JSON format'
+complete -c you-get -s n -l no-merge -d 'do not merge video parts'
+complete -c you-get -l no-caption -d 'do not download captions'
+complete -c you-get -s f -l force -d 'force overwrite existing files'
+complete -c you-get -s F -l format -x -d 'set video format to the specified stream id'
+complete -c you-get -s O -l output-filename -d 'set output filename' \
+         -x -a '(__fish_complete_path (commandline -ct) "output filename")'
+complete -c you-get -s o -l output-dir  -d 'set output directory' \
+         -x -a '(__fish_complete_directories (commandline -ct) "output directory")'
+complete -c you-get -s p -l player -x -d 'stream extracted URL to the specified player'
+complete -c you-get -s c -l cookies -d 'load cookies.txt or cookies.sqlite' \
+         -x -a '(__fish_complete_path (commandline -ct) "cookies.txt or cookies.sqlite")'
+complete -c you-get -s x -l http-proxy -x -d 'use the specified HTTP proxy for downloading'
+complete -c you-get -s y -l extractor-proxy -x -d 'use the specified HTTP proxy for extraction only'
+complete -c you-get -l no-proxy -d 'do not use a proxy'
+complete -c you-get -s t -l timeout -x -d 'set socket timeout'
+complete -c you-get -s d -l debug -d 'show traceback and other debug info'

From c372b09e5957b6cfe143d9ce58c56ce224b6128c Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Sun, 29 May 2016 17:29:15 -0400
Subject: [PATCH 047/149] [Readme] Add pip install from Github

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index 60cb125a..a972df23 100644
--- a/README.md
+++ b/README.md
@@ -107,6 +107,12 @@ or download the latest release via:
 $ you-get https://github.com/soimort/you-get/archive/master.zip
 ```
 
+In order to get the latest ```develop``` branch without messing up the PIP, you can try:
+
+```
+$ pip3 install --upgrade git+https://github.com/soimort/you-get@develop
+```
+
 ## Getting Started
 
 ### Download a video

From aaf00414462cf0a7c0d7bb6a131f54e2c16891fc Mon Sep 17 00:00:00 2001
From: zhuzhichao <me@zhuzhichao.com>
Date: Sat, 4 Jun 2016 14:20:42 +0800
Subject: [PATCH 048/149] Install via brew (Mac only)

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index a972df23..d915b020 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,14 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
+### Option 6: Homebrew (Mac only)
+
+You can install `you-get` easily via:
+
+```
+$ brew install you-get
+```
+
 ## Upgrading
 
 Based on which option you chose to install `you-get`, you may upgrade it via:

From b05fa57a36bb153b847f2c70c00c02ab93f17415 Mon Sep 17 00:00:00 2001
From: Yi Zhi <zhiyi523@gmail.com>
Date: Sun, 5 Jun 2016 10:39:03 +0800
Subject: [PATCH 049/149] [Bilibili] download specified P according to the link

---
 src/you_get/extractors/bilibili.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 55293533..d355eabd 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -11,12 +11,14 @@ from .youku import youku_download_by_vid
 import hashlib
 import re
 
-appkey='f3bb208b3d081dc8'
+appkey = 'f3bb208b3d081dc8'
+
 
 def get_srt_xml(id):
     url = 'http://comment.bilibili.com/%s.xml' % id
     return get_html(url)
 
+
 def parse_srt_p(p):
     fields = p.split(',')
     assert len(fields) == 8, fields
@@ -44,12 +46,14 @@ def parse_srt_p(p):
 
     return pool, mode, font_size, font_color
 
+
 def parse_srt_xml(xml):
     d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
     for x, y in d:
         p = parse_srt_p(x)
     raise NotImplementedError()
 
+
 def parse_cid_playurl(xml):
     from xml.dom.minidom import parseString
     try:
@@ -59,14 +63,15 @@ def parse_cid_playurl(xml):
     except:
         return []
 
+
 def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
     urls = []
     for cid in cids:
         url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
         urls += [i
-                if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-                else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                for i in parse_cid_playurl(get_content(url))]
+                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+                 for i in parse_cid_playurl(get_content(url))]
 
     type_ = ''
     size = 0
@@ -78,6 +83,7 @@ def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
 def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
     url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
     urls = [i
@@ -98,6 +104,7 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
 def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
     api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
     urls = parse_cid_playurl(get_content(api_url))
@@ -109,6 +116,7 @@ def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_o
         if not info_only:
             download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
@@ -118,9 +126,10 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         title = unescape_html(title)
         title = escape_file_path(title)
 
-    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
+                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
     assert flashvars
-    flashvars = flashvars.replace(': ','=')
+    flashvars = flashvars.replace(': ', '=')
     t, cid = flashvars.split('=', 1)
     cid = cid.split('&')[0]
     if t == 'cid':
@@ -133,7 +142,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
             titles = re.findall('<option value=.*>(.+)</option>', html)
-            for page in pages:
+            for i, page in enumerate(pages):
                 html = get_html("http://www.bilibili.com%s" % page)
                 flashvars = r1_of([r'(cid=\d+)',
                                    r'flashvars="([^"]+)"',
@@ -141,6 +150,10 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
                 if flashvars:
                     t, cid = flashvars.split('=', 1)
                     cids.append(cid.split('&')[0])
+                if url.endswith(page):
+                    cids = [cid.split('&')[0]]
+                    titles = [titles[i]]
+                    break
 
             # no multi-P
             if not pages:
@@ -173,6 +186,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
+
 site_info = "bilibili.com"
 download = bilibili_download
 download_playlist = bilibili_download

From fc83718e5870829dcc2ed1b0db5b01e3df7e277e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 23 Jun 2016 21:23:17 +0200
Subject: [PATCH 050/149] [miaopai] fix #1194

---
 src/you_get/extractors/miaopai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 9f195baa..48a71239 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -37,7 +37,7 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
         miaopai_download_by_url(url, output_dir, merge, info_only)
     elif re.match(r'http://weibo.com/p/230444\w+', url):
         _fid = match1(url, r'http://weibo.com/p/230444(\w+)')
-        miaopai_download_by_url('http://video.weibo.com/show?fid=1034:{_fid}'.format(_fid = _fid))
+        miaopai_download_by_url('http://video.weibo.com/show?fid=1034:{_fid}'.format(_fid = _fid), output_dir, merge, info_only)
 
 site_info = "miaopai"
 download = miaopai_download

From 13dc2e9d61dee1db5a7e843a5c1d24c4dca4309d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 23 Jun 2016 22:13:15 +0200
Subject: [PATCH 051/149] [facebook] fix #1203

---
 src/you_get/extractors/facebook.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 4ed1e2af..72ce8f31 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -5,24 +5,21 @@ __all__ = ['facebook_download']
 from ..common import *
 import json
 
-
 def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
 
-    title = r1(r'<title id="pageTitle">(.+) \| Facebook</title>', html)
-    s2 = parse.unquote(unicodize(r1(r'\["params","([^"]*)"\]', html)))
-    data = json.loads(s2)
-    video_data = data["video_data"]["progressive"]
-    for fmt in ["hd_src", "sd_src"]:
-        src = video_data[0][fmt]
-        if src:
-            break
+    title = r1(r'<title id="pageTitle">(.+)</title>', html)
+    sd_urls = [
+        unicodize(str.replace(i, '\\/', '/'))
+        for i in re.findall(r'"sd_src_no_ratelimit":"([^"]*)"', html)
+    ]
 
-    type, ext, size = url_info(src, True)
+    type, ext, size = url_info(sd_urls[0], True)
+    size = urls_size(sd_urls)
 
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([src], title, ext, size, output_dir, merge=merge)
+        download_urls(sd_urls, title, ext, size, output_dir, merge=False)
 
 site_info = "Facebook.com"
 download = facebook_download

From a256b44e3ecbcbcf41e82cf03977dd22d615cc96 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 23 Jun 2016 22:15:36 +0200
Subject: [PATCH 052/149] version 0.4.455

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 5a9d5581..61e75ead 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.424'
+__version__ = '0.4.455'

From 9c8d8b0023d8399e047de0ab503e29f12881118b Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Sat, 25 Jun 2016 15:11:43 +0800
Subject: [PATCH 053/149] iqiyi: use html5 to fix #1211 , but lose some stream
 quality

thanks @yan12125

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py    | 51 ++++++++---------
 src/you_get/extractors/iqiyi_sc.py | 92 ++++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+), 28 deletions(-)
 create mode 100644 src/you_get/extractors/iqiyi_sc.py

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 7431b91f..320520fa 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -9,6 +9,10 @@ from math import floor
 from zlib import decompress
 import hashlib
 
+import time
+
+from .iqiyi_sc import gen_sc
+
 '''
 Changelog:
 -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf
@@ -43,6 +47,7 @@ bid meaning for quality
 10 4k
 96 topspeed
 
+'''
 '''
 def mix(tvid):
     salt = '4a1caba4b4465345366f28da7c117d20'
@@ -75,42 +80,32 @@ def getDispathKey(rid):
     time=json.loads(get_content("http://data.video.qiyi.com/t?tn="+str(random())))["t"]
     t=str(int(floor(int(time)/(10*60.0))))
     return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest()
+'''
 
 class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': '4k', 'container': 'f4v', 'video_profile': '4K'},
-        {'id': 'fullhd', 'container': 'f4v', 'video_profile': '全高清'},
-        {'id': 'suprt-high', 'container': 'f4v', 'video_profile': '超高清'},
-        {'id': 'super', 'container': 'f4v', 'video_profile': '超清'},
-        {'id': 'high', 'container': 'f4v', 'video_profile': '高清'},
-        {'id': 'standard', 'container': 'f4v', 'video_profile': '标清'},
-        {'id': 'topspeed', 'container': 'f4v', 'video_profile': '最差'},
+        {'id': 'high', 'container': 'mp4', 'video_profile': '高清'},
+        {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'},
     ]
 
+    supported_stream_types = [ 'high', 'standard']
+
+
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
 
-    stream_urls = {  '4k': [] , 'fullhd' : [], 'suprt-high' : [], 'super' : [], 'high' : [], 'standard' :[], 'topspeed' :[]}
-
-    baseurl = ''
-
-    gen_uid = ''
-    def getVMS(self):
+    def getVMS(self,rate):
         #tm ->the flash run time for md5 usage
         #um -> vip 1 normal 0
         #authkey -> for password protected video ,replace '' with your password
         #puid user.passportid may empty?
         #TODO: support password protected video
         tvid, vid = self.vid
-        tm, sc, src = mix(tvid)
-        uid = self.gen_uid
-        vmsreq='http://cache.video.qiyi.com/vms?key=fvip&src=1702633101b340d8917a69cf8a4b8c7' +\
-                "&tvId="+tvid+"&vid="+vid+"&vinfo=1&tm="+tm+\
-                "&enc="+sc+\
-                "&qyid="+uid+"&tn="+str(random()) +"&um=1" +\
-                "&authkey="+hashlib.new('md5',bytes(hashlib.new('md5', b'').hexdigest()+str(tm)+tvid,'utf-8')).hexdigest()
-        return json.loads(get_content(vmsreq))
+        t = int(time.time() * 1000)
+        sc = gen_sc(tvid, t).decode('utf-8')
+        vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7)
+        return json.loads(get_content(vmsreq)[13:])
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -134,13 +129,12 @@ class Iqiyi(VideoExtractor):
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
 
-        self.gen_uid = uuid4().hex
-        try:
-            info = self.getVMS()
-        except:
-            self.download_playlist_by_url(self.url, **kwargs)
-            exit(0)
-
+        for stream in self.supported_stream_types:
+            info = self.getVMS(self.stream_to_bid[stream])
+            if info["code"] == "A00000":
+                self.title = info['data']['playInfo']['vn']
+                self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])}
+'''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
             log.wtf("is your you-get up-to-date?")
@@ -208,6 +202,7 @@ class Iqiyi(VideoExtractor):
         #because the url is generated before start downloading
         #and the key may be expired after 10 minutes
         self.streams[stream_id]['src'] = urls
+'''
 
 site = Iqiyi()
 download = site.download_by_url
diff --git a/src/you_get/extractors/iqiyi_sc.py b/src/you_get/extractors/iqiyi_sc.py
new file mode 100644
index 00000000..4fa4ccdf
--- /dev/null
+++ b/src/you_get/extractors/iqiyi_sc.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import binascii
+import math
+import time
+
+M = [1732584193, -271733879]
+M.extend([~M[0], ~M[1]])
+I_table = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]
+C_base = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8388608, 432]
+
+
+def L(n, t):
+    if t is None:
+        t = 0
+    return trunc(((n >> 1) + (t >> 1) << 1) + (n & 1) + (t & 1))
+
+
+def rshift(val, n):
+    return val >> n if val >= 0 else (val+0x100000000) >> n
+
+
+def trunc(n):
+    n = n % 0x100000000
+    if n > 0x7fffffff:
+        n -= 0x100000000
+    return n
+
+
+def gen_sc(tvid, Z):
+    def transform(string, mod):
+        num = int(string, 16)
+        return (num >> 8 * (i % 4) & 255 ^ i % mod) << ((a & 3) << 3)
+
+    C = list(C_base)
+    o = list(M)
+    k = str(Z - 7)
+    for i in range(13):
+        a = i
+        C[a >> 2] |= ord(k[a]) << 8 * (a % 4)
+
+    for i in range(16):
+        a = i + 13
+        start = (i >> 2) * 8
+        r = '03967743b643f66763d623d637e30733'
+        C[a >> 2] |= transform(''.join(reversed(r[start:start + 8])), 7)
+
+    for i in range(16):
+        a = i + 29
+        start = (i >> 2) * 8
+        r = '7038766939776a32776a32706b337139'
+        C[a >> 2] |= transform(r[start:start + 8], 1)
+
+    for i in range(9):
+        a = i + 45
+        if i < len(tvid):
+            C[a >> 2] |= ord(tvid[i]) << 8 * (a % 4)
+
+    for a in range(64):
+        i = a
+        I = i >> 4
+        C_index = [i, 5 * i + 1, 3 * i + 5, 7 * i][I] % 16 + rshift(a, 6)
+        m = L(
+                L(
+                    o[0],
+                    [
+                        trunc(o[1] & o[2]) | trunc(~o[1] & o[3]),
+                        trunc(o[3] & o[1]) | trunc(~o[3] & o[2]),
+                        o[1] ^ o[2] ^ o[3],
+                        o[2] ^ trunc(o[1] | ~o[3])
+                    ][I]
+                ),
+                L(
+                    trunc(int(abs(math.sin(i + 1)) * 4294967296)),
+                    C[C_index] if C_index < len(C) else None
+                )
+            )
+        I = I_table[4 * I + i % 4]
+        o = [
+                o[3],
+                L(o[1], trunc(trunc(m << I) | rshift(m, 32 - I))),
+                o[1],
+                o[2],
+            ]
+
+    new_M = [L(o[0], M[0]), L(o[1], M[1]), L(o[2], M[2]), L(o[3], M[3])]
+    s = [new_M[a >> 3] >> (1 ^ a & 7) * 4 & 15 for a in range(32)]
+    return binascii.hexlify(bytes(s))[1::2]
+
+if __name__ == '__main__':
+    print(gen_sc("494496100", 1466495259194))
+    print(gen_sc("397768800", 1466795077775))
+    print(gen_sc("397768800", 1466796325746))

From d8aca8f5427d4e365da817288906cff9dd88bade Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 26 Jun 2016 16:50:28 +0200
Subject: [PATCH 054/149] [bilibili] quick hack for bangumi URLs (fix #1226)

---
 src/you_get/extractors/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index d355eabd..24782598 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -120,6 +120,11 @@ def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_o
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
+    if re.match(r'https?://bangumi\.bilibili\.com/', url):
+        # quick hack for bangumi URLs
+        url = r1(r'"([^"]+)" class="v-av-link"', html)
+        html = get_content(url)
+
     title = r1_of([r'<meta name="title" content="([^<>]{1,999})" />',
                    r'<h1[^>]*>([^<>]+)</h1>'], html)
     if title:

From d370e9952f46d4e56e2abb6a53c8955c7a2f0d7c Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Tue, 28 Jun 2016 16:44:54 +0800
Subject: [PATCH 055/149] enable --socks-proxy

---
 src/you_get/common.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 100f3869..119640d5 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1071,12 +1071,13 @@ def script_main(script_name, download, download_playlist, **kwargs):
     -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
          --no-proxy                     Never use a proxy.
+    -s | --socks-proxy <HOST:PORT>      Use an SOCKS5 proxy for downloading.
     -t | --timeout <SECONDS>            Set socket timeout.
     -d | --debug                        Show traceback and other debug info.
     '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:t:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
+    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:'
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
     if download_playlist:
         short_opts = 'l' + short_opts
         opts = ['playlist'] + opts
@@ -1104,6 +1105,7 @@ def script_main(script_name, download, download_playlist, **kwargs):
     lang = None
     output_dir = '.'
     proxy = None
+    socks_proxy = None
     extractor_proxy = None
     traceback = False
     timeout = 600
@@ -1176,6 +1178,8 @@ def script_main(script_name, download, download_playlist, **kwargs):
             caption = False
         elif o in ('-x', '--http-proxy'):
             proxy = a
+        elif o in ('-s', '--socks-proxy'):
+            socks_proxy = a
         elif o in ('-y', '--extractor-proxy'):
             extractor_proxy = a
         elif o in ('--lang',):
@@ -1189,7 +1193,21 @@ def script_main(script_name, download, download_playlist, **kwargs):
         print(help)
         sys.exit()
 
-    set_http_proxy(proxy)
+    if (socks_proxy):
+        try:
+          import socket
+          import socks
+          socks_proxy_addrs = socks_proxy.split(':')
+          socks.set_default_proxy(socks.SOCKS5, 
+                                  socks_proxy_addrs[0], 
+                                  int(socks_proxy_addrs[1]))
+          socket.socket = socks.socksocket
+        except ImportError:
+          log.w('Error importing PySocks library, socks proxy ignored.'
+                'In order to use use socks proxy, please install PySocks.')
+    else:
+        import socket
+        set_http_proxy(proxy)
 
     socket.setdefaulttimeout(timeout)
 

From 8bc81d9a70448e5078af86e07c35b12f3f8b1b6a Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Wed, 29 Jun 2016 20:18:18 +0800
Subject: [PATCH 056/149] iqiyi: support more stream quality

algorism form @ERioK
thank you @ERioK

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py | 45 +++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 320520fa..bda2c2e8 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -11,8 +11,6 @@ import hashlib
 
 import time
 
-from .iqiyi_sc import gen_sc
-
 '''
 Changelog:
 -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf
@@ -86,26 +84,31 @@ class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': 'high', 'container': 'mp4', 'video_profile': '高清'},
-        {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'},
+        {'id': 'BD', 'container': 'm3u8', 'video_profile': '全高清'},
+        {'id': 'FD', 'container': 'm3u8', 'video_profile': '超高清'},
+        {'id': 'TD', 'container': 'm3u8', 'video_profile': '超清'},
+        {'id': 'HD', 'container': 'm3u8', 'video_profile': '高清'},
+        {'id': 'SD', 'container': 'm3u8', 'video_profile': '标清'},
+        {'id': 'LD', 'container': 'm3u8', 'video_profile': '流畅'},
     ]
-
+    '''
     supported_stream_types = [ 'high', 'standard']
 
 
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
+    '''
+    ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD']
+    vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'}
+    vd_2_profile = {21: u'超清', 2: u'高清', 4: u'超高清', 17: u'全高清', 96: u'流畅', 1: u'标清'}
 
-    def getVMS(self,rate):
-        #tm ->the flash run time for md5 usage
-        #um -> vip 1 normal 0
-        #authkey -> for password protected video ,replace '' with your password
-        #puid user.passportid may empty?
-        #TODO: support password protected video
+    def getVMS(self):
         tvid, vid = self.vid
         t = int(time.time() * 1000)
-        sc = gen_sc(tvid, t).decode('utf-8')
-        vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7)
-        return json.loads(get_content(vmsreq)[13:])
+        src = '76f90cbd92f94a2e925d83e8ccd22cb7'
+        key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+        sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
+        vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
+        return json.loads(get_content(vmsreq))
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -128,12 +131,16 @@ class Iqiyi(VideoExtractor):
                       r1(r'vid=([^&]+)', self.url) or \
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
+            self.title = match1(html, '<title>([^<]+)').split('-')[0]
+
+        info = self.getVMS()
+        assert info['code'] == 'A00000', 'can\'t play this video'
+
+        for stream in info['data']['vidl']:
+           stream_id = self.vd_2_id[stream['vd']]
+           stream_profile = self.vd_2_profile[stream['vd']]
+           self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
 
-        for stream in self.supported_stream_types:
-            info = self.getVMS(self.stream_to_bid[stream])
-            if info["code"] == "A00000":
-                self.title = info['data']['playInfo']['vn']
-                self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])}
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")

From 5966a090f478b876b31632f2b654f27c382ebdba Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 15:56:39 -0400
Subject: [PATCH 057/149] [Common, ffmpeg]Exp: Add a ffmpeg downloader and
 player for any URL

---
 src/you_get/common.py             | 17 +++++++++
 src/you_get/processor/ffmpeg.py   | 61 +++++++++++++++++++++++++++++++
 src/you_get/processor/rtmpdump.py |  1 +
 3 files changed, 79 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 119640d5..c0097a4e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -898,6 +898,23 @@ def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', re
     assert has_rtmpdump_installed(), "RTMPDump not installed."
     download_rtmpdump_stream(url,  title, ext,params, output_dir)
 
+def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
+    assert url
+    if dry_run:
+        print('Real URL:\n%s\n' % [url])
+        if params.get("-y",False): #None or unset ->False
+            print('Real Playpath:\n%s\n' % [params.get("-y")])
+        return
+
+    if player:
+        from .processor.ffmpeg import ffmpeg_play_stream
+        ffmpeg_play_stream(player, url, params)
+        return
+
+    from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_streaming
+    assert has_ffmpeg_installed(), "FFmpeg not installed."
+    ffmpeg_download_stream(url, title, ext, params, output_dir)
+
 def playlist_not_supported(name):
     def f(*args, **kwargs):
         raise NotImplementedError('Playlist is not supported for ' + name)
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index e7ee35d6..1d5d850c 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -199,3 +199,64 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     for file in files:
         os.remove(file + '.ts')
     return True
+
+def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
+    """str, str->True
+    WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
+    You can basicly download anything with this function
+    but better leave it alone with 
+    """
+    output = title + '.' + ext
+    
+    if not (output_dir == '.'):
+        output = output_dir + output
+        
+    ffmpeg_params = []
+    #should these exist...
+    if len(params) > 0:
+        for k, v in params:
+            ffmpeg_params.append(k)
+            ffmpeg_params.append(v)
+        
+    print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...')
+    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    ffmpeg_params.append(files)  #not the same here!!!!
+    
+    if FFMPEG == 'avconv':  #who cares?
+        ffmpeg_params += ['-c', 'copy', output]
+    else:
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', output]
+    
+    ffmpeg_params.append(output)
+    
+    subprocess.call(ffmpeg_params)
+
+    return True
+
+#
+#To be refactor
+#Direct copy of rtmpdump.py
+#
+def ffmpeg_play_stream(player, url, params={}):
+    ffmpeg_params = []
+    #should these exist...
+    if len(params) > 0:
+        for k, v in params:
+            ffmpeg_params.append(k)
+            ffmpeg_params.append(v)
+        
+    print('Playing streaming content with FFmpeg, press Ctrl+C to stop recording...')
+    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    ffmpeg_params.append(url)  #not the same here!!!!
+    
+    if FFMPEG == 'avconv':  #who cares?
+        ffmpeg_params += ['-c', 'copy', '|']
+    else:
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', '|']
+    
+    ffmpeg_params += [player, '-']
+    
+    print(' '.join(ffmpeg_params))
+    
+    subprocess.call(ffmpeg_params)
+    return
diff --git a/src/you_get/processor/rtmpdump.py b/src/you_get/processor/rtmpdump.py
index aadb6887..cf5f822c 100644
--- a/src/you_get/processor/rtmpdump.py
+++ b/src/you_get/processor/rtmpdump.py
@@ -43,6 +43,7 @@ def download_rtmpdump_stream(url, title, ext,params={},output_dir='.'):
 
 #
 #To be refactor
+#To the future myself: Remember to refactor the same function in ffmpeg.py
 #
 def play_rtmpdump_stream(player, url, params={}):
     cmdline="rtmpdump -r '%s' "%url

From 0d06c260c0ca535d7652e50c60f676f0bdceba14 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 21:28:49 -0400
Subject: [PATCH 058/149] [iQiyi]try to fix CI complaint

---
 src/you_get/extractors/iqiyi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index bda2c2e8..4544552d 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -137,9 +137,9 @@ class Iqiyi(VideoExtractor):
         assert info['code'] == 'A00000', 'can\'t play this video'
 
         for stream in info['data']['vidl']:
-           stream_id = self.vd_2_id[stream['vd']]
-           stream_profile = self.vd_2_profile[stream['vd']]
-           self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            stream_id = self.vd_2_id[stream['vd']]
+            stream_profile = self.vd_2_profile[stream['vd']]
+            self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
 
 '''
         if info["code"] != "A000000":

From 2d542c2cd3b2b6a56886342f53ceb476a08818eb Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 21:54:46 -0400
Subject: [PATCH 059/149] [iQiyi]CI complaint CI complaint go away

---
 src/you_get/extractors/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 4544552d..de8756bf 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -99,7 +99,7 @@ class Iqiyi(VideoExtractor):
     '''
     ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD']
     vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'}
-    vd_2_profile = {21: u'超清', 2: u'高清', 4: u'超高清', 17: u'全高清', 96: u'流畅', 1: u'标清'}
+    vd_2_profile = {21: '超清', 2: '高清', 4: '超高清', 17: '全高清', 96: '流畅', 1: '标清'}
 
     def getVMS(self):
         tvid, vid = self.vid

From fd93b0380d0c04c4766e520e53a6256437e39067 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 23:15:28 -0400
Subject: [PATCH 060/149] [FFmpeg] Change arguments to record M3U

---
 src/you_get/processor/ffmpeg.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 1d5d850c..a387be0a 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -219,16 +219,18 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
             ffmpeg_params.append(v)
         
     print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...')
-    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    ffmpeg_params = [FFMPEG] + ['-y', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
     
     if FFMPEG == 'avconv':  #who cares?
         ffmpeg_params += ['-c', 'copy', output]
     else:
-        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', output]
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
     
     ffmpeg_params.append(output)
     
+    print(' '.join(ffmpeg_params))
+    
     subprocess.call(ffmpeg_params)
 
     return True
@@ -252,7 +254,7 @@ def ffmpeg_play_stream(player, url, params={}):
     if FFMPEG == 'avconv':  #who cares?
         ffmpeg_params += ['-c', 'copy', '|']
     else:
-        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', '|']
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|']
     
     ffmpeg_params += [player, '-']
     

From 18bc44ea4155d979f8e5fb864e2233d9b3f36eda Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 23:42:19 -0400
Subject: [PATCH 061/149] [FFmpeg] Fix stream corrupted if Ctrl+C

Use q instead.
---
 src/you_get/processor/ffmpeg.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index a387be0a..c23b0eaf 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -218,8 +218,8 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
             ffmpeg_params.append(k)
             ffmpeg_params.append(v)
         
-    print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...')
-    ffmpeg_params = [FFMPEG] + ['-y', '-i']
+    print('Downloading streaming content with FFmpeg, press q to stop recording...')
+    ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
     
     if FFMPEG == 'avconv':  #who cares?
@@ -231,7 +231,14 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     
     print(' '.join(ffmpeg_params))
     
-    subprocess.call(ffmpeg_params)
+    try:
+        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
+        a.communicate()
+    except KeyboardInterrupt:
+        try:
+            a.stdin.write('q'.encode('utf-8'))
+        except:
+            pass
 
     return True
 
@@ -247,8 +254,8 @@ def ffmpeg_play_stream(player, url, params={}):
             ffmpeg_params.append(k)
             ffmpeg_params.append(v)
         
-    print('Playing streaming content with FFmpeg, press Ctrl+C to stop recording...')
-    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    print('Playing streaming content with FFmpeg, press 1 to stop recording...')
+    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']
     ffmpeg_params.append(url)  #not the same here!!!!
     
     if FFMPEG == 'avconv':  #who cares?
@@ -260,5 +267,13 @@ def ffmpeg_play_stream(player, url, params={}):
     
     print(' '.join(ffmpeg_params))
     
-    subprocess.call(ffmpeg_params)
-    return
+    try:
+        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
+        a.communicate()
+    except KeyboardInterrupt:
+        try:
+            a.stdin.write('q'.encode('utf-8'))
+        except:
+            pass
+
+    return True
\ No newline at end of file

From 62a9e35ed0cc0aebb9ef13edb76394c9efd543e6 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 30 Jun 2016 02:36:07 -0400
Subject: [PATCH 062/149] [CNTV] Fix CCAV domain, Add pattern, fix #1193 ,#909
 , replace #1031

---
 src/you_get/common.py          | 1 +
 src/you_get/extractors/cntv.py | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 119640d5..d30397c7 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -9,6 +9,7 @@ SITES = {
     'bandcamp'         : 'bandcamp',
     'baomihua'         : 'baomihua',
     'bilibili'         : 'bilibili',
+    'cctv'             : 'cntv',
     'cntv'             : 'cntv',
     'cbs'              : 'cbs',
     'dailymotion'      : 'dailymotion',
diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index cfd96e59..a32808cb 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -7,6 +7,7 @@ from ..common import *
 import json
 import re
 
+
 def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
     assert id
     info = json.loads(get_html('http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=' + id))
@@ -31,7 +32,12 @@ def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
         id = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
-    elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url):
+    elif re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url):
+        html = get_content(url)
+        id = match1(html, r'guid = \"(.+)\"')
+    elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
+         re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
+         re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url): 
         id = r1(r'videoCenterId","(\w+)"', get_html(url))
     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
         id = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)

From 48159051de49b16b4d3b721baa076c9416dbfe75 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 30 Jun 2016 02:43:41 -0400
Subject: [PATCH 063/149] [CNTV] reduce code

---
 src/you_get/extractors/cntv.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index a32808cb..e25fa961 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -32,12 +32,11 @@ def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
         id = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
-    elif re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url):
-        html = get_content(url)
-        id = match1(html, r'guid = \"(.+)\"')
     elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
-         re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url): 
+         re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url) or \
+         re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or \
+         re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url): 
         id = r1(r'videoCenterId","(\w+)"', get_html(url))
     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
         id = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)

From 78ee9fffe32f4738264554b854c99a572b7fa4f4 Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Wed, 29 Jun 2016 20:18:18 +0800
Subject: [PATCH 064/149] iqiyi: support more stream quality

algorism form @ERioK
thank you @ERioK

v2. remove iqiyi_sc.py, and remove Unicode literal syntax
v3. indent mistake
v4. support vip streams
v5. combine stream profile by size
v6. support 4k profiles

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py    | 81 +++++++++++++++++++-------
 src/you_get/extractors/iqiyi_sc.py | 92 ------------------------------
 2 files changed, 60 insertions(+), 113 deletions(-)
 delete mode 100644 src/you_get/extractors/iqiyi_sc.py

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 320520fa..9761f3d1 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -8,11 +8,10 @@ import json
 from math import floor
 from zlib import decompress
 import hashlib
+from ..util import log
 
 import time
 
-from .iqiyi_sc import gen_sc
-
 '''
 Changelog:
 -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf
@@ -81,31 +80,36 @@ def getDispathKey(rid):
     t=str(int(floor(int(time)/(10*60.0))))
     return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest()
 '''
+def getVMS(tvid, vid):
+    t = int(time.time() * 1000)
+    src = '76f90cbd92f94a2e925d83e8ccd22cb7'
+    key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+    sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
+    vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
+    return json.loads(get_content(vmsreq))
 
 class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': 'high', 'container': 'mp4', 'video_profile': '高清'},
-        {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'},
+        {'id': '4k', 'container': 'm3u8', 'video_profile': '4k'},
+        {'id': 'BD', 'container': 'm3u8', 'video_profile': '1080p'},
+        {'id': 'TD', 'container': 'm3u8', 'video_profile': '720p'},
+        {'id': 'HD', 'container': 'm3u8', 'video_profile': '540p'},
+        {'id': 'SD', 'container': 'm3u8', 'video_profile': '360p'},
+        {'id': 'LD', 'container': 'm3u8', 'video_profile': '210p'},
     ]
-
+    '''
     supported_stream_types = [ 'high', 'standard']
 
 
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
+    '''
+    ids = ['4k','BD', 'TD', 'HD', 'SD', 'LD']
+    vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD', 2: 'HD', 4: 'TD', 17: 'TD', 96: 'LD', 1: 'SD'}
+    id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p'}
+
 
-    def getVMS(self,rate):
-        #tm ->the flash run time for md5 usage
-        #um -> vip 1 normal 0
-        #authkey -> for password protected video ,replace '' with your password
-        #puid user.passportid may empty?
-        #TODO: support password protected video
-        tvid, vid = self.vid
-        t = int(time.time() * 1000)
-        sc = gen_sc(tvid, t).decode('utf-8')
-        vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7)
-        return json.loads(get_content(vmsreq)[13:])
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -128,12 +132,47 @@ class Iqiyi(VideoExtractor):
                       r1(r'vid=([^&]+)', self.url) or \
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
+            self.title = match1(html, '<title>([^<]+)').split('-')[0]
+        tvid, videoid = self.vid
+        info = getVMS(tvid, videoid)
+        assert info['code'] == 'A00000', 'can\'t play this video'
 
-        for stream in self.supported_stream_types:
-            info = self.getVMS(self.stream_to_bid[stream])
-            if info["code"] == "A00000":
-                self.title = info['data']['playInfo']['vn']
-                self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])}
+        for stream in info['data']['vidl']:
+            try:
+                stream_id = self.vd_2_id[stream['vd']]
+                if stream_id in self.stream_types:
+                    continue
+                stream_profile = self.id_2_profile[stream_id]
+                self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            except:
+                log.i("vd: {} is not handled".format(stream['vd']))
+                log.i("info is {}".format(stream))
+        # why I need do below???
+        if not 'BD' in self.stream_types:
+            p1080_vids = []
+            if 18 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['18']['vid'])
+            if 5 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['5']['vid'])
+            for v in p1080_vids:
+                p1080_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    p1080_url = p1080_info['data']['m3u']
+                    self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
+                    break
+
+        if not '4k' in self.stream_types:
+            k4_vids = []
+            if 19 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['19']['vid'])
+            if 10 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['10']['vid'])
+            for v in k4_vids:
+                k4_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    k4_url = k4_info['data']['m3u']
+                    self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0}
+                    break
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
diff --git a/src/you_get/extractors/iqiyi_sc.py b/src/you_get/extractors/iqiyi_sc.py
deleted file mode 100644
index 4fa4ccdf..00000000
--- a/src/you_get/extractors/iqiyi_sc.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-import binascii
-import math
-import time
-
-M = [1732584193, -271733879]
-M.extend([~M[0], ~M[1]])
-I_table = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]
-C_base = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8388608, 432]
-
-
-def L(n, t):
-    if t is None:
-        t = 0
-    return trunc(((n >> 1) + (t >> 1) << 1) + (n & 1) + (t & 1))
-
-
-def rshift(val, n):
-    return val >> n if val >= 0 else (val+0x100000000) >> n
-
-
-def trunc(n):
-    n = n % 0x100000000
-    if n > 0x7fffffff:
-        n -= 0x100000000
-    return n
-
-
-def gen_sc(tvid, Z):
-    def transform(string, mod):
-        num = int(string, 16)
-        return (num >> 8 * (i % 4) & 255 ^ i % mod) << ((a & 3) << 3)
-
-    C = list(C_base)
-    o = list(M)
-    k = str(Z - 7)
-    for i in range(13):
-        a = i
-        C[a >> 2] |= ord(k[a]) << 8 * (a % 4)
-
-    for i in range(16):
-        a = i + 13
-        start = (i >> 2) * 8
-        r = '03967743b643f66763d623d637e30733'
-        C[a >> 2] |= transform(''.join(reversed(r[start:start + 8])), 7)
-
-    for i in range(16):
-        a = i + 29
-        start = (i >> 2) * 8
-        r = '7038766939776a32776a32706b337139'
-        C[a >> 2] |= transform(r[start:start + 8], 1)
-
-    for i in range(9):
-        a = i + 45
-        if i < len(tvid):
-            C[a >> 2] |= ord(tvid[i]) << 8 * (a % 4)
-
-    for a in range(64):
-        i = a
-        I = i >> 4
-        C_index = [i, 5 * i + 1, 3 * i + 5, 7 * i][I] % 16 + rshift(a, 6)
-        m = L(
-                L(
-                    o[0],
-                    [
-                        trunc(o[1] & o[2]) | trunc(~o[1] & o[3]),
-                        trunc(o[3] & o[1]) | trunc(~o[3] & o[2]),
-                        o[1] ^ o[2] ^ o[3],
-                        o[2] ^ trunc(o[1] | ~o[3])
-                    ][I]
-                ),
-                L(
-                    trunc(int(abs(math.sin(i + 1)) * 4294967296)),
-                    C[C_index] if C_index < len(C) else None
-                )
-            )
-        I = I_table[4 * I + i % 4]
-        o = [
-                o[3],
-                L(o[1], trunc(trunc(m << I) | rshift(m, 32 - I))),
-                o[1],
-                o[2],
-            ]
-
-    new_M = [L(o[0], M[0]), L(o[1], M[1]), L(o[2], M[2]), L(o[3], M[3])]
-    s = [new_M[a >> 3] >> (1 ^ a & 7) * 4 & 15 for a in range(32)]
-    return binascii.hexlify(bytes(s))[1::2]
-
-if __name__ == '__main__':
-    print(gen_sc("494496100", 1466495259194))
-    print(gen_sc("397768800", 1466795077775))
-    print(gen_sc("397768800", 1466796325746))

From 7452a4bb0e7aac32db89d871e7636741fddf641a Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 1 Jul 2016 00:38:26 -0400
Subject: [PATCH 065/149] [iQiyi] fix for PR conflict

---
 src/you_get/extractors/iqiyi.py | 78 +++++++++++++++++++++++----------
 1 file changed, 55 insertions(+), 23 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/iqiyi.py

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
old mode 100644
new mode 100755
index de8756bf..a1552fe4
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -8,6 +8,7 @@ import json
 from math import floor
 from zlib import decompress
 import hashlib
+from ..util import log
 
 import time
 
@@ -79,17 +80,24 @@ def getDispathKey(rid):
     t=str(int(floor(int(time)/(10*60.0))))
     return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest()
 '''
+def getVMS(tvid, vid):
+    t = int(time.time() * 1000)
+    src = '76f90cbd92f94a2e925d83e8ccd22cb7'
+    key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+    sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
+    vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
+    return json.loads(get_content(vmsreq))
 
 class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': 'BD', 'container': 'm3u8', 'video_profile': '全高清'},
-        {'id': 'FD', 'container': 'm3u8', 'video_profile': '超高清'},
-        {'id': 'TD', 'container': 'm3u8', 'video_profile': '超清'},
-        {'id': 'HD', 'container': 'm3u8', 'video_profile': '高清'},
-        {'id': 'SD', 'container': 'm3u8', 'video_profile': '标清'},
-        {'id': 'LD', 'container': 'm3u8', 'video_profile': '流畅'},
+        {'id': '4k', 'container': 'm3u8', 'video_profile': '4k'},
+        {'id': 'BD', 'container': 'm3u8', 'video_profile': '1080p'},
+        {'id': 'TD', 'container': 'm3u8', 'video_profile': '720p'},
+        {'id': 'HD', 'container': 'm3u8', 'video_profile': '540p'},
+        {'id': 'SD', 'container': 'm3u8', 'video_profile': '360p'},
+        {'id': 'LD', 'container': 'm3u8', 'video_profile': '210p'},
     ]
     '''
     supported_stream_types = [ 'high', 'standard']
@@ -97,18 +105,11 @@ class Iqiyi(VideoExtractor):
 
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
     '''
-    ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD']
-    vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'}
-    vd_2_profile = {21: '超清', 2: '高清', 4: '超高清', 17: '全高清', 96: '流畅', 1: '标清'}
+    ids = ['4k','BD', 'TD', 'HD', 'SD', 'LD']
+    vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD', 2: 'HD', 4: 'TD', 17: 'TD', 96: 'LD', 1: 'SD'}
+    id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p'}
+
 
-    def getVMS(self):
-        tvid, vid = self.vid
-        t = int(time.time() * 1000)
-        src = '76f90cbd92f94a2e925d83e8ccd22cb7'
-        key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
-        sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
-        vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
-        return json.loads(get_content(vmsreq))
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -132,15 +133,46 @@ class Iqiyi(VideoExtractor):
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
             self.title = match1(html, '<title>([^<]+)').split('-')[0]
-
-        info = self.getVMS()
+        tvid, videoid = self.vid
+        info = getVMS(tvid, videoid)
         assert info['code'] == 'A00000', 'can\'t play this video'
 
         for stream in info['data']['vidl']:
-            stream_id = self.vd_2_id[stream['vd']]
-            stream_profile = self.vd_2_profile[stream['vd']]
-            self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            try:
+                stream_id = self.vd_2_id[stream['vd']]
+                if stream_id in self.stream_types:
+                    continue
+                stream_profile = self.id_2_profile[stream_id]
+                self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            except:
+                log.i("vd: {} is not handled".format(stream['vd']))
+                log.i("info is {}".format(stream))
+        # why I need do below???
+        if not 'BD' in self.stream_types:
+            p1080_vids = []
+            if 18 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['18']['vid'])
+            if 5 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['5']['vid'])
+            for v in p1080_vids:
+                p1080_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    p1080_url = p1080_info['data']['m3u']
+                    self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
+                    break
 
+        if not '4k' in self.stream_types:
+            k4_vids = []
+            if 19 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['19']['vid'])
+            if 10 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['10']['vid'])
+            for v in k4_vids:
+                k4_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    k4_url = k4_info['data']['m3u']
+                    self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0}
+                    break
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
@@ -214,4 +246,4 @@ class Iqiyi(VideoExtractor):
 site = Iqiyi()
 download = site.download_by_url
 iqiyi_download_by_vid = site.download_by_vid
-download_playlist = site.download_playlist_by_url
+download_playlist = site.download_playlist_by_url
\ No newline at end of file

From 5a99b2f42f1157c6b51be768d69fb771b8761405 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 1 Jul 2016 01:07:32 -0400
Subject: [PATCH 066/149] [Showroom] Add support via HLS M3U

---
 src/you_get/common.py              |  3 +-
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/showroom.py | 67 ++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100755 src/you_get/extractors/showroom.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 99ce14c7..c8843ddf 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -62,6 +62,7 @@ SITES = {
     'pptv'             : 'pptv',
     'qianmo'           : 'qianmo',
     'qq'               : 'qq',
+    'showroom-live'    : 'showroom',
     'sina'             : 'sina',
     'smgbb'            : 'bilibili',
     'sohu'             : 'sohu',
@@ -912,7 +913,7 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
         ffmpeg_play_stream(player, url, params)
         return
 
-    from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_streaming
+    from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
     assert has_ffmpeg_installed(), "FFmpeg not installed."
     ffmpeg_download_stream(url, title, ext, params, output_dir)
 
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 20a7f7cf..b879b83e 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -55,6 +55,7 @@ from .pptv import *
 from .qianmo import *
 from .qie import *
 from .qq import *
+from .showroom import *
 from .sina import *
 from .sohu import *
 from .soundcloud import *
diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
new file mode 100755
index 00000000..43d7b192
--- /dev/null
+++ b/src/you_get/extractors/showroom.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+__all__ = ['showroom_download']
+
+from ..common import *
+import urllib.error
+from json import loads
+from time import time
+
+#----------------------------------------------------------------------
+def showroom_get_roomid_by_room_url_key(room_url_key):
+    """str->str"""
+    fake_headers_mobile = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
+    }
+    webpage_url = 'https://www.showroom-live.com/' + room_url_key
+    html = get_content(webpage_url, headers = fake_headers_mobile)
+    roomid = match1(html, r'room\?room_id\=(\d+)')
+    assert roomid
+    return roomid
+
+def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_only = False, **kwargs):
+    '''Source: Android mobile'''
+    timestamp = str(int(time() * 1000))
+    api_endpoint = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}&_={timestamp}'.format(room_id = room_id, timestamp = timestamp)
+    html = get_content(api_endpoint)
+    html = json.loads(html)
+    #{'streaming_url_list': [{'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 1, 'label': 'original spec(low latency)', 'is_default': True, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed/playlist.m3u8', 'is_default': True, 'id': 2, 'type': 'hls', 'label': 'original spec'}, {'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 3, 'label': 'low spec(low latency)', 'is_default': False, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low/playlist.m3u8', 'is_default': False, 'id': 4, 'type': 'hls', 'label': 'low spec'}]}
+    if len(html) < 1:
+        log.wtf('Cannot find any live URL! Maybe the live have ended or haven\'t start yet?')
+        
+    #This is mainly for testing the M3U FFmpeg parser so I would ignore any non-m3u ones
+    stream_url = [i['url'] for i in html['streaming_url_list'] if i['is_default'] and i['type'] == 'hls'][0]
+    
+    assert stream_url
+    
+    #title
+    title = ''
+    profile_api = 'https://www.showroom-live.com/api/room/profile?room_id={room_id}'.format(room_id = room_id)
+    html = loads(get_content(profile_api))
+    try:
+        title = html['main_name']
+    except KeyError:
+        title = 'Showroom_{room_id}'.format(room_id = room_id)
+    
+    type_, ext, size = url_info(stream_url)
+    print_info(site_info, title, type_, size)
+    if not info_only:
+        download_url_ffmpeg(url=stream_url, title=title, ext= 'mp4', output_dir=output_dir)
+    
+
+#----------------------------------------------------------------------
+def showroom_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    """"""
+    if re.match( r'(\w+)://www.showroom-live.com/(\w*)', url):
+        room_url_key = match1(url, r'\w+://www.showroom-live.com/(\w*)')
+        room_id = showroom_get_roomid_by_room_url_key(room_url_key)
+        showroom_download_by_room_id(room_id, output_dir, merge, 
+                                    info_only)
+
+site_info = "Showroom"
+download = showroom_download
+download_playlist = playlist_not_supported('showroom')
\ No newline at end of file

From bbe1376a8670060f51803ed11d0027a3ab9792da Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 15:35:02 +0200
Subject: [PATCH 067/149] [showroom] support room URLs with dashes (-)

---
 src/you_get/extractors/showroom.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
index 43d7b192..d0f636bc 100755
--- a/src/you_get/extractors/showroom.py
+++ b/src/you_get/extractors/showroom.py
@@ -56,8 +56,8 @@ def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_
 #----------------------------------------------------------------------
 def showroom_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     """"""
-    if re.match( r'(\w+)://www.showroom-live.com/(\w*)', url):
-        room_url_key = match1(url, r'\w+://www.showroom-live.com/(\w*)')
+    if re.match( r'(\w+)://www.showroom-live.com/([-\w]+)', url):
+        room_url_key = match1(url, r'\w+://www.showroom-live.com/([-\w]+)')
         room_id = showroom_get_roomid_by_room_url_key(room_url_key)
         showroom_download_by_room_id(room_id, output_dir, merge, 
                                     info_only)

From 7516028dd860d642bb1620fef0d78ed6a06153d8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 15:58:57 +0200
Subject: [PATCH 068/149] =?UTF-8?q?extractors:=20revert=20permissions=20(1?=
 =?UTF-8?q?00755=20=E2=86=92=20100644)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/__init__.py | 0
 src/you_get/extractors/acfun.py    | 0
 src/you_get/extractors/baidu.py    | 0
 src/you_get/extractors/baomihua.py | 0
 src/you_get/extractors/dilidili.py | 0
 src/you_get/extractors/funshion.py | 0
 src/you_get/extractors/iqiyi.py    | 0
 src/you_get/extractors/khan.py     | 0
 src/you_get/extractors/miomio.py   | 0
 src/you_get/extractors/showroom.py | 0
 src/you_get/extractors/yixia.py    | 0
 11 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/you_get/extractors/__init__.py
 mode change 100755 => 100644 src/you_get/extractors/acfun.py
 mode change 100755 => 100644 src/you_get/extractors/baidu.py
 mode change 100755 => 100644 src/you_get/extractors/baomihua.py
 mode change 100755 => 100644 src/you_get/extractors/dilidili.py
 mode change 100755 => 100644 src/you_get/extractors/funshion.py
 mode change 100755 => 100644 src/you_get/extractors/iqiyi.py
 mode change 100755 => 100644 src/you_get/extractors/khan.py
 mode change 100755 => 100644 src/you_get/extractors/miomio.py
 mode change 100755 => 100644 src/you_get/extractors/showroom.py
 mode change 100755 => 100644 src/you_get/extractors/yixia.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/baomihua.py b/src/you_get/extractors/baomihua.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/funshion.py b/src/you_get/extractors/funshion.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/khan.py b/src/you_get/extractors/khan.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/miomio.py b/src/you_get/extractors/miomio.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
old mode 100755
new mode 100644

From e84810c4d2c1242bbe14265919f8627b9c1e7e54 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 16:42:04 +0200
Subject: [PATCH 069/149] [facebook] use hd_src whenever possible and remove
 duplicates

---
 src/you_get/extractors/facebook.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 72ce8f31..2a96fcb0 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -9,17 +9,22 @@ def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     html = get_html(url)
 
     title = r1(r'<title id="pageTitle">(.+)</title>', html)
-    sd_urls = [
+    sd_urls = list(set([
         unicodize(str.replace(i, '\\/', '/'))
         for i in re.findall(r'"sd_src_no_ratelimit":"([^"]*)"', html)
-    ]
+    ]))
+    hd_urls = list(set([
+        unicodize(str.replace(i, '\\/', '/'))
+        for i in re.findall(r'"hd_src_no_ratelimit":"([^"]*)"', html)
+    ]))
+    urls = hd_urls if hd_urls else sd_urls
 
-    type, ext, size = url_info(sd_urls[0], True)
-    size = urls_size(sd_urls)
+    type, ext, size = url_info(urls[0], True)
+    size = urls_size(urls)
 
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls(sd_urls, title, ext, size, output_dir, merge=False)
+        download_urls(urls, title, ext, size, output_dir, merge=False)
 
 site_info = "Facebook.com"
 download = facebook_download

From 17386968c7315347ef7384f1a895afbf44d0b6fe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 20:28:03 +0200
Subject: [PATCH 070/149] [tumblr] fix #1232

---
 src/you_get/extractors/tumblr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index fea061ce..0c59f25a 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -68,7 +68,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         real_url = r1(r'<source src="([^"]*)"', html)
     if not real_url:
         iframe_url = r1(r'<[^>]+tumblr_video_container[^>]+><iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
-        if len(iframe_url) > 0:
+        if iframe_url:
             iframe_html = get_content(iframe_url, headers=fake_headers)
             real_url = r1(r'<video[^>]*>[\n ]*<source[^>]+src=[\'"]([^\'"]*)[\'"]', iframe_html)
         else:

From a4ed7859661e8bc9d7c9eab1a0c04eaf8fc7836d Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Sat, 2 Jul 2016 06:54:35 +0800
Subject: [PATCH 071/149] iqiyi: fix when no vip info

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index a1552fe4..dc9b05fd 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -148,12 +148,17 @@ class Iqiyi(VideoExtractor):
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
         # why I need do below???
-        if not 'BD' in self.stream_types:
+        try:
+            vip_vds = info['data']['ctl']['vip']['bids']
+            vip_conf = info['data']['ctl']['configs']
+        except:
+            return
+        if not 'BD' in self.streams.keys():
             p1080_vids = []
-            if 18 in info['data']['ctl']['vip']['bids']:
-                p1080_vids.append(info['data']['ctl']['configs']['18']['vid'])
-            if 5 in info['data']['ctl']['vip']['bids']:
-                p1080_vids.append(info['data']['ctl']['configs']['5']['vid'])
+            if 18 in vip_vds:
+                p1080_vids.append(vip_conf['18']['vid'])
+            if 5 in vip_vds:
+                p1080_vids.append(vip_conf['5']['vid'])
             for v in p1080_vids:
                 p1080_info = getVMS(tvid, v)
                 if info['code'] == 'A00000':
@@ -161,12 +166,12 @@ class Iqiyi(VideoExtractor):
                     self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
                     break
 
-        if not '4k' in self.stream_types:
+        if not '4k' in self.streams.keys():
             k4_vids = []
-            if 19 in info['data']['ctl']['vip']['bids']:
-                k4_vids.append(info['data']['ctl']['configs']['19']['vid'])
-            if 10 in info['data']['ctl']['vip']['bids']:
-                k4_vids.append(info['data']['ctl']['configs']['10']['vid'])
+            if 19 in vip_vds:
+                k4_vids.append(vip_conf['19']['vid'])
+            if 10 in vip_vds:
+                k4_vids.append(vip_conf['10']['vid'])
             for v in k4_vids:
                 k4_info = getVMS(tvid, v)
                 if info['code'] == 'A00000':
@@ -246,4 +251,4 @@ class Iqiyi(VideoExtractor):
 site = Iqiyi()
 download = site.download_by_url
 iqiyi_download_by_vid = site.download_by_vid
-download_playlist = site.download_playlist_by_url
\ No newline at end of file
+download_playlist = site.download_playlist_by_url

From 3b2b82c2cb6e56cc59228818cf8c2879bdf2e6a9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Jul 2016 14:04:04 +0200
Subject: [PATCH 072/149] [iqiyi] remove invalid VIP profiles

See: https://github.com/soimort/you-get/commit/7452a4bb0e7aac32db89d871e7636741fddf641a#commitcomment-18107708
---
 src/you_get/extractors/iqiyi.py | 30 ------------------------------
 1 file changed, 30 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index dc9b05fd..e9ee5afb 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -147,37 +147,7 @@ class Iqiyi(VideoExtractor):
             except:
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
-        # why I need do below???
-        try:
-            vip_vds = info['data']['ctl']['vip']['bids']
-            vip_conf = info['data']['ctl']['configs']
-        except:
-            return
-        if not 'BD' in self.streams.keys():
-            p1080_vids = []
-            if 18 in vip_vds:
-                p1080_vids.append(vip_conf['18']['vid'])
-            if 5 in vip_vds:
-                p1080_vids.append(vip_conf['5']['vid'])
-            for v in p1080_vids:
-                p1080_info = getVMS(tvid, v)
-                if info['code'] == 'A00000':
-                    p1080_url = p1080_info['data']['m3u']
-                    self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
-                    break
 
-        if not '4k' in self.streams.keys():
-            k4_vids = []
-            if 19 in vip_vds:
-                k4_vids.append(vip_conf['19']['vid'])
-            if 10 in vip_vds:
-                k4_vids.append(vip_conf['10']['vid'])
-            for v in k4_vids:
-                k4_info = getVMS(tvid, v)
-                if info['code'] == 'A00000':
-                    k4_url = k4_info['data']['m3u']
-                    self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0}
-                    break
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")

From 04fe6cd5a406bb0767314d3a66cdb75b479c050d Mon Sep 17 00:00:00 2001
From: misha shelemetyev <misha.shelemetyev@onforce.com>
Date: Thu, 7 Jul 2016 16:57:38 -0400
Subject: [PATCH 073/149] you get vk photos

---
 src/you_get/extractors/vk.py | 38 +++++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index c83dc48e..6ad6624d 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -4,7 +4,8 @@ __all__ = ['vk_download']
 
 from ..common import *
 
-def vk_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+
+def get_video_info(url):
     video_page = get_content(url)
     title = unescape_html(r1(r'"title":"([^"]+)"', video_page))
     info = dict(re.findall(r'\\"url(\d+)\\":\\"([^"]+)\\"', video_page))
@@ -13,12 +14,39 @@ def vk_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             url = re.sub(r'\\\\\\/', r'/', info[quality])
             break
     assert url
-
     type, ext, size = url_info(url)
-
     print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge=merge)
+
+    return url, title, ext, size
+
+
+def get_image_info(url):
+    image_page = get_content(url)
+    # used for title - vk page owner
+    page_of = re.findall(r'Sender:</dt><dd><a href=.*>(.[^>]+?)</a', image_page)
+    # used for title - date when photo was uploaded
+    photo_date = re.findall(r'<span class="item_date">(.[^>]+?)</span', image_page)
+
+    title = (' ').join(page_of + photo_date)
+    image_link = r1(r'href="([^"]+)" class=\"mva_item\" target="_blank">Download full size', image_page)
+    type, ext, size = url_info(image_link)
+    print_info(site_info, title, type, size)
+
+    return image_link, title, ext, size
+
+
+def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
+    link = None
+    if re.match(r'vk.com/photo', url):
+        link, title, ext, size = get_video_info(url)
+    elif re.match(r'(.+)vk\.com\/photo(.+)', url):
+        link, title, ext, size = get_image_info(url)
+    else:
+        raise NotImplementedError('Nothing to download here')
+
+    if not info_only and link is not None:
+        download_urls([link], title, ext, size, output_dir, merge=merge)
+
 
 site_info = "VK.com"
 download = vk_download

From 3aa73fc1816e6f76b007599730f2ec5315896370 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 7 Jul 2016 20:09:56 -0400
Subject: [PATCH 074/149] [iQiyi] Use FFmpeg to record M3U file

---
 src/you_get/extractors/iqiyi.py | 65 +++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index e9ee5afb..248446a7 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -147,6 +147,71 @@ class Iqiyi(VideoExtractor):
             except:
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
+    
+
+    def download(self, **kwargs):
+        """Override the original one
+        Ugly ugly dirty hack"""
+        if 'json_output' in kwargs and kwargs['json_output']:
+            json_output.output(self)
+        elif 'info_only' in kwargs and kwargs['info_only']:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Display the stream
+                stream_id = kwargs['stream_id']
+                if 'index' not in kwargs:
+                    self.p(stream_id)
+                else:
+                    self.p_i(stream_id)
+            else:
+                # Display all available streams
+                if 'index' not in kwargs:
+                    self.p([])
+                else:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                    self.p_i(stream_id)
+
+        else:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Download the stream
+                stream_id = kwargs['stream_id']
+            else:
+                # Download stream with the best quality
+                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+
+            if 'index' not in kwargs:
+                self.p(stream_id)
+            else:
+                self.p_i(stream_id)
+
+            if stream_id in self.streams:
+                urls = self.streams[stream_id]['src']
+                ext = self.streams[stream_id]['container']
+                total_size = self.streams[stream_id]['size']
+            else:
+                urls = self.dash_streams[stream_id]['src']
+                ext = self.dash_streams[stream_id]['container']
+                total_size = self.dash_streams[stream_id]['size']
+
+            if not urls:
+                log.wtf('[Failed] Cannot extract video source.')
+            # For legacy main()
+            
+            #Here's the change!!
+            download_url_ffmpeg(urls[0], self.title, 'mp4',
+                          output_dir=kwargs['output_dir'],
+                          merge=kwargs['merge'],)
+
+            if not kwargs['caption']:
+                print('Skipping captions.')
+                return
+            for lang in self.caption_tracks:
+                filename = '%s.%s.srt' % (get_filename(self.title), lang)
+                print('Saving %s ... ' % filename, end="", flush=True)
+                srt = self.caption_tracks[lang]
+                with open(os.path.join(kwargs['output_dir'], filename),
+                          'w', encoding='utf-8') as x:
+                    x.write(srt)
+                print('Done.')    
 
 '''
         if info["code"] != "A000000":

From 13b63aa7733f713da01b63c958e204744ee71328 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Jul 2016 19:12:03 +0200
Subject: [PATCH 075/149] version 0.4.486

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 61e75ead..46850c8a 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.455'
+__version__ = '0.4.486'

From 50c911507b6d32522d2856cff6cf4460bd9893f6 Mon Sep 17 00:00:00 2001
From: misha shelemetyev <misha.shelemetyev@onforce.com>
Date: Fri, 8 Jul 2016 15:12:21 -0400
Subject: [PATCH 076/149] Lets fix vk videos

---
 README.md                    |  2 +-
 src/you_get/extractors/vk.py | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index aa7f61a0..d3d44b33 100644
--- a/README.md
+++ b/README.md
@@ -318,7 +318,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | :--: | :-- | :-----: | :-----: | :-----: |
 | **YouTube** | <https://www.youtube.com/>    |✓| | |
 | **Twitter** | <https://twitter.com/>        |✓|✓| |
-| VK          | <http://vk.com/>              |✓| | |
+| VK          | <http://vk.com/>              |✓|✓| |
 | Vine        | <https://vine.co/>            |✓| | |
 | Vimeo       | <https://vimeo.com/>          |✓| | |
 | Vidto       | <http://vidto.me/>            |✓| | |
diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index 6ad6624d..ea3e3851 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -7,12 +7,14 @@ from ..common import *
 
 def get_video_info(url):
     video_page = get_content(url)
-    title = unescape_html(r1(r'"title":"([^"]+)"', video_page))
-    info = dict(re.findall(r'\\"url(\d+)\\":\\"([^"]+)\\"', video_page))
+    title = r1(r'<div class="vv_summary">(.[^>]+?)</div', video_page)
+    sources = re.findall(r'<source src=\"(.[^>]+?)"', video_page)
+
     for quality in ['1080', '720', '480', '360', '240']:
-        if quality in info:
-            url = re.sub(r'\\\\\\/', r'/', info[quality])
-            break
+        for source in sources:
+            if source.find(quality) != -1:
+                url = source
+                break
     assert url
     type, ext, size = url_info(url)
     print_info(site_info, title, type, size)
@@ -37,7 +39,7 @@ def get_image_info(url):
 
 def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
     link = None
-    if re.match(r'vk.com/photo', url):
+    if re.match(r'(.+)z\=video(.+)', url):
         link, title, ext, size = get_video_info(url)
     elif re.match(r'(.+)vk\.com\/photo(.+)', url):
         link, title, ext, size = get_image_info(url)

From 4793bc30d03bf104458cadadbed30ed30f96901a Mon Sep 17 00:00:00 2001
From: misha shelemetyev <misha.shelemetyev@onforce.com>
Date: Fri, 8 Jul 2016 15:35:07 -0400
Subject: [PATCH 077/149] 1248 surround quality by dots to avoid issues with
 having q value in uri

---
 src/you_get/extractors/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index ea3e3851..98f3471b 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -10,7 +10,7 @@ def get_video_info(url):
     title = r1(r'<div class="vv_summary">(.[^>]+?)</div', video_page)
     sources = re.findall(r'<source src=\"(.[^>]+?)"', video_page)
 
-    for quality in ['1080', '720', '480', '360', '240']:
+    for quality in ['.1080.', '.720.', '.480.', '.360.', '.240.']:
         for source in sources:
             if source.find(quality) != -1:
                 url = source

From 01cc77c96acf6f961111732d345d0ad23eaa3d40 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 10 Jul 2016 07:50:51 +0200
Subject: [PATCH 078/149] [showroom] do polling on offline broadcasts

---
 src/you_get/extractors/showroom.py | 33 ++++++++++++++++--------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
index d0f636bc..606dc806 100644
--- a/src/you_get/extractors/showroom.py
+++ b/src/you_get/extractors/showroom.py
@@ -5,7 +5,7 @@ __all__ = ['showroom_download']
 from ..common import *
 import urllib.error
 from json import loads
-from time import time
+from time import time, sleep
 
 #----------------------------------------------------------------------
 def showroom_get_roomid_by_room_url_key(room_url_key):
@@ -25,19 +25,22 @@ def showroom_get_roomid_by_room_url_key(room_url_key):
 
 def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
-    timestamp = str(int(time() * 1000))
-    api_endpoint = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}&_={timestamp}'.format(room_id = room_id, timestamp = timestamp)
-    html = get_content(api_endpoint)
-    html = json.loads(html)
-    #{'streaming_url_list': [{'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 1, 'label': 'original spec(low latency)', 'is_default': True, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed/playlist.m3u8', 'is_default': True, 'id': 2, 'type': 'hls', 'label': 'original spec'}, {'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 3, 'label': 'low spec(low latency)', 'is_default': False, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low/playlist.m3u8', 'is_default': False, 'id': 4, 'type': 'hls', 'label': 'low spec'}]}
-    if len(html) < 1:
-        log.wtf('Cannot find any live URL! Maybe the live have ended or haven\'t start yet?')
-        
+    while True:
+        timestamp = str(int(time() * 1000))
+        api_endpoint = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}&_={timestamp}'.format(room_id = room_id, timestamp = timestamp)
+        html = get_content(api_endpoint)
+        html = json.loads(html)
+        #{'streaming_url_list': [{'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 1, 'label': 'original spec(low latency)', 'is_default': True, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed/playlist.m3u8', 'is_default': True, 'id': 2, 'type': 'hls', 'label': 'original spec'}, {'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 3, 'label': 'low spec(low latency)', 'is_default': False, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low/playlist.m3u8', 'is_default': False, 'id': 4, 'type': 'hls', 'label': 'low spec'}]}
+        if len(html) >= 1:
+            break
+        log.w('The live show is currently offline.')
+        sleep(1)
+
     #This is mainly for testing the M3U FFmpeg parser so I would ignore any non-m3u ones
     stream_url = [i['url'] for i in html['streaming_url_list'] if i['is_default'] and i['type'] == 'hls'][0]
-    
+
     assert stream_url
-    
+
     #title
     title = ''
     profile_api = 'https://www.showroom-live.com/api/room/profile?room_id={room_id}'.format(room_id = room_id)
@@ -46,12 +49,12 @@ def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_
         title = html['main_name']
     except KeyError:
         title = 'Showroom_{room_id}'.format(room_id = room_id)
-    
+
     type_, ext, size = url_info(stream_url)
     print_info(site_info, title, type_, size)
     if not info_only:
         download_url_ffmpeg(url=stream_url, title=title, ext= 'mp4', output_dir=output_dir)
-    
+
 
 #----------------------------------------------------------------------
 def showroom_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
@@ -59,9 +62,9 @@ def showroom_download(url, output_dir = '.', merge = False, info_only = False, *
     if re.match( r'(\w+)://www.showroom-live.com/([-\w]+)', url):
         room_url_key = match1(url, r'\w+://www.showroom-live.com/([-\w]+)')
         room_id = showroom_get_roomid_by_room_url_key(room_url_key)
-        showroom_download_by_room_id(room_id, output_dir, merge, 
+        showroom_download_by_room_id(room_id, output_dir, merge,
                                     info_only)
 
 site_info = "Showroom"
 download = showroom_download
-download_playlist = playlist_not_supported('showroom')
\ No newline at end of file
+download_playlist = playlist_not_supported('showroom')

From e5c606503cbeb67656155695155da15c91f2d7fe Mon Sep 17 00:00:00 2001
From: Cheng Zhang <13501393281@163.com>
Date: Sun, 10 Jul 2016 15:48:34 -0400
Subject: [PATCH 079/149] add the option to use choco to install on windows

---
 README.md | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index aa7f61a0..73af5dc3 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ Are you a Python programmer? Then check out [the source](https://github.com/soim
 
 ### Prerequisites
 
-The following dependencies are required and must be installed separately, unless you are using a pre-built package on Windows:
+The following dependencies are required and must be installed separately, unless you are using a pre-built package or chocolatey on Windows:
 
 * **[Python 3](https://www.python.org/downloads/)**
 * **[FFmpeg](https://www.ffmpeg.org/)** (strongly recommended) or [Libav](https://libav.org/)
@@ -61,11 +61,19 @@ Add the following line to your `.zshrc`:
 
     antigen bundle soimort/you-get
 
-### Option 3: Use a pre-built package (Windows only)
+### Option 3: Using [Chocolatey](https://chocolatey.org/) (Windows only)
+
+    choco install you-get
+
+The chocolatey package source can be found [here](https://github.com/chantisnake/you-get-choco)
+
+The chocolatey package page can be found [here](https://chocolatey.org/packages/you-get/0.4.486)
+
+### Option 4: Use a pre-built package (Windows only)
 
 Download the `exe` (standalone) or `7z` (all dependencies included) from: <https://github.com/soimort/you-get/releases/latest>.
 
-### Option 4: Download from GitHub
+### Option 5: Download from GitHub
 
 You may either download the [stable](https://github.com/soimort/you-get/archive/master.zip) (identical with the latest release on PyPI) or the [develop](https://github.com/soimort/you-get/archive/develop.zip) (more hotfixes, unstable features) branch of `you-get`. Unzip it, and put the directory containing the `you-get` script into your `PATH`.
 
@@ -83,7 +91,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option 5: Git clone
+### Option6: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 
@@ -93,7 +101,7 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
-### Option 6: Homebrew (Mac only)
+### Option 7: Homebrew (Mac only)
 
 You can install `you-get` easily via:
 
@@ -119,6 +127,12 @@ or download the latest release via:
 $ you-get https://github.com/soimort/you-get/archive/master.zip
 ```
 
+or use [chocolatey package manager](https://chocolatey.org):
+
+```
+> choco upgrade you-get 
+```
+
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
 
 ```

From 06252f0b6ee151944bdd8e29e5feea71d88328c4 Mon Sep 17 00:00:00 2001
From: Cheng Zhang <13501393281@163.com>
Date: Sun, 10 Jul 2016 15:51:33 -0400
Subject: [PATCH 080/149] fix an unfortunate typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 73af5dc3..ff85a69d 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option6: Git clone
+### Option 6: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 

From c43bb6006dabf5e7d2f22bb96fff5058b765dc01 Mon Sep 17 00:00:00 2001
From: Cheng Zhang <13501393281@163.com>
Date: Sun, 10 Jul 2016 20:02:41 -0400
Subject: [PATCH 081/149] fix the order

let choco option more close to home brew since they are alike
---
 README.md | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index ff85a69d..8afc84ea 100644
--- a/README.md
+++ b/README.md
@@ -61,19 +61,11 @@ Add the following line to your `.zshrc`:
 
     antigen bundle soimort/you-get
 
-### Option 3: Using [Chocolatey](https://chocolatey.org/) (Windows only)
-
-    choco install you-get
-
-The chocolatey package source can be found [here](https://github.com/chantisnake/you-get-choco)
-
-The chocolatey package page can be found [here](https://chocolatey.org/packages/you-get/0.4.486)
-
-### Option 4: Use a pre-built package (Windows only)
+### Option 3: Use a pre-built package (Windows only)
 
 Download the `exe` (standalone) or `7z` (all dependencies included) from: <https://github.com/soimort/you-get/releases/latest>.
 
-### Option 5: Download from GitHub
+### Option 4: Download from GitHub
 
 You may either download the [stable](https://github.com/soimort/you-get/archive/master.zip) (identical with the latest release on PyPI) or the [develop](https://github.com/soimort/you-get/archive/develop.zip) (more hotfixes, unstable features) branch of `you-get`. Unzip it, and put the directory containing the `you-get` script into your `PATH`.
 
@@ -91,7 +83,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option 6: Git clone
+### Option 5: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 
@@ -101,6 +93,12 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
+### Option 6: Using [Chocolatey](https://chocolatey.org/) (Windows only)
+
+```
+> choco install you-get
+```
+
 ### Option 7: Homebrew (Mac only)
 
 You can install `you-get` easily via:

From d5e801b7c109c7f5b7166947d01ce7630f236cc1 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Sun, 17 Jul 2016 02:24:30 -0400
Subject: [PATCH 082/149] [iQiyi]not import json_output module

From: @wwqgtxx , #1282, as requested.
---
 src/you_get/extractors/iqiyi.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 248446a7..80fa5f93 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -2,14 +2,15 @@
 
 from ..common import *
 from ..extractor import VideoExtractor
+from ..util import log
+from .. import json_output
+
 from uuid import uuid4
 from random import random,randint
 import json
 from math import floor
 from zlib import decompress
 import hashlib
-from ..util import log
-
 import time
 
 '''

From b95b1a10ee01299358fd23e9ef85e1738cdfacd2 Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Wed, 20 Jul 2016 12:04:15 +0800
Subject: [PATCH 083/149] fix playlist download

---
 src/you_get/extractors/youku.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index fefaf5ee..e04a51fc 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -76,7 +76,7 @@ class Youku(VideoExtractor):
         for x in xs:
             if x not in mem:
                 mem.add(x)
-                yield(x)
+        return mem
 
     def get_vid_from_url(url):
         """Extracts video ID from URL.
@@ -89,7 +89,7 @@ class Youku(VideoExtractor):
     def get_playlist_id_from_url(url):
         """Extracts playlist ID from URL.
         """
-        return match1(url, r'youku\.com/playlist_show/id_([a-zA-Z0-9=]+)')
+        return match1(url, r'youku\.com/albumlist/show\?id=([a-zA-Z0-9=]+)')
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -97,16 +97,19 @@ class Youku(VideoExtractor):
         try:
             playlist_id = self.__class__.get_playlist_id_from_url(self.url)
             assert playlist_id
-
-            video_page = get_content('http://www.youku.com/playlist_show/id_%s' % playlist_id)
+            video_page = get_content('http://list.youku.com/albumlist/show?id=%s' % playlist_id)
             videos = Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', video_page))
-
             # Parse multi-page playlists
-            for extra_page_url in Youku.oset(re.findall('href="(http://www\.youku\.com/playlist_show/id_%s_[^?"]+)' % playlist_id, video_page)):
-                extra_page = get_content(extra_page_url)
-                videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
-
-        except:
+            last_page_url = re.findall(r'href="(/albumlist/show\?id=%s[^"]+)" title="末页"' % playlist_id, video_page)[0]
+            num_pages = int(re.findall(r'page=([0-9]+)\.htm', last_page_url)[0])
+            if (num_pages > 0):
+                # download one by one
+                for pn in range(1, num_pages + 1):
+                    extra_page_url = re.sub(r'page=([0-9]+)\.htm', r'page=%s.htm' % pn, last_page_url)
+                    extra_page = get_content('http://list.youku.com' + extra_page_url)
+                    videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
+        except Exception as e:
+            print(e)
             # Show full list of episodes
             if match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)'):
                 ep_id = match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)')

From cb2878b8cfd974605112cc6c4fff2c405c92f39f Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Wed, 20 Jul 2016 12:09:20 +0800
Subject: [PATCH 084/149] avoid unnecessary downloading page 1

---
 src/you_get/extractors/youku.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index e04a51fc..345347d0 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -104,12 +104,11 @@ class Youku(VideoExtractor):
             num_pages = int(re.findall(r'page=([0-9]+)\.htm', last_page_url)[0])
             if (num_pages > 0):
                 # download one by one
-                for pn in range(1, num_pages + 1):
+                for pn in range(2, num_pages + 1):
                     extra_page_url = re.sub(r'page=([0-9]+)\.htm', r'page=%s.htm' % pn, last_page_url)
                     extra_page = get_content('http://list.youku.com' + extra_page_url)
                     videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
-        except Exception as e:
-            print(e)
+        except:
             # Show full list of episodes
             if match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)'):
                 ep_id = match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)')

From fdb1809455247746e76aa164f08485bc21de7426 Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Wed, 20 Jul 2016 15:18:28 +0800
Subject: [PATCH 085/149] fix socks DNS resolve issue

---
 src/you_get/common.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c8843ddf..a9b12b08 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1214,15 +1214,18 @@ def script_main(script_name, download, download_playlist, **kwargs):
 
     if (socks_proxy):
         try:
-          import socket
-          import socks
-          socks_proxy_addrs = socks_proxy.split(':')
-          socks.set_default_proxy(socks.SOCKS5, 
-                                  socks_proxy_addrs[0], 
-                                  int(socks_proxy_addrs[1]))
-          socket.socket = socks.socksocket
+            import socket
+            import socks
+            socks_proxy_addrs = socks_proxy.split(':')
+            socks.set_default_proxy(socks.SOCKS5, 
+                                    socks_proxy_addrs[0], 
+                                    int(socks_proxy_addrs[1]))
+            socket.socket = socks.socksocket
+            def getaddrinfo(*args):
+                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+            socket.getaddrinfo = getaddrinfo
         except ImportError:
-          log.w('Error importing PySocks library, socks proxy ignored.'
+            log.w('Error importing PySocks library, socks proxy ignored.'
                 'In order to use use socks proxy, please install PySocks.')
     else:
         import socket

From 09bfe2847243f58a3e11bfde3ec1bf1fd84ce5ec Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 21 Jul 2016 00:50:21 -0400
Subject: [PATCH 086/149] [Processor] Fix rtmpdump not portable under Windows

---
 src/you_get/processor/rtmpdump.py | 32 +++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/src/you_get/processor/rtmpdump.py b/src/you_get/processor/rtmpdump.py
index cf5f822c..f1ab9a4b 100644
--- a/src/you_get/processor/rtmpdump.py
+++ b/src/you_get/processor/rtmpdump.py
@@ -41,16 +41,32 @@ def download_rtmpdump_stream(url, title, ext,params={},output_dir='.'):
     subprocess.call(cmdline)
     return
 
-#
-#To be refactor
-#To the future myself: Remember to refactor the same function in ffmpeg.py
 #
 def play_rtmpdump_stream(player, url, params={}):
-    cmdline="rtmpdump -r '%s' "%url
+    
+    #construct left side of pipe
+    cmdline = [RTMPDUMP, '-r']
+    cmdline.append(url)
+    
+    #append other params if exist
     for key in params.keys():
-        cmdline+=key+" "+params[key] if params[key]!=None else ""+" "
-    cmdline+=" -o - | %s -"%player
-    print(cmdline)
-    os.system(cmdline)
+        cmdline.append(key)
+        if params[key]!=None:
+            cmdline.append(params[key])
+
+    cmdline.append('-o')
+    cmdline.append('-')
+
+    #pipe start
+    cmdline.append('|')
+    cmdline.append(player)
+    cmdline.append('-')
+
+    #logging
+    print("Call rtmpdump:\n"+" ".join(cmdline)+"\n")
+
+    #call RTMPDump!
+    subprocess.call(cmdline)
+    
     # os.system("rtmpdump -r '%s' -y '%s' -o - | %s -" % (url, playpath, player))
     return

From 80a8265a49a6d5b96d885023d72c3710681d19cc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Jul 2016 00:04:39 +0200
Subject: [PATCH 087/149] [extractors] import zhanqi (close #1297)

---
 src/you_get/extractors/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index b879b83e..2f148aca 100644
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -79,3 +79,4 @@ from .youku import *
 from .youtube import *
 from .ted import *
 from .khan import *
+from .zhanqi import *

From fbf5c491aa8f20263c5fa0c36f7bb01c468f8c91 Mon Sep 17 00:00:00 2001
From: chinat <chinaxiahaifeng@gmail.com>
Date: Wed, 27 Jul 2016 13:42:06 +0800
Subject: [PATCH 088/149] [bilibili] video title may have space around

---
 src/you_get/extractors/bilibili.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 24782598..5c010a80 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -125,8 +125,8 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         url = r1(r'"([^"]+)" class="v-av-link"', html)
         html = get_content(url)
 
-    title = r1_of([r'<meta name="title" content="([^<>]{1,999})" />',
-                   r'<h1[^>]*>([^<>]+)</h1>'], html)
+    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
+                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
     if title:
         title = unescape_html(title)
         title = escape_file_path(title)
@@ -139,14 +139,14 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     cid = cid.split('&')[0]
     if t == 'cid':
         if re.match(r'https?://live\.bilibili\.com/', url):
-            title = r1(r'<title>([^<>]+)</title>', html)
+            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
             bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
 
         else:
             # multi-P
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>(.+)</option>', html)
+            titles = re.findall('<option value=.*>\s*(.+)\s*</option>', html)
             for i, page in enumerate(pages):
                 html = get_html("http://www.bilibili.com%s" % page)
                 flashvars = r1_of([r'(cid=\d+)',
@@ -163,7 +163,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             # no multi-P
             if not pages:
                 cids = [cid]
-                titles = [r1(r'<option value=.* selected>(.+)</option>', html) or title]
+                titles = [r1(r'<option value=.* selected>\s*(.+)\s*</option>', html) or title]
 
             for i in range(len(cids)):
                 bilibili_download_by_cid(cids[i],

From 056082c36cb3e93f9858ee132ff7e9979a88af96 Mon Sep 17 00:00:00 2001
From: chinat <chinaxiahaifeng@gmail.com>
Date: Wed, 27 Jul 2016 14:16:32 +0800
Subject: [PATCH 089/149] [bilibili] video title may have space around

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 5c010a80..3fbf946f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -146,7 +146,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             # multi-P
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>\s*(.+)\s*</option>', html)
+            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
             for i, page in enumerate(pages):
                 html = get_html("http://www.bilibili.com%s" % page)
                 flashvars = r1_of([r'(cid=\d+)',
@@ -163,7 +163,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             # no multi-P
             if not pages:
                 cids = [cid]
-                titles = [r1(r'<option value=.* selected>\s*(.+)\s*</option>', html) or title]
+                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
 
             for i in range(len(cids)):
                 bilibili_download_by_cid(cids[i],

From 8e8506790fb033f166d898d24676fdcf2b57fe82 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 28 Jul 2016 04:23:44 -0400
Subject: [PATCH 090/149] [Youku COOP] Add embsig to parasing

---
 src/you_get/extractors/youku.py | 35 ++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 345347d0..853a75ba 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -330,22 +330,51 @@ class Youku(VideoExtractor):
 
     def open_download_by_vid(self, client_id, vid, **kwargs):
         """self, str, str, **kwargs->None
+        
+        Arguments:
+        client_id:        An ID per client. For now we only know Acfun's
+                          such ID.
+        
+        vid:              An video ID for each video, starts with "C".
+        
+        kwargs['embsig']: Youku COOP's anti hotlinking.
+                          For Acfun, an API call must be done to Acfun's
+                          server, or the "playsign" of the content of sign_url
+                          shall be empty.
+        
+        Misc:
         Override the original one with VideoExtractor.
-        Most of the credit are to @ERioK, who gave his POC."""
+        
+        Author:
+        Most of the credit are to @ERioK, who gave his POC.
+        
+        History:
+        Jul.28.2016 Youku COOP now have anti hotlinking via embsig. """
         self.f_code_1 = '10ehfkbv'  #can be retrived by running r.translate with the keys and the list e
         self.f_code_2 = 'msjv7h2b'
+        
+        # as in VideoExtractor
         self.url = None
         self.vid = vid
         self.name = "优酷开放平台 (Youku COOP)"
 
         #A little bit of work before self.prepare
-        sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
+        
+        #Change as Jul.28.2016 Youku COOP updates its platform to add ant hotlinking
+        if kwargs['embsig']:
+            sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}&embsig={embsig}".format(client_id = client_id, video_id = vid, embsig = kwargs['embsig'])
+        else:
+            sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
+
         playsign = json.loads(get_content(sign_url))['playsign']
-    
+
+        #to be injected and replace ct10 and 12
         api85_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=85&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
         api86_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=86&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
         
         self.prepare(api_url = api85_url, api12_url = api86_url, ctype = 86, **kwargs)
+        
+        #exact copy from original VideoExtractor
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             unset_proxy()
 

From 063d4b1457e132ef2dc1e3be6154f10ee21a5cef Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 28 Jul 2016 04:24:12 -0400
Subject: [PATCH 091/149] [Acfun] Update embsig with Youku COOP to bypass anti
 hot linking

---
 src/you_get/extractors/acfun.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 983606d4..2ccc4718 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -17,10 +17,24 @@ def get_srt_json(id):
     return get_html(url)
 
 def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
+    """str, str, str, bool, bool ->None
+    
+    Download Acfun video by vid.
+    
+    Call Acfun API, decide which site to use, and pass the job to its
+    extractor.
+    """
+
+    #first call the main parasing API
     info = json.loads(get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
+
     sourceType = info['sourceType']
+
+    #decide sourceId to know which extractor to use
     if 'sourceId' in info: sourceId = info['sourceId']
     # danmakuId = info['danmakuId']
+
+    #call extractor decided by sourceId
     if sourceType == 'sina':
         sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'youku':
@@ -32,11 +46,13 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     elif sourceType == 'letv':
         letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'zhuzhan':
+        #As in Jul.28.2016, Acfun is using embsig to anti hotlink so we need to pass this
+        embsig =  info['encode']
         a = 'http://api.aixifan.com/plays/%s' % vid
         s = json.loads(get_content(a, headers={'deviceType': '2'}))
         if s['data']['source'] == "zhuzhan-youku":
             sourceId = s['data']['sourceId']
-            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir,merge=merge, info_only=info_only, embsig = embsig, **kwargs)
     else:
         raise NotImplementedError(sourceType)
 

From 468fb6f403d98d6870fab4d6670a6da06f5f9238 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 29 Jul 2016 01:27:22 -0400
Subject: [PATCH 092/149] [Processor]FFmpeg: check arguments not None

---
 src/you_get/processor/ffmpeg.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index c23b0eaf..deb13bd2 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -213,10 +213,12 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
         
     ffmpeg_params = []
     #should these exist...
-    if len(params) > 0:
-        for k, v in params:
-            ffmpeg_params.append(k)
-            ffmpeg_params.append(v)
+    if params is not None:
+        if len(params) > 0:
+            for k, v in params:
+                ffmpeg_params.append(k)
+                ffmpeg_params.append(v)
+
         
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
     ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
@@ -249,10 +251,12 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
 def ffmpeg_play_stream(player, url, params={}):
     ffmpeg_params = []
     #should these exist...
-    if len(params) > 0:
-        for k, v in params:
-            ffmpeg_params.append(k)
-            ffmpeg_params.append(v)
+    if params is not None:
+        if len(params) > 0:
+            for k, v in params:
+                ffmpeg_params.append(k)
+                ffmpeg_params.append(v)
+
         
     print('Playing streaming content with FFmpeg, press 1 to stop recording...')
     ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']

From 6689f45469f5f14dd338ae7743313f7b8c6c5174 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 13:35:05 +0800
Subject: [PATCH 093/149] [Douyutv]Update api

Thanks #1299 @cnbeining #1235 @hellofwy
---
 src/you_get/extractors/douyutv.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 449022ca..29bcedd4 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -6,27 +6,44 @@ from ..common import *
 import json
 import hashlib
 import time
+import random
+import string
+import requests
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     room_id = url[url.rfind('/')+1:]
-    #Thanks to @yan12125 for providing decoding method!!
-    suffix = 'room/%s?aid=android&client_sys=android&time=%d' % (room_id, int(time.time()))
-    sign = hashlib.md5((suffix + '1231').encode('ascii')).hexdigest()
-    json_request_url = "http://www.douyu.com/api/v1/%s&auth=%s" % (suffix, sign)
+
+    json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_html(json_request_url)
     data = json.loads(content)['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
+
     title = data.get('room_name')
     show_status = data.get('show_status')
     if show_status is not "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
+
+    tt = int(time.time() / 60)
+    did = ''.join([random.choice(string.ascii_uppercase + string.digits) for n in range(32)])
+    sign = hashlib.md5((room_id + did + 'A12Svb&%1UUmf@hC' + "%d" % tt).encode("utf-8")).hexdigest()
+    json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
+
+    payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
+    r = requests.post(json_request_url, data=payload)
+    content = r.json()
+    data = content['data']
+
+    server_status = data.get('error',0)
+    if server_status is not 0:
+        raise ValueError("Server returned error:%s" % server_status)
+
     real_url = data.get('rtmp_url')+'/'+data.get('rtmp_live')
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+        download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
 
 site_info = "douyu.com"
 download = douyutv_download

From 06b57e2db7cce08ff0b6f2a5e0ad6d2723f1d017 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 18:25:40 +0800
Subject: [PATCH 094/149] fix CI failure by using 3rd packages

---
 src/you_get/extractors/douyutv.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 29bcedd4..38b2d48d 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -8,13 +8,13 @@ import hashlib
 import time
 import random
 import string
-import requests
+import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     room_id = url[url.rfind('/')+1:]
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
-    content = get_html(json_request_url)
+    content = get_content(json_request_url)
     data = json.loads(content)['data']
     server_status = data.get('error',0)
     if server_status is not 0:
@@ -27,14 +27,19 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     tt = int(time.time() / 60)
     did = ''.join([random.choice(string.ascii_uppercase + string.digits) for n in range(32)])
-    sign = hashlib.md5((room_id + did + 'A12Svb&%1UUmf@hC' + "%d" % tt).encode("utf-8")).hexdigest()
+    sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
+    sign = hashlib.md5(sign_content.encode('utf-8')).hexdigest()
+
     json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
-
     payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
-    r = requests.post(json_request_url, data=payload)
-    content = r.json()
-    data = content['data']
+    json_data = json.dumps(payload).encode('utf8')
+    postdata = urllib.parse.urlencode(payload)
+    req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
+    with urllib.request.urlopen(req) as response:
+        the_page = response.read()
 
+    content = json.loads(the_page.decode('utf-8'))
+    data = content['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
@@ -43,6 +48,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
+        print(real_url)
         download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
 
 site_info = "douyu.com"

From ddbc2caa23ed96a38146aa735dd83a48cd855f0a Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 20:50:15 +0800
Subject: [PATCH 095/149] add support for non digit channel

event page is still not support because the rule is complicated
---
 src/you_get/extractors/douyutv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 38b2d48d..8cdb8625 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -11,7 +11,9 @@ import string
 import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    room_id = url[url.rfind('/')+1:]
+    html = get_content(url)
+    room_id_patt = r'"room_id"\s*:\s*(\d+),'
+    room_id = match1(html, room_id_patt)
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_content(json_request_url)
@@ -48,7 +50,6 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        print(real_url)
         download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
 
 site_info = "douyu.com"

From 2322a16ca829e110743d037e0fb4a6cc2cfb4799 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 21:30:43 +0800
Subject: [PATCH 096/149] delete useless line

---
 src/you_get/extractors/douyutv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 8cdb8625..6587efe9 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -34,7 +34,6 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
     payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
-    json_data = json.dumps(payload).encode('utf8')
     postdata = urllib.parse.urlencode(payload)
     req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
     with urllib.request.urlopen(req) as response:

From d0215aa23882d0d8c63cfaf856f6e8e7b73d4b1b Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Mon, 1 Aug 2016 05:39:38 -0400
Subject: [PATCH 097/149] [Bigthink]Add Bigthink

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/bigthink.py | 76 ++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)
 mode change 100644 => 100755 src/you_get/extractors/__init__.py
 create mode 100644 src/you_get/extractors/bigthink.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a9b12b08..3cbee373 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -8,6 +8,7 @@ SITES = {
     'baidu'            : 'baidu',
     'bandcamp'         : 'bandcamp',
     'baomihua'         : 'baomihua',
+    'bigthink'         : 'bigthink',
     'bilibili'         : 'bilibili',
     'cctv'             : 'cntv',
     'cntv'             : 'cntv',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
old mode 100644
new mode 100755
index 2f148aca..97ab0b41
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -5,6 +5,7 @@ from .alive import *
 from .archive import *
 from .baidu import *
 from .bandcamp import *
+from .bigthink import *
 from .bilibili import *
 from .cbs import *
 from .ckplayer import *
diff --git a/src/you_get/extractors/bigthink.py b/src/you_get/extractors/bigthink.py
new file mode 100644
index 00000000..1dd196d5
--- /dev/null
+++ b/src/you_get/extractors/bigthink.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+import json
+
+class Bigthink(VideoExtractor):
+    name = "Bigthink"
+
+    stream_types = [  #this is just a sample. Will make it in prepare()
+        # {'id': '1080'},
+        # {'id': '720'},
+        # {'id': '360'},
+        # {'id': '288'},
+        # {'id': '190'},
+        # {'id': '180'},
+        
+    ]
+
+    @staticmethod
+    def get_streams_by_id(account_number, video_id):
+        """
+        int, int->list
+        
+        Get the height of the videos.
+        
+        Since brightcove is using 3 kinds of links: rtmp, http and https,
+        we will be using the HTTPS one to make it secure.
+        
+        If somehow akamaihd.net is blocked by the Great Fucking Wall,
+        change the "startswith https" to http.
+        """
+        endpoint = 'https://edge.api.brightcove.com/playback/v1/accounts/{account_number}/videos/{video_id}'.format(account_number = account_number, video_id = video_id)
+        fake_header_id = fake_headers
+        #is this somehow related to the time? Magic....
+        fake_header_id['Accept'] ='application/json;pk=BCpkADawqM1cc6wmJQC2tvoXZt4mrB7bFfi6zGt9QnOzprPZcGLE9OMGJwspQwKfuFYuCjAAJ53JdjI8zGFx1ll4rxhYJ255AXH1BQ10rnm34weknpfG-sippyQ'
+
+        html = get_content(endpoint, headers= fake_header_id)
+        html_json = json.loads(html)
+
+        link_list = []
+
+        for i in html_json['sources']:
+            if 'src' in i:  #to avoid KeyError
+                if i['src'].startswith('https'):
+                    link_list.append((str(i['height']), i['src']))
+
+        return link_list
+
+    def prepare(self, **kwargs):
+
+        html = get_content(self.url)
+
+        self.title = match1(html, r'<meta property="og:title" content="([^"]*)"')
+
+        account_number = match1(html, r'data-account="(\d+)"')
+
+        video_id = match1(html, r'data-brightcove-id="(\d+)"')
+        
+        assert account_number, video_id
+
+        link_list = self.get_streams_by_id(account_number, video_id)
+
+        for i in link_list:
+            self.stream_types.append({'id': str(i[0])})
+            self.streams[i[0]] = {'url': i[1]}
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+
+site = Bigthink()
+download = site.download_by_url

From f8939128e6a17304aee641da2f1eb15535693dfb Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 3 Aug 2016 21:39:10 -0400
Subject: [PATCH 098/149] [QQ]Add QQ Daxue

---
 src/you_get/extractors/qq.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 783bb2f5..80f8d6b7 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -5,6 +5,7 @@ __all__ = ['qq_download']
 from ..common import *
 from .qie import download as qieDownload
 from urllib.parse import urlparse,parse_qs
+
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
     content = get_html(api)
@@ -23,7 +24,9 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     if not info_only:
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
+
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    """"""
     if 'live.qq.com' in url:
         qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
         return 
@@ -35,7 +38,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         content = get_html(url)
         url = match1(content,r'window\.location\.href="(.*?)"')
         
-    if 'kuaibao.qq.com' in url:
+    if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')

From af039eae5f9418806b3d0297baec599df91c6986 Mon Sep 17 00:00:00 2001
From: johan <johan@ximalaya.com>
Date: Thu, 4 Aug 2016 15:32:22 +0800
Subject: [PATCH 099/149] [ifeng] fix video info xml url

---
 src/you_get/extractors/ifeng.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
index 75078177..20f38743 100644
--- a/src/you_get/extractors/ifeng.py
+++ b/src/you_get/extractors/ifeng.py
@@ -6,7 +6,7 @@ from ..common import *
 
 def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
     assert r1(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', id), id
-    url = 'http://v.ifeng.com/video_info_new/%s/%s/%s.xml' % (id[-2], id[-2:], id)
+    url = 'http://vxml.ifengimg.com/video_info_new/%s/%s/%s.xml' % (id[-2], id[-2:], id)
     xml = get_html(url, 'utf-8')
     title = r1(r'Name="([^"]+)"', xml)
     title = unescape_html(title)
@@ -15,7 +15,7 @@ def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_
     r = randint(10, 19)
     url = url.replace('http://video.ifeng.com/', 'http://video%s.ifeng.com/' % r)
     type, ext, size = url_info(url)
-    
+
     print_info(site_info, title, ext, size)
     if not info_only:
         download_urls([url], title, ext, size, output_dir, merge = merge)
@@ -24,7 +24,7 @@ def ifeng_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     id = r1(r'/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.shtml$', url)
     if id:
         return ifeng_download_by_id(id, None, output_dir = output_dir, merge = merge, info_only = info_only)
-    
+
     html = get_html(url)
     id = r1(r'var vid="([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"', html)
     assert id, "can't find video info"

From 021a2f62a3764fc42d141aaa461b74fd7d12c13b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 5 Aug 2016 01:24:12 +0200
Subject: [PATCH 100/149] version 0.4.523

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 46850c8a..88a85141 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.486'
+__version__ = '0.4.523'

From a8015ec92ebcb0f31036c44eb459a8e2bcdc522c Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Tue, 9 Aug 2016 02:27:59 +0800
Subject: [PATCH 101/149] [Douyutv]fix channel on event

some channel on event will redirect to event page
exp. http://www.douyu.com/65251 redirect to http://www.douyu.com/cms/zt/RioOlympics.html#a
if channel is on event, use entering url to get room_id, and non-digital room_id will not work
---
 src/you_get/extractors/douyutv.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 6587efe9..61b6d324 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -6,14 +6,15 @@ from ..common import *
 import json
 import hashlib
 import time
-import random
-import string
+import uuid
 import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_content(url)
     room_id_patt = r'"room_id"\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)
+    if room_id == "0":
+        room_id = url[url.rfind('/')+1:]
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_content(json_request_url)
@@ -28,7 +29,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
 
     tt = int(time.time() / 60)
-    did = ''.join([random.choice(string.ascii_uppercase + string.digits) for n in range(32)])
+    did = uuid.uuid4().hex.upper()
     sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
     sign = hashlib.md5(sign_content.encode('utf-8')).hexdigest()
 
@@ -37,10 +38,9 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     postdata = urllib.parse.urlencode(payload)
     req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
     with urllib.request.urlopen(req) as response:
-        the_page = response.read()
+        content = response.read()
 
-    content = json.loads(the_page.decode('utf-8'))
-    data = content['data']
+    data = json.loads(content.decode('utf-8'))['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)

From e765f328057fcf340e580fcfb09ac257928fd295 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 10 Aug 2016 13:56:52 +0200
Subject: [PATCH 102/149] [netease] fix #1341, also do not create new directory
 when info_only

---
 src/you_get/extractors/netease.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 332d48ec..b37ee003 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -4,6 +4,7 @@
 __all__ = ['netease_download']
 
 from ..common import *
+from ..util import fs
 from json import loads
 import hashlib
 import base64
@@ -28,10 +29,10 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
 
         artist_name = j['album']['artists'][0]['name']
         album_name = j['album']['name']
-        new_dir = output_dir + '/' + "%s - %s" % (artist_name, album_name)
-        if not os.path.exists(new_dir):
-            os.mkdir(new_dir)
+        new_dir = output_dir + '/' + fs.legitimize("%s - %s" % (artist_name, album_name))
         if not info_only:
+            if not os.path.exists(new_dir):
+                os.mkdir(new_dir)
             cover_url = j['album']['picUrl']
             download_urls([cover_url], "cover", "jpg", 0, new_dir)
 
@@ -46,10 +47,10 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
     elif "playlist" in url:
         j = loads(get_content("http://music.163.com/api/playlist/detail?id=%s&csrf_token=" % rid, headers={"Referer": "http://music.163.com/"}))
 
-        new_dir = output_dir + '/' + j['result']['name']
-        if not os.path.exists(new_dir):
-            os.mkdir(new_dir)
+        new_dir = output_dir + '/' + fs.legitimize(j['result']['name'])
         if not info_only:
+            if not os.path.exists(new_dir):
+                os.mkdir(new_dir)
             cover_url = j['result']['coverImgUrl']
             download_urls([cover_url], "cover", "jpg", 0, new_dir)
 
@@ -61,7 +62,7 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
                 netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
             except: pass
 
-    elif "song" in url: 
+    elif "song" in url:
         j = loads(get_content("http://music.163.com/api/song/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_song_download(j["songs"][0], output_dir=output_dir, info_only=info_only)
         try: # download lyrics

From 0c50e0466650b184c1af15fdcc822d1f6489a09a Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 11 Aug 2016 03:29:59 -0400
Subject: [PATCH 103/149] [DouyuTV] Fix discarding output_dir

---
 src/you_get/extractors/douyutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 6587efe9..dd6aa122 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -49,7 +49,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
+        download_url_ffmpeg(real_url, title, 'flv', None, output_dir = output_dir, merge = merge)
 
 site_info = "douyu.com"
 download = douyutv_download

From bb485a111074a6571c10d7e26e8ed048847eddb6 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 11 Aug 2016 03:30:21 -0400
Subject: [PATCH 104/149] [ffmpeg] Fix wrong format of output dir

---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index deb13bd2..96e1348b 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -209,7 +209,7 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     output = title + '.' + ext
     
     if not (output_dir == '.'):
-        output = output_dir + output
+        output = output_dir + '/' + output
         
     ffmpeg_params = []
     #should these exist...

From 3a8ce8819df02a090dcf6b041ea0e1f49faf48a4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 14 Aug 2016 03:52:02 +0200
Subject: [PATCH 105/149] [ffmpeg] remove unuseful ffmpeg_play_stream(), fix
 #1336

---
 src/you_get/common.py           | 11 +++----
 src/you_get/processor/ffmpeg.py | 54 +++++----------------------------
 2 files changed, 13 insertions(+), 52 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 3cbee373..2ff61d55 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -910,8 +910,7 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
         return
 
     if player:
-        from .processor.ffmpeg import ffmpeg_play_stream
-        ffmpeg_play_stream(player, url, params)
+        launch_player(player, [url])
         return
 
     from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
@@ -1218,12 +1217,12 @@ def script_main(script_name, download, download_playlist, **kwargs):
             import socket
             import socks
             socks_proxy_addrs = socks_proxy.split(':')
-            socks.set_default_proxy(socks.SOCKS5, 
-                                    socks_proxy_addrs[0], 
+            socks.set_default_proxy(socks.SOCKS5,
+                                    socks_proxy_addrs[0],
                                     int(socks_proxy_addrs[1]))
             socket.socket = socks.socksocket
-            def getaddrinfo(*args):
-                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+            def getaddrinfo(*args):
+                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
             socket.getaddrinfo = getaddrinfo
         except ImportError:
             log.w('Error importing PySocks library, socks proxy ignored.'
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 96e1348b..c34f43c2 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -204,13 +204,13 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     """str, str->True
     WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
     You can basicly download anything with this function
-    but better leave it alone with 
+    but better leave it alone with
     """
     output = title + '.' + ext
-    
+
     if not (output_dir == '.'):
         output = output_dir + '/' + output
-        
+
     ffmpeg_params = []
     #should these exist...
     if params is not None:
@@ -219,20 +219,20 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
                 ffmpeg_params.append(k)
                 ffmpeg_params.append(v)
 
-        
+
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
     ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
-    
+
     if FFMPEG == 'avconv':  #who cares?
         ffmpeg_params += ['-c', 'copy', output]
     else:
         ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
-    
+
     ffmpeg_params.append(output)
-    
+
     print(' '.join(ffmpeg_params))
-    
+
     try:
         a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
         a.communicate()
@@ -243,41 +243,3 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
             pass
 
     return True
-
-#
-#To be refactor
-#Direct copy of rtmpdump.py
-#
-def ffmpeg_play_stream(player, url, params={}):
-    ffmpeg_params = []
-    #should these exist...
-    if params is not None:
-        if len(params) > 0:
-            for k, v in params:
-                ffmpeg_params.append(k)
-                ffmpeg_params.append(v)
-
-        
-    print('Playing streaming content with FFmpeg, press 1 to stop recording...')
-    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']
-    ffmpeg_params.append(url)  #not the same here!!!!
-    
-    if FFMPEG == 'avconv':  #who cares?
-        ffmpeg_params += ['-c', 'copy', '|']
-    else:
-        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|']
-    
-    ffmpeg_params += [player, '-']
-    
-    print(' '.join(ffmpeg_params))
-    
-    try:
-        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
-        a.communicate()
-    except KeyboardInterrupt:
-        try:
-            a.stdin.write('q'.encode('utf-8'))
-        except:
-            pass
-
-    return True
\ No newline at end of file

From 326bae23e15e1508dd4d1398f2e746ccdafdec41 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Aug 2016 18:54:58 +0200
Subject: [PATCH 106/149] [qq] fix #1355

---
 src/you_get/extractors/qq.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 80f8d6b7..59460e1d 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -29,7 +29,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
     if 'live.qq.com' in url:
         qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
-        return 
+        return
 
     #do redirect
     if 'v.qq.com/page' in url:
@@ -37,7 +37,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         # http://v.qq.com/page/k/9/7/k0194pwgw97.html
         content = get_html(url)
         url = match1(content,r'window\.location\.href="(.*?)"')
-        
+
     if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
@@ -49,10 +49,11 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = vid
     else:
         content = get_html(url)
-        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5 
+        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
         vid = vid[0] if vid else match1(content, r'vid\s*:\s*"\s*([^"]+)"') #general fallback
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title
+        title = match1(content, r'"title":"([^"]+)"') if not title else title
         title = vid if not title else title #general fallback
 
 

From 7cecd0368dea3e8e7318f48908e39d3d8405c95f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Aug 2016 21:44:05 +0200
Subject: [PATCH 107/149] [bilibili] close #1340

---
 src/you_get/extractors/bilibili.py | 206 +++++++----------------------
 1 file changed, 48 insertions(+), 158 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 3fbf946f..c7c4fac9 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -2,94 +2,17 @@
 
 __all__ = ['bilibili_download']
 
+import json
+import re
 from ..common import *
 
-from .sina import sina_download_by_vid
-from .tudou import tudou_download_by_id
-from .youku import youku_download_by_vid
+def get_srt_xml(cid):
+    return get_html('http://comment.bilibili.com/%s.xml' % cid)
 
-import hashlib
-import re
-
-appkey = 'f3bb208b3d081dc8'
-
-
-def get_srt_xml(id):
-    url = 'http://comment.bilibili.com/%s.xml' % id
-    return get_html(url)
-
-
-def parse_srt_p(p):
-    fields = p.split(',')
-    assert len(fields) == 8, fields
-    time, mode, font_size, font_color, pub_time, pool, user_id, history = fields
-    time = float(time)
-
-    mode = int(mode)
-    assert 1 <= mode <= 8
-    # mode 1~3: scrolling
-    # mode 4: bottom
-    # mode 5: top
-    # mode 6: reverse?
-    # mode 7: position
-    # mode 8: advanced
-
-    pool = int(pool)
-    assert 0 <= pool <= 2
-    # pool 0: normal
-    # pool 1: srt
-    # pool 2: special?
-
-    font_size = int(font_size)
-
-    font_color = '#%06x' % int(font_color)
-
-    return pool, mode, font_size, font_color
-
-
-def parse_srt_xml(xml):
-    d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
-    for x, y in d:
-        p = parse_srt_p(x)
-    raise NotImplementedError()
-
-
-def parse_cid_playurl(xml):
-    from xml.dom.minidom import parseString
-    try:
-        doc = parseString(xml.encode('utf-8'))
-        urls = [durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl')]
-        return urls
-    except:
-        return []
-
-
-def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
-    urls = []
-    for cid in cids:
-        url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
-        urls += [i
-                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                 for i in parse_cid_playurl(get_content(url))]
-
-    type_ = ''
-    size = 0
-    for url in urls:
-        _, type_, temp = url_info(url)
-        size += temp
-
-    print_info(site_info, title, type_, size)
-    if not info_only:
-        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
-
-def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
-    urls = [i
-            if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-            else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-            for i in parse_cid_playurl(get_content(url))]
+def bilibili_download_by_api(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    title = r1(r'cid=(\d+)', url)
+    info = json.loads(get_content(url))
+    urls = [i['url'] for i in info['durl']]
 
     type_ = ''
     size = 0
@@ -104,82 +27,50 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if re.match(r'https?://interface\.bilibili\.com/', url):
+        # quick hack for explicit API
+        bilibili_download_by_api(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
 
-def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
-    urls = parse_cid_playurl(get_content(api_url))
+    html = get_content(url)
+    main_title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
+                        r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
+    cid = r1(r'cid=(\d+)', html)
 
-    for url in urls:
-        _, type_, _ = url_info(url)
-        size = 0
+    aid = r1(r'av(\d+)', url)
+    page = r1(r'index_(\d+)', url)
+    sub_titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
+    if page is None and sub_titles: # download all
+        for t in enumerate(sub_titles):
+            page, sub_title = t[0] + 1, t[1]
+            title = main_title + ' - ' + sub_title
+
+            api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
+            info = json.loads(get_content(api))
+            src = info['src']
+            _, type_, size = url_info(src)
+            print_info(site_info, title, type_, size)
+            if not info_only:
+                download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
+
+    else: # download selected
+        if page is None: page = 1
+        sub_title = r1('<option value=.* selected>\s*([^<>]+)\s*</option>', html)
+        if sub_title is None:
+            sub_title = r1('<option value=.*>\s*([^<>]+)\s*</option>', html)
+        if sub_title:
+            title = main_title + ' - ' + sub_title
+        else:
+            title = main_title
+
+        api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
+        info = json.loads(get_content(api))
+        src = info['src']
+        _, type_, size = url_info(src)
         print_info(site_info, title, type_, size)
         if not info_only:
-            download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
-
-def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_content(url)
-
-    if re.match(r'https?://bangumi\.bilibili\.com/', url):
-        # quick hack for bangumi URLs
-        url = r1(r'"([^"]+)" class="v-av-link"', html)
-        html = get_content(url)
-
-    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
-                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
-    if title:
-        title = unescape_html(title)
-        title = escape_file_path(title)
-
-    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
-                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-    assert flashvars
-    flashvars = flashvars.replace(': ', '=')
-    t, cid = flashvars.split('=', 1)
-    cid = cid.split('&')[0]
-    if t == 'cid':
-        if re.match(r'https?://live\.bilibili\.com/', url):
-            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
-            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
-
-        else:
-            # multi-P
-            cids = []
-            pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-            for i, page in enumerate(pages):
-                html = get_html("http://www.bilibili.com%s" % page)
-                flashvars = r1_of([r'(cid=\d+)',
-                                   r'flashvars="([^"]+)"',
-                                   r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-                if flashvars:
-                    t, cid = flashvars.split('=', 1)
-                    cids.append(cid.split('&')[0])
-                if url.endswith(page):
-                    cids = [cid.split('&')[0]]
-                    titles = [titles[i]]
-                    break
-
-            # no multi-P
-            if not pages:
-                cids = [cid]
-                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
-
-            for i in range(len(cids)):
-                bilibili_download_by_cid(cids[i],
-                                         titles[i],
-                                         output_dir=output_dir,
-                                         merge=merge,
-                                         info_only=info_only)
-
-    elif t == 'vid':
-        sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'ykid':
-        youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'uid':
-        tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
-    else:
-        raise NotImplementedError(flashvars)
+            download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
 
     if not info_only and not dry_run:
         if not kwargs['caption']:
@@ -191,7 +82,6 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
-
 site_info = "bilibili.com"
 download = bilibili_download
 download_playlist = bilibili_download

From 550ef5aeb24bfc799a23091afc60ebbba7808793 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Aug 2016 23:35:33 +0200
Subject: [PATCH 108/149] [acfun] fix #1353 (get and only get the specified
 part)

---
 src/you_get/extractors/acfun.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 2ccc4718..5c6dc987 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -18,9 +18,9 @@ def get_srt_json(id):
 
 def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
     """str, str, str, bool, bool ->None
-    
+
     Download Acfun video by vid.
-    
+
     Call Acfun API, decide which site to use, and pass the job to its
     extractor.
     """
@@ -78,15 +78,14 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = escape_file_path(title)
     assert title
 
-    videos = re.findall("data-vid=\"(\d+)\".*href=\"[^\"]+\".*title=\"([^\"]+)\"", html)
-    for video in videos:
-        p_vid = video[0]
-        p_title = title + " - " + video[1] if video[1] != '删除标签' else title
-        acfun_download_by_vid(p_vid, p_title,
-                              output_dir=output_dir,
-                              merge=merge,
-                              info_only=info_only,
-                              **kwargs)
+    video = re.search('data-vid="(\d+)"\s*data-scode="".*>([^<]+)</a>', html)
+    vid = video.group(1)
+    title = title + ' - ' + video.group(2)
+    acfun_download_by_vid(vid, title,
+                          output_dir=output_dir,
+                          merge=merge,
+                          info_only=info_only,
+                          **kwargs)
 
 site_info = "AcFun.tv"
 download = acfun_download

From ce759590d3d72de226b6b1e692be86d658742ac0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 22 Aug 2016 11:41:26 +0200
Subject: [PATCH 109/149] [ifeng] fix #1358

---
 src/you_get/extractors/ifeng.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
index 20f38743..6ca45b30 100644
--- a/src/you_get/extractors/ifeng.py
+++ b/src/you_get/extractors/ifeng.py
@@ -13,7 +13,7 @@ def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_
     url = r1(r'VideoPlayUrl="([^"]+)"', xml)
     from random import randint
     r = randint(10, 19)
-    url = url.replace('http://video.ifeng.com/', 'http://video%s.ifeng.com/' % r)
+    url = url.replace('http://wideo.ifeng.com/', 'http://ips.ifeng.com/wideo.ifeng.com/')
     type, ext, size = url_info(url)
 
     print_info(site_info, title, ext, size)

From f8bf3e6cf3c96786523180d0568c72e0aa1d332e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 22 Aug 2016 23:58:48 +0200
Subject: [PATCH 110/149] [sina] fix #1361, close #1332

---
 src/you_get/extractors/sina.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/sina.py b/src/you_get/extractors/sina.py
index 88f819c0..fd458335 100644
--- a/src/you_get/extractors/sina.py
+++ b/src/you_get/extractors/sina.py
@@ -14,7 +14,7 @@ def get_k(vid, rand):
 
 def video_info_xml(vid):
     rand = "0.{0}{1}".format(randint(10000, 10000000), randint(10000, 10000000))
-    url = 'http://v.iask.com/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
+    url = 'http://ask.ivideo.sina.com.cn/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
     xml = get_content(url, headers=fake_headers, decoded=True)
     return xml
 
@@ -71,7 +71,7 @@ def sina_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             vid = vids[-1]
 
     if vid is None:
-        vid = match1(video_page, r'vid:(\d+)')
+        vid = match1(video_page, r'vid:"?(\d+)"?')
     if vid:
         title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
         sina_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)

From 16015a2db0575756e24856d1e17865e01e6b73c4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 23 Aug 2016 00:54:33 +0200
Subject: [PATCH 111/149] version 0.4.536

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 88a85141..a877863a 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.523'
+__version__ = '0.4.536'

From 315634332a0303297905deaf27e6779ac38a1cf6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 23 Aug 2016 14:39:43 +0200
Subject: [PATCH 112/149] [twitter] fix #1357

---
 src/you_get/extractors/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 09956141..db01daa2 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -73,10 +73,10 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             scribe_playlist_url = i['scribe_playlist_url']
             return vine_download(scribe_playlist_url, output_dir, merge=merge, info_only=info_only)
 
-        if source.endswith('.mp4'):
-            urls = [source]
-        else:
+        try:
             urls = extract_m3u(source)
+        except:
+            urls = [source]
         size = urls_size(urls)
         mime, ext = 'video/mp4', 'mp4'
 

From a3f3d880f9126ce003d94fe4105ca11e5862e127 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Aug 2016 02:04:05 +0200
Subject: [PATCH 113/149] [bilibili] quick hack for bangumi URLs again (fix
 #1362)

---
 src/you_get/extractors/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index c7c4fac9..a9caf8ed 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -33,6 +33,11 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         bilibili_download_by_api(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
         return
 
+    if re.match(r'https?://bangumi\.bilibili\.com/', url):
+        # quick hack for bangumi URLs
+        html = get_content(url)
+        url = r1(r'"([^"]+)" class="v-av-link"', html)
+
     html = get_content(url)
     main_title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
                         r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)

From ebd3db9acb34f40e84f75b81db00ad25573b7602 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 24 Aug 2016 23:00:48 -0400
Subject: [PATCH 114/149] [BokeCC] Add support

---
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/bokecc.py   | 95 ++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 src/you_get/extractors/bokecc.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 97ab0b41..d283e30c 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -7,6 +7,7 @@ from .baidu import *
 from .bandcamp import *
 from .bigthink import *
 from .bilibili import *
+from .bokecc import *
 from .cbs import *
 from .ckplayer import *
 from .cntv import *
diff --git a/src/you_get/extractors/bokecc.py b/src/you_get/extractors/bokecc.py
new file mode 100644
index 00000000..8566e828
--- /dev/null
+++ b/src/you_get/extractors/bokecc.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+import xml.etree.ElementTree as ET
+
+class BokeCC(VideoExtractor):
+    name = "BokeCC"
+
+    stream_types = [  # we do now know for now, as we have to check the
+                      # output from the API
+    ]
+
+    API_ENDPOINT = 'http://p.bokecc.com/'
+
+
+    def download_by_id(self, vid = '', title = None, output_dir='.', merge=True, info_only=False,**kwargs):
+        """self, str->None
+        
+        Keyword arguments:
+        self: self
+        vid: The video ID for BokeCC cloud, something like
+        FE3BB999594978049C33DC5901307461
+        
+        Calls the prepare() to download the video.
+        
+        If no title is provided, this method shall try to find a proper title
+        with the information providin within the
+        returned content of the API."""
+
+        assert vid
+
+        self.prepare(vid = vid, title = title, **kwargs)
+
+        self.extract(**kwargs)
+
+        self.download(output_dir = output_dir, 
+                    merge = merge, 
+                    info_only = info_only, **kwargs)
+
+    def prepare(self, vid = '', title = None, **kwargs):
+        assert vid
+
+        api_url = self.API_ENDPOINT + \
+            'servlet/playinfo?vid={vid}&m=0'.format(vid = vid)  #return XML
+
+        html = get_content(api_url)
+        self.tree = ET.ElementTree(ET.fromstring(html))
+
+        if self.tree.find('result').text != '1':
+            log.wtf('API result says failed!')
+            raise 
+
+        if title is None:
+            self.title = '_'.join([i.text for i in tree.iterfind('video/videomarks/videomark/markdesc')])
+        else:
+            self.title = title
+
+        for i in self.tree.iterfind('video/quality'):
+            quality = i.attrib ['value']
+            url = i[0].attrib['playurl']
+            self.stream_types.append({'id': quality,
+                                      'video_profile': i.attrib ['desp']})
+            self.streams[quality] = {'url': url,
+                                     'video_profile': i.attrib ['desp']}
+            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # Extract the stream
+            stream_id = kwargs['stream_id']
+
+            if stream_id not in self.streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # Extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+
+site = BokeCC()
+
+# I don't know how to call the player directly so I just put it here
+# just in case anyone touchs it -- Beining@Aug.24.2016
+#download = site.download_by_url
+#download_playlist = site.download_by_url
+
+bokecc_download_by_id = site.download_by_id

From e60981f91e070c222923ea68a7a6aff79a650e5f Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 24 Aug 2016 23:12:06 -0400
Subject: [PATCH 115/149] [Wanmen]Add support

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/wanmen.py   | 123 +++++++++++++++++++++++++++++
 3 files changed, 125 insertions(+)
 create mode 100755 src/you_get/extractors/wanmen.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2ff61d55..a5a0fbab 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -79,6 +79,7 @@ SITES = {
     'videomega'        : 'videomega',
     'vidto'            : 'vidto',
     'vimeo'            : 'vimeo',
+    'wanmen'           : 'wanmen',
     'weibo'            : 'miaopai',
     'veoh'             : 'veoh',
     'vine'             : 'vine',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index d283e30c..e69bc2fd 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -74,6 +74,7 @@ from .vimeo import *
 from .vine import *
 from .vk import *
 from .w56 import *
+from .wanmen import *
 from .xiami import *
 from .yinyuetai import *
 from .yixia import *
diff --git a/src/you_get/extractors/wanmen.py b/src/you_get/extractors/wanmen.py
new file mode 100755
index 00000000..20c543c1
--- /dev/null
+++ b/src/you_get/extractors/wanmen.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+__all__ = ['wanmen_download', 'wanmen_download_by_course', 'wanmen_download_by_course_topic', 'wanmen_download_by_course_topic_part']
+
+from ..common import *
+from .bokecc import bokecc_download_by_id
+from json import loads
+
+
+##Helper functions
+def _wanmen_get_json_api_content_by_courseID(courseID):
+    """int->JSON
+    
+    Return a parsed JSON tree of WanMen's API."""
+
+    return loads(get_content('http://api.wanmen.org/course/getCourseNested/{courseID}'.format(courseID = courseID)))
+
+def _wanmen_get_title_by_json_topic_part(json_content, tIndex, pIndex):
+    """JSON, int, int, int->str
+    
+    Get a proper title with courseid+topicID+partID."""
+
+    return '_'.join([json_content[0]['name'],
+                    json_content[0]['Topics'][tIndex]['name'],
+                    json_content[0]['Topics'][tIndex]['Parts'][pIndex]['name']])
+
+
+def _wanmen_get_boke_id_by_json_topic_part(json_content, tIndex, pIndex):
+    """JSON, int, int, int->str
+    
+    Get one BokeCC video ID with courseid+topicID+partID."""
+
+    return json_content[0]['Topics'][tIndex]['Parts'][pIndex]['ccVideoLink']
+
+
+##Parsers
+def wanmen_download_by_course(json_api_content, output_dir='.', merge=True, info_only=False, **kwargs):
+    """int->None
+    
+    Download a WHOLE course.
+    Reuse the API call to save time."""
+
+    for tIndex in range(len(json_api_content[0]['Topics'])):
+        for pIndex in range(len(json_api_content[0]['Topics'][tIndex]['Parts'])):
+            wanmen_download_by_course_topic_part(json_api_content,
+                                                 tIndex,
+                                                 pIndex,
+                                                 output_dir=output_dir,
+                                                 merge=merge,
+                                                 info_only=info_only,
+                                                 **kwargs)
+
+
+def wanmen_download_by_course_topic(json_api_content, tIndex, output_dir='.', merge=True, info_only=False, **kwargs):
+    """int, int->None
+    
+    Download a TOPIC of a course.
+    Reuse the API call to save time."""
+
+    for pIndex in range(len(json_api_content[0]['Topics'][tIndex]['Parts'])):
+        wanmen_download_by_course_topic_part(json_api_content,
+                                             tIndex,
+                                             pIndex, 
+                                            output_dir=output_dir,
+                                            merge=merge,
+                                            info_only=info_only,
+                                            **kwargs)
+
+def wanmen_download_by_course_topic_part(json_api_content, tIndex, pIndex, output_dir='.', merge=True, info_only=False, **kwargs):
+    """int, int, int->None
+    
+    Download ONE PART of the course."""
+
+    html = json_api_content
+
+    title = _wanmen_get_title_by_json_topic_part(html, 
+                                                  tIndex, 
+                                                  pIndex)
+
+    bokeccID = _wanmen_get_boke_id_by_json_topic_part(html,
+                                                      tIndex, 
+                                                     pIndex)
+
+    bokecc_download_by_id(vid = bokeccID, title = title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+
+
+##Main entrance
+def wanmen_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+
+    if not 'wanmen.org' in url:
+        log.wtf('You are at the wrong place dude. This is for WanMen University!')
+        raise
+
+    courseID = int(match1(url, r'course\/(\d+)'))
+    assert courseID > 0  #without courseID we cannot do anything
+
+    tIndex = int(match1(url, r'tIndex=(\d+)'))
+
+    pIndex = int(match1(url, r'pIndex=(\d+)'))
+
+    json_api_content = _wanmen_get_json_api_content_by_courseID(courseID)
+
+    if pIndex:  #only download ONE single part
+        assert tIndex >= 0
+        wanmen_download_by_course_topic_part(json_api_content, tIndex, pIndex, 
+                                            output_dir = output_dir, 
+                                            merge = merge, 
+                                            info_only = info_only)
+    elif tIndex:  #download a topic
+        wanmen_download_by_course_topic(json_api_content, tIndex, 
+                                       output_dir = output_dir, 
+                                       merge = merge, 
+                                       info_only = info_only)
+    else:  #download the whole course
+        wanmen_download_by_course(json_api_content,
+                                 output_dir = output_dir, 
+                                 merge = merge, 
+                                 info_only = info_only)
+
+
+site_info = "WanMen University"
+download = wanmen_download
+download_playlist = wanmen_download_by_course
\ No newline at end of file

From 6fc2cc375ea4bb2dcb6bab74ce67688af0efde59 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 26 Aug 2016 19:27:40 +0200
Subject: [PATCH 116/149] [universal] call embed_download only if content_type
 is text/html (#1369)

---
 src/you_get/extractors/universal.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index b0d929c9..ebab70f8 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -6,20 +6,20 @@ from ..common import *
 from .embed import *
 
 def universal_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    try:
-        embed_download(url, output_dir, merge=merge, info_only=info_only)
-    except: pass
-    else: return
+    content_type = get_head(url, headers=fake_headers)['Content-Type']
+    if content_type.startswith('text/html'):
+        try:
+            embed_download(url, output_dir, merge=merge, info_only=info_only)
+        except: pass
+        else: return
 
     domains = url.split('/')[2].split('.')
     if len(domains) > 2: domains = domains[1:]
     site_info = '.'.join(domains)
 
-    response = get_response(url, faker=True)
-    content_type = response.headers['Content-Type']
-
     if content_type.startswith('text/html'):
         # extract an HTML page
+        response = get_response(url, faker=True)
         page = str(response.data)
 
         page_title = r1(r'<title>([^<]*)', page)

From d66749801f6fbb0aee4e2030e93448e9d46e13f8 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Sun, 28 Aug 2016 23:39:19 -0400
Subject: [PATCH 117/149] [Bilibili] Change back to extract via interface API
 with Miniloader s key

---
 src/you_get/extractors/bilibili.py | 203 ++++++++++++++++++++++-------
 1 file changed, 155 insertions(+), 48 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a9caf8ed..e9d3e7ad 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -2,17 +2,96 @@
 
 __all__ = ['bilibili_download']
 
-import json
-import re
 from ..common import *
 
-def get_srt_xml(cid):
-    return get_html('http://comment.bilibili.com/%s.xml' % cid)
+from .sina import sina_download_by_vid
+from .tudou import tudou_download_by_id
+from .youku import youku_download_by_vid
 
-def bilibili_download_by_api(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    title = r1(r'cid=(\d+)', url)
-    info = json.loads(get_content(url))
-    urls = [i['url'] for i in info['durl']]
+import hashlib
+import re
+
+appkey = 'f3bb208b3d081dc8'
+SECRETKEY_MINILOADER = '1c15888dc316e05a15fdd0a02ed6584f'
+
+def get_srt_xml(id):
+    url = 'http://comment.bilibili.com/%s.xml' % id
+    return get_html(url)
+
+
+def parse_srt_p(p):
+    fields = p.split(',')
+    assert len(fields) == 8, fields
+    time, mode, font_size, font_color, pub_time, pool, user_id, history = fields
+    time = float(time)
+
+    mode = int(mode)
+    assert 1 <= mode <= 8
+    # mode 1~3: scrolling
+    # mode 4: bottom
+    # mode 5: top
+    # mode 6: reverse?
+    # mode 7: position
+    # mode 8: advanced
+
+    pool = int(pool)
+    assert 0 <= pool <= 2
+    # pool 0: normal
+    # pool 1: srt
+    # pool 2: special?
+
+    font_size = int(font_size)
+
+    font_color = '#%06x' % int(font_color)
+
+    return pool, mode, font_size, font_color
+
+
+def parse_srt_xml(xml):
+    d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
+    for x, y in d:
+        p = parse_srt_p(x)
+    raise NotImplementedError()
+
+
+def parse_cid_playurl(xml):
+    from xml.dom.minidom import parseString
+    try:
+        doc = parseString(xml.encode('utf-8'))
+        urls = [durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl')]
+        return urls
+    except:
+        return []
+
+
+def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
+    urls = []
+    for cid in cids:
+        sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
+        url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
+        urls += [i
+                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+                 for i in parse_cid_playurl(get_content(url))]
+
+    type_ = ''
+    size = 0
+    for url in urls:
+        _, type_, temp = url_info(url)
+        size += temp
+
+    print_info(site_info, title, type_, size)
+    if not info_only:
+        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
+
+
+def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
+    sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
+    url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
+    urls = [i
+            if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+            else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+            for i in parse_cid_playurl(get_content(url))]
 
     type_ = ''
     size = 0
@@ -27,55 +106,82 @@ def bilibili_download_by_api(url, output_dir='.', merge=True, info_only=False, *
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
+def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
+    api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
+    urls = parse_cid_playurl(get_content(api_url))
+
+    for url in urls:
+        _, type_, _ = url_info(url)
+        size = 0
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
+
+
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    if re.match(r'https?://interface\.bilibili\.com/', url):
-        # quick hack for explicit API
-        bilibili_download_by_api(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
-        return
+    html = get_content(url)
 
     if re.match(r'https?://bangumi\.bilibili\.com/', url):
         # quick hack for bangumi URLs
-        html = get_content(url)
         url = r1(r'"([^"]+)" class="v-av-link"', html)
+        html = get_content(url)
 
-    html = get_content(url)
-    main_title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
-                        r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
-    cid = r1(r'cid=(\d+)', html)
+    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
+                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
+    if title:
+        title = unescape_html(title)
+        title = escape_file_path(title)
 
-    aid = r1(r'av(\d+)', url)
-    page = r1(r'index_(\d+)', url)
-    sub_titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-    if page is None and sub_titles: # download all
-        for t in enumerate(sub_titles):
-            page, sub_title = t[0] + 1, t[1]
-            title = main_title + ' - ' + sub_title
+    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
+                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+    assert flashvars
+    flashvars = flashvars.replace(': ', '=')
+    t, cid = flashvars.split('=', 1)
+    cid = cid.split('&')[0]
+    if t == 'cid':
+        if re.match(r'https?://live\.bilibili\.com/', url):
+            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
+            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
 
-            api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
-            info = json.loads(get_content(api))
-            src = info['src']
-            _, type_, size = url_info(src)
-            print_info(site_info, title, type_, size)
-            if not info_only:
-                download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
-
-    else: # download selected
-        if page is None: page = 1
-        sub_title = r1('<option value=.* selected>\s*([^<>]+)\s*</option>', html)
-        if sub_title is None:
-            sub_title = r1('<option value=.*>\s*([^<>]+)\s*</option>', html)
-        if sub_title:
-            title = main_title + ' - ' + sub_title
         else:
-            title = main_title
+            # multi-P
+            cids = []
+            pages = re.findall('<option value=\'([^\']*)\'', html)
+            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
+            for i, page in enumerate(pages):
+                html = get_html("http://www.bilibili.com%s" % page)
+                flashvars = r1_of([r'(cid=\d+)',
+                                   r'flashvars="([^"]+)"',
+                                   r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+                if flashvars:
+                    t, cid = flashvars.split('=', 1)
+                    cids.append(cid.split('&')[0])
+                if url.endswith(page):
+                    cids = [cid.split('&')[0]]
+                    titles = [titles[i]]
+                    break
 
-        api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
-        info = json.loads(get_content(api))
-        src = info['src']
-        _, type_, size = url_info(src)
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
+            # no multi-P
+            if not pages:
+                cids = [cid]
+                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
+
+            for i in range(len(cids)):
+                bilibili_download_by_cid(cids[i],
+                                         titles[i],
+                                         output_dir=output_dir,
+                                         merge=merge,
+                                         info_only=info_only)
+
+    elif t == 'vid':
+        sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif t == 'ykid':
+        youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif t == 'uid':
+        tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+    else:
+        raise NotImplementedError(flashvars)
 
     if not info_only and not dry_run:
         if not kwargs['caption']:
@@ -87,6 +193,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
+
 site_info = "bilibili.com"
 download = bilibili_download
-download_playlist = bilibili_download
+download_playlist = bilibili_download
\ No newline at end of file

From 5e612e870ce4cd9971675fb5420cb4b07620c28b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 30 Aug 2016 12:04:12 +0200
Subject: [PATCH 118/149] [pptv] fix #447 (per discussion in #1381)

---
 src/you_get/extractors/pptv.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index f9da64a5..b4ae78d1 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -37,14 +37,14 @@ def constructKey(arg):
             l4+=1
         return l3
         pass
-    
+
     def rot(k,b): ##>>> in as3
         if k>=0:
             return k>>b
         elif k<0:
             return (2**32+k)>>b
         pass
-     
+
     def lot(k,b):
         return (k<<b)%(2**32)
 
@@ -72,10 +72,10 @@ def constructKey(arg):
         l18=ord(l8[l12+5])<<8
         l19=ord(l8[l12+6])<<16
         l20=ord(l8[l12+7])<<24
-     
+
         l21=(((0|l13)|l14)|l15)|l16
         l22=(((0|l17)|l18)|l19)|l20
-     
+
         l23=0
         l24=0
         while l24<32:
@@ -91,9 +91,9 @@ def constructKey(arg):
             l40=(l39+l7)%(2**32)
             l41=((l37^l38)%(2**32)^l40)%(2**32)
             l22=(l22+l41)%(2**32)
-     
+
             l24+=1
-     
+
         l11+=chr(rot(l21,0)&0xff)
         l11+=chr(rot(l21,8)&0xff)
         l11+=chr(rot(l21,16)&0xff)
@@ -102,9 +102,9 @@ def constructKey(arg):
         l11+=chr(rot(l22,8)&0xff)
         l11+=chr(rot(l22,16)&0xff)
         l11+=chr(rot(l22,24)&0xff)
-     
-        return l11        
-            
+
+        return l11
+
 
     loc1=hex(int(arg))[2:]+(16-len(hex(int(arg))[2:]))*"\x00"
     SERVER_KEY="qqqqqww"+"\x00"*9
@@ -129,7 +129,7 @@ def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 
     pieces = re.findall('<sgm no="(\d+)"[^<>]+fs="(\d+)"', xml)
     numbers, fs = zip(*pieces)
-    urls=[ "http://ccf.pptv.com/{}/{}?key={}&fpp.ver=1.3.0.4&k={}&type=web.fpp".format(i,rid,key,k)  for i in range(max(map(int,numbers))+1)]
+    urls=["http://{}/{}/{}?key={}&fpp.ver=1.3.0.4&k={}&type=web.fpp".format(host,i,rid,key,k) for i in range(max(map(int,numbers))+1)]
 
     total_size = sum(map(int, fs))
     assert rid.endswith('.mp4')

From 44c09c542f4e15192283cc34ee201b878b292e82 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 30 Aug 2016 12:42:50 +0200
Subject: [PATCH 119/149] [funshion] fix (partly) #1304 and close #1378

---
 src/you_get/extractors/funshion.py | 35 ++++++++++++++----------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/funshion.py b/src/you_get/extractors/funshion.py
index e50b4bd7..6a24b464 100644
--- a/src/you_get/extractors/funshion.py
+++ b/src/you_get/extractors/funshion.py
@@ -10,9 +10,9 @@ import json
 def funshion_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     """"""
     if re.match(r'http://www.fun.tv/vplay/v-(\w+)', url):  #single video
-        funshion_download_by_url(url, output_dir = '.', merge = False, info_only = False)
-    elif re.match(r'http://www.fun.tv/vplay/g-(\w+)', url):  #whole drama
-        funshion_download_by_drama_url(url, output_dir = '.', merge = False, info_only = False)
+        funshion_download_by_url(url, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif re.match(r'http://www.fun.tv/vplay/.*g-(\w+)', url):  #whole drama
+        funshion_download_by_drama_url(url, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
         return
 
@@ -25,7 +25,7 @@ def funshion_download_by_url(url, output_dir = '.', merge = False, info_only = F
     if re.match(r'http://www.fun.tv/vplay/v-(\w+)', url):
         match = re.search(r'http://www.fun.tv/vplay/v-(\d+)(.?)', url)
     vid = match.group(1)
-    funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = False)
+    funshion_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
 
 #----------------------------------------------------------------------
 def funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = False):
@@ -34,11 +34,11 @@ def funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = F
     """
     title = funshion_get_title_by_vid(vid)
     url_list = funshion_vid_to_urls(vid)
-    
+
     for url in url_list:
         type, ext, size = url_info(url)
         print_info(site_info, title, type, size)
-    
+
     if not info_only:
         download_urls(url_list, title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
@@ -63,14 +63,11 @@ def funshion_download_by_drama_url(url, output_dir = '.', merge = False, info_on
     """str->None
     url = 'http://www.fun.tv/vplay/g-95785/'
     """
-    if re.match(r'http://www.fun.tv/vplay/g-(\w+)', url):
-        match = re.search(r'http://www.fun.tv/vplay/g-(\d+)(.?)', url)
-    id = match.group(1)
-    
+    id = r1(r'http://www.fun.tv/vplay/.*g-(\d+)', url)
     video_list = funshion_drama_id_to_vid(id)
-    
+
     for video in video_list:
-        funshion_download_by_id((video[0], id), output_dir = '.', merge = False, info_only = False)
+        funshion_download_by_id((video[0], id), output_dir=output_dir, merge=merge, info_only=info_only)
         # id is for drama, vid not the same as the ones used in single video
 
 #----------------------------------------------------------------------
@@ -81,11 +78,11 @@ def funshion_download_by_id(vid_id_tuple, output_dir = '.', merge = False, info_
     (vid, id) = vid_id_tuple
     title = funshion_get_title_by_id(vid, id)
     url_list = funshion_id_to_urls(vid)
-    
+
     for url in url_list:
         type, ext, size = url_info(url)
         print_info(site_info, title, type, size)
-    
+
     if not info_only:
         download_urls(url_list, title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
@@ -95,7 +92,7 @@ def funshion_drama_id_to_vid(episode_id):
     id: 95785
     ->[('626464', '1'), ('626466', '2'), ('626468', '3'),...
     Drama ID to vids used in drama.
-    
+
     **THIS VID IS NOT THE SAME WITH THE ONES USED IN SINGLE VIDEO!!**
     """
     html = get_content('http://pm.funshion.com/v5/media/episode?id={episode_id}&cl=aphone&uc=5'.format(episode_id = episode_id))
@@ -118,7 +115,7 @@ def funshion_get_title_by_id(single_episode_id, drama_id):
     Get title for single drama video."""
     html = get_content('http://pm.funshion.com/v5/media/episode?id={id}&cl=aphone&uc=5'.format(id = drama_id))
     c = json.loads(html)
-    
+
     for i in c['episodes']:
         if i['id'] == str(single_episode_id):
             return c['name'] + ' - ' + i['name']
@@ -127,11 +124,11 @@ def funshion_get_title_by_id(single_episode_id, drama_id):
 #----------------------------------------------------------------------
 def select_url_from_video_api(html):
     """str(html)->str(url)
-    
+
     Choose the best one.
-    
+
     Used in both single and drama download.
-    
+
     code definition:
     {'tv': 'liuchang',
     'dvd': 'biaoqing',

From 2baeed762ae90f8f322f687859fcf6dac1d1ff9e Mon Sep 17 00:00:00 2001
From: "H.G" <helloworld@outlook.co.nz>
Date: Mon, 5 Sep 2016 10:50:58 +1200
Subject: [PATCH 120/149] [universal] fix issue: "flush" is not available in
 Python under 3.3

---
 src/you_get/common.py             | 16 ++++++++++++++++
 src/you_get/extractor.py          |  1 +
 src/you_get/extractors/iqiyi.py   |  1 +
 src/you_get/extractors/netease.py |  1 +
 src/you_get/processor/ffmpeg.py   |  1 +
 5 files changed, 20 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a5a0fbab..7f76aaac 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1046,6 +1046,22 @@ def set_http_proxy(proxy):
     opener = request.build_opener(proxy_support)
     request.install_opener(opener)
 
+def print_more_compatible(*args, **kwargs):
+    import builtins as __builtin__
+    """Overload default print function as py (<3.3) does not support 'flush' keyword.
+    Although the function name can be same as print to get itself overloaded automatically,
+    I'd rather leave it with a different name and only overload it when importing to make less confusion. """
+    # nothing happens on py3.3 and later
+    if sys.version_info[:2] >= (3, 3):
+        return __builtin__.print(*args, **kwargs)
+
+    # in lower pyver (e.g. 3.2.x), remove 'flush' keyword and flush it as requested
+    doFlush = kwargs.pop('flush', False)
+    ret = __builtin__.print(*args, **kwargs)
+    if doFlush:
+        kwargs.get('file', sys.stdout).flush()
+    return ret
+
 
 
 def download_main(download, download_playlist, urls, playlist, **kwargs):
diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 3cc78289..594b908e 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy
+from .common import print_more_compatible as print
 from .util import log
 from . import json_output
 import os
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 80fa5f93..91329708 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from ..common import *
+from ..common import print_more_compatible as print
 from ..extractor import VideoExtractor
 from ..util import log
 from .. import json_output
diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index b37ee003..63ee59b8 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -4,6 +4,7 @@
 __all__ = ['netease_download']
 
 from ..common import *
+from ..common import print_more_compatible as print
 from ..util import fs
 from json import loads
 import hashlib
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index c34f43c2..1c0ba1a3 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -3,6 +3,7 @@
 import os.path
 import subprocess
 from ..util.strings import parameterize
+from ..common import print_more_compatible as print
 
 def get_usable_ffmpeg(cmd):
     try:

From 24b2b4cb55119ab0198c3bc5538854883d794828 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 7 Sep 2016 10:44:05 +0200
Subject: [PATCH 121/149] [bilibili] do not hide any URLError (better for
 debugging)

---
 src/you_get/extractors/bilibili.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e9d3e7ad..c18290b8 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -95,12 +95,9 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
 
     type_ = ''
     size = 0
-    try:
-        for url in urls:
-            _, type_, temp = url_info(url)
-            size += temp or 0
-    except error.URLError:
-        log.wtf('[Failed] DNS not resolved. Please change your DNS server settings.')
+    for url in urls:
+        _, type_, temp = url_info(url)
+        size += temp or 0
 
     print_info(site_info, title, type_, size)
     if not info_only:
@@ -196,4 +193,4 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
 
 site_info = "bilibili.com"
 download = bilibili_download
-download_playlist = bilibili_download
\ No newline at end of file
+download_playlist = bilibili_download

From 47011101a940f2884aaa1a41c3a72ac7e6f37cda Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Wed, 7 Sep 2016 10:47:38 +0200
Subject: [PATCH 122/149] added fix for nonstandard plflags on panda.tv

---
 src/you_get/extractors/panda.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
index 74064f9f..3f9ceade 100644
--- a/src/you_get/extractors/panda.py
+++ b/src/you_get/extractors/panda.py
@@ -18,10 +18,11 @@ def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     data = json.loads(content)['data']
     title = data.get('roominfo')['name']
     room_key = data.get('videoinfo')['room_key']
+    plflag = data.get('videoinfo')['plflag'].split('_')
     status = data.get('videoinfo')['status']
     if status is not "2":
         raise ValueError("The live stream is not online! (status:%s)" % status)
-    real_url = 'http://pl3.live.panda.tv/live_panda/{}.flv'.format(room_key)
+    real_url = 'http://pl{}.live.panda.tv/live_panda/{}.flv'.format(plflag[1],room_key)
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From 7baa1f029e6c79a5741c07e428930defefd767db Mon Sep 17 00:00:00 2001
From: zhangchao <zhangchaolhr@qq.com>
Date: Mon, 29 Aug 2016 19:14:29 +0800
Subject: [PATCH 123/149] qq compatible with new urls and modify baomihua
 getvidourl address

---
 src/you_get/extractors/baomihua.py | 2 +-
 src/you_get/extractors/qq.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/baomihua.py b/src/you_get/extractors/baomihua.py
index f8be6fa9..4c4febb7 100644
--- a/src/you_get/extractors/baomihua.py
+++ b/src/you_get/extractors/baomihua.py
@@ -7,7 +7,7 @@ from ..common import *
 import urllib
 
 def baomihua_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html('http://play.baomihua.com/getvideourl.aspx?flvid=%s' % id)
+    html = get_html('http://play.baomihua.com/getvideourl.aspx?flvid=%s&devicetype=phone_app' % id)
     host = r1(r'host=([^&]*)', html)
     assert host
     type = r1(r'videofiletype=([^&]*)', html)
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 59460e1d..9ca8af82 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -50,7 +50,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     else:
         content = get_html(url)
         vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        vid = vid[0] if vid else match1(content, r'vid\s*:\s*"\s*([^"]+)"') #general fallback
+        vid = vid[0] if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title
         title = match1(content, r'"title":"([^"]+)"') if not title else title

From 0984190f93bd0b5c55748c41ca657d1ba6bf5a6b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 14 Sep 2016 19:59:55 +0200
Subject: [PATCH 124/149] [acfun] improve regex for getting video part title

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 5c6dc987..4638cb8f 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -78,7 +78,7 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = escape_file_path(title)
     assert title
 
-    video = re.search('data-vid="(\d+)"\s*data-scode="".*>([^<]+)</a>', html)
+    video = re.search('data-vid="(\d+)"\s*data-scode=""[^<]*title="([^"]+)"', html)
     vid = video.group(1)
     title = title + ' - ' + video.group(2)
     acfun_download_by_vid(vid, title,

From 38ba0dbe48ecac4b7a354e4cf5766cf9415fb3c9 Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Mon, 19 Sep 2016 13:15:25 +0200
Subject: [PATCH 125/149] fix for #1405

---
 src/you_get/extractors/youku.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 853a75ba..1fb09e8c 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -314,6 +314,9 @@ class Youku(VideoExtractor):
                                 q         = q
                             )
                         ksegs += [i['server'] for i in json.loads(get_content(u))]
+
+                        if (parse_host(ksegs[len(ksegs)-1])[0] == "vali.cp31.ott.cibntv.net"):
+                            ksegs.pop(len(ksegs)-1)
             except error.HTTPError as e:
                 # Use fallback stream data in case of HTTP 404
                 log.e('[Error] ' + str(e))

From 7e5abc937153791ea66b7fc8233ab8e78aa85e80 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 23 Sep 2016 08:39:25 +0200
Subject: [PATCH 126/149] version 0.4.555

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index a877863a..6d91656c 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.536'
+__version__ = '0.4.555'

From fb97d7d0c3a4b5c4db28b2f08dee82faa914b7c4 Mon Sep 17 00:00:00 2001
From: Bochun Bai <bbc@sinofool.net>
Date: Mon, 3 Oct 2016 00:18:50 +0800
Subject: [PATCH 127/149] Tencent Video use best quality

Prefer 1080p and 720p if available
---
 src/you_get/extractors/qq.py | 75 ++++++++++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 9ca8af82..f1707527 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -7,22 +7,67 @@ from .qie import download as qieDownload
 from urllib.parse import urlparse,parse_qs
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
-    content = get_html(api)
-    output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
-    url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']
-    fvkey = output_json['vl']['vi'][0]['fvkey']
-    mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
-    if mp4:
-        mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
-    else:
-        mp4 = output_json['vl']['vi'][0]['fn']
-    url = '%s/%s?vkey=%s' % ( url, mp4, fvkey )
-    _, ext, size = url_info(url, faker=True)
+    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3%2E2%2E19%2E333&platform=11&defnpayver=1&vid=' + vid
+    info = get_html(info_api)
+    video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
+    parts_vid = video_json['vl']['vi'][0]['vid']
+    parts_ti = video_json['vl']['vi'][0]['ti']
+    parts_prefix = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
+    parts_formats = video_json['fl']['fi']
+    # find best quality
+    # only looking for fhd(1080p) and shd(720p) here.
+    # 480p usually come with a single file, will be downloaded as fallback.
+    best_quality = ''
+    for part_format in parts_formats:
+        if part_format['name'] == 'fhd':
+            best_quality = 'fhd'
+            break
 
-    print_info(site_info, title, ext, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
+        if part_format['name'] == 'shd':
+            best_quality = 'shd'
+
+    for part_format in parts_formats:
+        if (not best_quality == '') and (not part_format['name'] == best_quality):
+            continue
+        part_format_id = part_format['id']
+        part_format_sl = part_format['sl']
+        if part_format_sl == 0:
+            part_urls= []
+            total_size = 0
+            try:
+                # For fhd(1080p), every part is about 100M and 6 minutes
+                # try 100 parts here limited download longest single video of 10 hours.
+                for part in range(1,100):
+                    filename = vid + '.p' + str(part_format_id % 1000) + '.' + str(part) + '.mp4'
+                    key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format=%s&vid=%s&filename=%s" % (part_format_id, parts_vid, filename)
+                    #print(filename)
+                    #print(key_api)
+                    part_info = get_html(key_api)
+                    key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
+                    #print(key_json)
+                    vkey = key_json['key']
+                    url = '%s/%s?vkey=%s' % (parts_prefix, filename, vkey)
+                    part_urls.append(url)
+                    _, ext, size = url_info(url, faker=True)
+                    total_size += size
+            except:
+                pass
+            print_info(site_info, parts_ti, ext, total_size)
+            if not info_only:
+                download_urls(part_urls, parts_ti, ext, total_size, output_dir=output_dir, merge=merge)
+        else:
+            fvkey = output_json['vl']['vi'][0]['fvkey']
+            mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
+            if mp4:
+                mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
+            else:
+                mp4 = output_json['vl']['vi'][0]['fn']
+            url = '%s/%s?vkey=%s' % ( parts_prefix, mp4, fvkey )
+            _, ext, size = url_info(url, faker=True)
+
+            print_info(site_info, title, ext, size)
+            if not info_only:
+                download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):

From 3b3e5cfe38fde46afe0ebf2717802c44e8028706 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 5 Oct 2016 14:38:02 +0200
Subject: [PATCH 128/149] update README (close #1422)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index a99e57fd..1b653308 100644
--- a/README.md
+++ b/README.md
@@ -339,6 +339,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **Tumblr**  | <https://www.tumblr.com/>     |✓|✓|✓|
 | TED         | <http://www.ted.com/>         |✓| | |
 | SoundCloud  | <https://soundcloud.com/>     | | |✓|
+| SHOWROOM    | <https://www.showroom-live.com/> |✓| | |
 | Pinterest   | <https://www.pinterest.com/>  | |✓| |
 | MusicPlayOn | <http://en.musicplayon.com/>  |✓| | |
 | MTV81       | <http://www.mtv81.com/>       |✓| | |

From c9ffae970e3ebc6131c4b4a6593320ab6fe65675 Mon Sep 17 00:00:00 2001
From: Vicent Tsai <vayn@vayn.de>
Date: Mon, 10 Oct 2016 00:13:01 +0800
Subject: [PATCH 129/149] [AcFun] fix #1429

---
 src/you_get/extractors/acfun.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 4638cb8f..87e005fb 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -73,14 +73,14 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     assert re.match(r'http://[^\.]+.acfun.[^\.]+/\D/\D\D(\d+)', url)
     html = get_html(url)
 
-    title = r1(r'<h1 id="txt-title-view">([^<>]+)<', html)
+    title = r1(r'data-title="([^"]+)"', html)
     title = unescape_html(title)
     title = escape_file_path(title)
     assert title
 
-    video = re.search('data-vid="(\d+)"\s*data-scode=""[^<]*title="([^"]+)"', html)
-    vid = video.group(1)
-    title = title + ' - ' + video.group(2)
+    vid = r1('data-vid="(\d+)"', html)
+    up = r1('data-name="([^"]+)"', html)
+    title = title + ' - ' + up
     acfun_download_by_vid(vid, title,
                           output_dir=output_dir,
                           merge=merge,

From 21fc4d4a0999e70b7886d3abd0cfaa6e4244dbae Mon Sep 17 00:00:00 2001
From: chinat <chinaxiahaifeng@gmail.com>
Date: Mon, 10 Oct 2016 11:28:45 +0800
Subject: [PATCH 130/149] translate char | to - in filename for ffmpeg concat

---
 src/you_get/util/fs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py
index 36e0b29d..4f415bf0 100644
--- a/src/you_get/util/fs.py
+++ b/src/you_get/util/fs.py
@@ -10,6 +10,7 @@ def legitimize(text, os=platform.system()):
     text = text.translate({
         0: None,
         ord('/'): '-',
+        ord('|'): '-',
     })
 
     if os == 'Windows':
@@ -20,7 +21,6 @@ def legitimize(text, os=platform.system()):
             ord('*'): '-',
             ord('?'): '-',
             ord('\\'): '-',
-            ord('|'): '-',
             ord('\"'): '\'',
             # Reserved in Windows VFAT
             ord('+'): '-',

From e2f86641ab5354fa103f29709a21ffdb4cf574f8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 14 Oct 2016 20:32:05 +0200
Subject: [PATCH 131/149] update README: use https

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1b653308..b994ebd1 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ Interested? [Install it](#installation) now and [get started by examples](#getti
 
 Are you a Python programmer? Then check out [the source](https://github.com/soimort/you-get) and fork it!
 
-![](http://i.imgur.com/GfthFAz.png)
+![](https://i.imgur.com/GfthFAz.png)
 
 ## Installation
 

From 669d7b558655f4ffa530ad24573936f90119ced2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 19 Oct 2016 20:47:17 +0200
Subject: [PATCH 132/149] [youtube] unescape HTML entities, fix #1462

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 0ef390ed..33e3923e 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -236,7 +236,7 @@ class YouTube(VideoExtractor):
                     start = '{:0>2}:{:0>2}:{:06.3f}'.format(int(h), int(m), s).replace('.', ',')
                     m, s = divmod(finish, 60); h, m = divmod(m, 60)
                     finish = '{:0>2}:{:0>2}:{:06.3f}'.format(int(h), int(m), s).replace('.', ',')
-                    content = text.firstChild.nodeValue
+                    content = unescape_html(text.firstChild.nodeValue)
 
                     srt += '%s\n' % str(seq)
                     srt += '%s --> %s\n' % (start, finish)

From 95a8d1e8afdd6df60b3e87ac739cf836b0d0d837 Mon Sep 17 00:00:00 2001
From: Cheng Gu <guchengf@gmail.com>
Date: Thu, 20 Oct 2016 14:19:45 +0800
Subject: [PATCH 133/149] [huomaotv] add huomao.com suppport

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/huomaotv.py | 36 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 src/you_get/extractors/huomaotv.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7f76aaac..3a60bf12 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -27,6 +27,7 @@ SITES = {
     'google'           : 'google',
     'heavy-music'      : 'heavymusic',
     'huaban'           : 'huaban',
+    'huomao'           : 'huomaotv',
     'iask'             : 'sina',
     'ifeng'            : 'ifeng',
     'imgur'            : 'imgur',
diff --git a/src/you_get/extractors/huomaotv.py b/src/you_get/extractors/huomaotv.py
new file mode 100644
index 00000000..4852ff06
--- /dev/null
+++ b/src/you_get/extractors/huomaotv.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+__all__ = ['huomaotv_download']
+
+from ..common import *
+
+
+def get_mobile_room_url(room_id):
+    return 'http://www.huomao.com/mobile/mob_live?cid=%s' % room_id
+
+
+def get_m3u8_url(stream_id):
+    return 'http://live-ws.huomaotv.cn/live/%s/playlist.m3u8' % stream_id
+
+
+def huomaotv_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    room_id_pattern = r'huomao.com/(\d+)'
+    room_id = match1(url, room_id_pattern)
+    html = get_content(get_mobile_room_url(room_id))
+
+    stream_id_pattern = r'id="html_stream" value="(\w+)"'
+    stream_id = match1(html, stream_id_pattern)
+
+    m3u8_url = get_m3u8_url(stream_id)
+
+    title = match1(html, r'<title>([^<]{1,9999})</title>')
+
+    print_info(site_info, title, 'm3u8', float('inf'))
+
+    if not info_only:
+        download_url_ffmpeg(m3u8_url, title, 'm3u8', None, output_dir=output_dir, merge=merge)
+
+
+site_info = 'huomao.com'
+download = huomaotv_download
+download_playlist = playlist_not_supported('huomao')

From 78fa1161310216e102f84f950b01d43c08899550 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Fri, 14 Oct 2016 00:03:56 -0600
Subject: [PATCH 134/149] Add POST method to common.py

---
 src/you_get/common.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6c65bd49..3e152732 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -327,6 +327,45 @@ def get_content(url, headers={}, decoded=True):
 
     return data
 
+def post_content(url, headers={}, post_data={}, decoded=True):
+    """Post the content of a URL via sending a HTTP POST request.
+
+    Args:
+        url: A URL.
+        headers: Request headers used by the client.
+        decoded: Whether decode the response body using UTF-8 or the charset specified in Content-Type.
+
+    Returns:
+        The content as a string.
+    """
+
+    logging.debug('post_content: %s \n post_data: %s' % (url, post_data))
+
+    req = request.Request(url, headers=headers)
+    if cookies:
+        cookies.add_cookie_header(req)
+        req.headers.update(req.unredirected_hdrs)
+    post_data_enc = bytes(parse.urlencode(post_data), 'utf-8')
+    response = request.urlopen(req, data = post_data_enc)
+    data = response.read()
+
+    # Handle HTTP compression for gzip and deflate (zlib)
+    content_encoding = response.getheader('Content-Encoding')
+    if content_encoding == 'gzip':
+        data = ungzip(data)
+    elif content_encoding == 'deflate':
+        data = undeflate(data)
+
+    # Decode the response body
+    if decoded:
+        charset = match1(response.getheader('Content-Type'), r'charset=([\w-]+)')
+        if charset is not None:
+            data = data.decode(charset)
+        else:
+            data = data.decode('utf-8')
+
+    return data
+
 def url_size(url, faker = False, headers = {}):
     if faker:
         response = request.urlopen(request.Request(url, headers = fake_headers), None)

From aef17dcb9926f9e2d056a92796dcf58537a42d63 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Fri, 14 Oct 2016 00:03:56 -0600
Subject: [PATCH 135/149] Add Baidu Cloud support

---
 src/you_get/extractors/baidu.py | 225 ++++++++++++++++++++++++++++----
 1 file changed, 197 insertions(+), 28 deletions(-)
 mode change 100755 => 100644 src/you_get/extractors/baidu.py

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
old mode 100755
new mode 100644
index aa9caa0c..d5efaf0b
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -7,8 +7,10 @@ from ..common import *
 from .embed import *
 from .universal import *
 
+
 def baidu_get_song_data(sid):
-    data = json.loads(get_html('http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker = True))['data']
+    data = json.loads(get_html(
+        'http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker=True))['data']
 
     if data['xcode'] != '':
         # inside china mainland
@@ -17,22 +19,28 @@ def baidu_get_song_data(sid):
         # outside china mainland
         return None
 
+
 def baidu_get_song_url(data):
     return data['songLink']
 
+
 def baidu_get_song_artist(data):
     return data['artistName']
 
+
 def baidu_get_song_album(data):
     return data['albumName']
 
+
 def baidu_get_song_title(data):
     return data['songName']
 
+
 def baidu_get_song_lyric(data):
     lrc = data['lrcLink']
     return None if lrc is '' else "http://music.baidu.com%s" % lrc
 
+
 def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
     data = baidu_get_song_data(sid)
     if data is not None:
@@ -51,7 +59,8 @@ def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
     type, ext, size = url_info(url, faker=True)
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([url], file_name, ext, size, output_dir, merge=merge, faker=True)
+        download_urls([url], file_name, ext, size,
+                      output_dir, merge=merge, faker=True)
 
     try:
         type, ext, size = url_info(lrc, faker=True)
@@ -61,12 +70,14 @@ def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
     except:
         pass
 
-def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False):
-    html = get_html('http://music.baidu.com/album/%s' % aid, faker = True)
+
+def baidu_download_album(aid, output_dir='.', merge=True, info_only=False):
+    html = get_html('http://music.baidu.com/album/%s' % aid, faker=True)
     album_name = r1(r'<h2 class="album-name">(.+?)<\/h2>', html)
     artist = r1(r'<span class="author_list" title="(.+?)">', html)
     output_dir = '%s/%s - %s' % (output_dir, artist, album_name)
-    ids = json.loads(r1(r'<span class="album-add" data-adddata=\'(.+?)\'>', html).replace('&quot', '').replace(';', '"'))['ids']
+    ids = json.loads(r1(r'<span class="album-add" data-adddata=\'(.+?)\'>',
+                        html).replace('&quot', '').replace(';', '"'))['ids']
     track_nr = 1
     for id in ids:
         song_data = baidu_get_song_data(id)
@@ -75,38 +86,29 @@ def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False)
         song_lrc = baidu_get_song_lyric(song_data)
         file_name = '%02d.%s' % (track_nr, song_title)
 
-        type, ext, size = url_info(song_url, faker = True)
+        type, ext, size = url_info(song_url, faker=True)
         print_info(site_info, song_title, type, size)
         if not info_only:
-            download_urls([song_url], file_name, ext, size, output_dir, merge = merge, faker = True)
+            download_urls([song_url], file_name, ext, size,
+                          output_dir, merge=merge, faker=True)
 
         if song_lrc:
-            type, ext, size = url_info(song_lrc, faker = True)
+            type, ext, size = url_info(song_lrc, faker=True)
             print_info(site_info, song_title, type, size)
             if not info_only:
-                download_urls([song_lrc], file_name, ext, size, output_dir, faker = True)
+                download_urls([song_lrc], file_name, ext,
+                              size, output_dir, faker=True)
 
         track_nr += 1
 
-def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False, **kwargs):
-    if re.match(r'http://imgsrc.baidu.com', url):
-        universal_download(url, output_dir, merge=merge, info_only=info_only)
-        return
 
-    elif re.match(r'http://pan.baidu.com', url):
-        html = get_html(url)
+def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
 
-        title = r1(r'server_filename="([^"]+)"', html)
-        if len(title.split('.')) > 1:
-            title = ".".join(title.split('.')[:-1])
-
-        real_url = r1(r'\\"dlink\\":\\"([^"]*)\\"', html).replace('\\\\/', '/')
-        type, ext, size = url_info(real_url, faker = True)
-
-        print_info(site_info, title, ext, size)
+    if re.match(r'http://pan.baidu.com', url):
+        real_url, title, ext, size = baidu_pan_download(url)
         if not info_only:
-            download_urls([real_url], title, ext, size, output_dir, merge = merge)
-
+            download_urls([real_url], title, ext, size,
+                          output_dir, url, merge=merge, faker=True)
     elif re.match(r'http://music.baidu.com/album/\d+', url):
         id = r1(r'http://music.baidu.com/album/(\d+)', url)
         baidu_download_album(id, output_dir, merge, info_only)
@@ -124,17 +126,20 @@ def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
-            items = re.findall(r'//imgsrc.baidu.com/forum/w[^"]+/([^/"]+)', html)
+            items = re.findall(
+                r'//imgsrc.baidu.com/forum/w[^"]+/([^/"]+)', html)
             urls = ['http://imgsrc.baidu.com/forum/pic/item/' + i
                     for i in set(items)]
 
             # handle albums
             kw = r1(r'kw=([^&]+)', html) or r1(r"kw:'([^']+)'", html)
             tid = r1(r'tid=(\d+)', html) or r1(r"tid:'([^']+)'", html)
-            album_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=%s&tid=%s' % (kw, tid)
+            album_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=%s&tid=%s' % (
+                kw, tid)
             album_info = json.loads(get_content(album_url))
             for i in album_info['data']['pic_list']:
-                urls.append('http://imgsrc.baidu.com/forum/pic/item/' + i['pic_id'] + '.jpg')
+                urls.append(
+                    'http://imgsrc.baidu.com/forum/pic/item/' + i['pic_id'] + '.jpg')
 
             ext = 'jpg'
             size = float('Inf')
@@ -144,6 +149,170 @@ def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info
                 download_urls(urls, title, ext, size,
                               output_dir=output_dir, merge=False)
 
+
+def baidu_pan_download(url):
+    errno_patt = r'errno":([^"]+),'
+    refer_url = ""
+    fake_headers = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'Host': 'pan.baidu.com',
+        'Origin': 'http://pan.baidu.com',
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2500.0 Safari/537.36',
+        'Referer': refer_url
+    }
+    if cookies:
+        print('Use user specified cookies')
+    else:
+        print('Generating cookies...')
+        fake_headers['Cookie'] = baidu_pan_gen_cookies(url)
+    refer_url = "http://pan.baidu.com"
+    html = get_content(url, fake_headers, decoded=True)
+    isprotected = False
+    sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
+        html)
+    if sign == None:
+        if re.findall(r'\baccess-code\b', html):
+            isprotected = True
+            sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, fake_headers, psk = baidu_pan_protected_share(
+                url)
+            # raise NotImplementedError("Password required!")
+        if isprotected != True:
+            raise AssertionError("Share not found or canceled: %s" % url)
+    if bdstoken == None:
+        bdstoken = ""
+    if isprotected != True:
+        sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
+            html)
+    request_url = "http://pan.baidu.com/api/sharedownload?sign=%s&timestamp=%s&bdstoken=%s&channel=chunlei&clienttype=0&web=1&app_id=%s" % (
+        sign, timestamp, bdstoken, appid)
+    refer_url = url
+    post_data = {
+        'encrypt': 0,
+        'product': 'share',
+        'uk': uk,
+        'primaryid': primary_id,
+        'fid_list': '[' + fs_id + ']'
+    }
+    if isprotected == True:
+        post_data['sekey'] = psk
+    response_content = post_content(request_url, fake_headers, post_data, True)
+    errno = match1(response_content, errno_patt)
+    if errno != "0":
+        raise AssertionError(
+            "Server refused to provide download link! (Errno:%s)" % errno)
+    real_url = r1(r'dlink":"([^"]+)"', response_content).replace('\\/', '/')
+    title = r1(r'server_filename":"([^"]+)"', response_content)
+    assert real_url
+    type, ext, size = url_info(real_url, faker=True)
+    title_wrapped = json.loads('{"wrapper":"%s"}' % title)
+    title = title_wrapped['wrapper']
+    logging.debug(real_url)
+    print_info(site_info, title, ext, size)
+    print('Hold on...')
+    time.sleep(5)
+    return real_url, title, ext, size
+
+
+def baidu_pan_parse(html):
+    sign_patt = r'sign":"([^"]+)"'
+    timestamp_patt = r'timestamp":([^"]+),'
+    appid_patt = r'app_id":"([^"]+)"'
+    bdstoken_patt = r'bdstoken":"([^"]+)"'
+    fs_id_patt = r'fs_id":([^"]+),'
+    uk_patt = r'uk":([^"]+),'
+    errno_patt = r'errno":([^"]+),'
+    primary_id_patt = r'shareid":([^"]+),'
+    sign = match1(html, sign_patt)
+    timestamp = match1(html, timestamp_patt)
+    appid = match1(html, appid_patt)
+    bdstoken = match1(html, bdstoken_patt)
+    fs_id = match1(html, fs_id_patt)
+    uk = match1(html, uk_patt)
+    primary_id = match1(html, primary_id_patt)
+    return sign, timestamp, bdstoken, appid, primary_id, fs_id, uk
+
+
+def baidu_pan_gen_cookies(url, post_data=None):
+    from http import cookiejar
+    cookiejar = cookiejar.CookieJar()
+    opener = request.build_opener(request.HTTPCookieProcessor(cookiejar))
+    resp = opener.open('http://pan.baidu.com')
+    if post_data != None:
+        resp = opener.open(url, bytes(parse.urlencode(post_data), 'utf-8'))
+    return cookjar2hdr(cookiejar)
+
+
+def baidu_pan_protected_share(url):
+    print('This share is protected by password!')
+    inpwd = input('Please provide unlock password: ')
+    inpwd = inpwd.replace(' ', '').replace('\t', '')
+    print('Please wait...')
+    post_pwd = {
+        'pwd': inpwd,
+        'vcode': None,
+        'vstr': None
+    }
+    from http import cookiejar
+    import time
+    cookiejar = cookiejar.CookieJar()
+    opener = request.build_opener(request.HTTPCookieProcessor(cookiejar))
+    resp = opener.open('http://pan.baidu.com')
+    resp = opener.open(url)
+    init_url = resp.geturl()
+    verify_url = 'http://pan.baidu.com/share/verify?%s&t=%s&channel=chunlei&clienttype=0&web=1' % (
+        init_url.split('?', 1)[1], int(time.time()))
+    refer_url = init_url
+    fake_headers = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'Host': 'pan.baidu.com',
+        'Origin': 'http://pan.baidu.com',
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2500.0 Safari/537.36',
+        'Referer': refer_url
+    }
+    opener.addheaders = dict2triplet(fake_headers)
+    pwd_resp = opener.open(verify_url, bytes(
+        parse.urlencode(post_pwd), 'utf-8'))
+    pwd_resp_str = ungzip(pwd_resp.read()).decode('utf-8')
+    pwd_res = json.loads(pwd_resp_str)
+    if pwd_res['errno'] != 0:
+        raise AssertionError(
+            'Server returned an error: %s (Incorrect password?)' % pwd_res['errno'])
+    pg_resp = opener.open('http://pan.baidu.com/share/link?%s' %
+                          init_url.split('?', 1)[1])
+    content = ungzip(pg_resp.read()).decode('utf-8')
+    sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
+        content)
+    psk = query_cookiejar(cookiejar, 'BDCLND')
+    psk = parse.unquote(psk)
+    fake_headers['Cookie'] = cookjar2hdr(cookiejar)
+    return sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, fake_headers, psk
+
+
+def cookjar2hdr(cookiejar):
+    cookie_str = ''
+    for i in cookiejar:
+        cookie_str = cookie_str + i.name + '=' + i.value + ';'
+    return cookie_str[:-1]
+
+
+def query_cookiejar(cookiejar, name):
+    for i in cookiejar:
+        if i.name == name:
+            return i.value
+
+
+def dict2triplet(dictin):
+    out_triplet = []
+    for i in dictin:
+        out_triplet.append((i, dictin[i]))
+    return out_triplet
+
 site_info = "Baidu.com"
 download = baidu_download
 download_playlist = playlist_not_supported("baidu")

From 4bbafeb9e48e76b7b622f2133685905b362a9096 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Thu, 20 Oct 2016 13:09:30 -0600
Subject: [PATCH 136/149] icourse: add supprt

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/icourses.py | 129 +++++++++++++++++++++++++++++
 3 files changed, 131 insertions(+)
 create mode 100644 src/you_get/extractors/icourses.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 948b0ca2..ca867673 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -29,6 +29,7 @@ SITES = {
     'huaban'           : 'huaban',
     'huomao'           : 'huomaotv',
     'iask'             : 'sina',
+    'icourses'         : 'icourses',
     'ifeng'            : 'ifeng',
     'imgur'            : 'imgur',
     'in'               : 'alive',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index e69bc2fd..61b6a0d1 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -24,6 +24,7 @@ from .funshion import *
 from .google import *
 from .heavymusic import *
 from .huaban import *
+from .icourses import *
 from .ifeng import *
 from .imgur import *
 from .infoq import *
diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
new file mode 100644
index 00000000..5f9b8edf
--- /dev/null
+++ b/src/you_get/extractors/icourses.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+from ..common import *
+from urllib import parse
+import xml.etree.ElementTree as ET
+import datetime
+import hashlib
+import base64
+import logging
+from urllib import error
+import re
+
+__all__ = ['icourses_download']
+
+
+def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
+    title, real_url = icourses_cn_url_parser(
+        url, info_only=info_only, **kwargs)
+    if real_url is not None:
+        for tries in range(0, 3):
+            try:
+                _, type_, size = url_info(real_url, faker=True)
+                break
+            except error.HTTPError:
+                logging.warning('Failed to fetch the video file! Retrying...')
+                title, real_url = icourses_cn_url_parser(url)
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls([real_url], title, 'flv',
+                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True)
+
+
+def icourses_playlist_download(url, **kwargs):
+    import random
+    from time import sleep
+    html = get_content(url)
+    page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
+    video_js_number = r'changeforvideo\((.*?)\)'
+    fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
+    page_navi_vars = re.search(pattern=page_type_patt, string=html)
+    dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
+        page_navi_vars.group(2), page_navi_vars.group(1))
+    html = get_content(dummy_page)
+    fs_status = match1(html, fs_flag)
+    video_list = re.findall(pattern=video_js_number, string=html)
+    for video in video_list:
+        video_args = video.replace('\'', '').split(',')
+        video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
+            video_args[0], video_args[1], fs_status or '1')
+        sleep(random.Random().randint(0, 5))  # Prevent from blockage
+        icourses_download(url=video_url, **kwargs)
+
+
+def icourses_cn_url_parser(url, **kwargs):
+    PLAYER_BASE_VER = '150606-1'
+    ENCRYPT_MOD_VER = '151020'
+    ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
+    html = get_content(url)
+    if re.search(pattern=r'showSectionNode\(.*\)', string=html):
+        logging.warning('Switching to playlist mode!')
+        return icourses_playlist_download(url, **kwargs)
+    flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
+    server_time_patt = r'MPlayer.swf\?v\=(\d+)'
+    uuid_patt = r'uuid:(\d+)'
+    other_args_patt = r'other:"(.*)"'
+    res_url_patt = r'IService:\'([^\']+)'
+    title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
+    title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
+    title_a = match1(html, title_a_patt).strip()
+    title_b = match1(html, title_b_patt).strip()
+    title = title_a + title_b  # WIP, FIXME
+    title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
+                   unescape_html(title).replace(' ', ''))
+    server_time = match1(html, server_time_patt)
+    flashvars = match1(html, flashvars_patt)
+    uuid = match1(flashvars, uuid_patt)
+    other_args = match1(flashvars, other_args_patt)
+    res_url = match1(flashvars, res_url_patt)
+    url_parts = {'v': server_time, 'other': other_args,
+                 'uuid': uuid, 'IService': res_url}
+    req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
+    logging.debug('Requesting video resource location...')
+    xml_resp = get_html(req_url)
+    xml_obj = ET.fromstring(xml_resp)
+    logging.debug('The result was {}'.format(xml_obj.get('status')))
+    if xml_obj.get('status') != 'success':
+        raise ValueError('Server returned error!')
+    common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
+                   'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
+                   'start': 0}
+    media_host = xml_obj.find(".//*[@name='host']").text
+    media_url = media_host + xml_obj.find(".//*[@name='url']").text
+    # This is what they called `SSLModule`... But obviously, just a kind of
+    # encryption, takes absolutely no effect in protecting data intergrity
+    if xml_obj.find(".//*[@name='ssl']").text != 'true':
+        logging.debug('The encryption mode is disabled')
+        # when the so-called `SSLMode` is not activated, the parameters, `h`
+        # and `p` can be found in response
+        arg_h = xml_obj.find(".//*[@name='h']").text
+        assert arg_h
+        arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
+        url_args = common_args.copy()
+        url_args.update({'h': arg_h, 'r': arg_r})
+        final_url = '{}?{}'.format(
+            media_url, parse.urlencode(url_args))
+        return title, final_url
+    # when the `SSLMode` is activated, we need to receive the timestamp and the
+    # time offset (?) value from the server
+    logging.debug('The encryption mode is in effect')
+    ssl_callback = get_html('{}/ssl/ssl.shtml'.format(media_host)).split(',')
+    ssl_timestamp = int(datetime.datetime.strptime(
+        ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
+    sign_this = ENCRYPT_SALT + \
+        parse.urlparse(media_url).path + str(ssl_timestamp)
+    arg_h = base64.b64encode(hashlib.md5(bytes(sign_this, 'utf-8')).digest())
+    # Post-processing, may subject to change, so leaving this alone...
+    arg_h = arg_h.decode('utf-8').strip('=').replace('+',
+                                                     '-').replace('/', '_')
+    arg_r = ssl_timestamp
+    url_args = common_args.copy()
+    url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
+    final_url = '{}?{}'.format(
+        media_url, parse.urlencode(url_args))
+    logging.debug('Concat`ed URL: {}'.format(final_url))
+    return title, final_url
+
+
+site_info = 'icourses.cn'
+download = icourses_download
+download_playlist = icourses_playlist_download

From 5351121186c2c8c94bc7b24419ea5ca305582462 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Tue, 25 Oct 2016 12:52:30 -0600
Subject: [PATCH 137/149] icouses: Code clean up

---
 src/you_get/extractors/icourses.py | 197 +++++++++++++++--------------
 1 file changed, 105 insertions(+), 92 deletions(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index 5f9b8edf..5c2f8cda 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -13,8 +13,9 @@ __all__ = ['icourses_download']
 
 
 def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
-    title, real_url = icourses_cn_url_parser(
-        url, info_only=info_only, **kwargs)
+    icourses_parser = ICousesExactor(url=url)
+    real_url = icourses_parser.icourses_cn_url_parser(**kwargs)
+    title = icourses_parser.title
     if real_url is not None:
         for tries in range(0, 3):
             try:
@@ -22,108 +23,120 @@ def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
                 break
             except error.HTTPError:
                 logging.warning('Failed to fetch the video file! Retrying...')
-                title, real_url = icourses_cn_url_parser(url)
+                real_url = icourses_parser.icourses_cn_url_parser()
+                title = icourses_parser.title
         print_info(site_info, title, type_, size)
         if not info_only:
             download_urls([real_url], title, 'flv',
                           total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True)
 
 
-def icourses_playlist_download(url, **kwargs):
-    import random
-    from time import sleep
-    html = get_content(url)
-    page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
-    video_js_number = r'changeforvideo\((.*?)\)'
-    fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
-    page_navi_vars = re.search(pattern=page_type_patt, string=html)
-    dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
-        page_navi_vars.group(2), page_navi_vars.group(1))
-    html = get_content(dummy_page)
-    fs_status = match1(html, fs_flag)
-    video_list = re.findall(pattern=video_js_number, string=html)
-    for video in video_list:
-        video_args = video.replace('\'', '').split(',')
-        video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
-            video_args[0], video_args[1], fs_status or '1')
-        sleep(random.Random().randint(0, 5))  # Prevent from blockage
-        icourses_download(url=video_url, **kwargs)
+# Why not using VideoExtractor: This site needs specical download method
+class ICousesExactor(object):
 
+    def __init__(self, url):
+        self.url = url
+        self.title = ''
+        return
 
-def icourses_cn_url_parser(url, **kwargs):
-    PLAYER_BASE_VER = '150606-1'
-    ENCRYPT_MOD_VER = '151020'
-    ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
-    html = get_content(url)
-    if re.search(pattern=r'showSectionNode\(.*\)', string=html):
-        logging.warning('Switching to playlist mode!')
-        return icourses_playlist_download(url, **kwargs)
-    flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
-    server_time_patt = r'MPlayer.swf\?v\=(\d+)'
-    uuid_patt = r'uuid:(\d+)'
-    other_args_patt = r'other:"(.*)"'
-    res_url_patt = r'IService:\'([^\']+)'
-    title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
-    title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
-    title_a = match1(html, title_a_patt).strip()
-    title_b = match1(html, title_b_patt).strip()
-    title = title_a + title_b  # WIP, FIXME
-    title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
-                   unescape_html(title).replace(' ', ''))
-    server_time = match1(html, server_time_patt)
-    flashvars = match1(html, flashvars_patt)
-    uuid = match1(flashvars, uuid_patt)
-    other_args = match1(flashvars, other_args_patt)
-    res_url = match1(flashvars, res_url_patt)
-    url_parts = {'v': server_time, 'other': other_args,
-                 'uuid': uuid, 'IService': res_url}
-    req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
-    logging.debug('Requesting video resource location...')
-    xml_resp = get_html(req_url)
-    xml_obj = ET.fromstring(xml_resp)
-    logging.debug('The result was {}'.format(xml_obj.get('status')))
-    if xml_obj.get('status') != 'success':
-        raise ValueError('Server returned error!')
-    common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
-                   'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
-                   'start': 0}
-    media_host = xml_obj.find(".//*[@name='host']").text
-    media_url = media_host + xml_obj.find(".//*[@name='url']").text
-    # This is what they called `SSLModule`... But obviously, just a kind of
-    # encryption, takes absolutely no effect in protecting data intergrity
-    if xml_obj.find(".//*[@name='ssl']").text != 'true':
-        logging.debug('The encryption mode is disabled')
-        # when the so-called `SSLMode` is not activated, the parameters, `h`
-        # and `p` can be found in response
-        arg_h = xml_obj.find(".//*[@name='h']").text
-        assert arg_h
-        arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
+    def icourses_playlist_download(self, **kwargs):
+        import random
+        from time import sleep
+        html = get_content(url)
+        page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
+        video_js_number = r'changeforvideo\((.*?)\)'
+        fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
+        page_navi_vars = re.search(pattern=page_type_patt, string=html)
+        dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
+            page_navi_vars.group(2), page_navi_vars.group(1))
+        html = get_content(dummy_page)
+        fs_status = match1(html, fs_flag)
+        video_list = re.findall(pattern=video_js_number, string=html)
+        for video in video_list:
+            video_args = video.replace('\'', '').split(',')
+            video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
+                video_args[0], video_args[1], fs_status or '1')
+            sleep(random.Random().randint(0, 5))  # Prevent from blockage
+            icourses_download(video_url, **kwargs)
+
+    def icourses_cn_url_parser(self, **kwargs):
+        PLAYER_BASE_VER = '150606-1'
+        ENCRYPT_MOD_VER = '151020'
+        ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
+        html = get_content(self.url)
+        if re.search(pattern=r'showSectionNode\(.*\)', string=html):
+            logging.warning('Switching to playlist mode!')
+            return self.icourses_playlist_download(**kwargs)
+        flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
+        server_time_patt = r'MPlayer.swf\?v\=(\d+)'
+        uuid_patt = r'uuid:(\d+)'
+        other_args_patt = r'other:"(.*)"'
+        res_url_patt = r'IService:\'([^\']+)'
+        title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
+        title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
+        title_a = match1(html, title_a_patt).strip()
+        title_b = match1(html, title_b_patt).strip()
+        title = title_a + title_b  # WIP, FIXME
+        title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
+                       unescape_html(title).replace(' ', ''))
+        server_time = match1(html, server_time_patt)
+        flashvars = match1(html, flashvars_patt)
+        uuid = match1(flashvars, uuid_patt)
+        other_args = match1(flashvars, other_args_patt)
+        res_url = match1(flashvars, res_url_patt)
+        url_parts = {'v': server_time, 'other': other_args,
+                     'uuid': uuid, 'IService': res_url}
+        req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
+        logging.debug('Requesting video resource location...')
+        xml_resp = get_html(req_url)
+        xml_obj = ET.fromstring(xml_resp)
+        logging.debug('The result was {}'.format(xml_obj.get('status')))
+        if xml_obj.get('status') != 'success':
+            raise ValueError('Server returned error!')
+        common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
+                       'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
+                       'start': 0}
+        media_host = xml_obj.find(".//*[@name='host']").text
+        media_url = media_host + xml_obj.find(".//*[@name='url']").text
+        # This is what they called `SSLModule`... But obviously, just a kind of
+        # encryption, takes absolutely no effect in protecting data intergrity
+        if xml_obj.find(".//*[@name='ssl']").text != 'true':
+            logging.debug('The encryption mode is disabled')
+            # when the so-called `SSLMode` is not activated, the parameters, `h`
+            # and `p` can be found in response
+            arg_h = xml_obj.find(".//*[@name='h']").text
+            assert arg_h
+            arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
+            url_args = common_args.copy()
+            url_args.update({'h': arg_h, 'r': arg_r})
+            final_url = '{}?{}'.format(
+                media_url, parse.urlencode(url_args))
+            self.title = title
+            return final_url
+        # when the `SSLMode` is activated, we need to receive the timestamp and the
+        # time offset (?) value from the server
+        logging.debug('The encryption mode is in effect')
+        ssl_callback = get_html(
+            '{}/ssl/ssl.shtml'.format(media_host)).split(',')
+        ssl_timestamp = int(datetime.datetime.strptime(
+            ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
+        sign_this = ENCRYPT_SALT + \
+            parse.urlparse(media_url).path + str(ssl_timestamp)
+        arg_h = base64.b64encode(hashlib.md5(
+            bytes(sign_this, 'utf-8')).digest())
+        # Post-processing, may subject to change, so leaving this alone...
+        arg_h = arg_h.decode('utf-8').strip('=').replace('+',
+                                                         '-').replace('/', '_')
+        arg_r = ssl_timestamp
         url_args = common_args.copy()
-        url_args.update({'h': arg_h, 'r': arg_r})
+        url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
         final_url = '{}?{}'.format(
             media_url, parse.urlencode(url_args))
-        return title, final_url
-    # when the `SSLMode` is activated, we need to receive the timestamp and the
-    # time offset (?) value from the server
-    logging.debug('The encryption mode is in effect')
-    ssl_callback = get_html('{}/ssl/ssl.shtml'.format(media_host)).split(',')
-    ssl_timestamp = int(datetime.datetime.strptime(
-        ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
-    sign_this = ENCRYPT_SALT + \
-        parse.urlparse(media_url).path + str(ssl_timestamp)
-    arg_h = base64.b64encode(hashlib.md5(bytes(sign_this, 'utf-8')).digest())
-    # Post-processing, may subject to change, so leaving this alone...
-    arg_h = arg_h.decode('utf-8').strip('=').replace('+',
-                                                     '-').replace('/', '_')
-    arg_r = ssl_timestamp
-    url_args = common_args.copy()
-    url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
-    final_url = '{}?{}'.format(
-        media_url, parse.urlencode(url_args))
-    logging.debug('Concat`ed URL: {}'.format(final_url))
-    return title, final_url
+        logging.debug('Crafted URL: {}'.format(final_url))
+        self.title = title
+        return final_url
 
 
 site_info = 'icourses.cn'
 download = icourses_download
-download_playlist = icourses_playlist_download
+# download_playlist = icourses_playlist_download

From ae4e533ec9d28fb1598fb91dfa87ce16cb06bc92 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Tue, 25 Oct 2016 14:03:21 -0600
Subject: [PATCH 138/149] common: add dynamic url support for
 `url_save_chunked`

---
 src/you_get/common.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 948b0ca2..0f7fd0e3 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -547,7 +547,11 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
         os.remove(filepath) # on Windows rename could fail if destination filepath exists
     os.rename(temp_filepath, filepath)
 
-def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}):
+def url_save_chunked(url, filepath, bar, dyn_callback=None, chunk_size=0, ignore_range=False, refer=None, is_part=False, faker=False, headers={}):
+    def dyn_update_url(received):
+        if callable(dyn_callback):
+            logging.debug('Calling callback %s for new URL from %s' % (dyn_callback.__name__, received))
+            return dyn_callback(received)
     if os.path.exists(filepath):
         if not force:
             if not is_part:
@@ -585,19 +589,26 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
     else:
         headers = {}
     if received:
-        headers['Range'] = 'bytes=' + str(received) + '-'
+        url = dyn_update_url(received)
+        if not ignore_range:
+            headers['Range'] = 'bytes=' + str(received) + '-'
     if refer:
         headers['Referer'] = refer
 
-    response = request.urlopen(request.Request(url, headers = headers), None)
+    response = request.urlopen(request.Request(url, headers=headers), None)
 
     with open(temp_filepath, open_mode) as output:
+        this_chunk = received
         while True:
             buffer = response.read(1024 * 256)
             if not buffer:
                 break
             output.write(buffer)
             received += len(buffer)
+            if chunk_size and (received - this_chunk) >= chunk_size:
+                url = dyn_callback(received)
+                this_chunk = received
+                response = request.urlopen(request.Request(url, headers=headers), None)
             if bar:
                 bar.update_received(len(buffer))
 
@@ -846,7 +857,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
 
     print()
 
-def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}):
+def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
     assert urls
     if dry_run:
         print('Real URLs:\n%s\n' % urls)
@@ -860,7 +871,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
 
     filename = '%s.%s' % (title, ext)
     filepath = os.path.join(output_dir, filename)
-    if total_size and ext in ('ts'):
+    if total_size:
         if not force and os.path.exists(filepath[:-3] + '.mkv'):
             print('Skipping %s: file already exists' % filepath[:-3] + '.mkv')
             print()
@@ -875,7 +886,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
         print('Downloading %s ...' % tr(filename))
         filepath = os.path.join(output_dir, filename)
         parts.append(filepath)
-        url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers)
+        url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers, **kwargs)
         bar.done()
 
         if not merge:

From 2183448c9098c1abd0e9cf47fa305e3775e1e098 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Tue, 25 Oct 2016 14:15:23 -0600
Subject: [PATCH 139/149] icourses: implement fake `keep connection alive`

---
 src/you_get/extractors/icourses.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index 5c2f8cda..cb2ff74a 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 from ..common import *
 from urllib import parse
+import random
+from time import sleep
 import xml.etree.ElementTree as ET
 import datetime
 import hashlib
@@ -12,23 +14,24 @@ import re
 __all__ = ['icourses_download']
 
 
-def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
+def icourses_download(url, merge=False, output_dir='.', **kwargs):
     icourses_parser = ICousesExactor(url=url)
     real_url = icourses_parser.icourses_cn_url_parser(**kwargs)
     title = icourses_parser.title
     if real_url is not None:
-        for tries in range(0, 3):
+        for tries in range(0, 5):
             try:
                 _, type_, size = url_info(real_url, faker=True)
                 break
             except error.HTTPError:
                 logging.warning('Failed to fetch the video file! Retrying...')
+                sleep(random.Random().randint(0, 5))  # Prevent from blockage
                 real_url = icourses_parser.icourses_cn_url_parser()
                 title = icourses_parser.title
         print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([real_url], title, 'flv',
-                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True)
+        if not kwargs['info_only']:
+            download_urls_chunked([real_url], title, 'flv',
+                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True, ignore_range=True, chunk_size=15000000, dyn_callback=icourses_parser.icourses_cn_url_parser)
 
 
 # Why not using VideoExtractor: This site needs specical download method
@@ -40,9 +43,7 @@ class ICousesExactor(object):
         return
 
     def icourses_playlist_download(self, **kwargs):
-        import random
-        from time import sleep
-        html = get_content(url)
+        html = get_content(self.url)
         page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
         video_js_number = r'changeforvideo\((.*?)\)'
         fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
@@ -59,7 +60,7 @@ class ICousesExactor(object):
             sleep(random.Random().randint(0, 5))  # Prevent from blockage
             icourses_download(video_url, **kwargs)
 
-    def icourses_cn_url_parser(self, **kwargs):
+    def icourses_cn_url_parser(self, received=0, **kwargs):
         PLAYER_BASE_VER = '150606-1'
         ENCRYPT_MOD_VER = '151020'
         ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
@@ -93,9 +94,14 @@ class ICousesExactor(object):
         logging.debug('The result was {}'.format(xml_obj.get('status')))
         if xml_obj.get('status') != 'success':
             raise ValueError('Server returned error!')
-        common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
+        if received:
+            play_type = 'seek'
+        else:
+            play_type = 'play'
+            received -= 1
+        common_args = {'lv': PLAYER_BASE_VER, 'ls': play_type,
                        'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
-                       'start': 0}
+                       'start': received + 1}
         media_host = xml_obj.find(".//*[@name='host']").text
         media_url = media_host + xml_obj.find(".//*[@name='url']").text
         # This is what they called `SSLModule`... But obviously, just a kind of

From ac33461c88344d86f74b69572f2f27d03fd708b5 Mon Sep 17 00:00:00 2001
From: Cheng Gu <guchengf@gmail.com>
Date: Thu, 27 Oct 2016 17:44:02 +0800
Subject: [PATCH 140/149] fix(huomao): adapt to new url format

---
 src/you_get/extractors/huomaotv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/huomaotv.py b/src/you_get/extractors/huomaotv.py
index 4852ff06..6e98c800 100644
--- a/src/you_get/extractors/huomaotv.py
+++ b/src/you_get/extractors/huomaotv.py
@@ -6,7 +6,7 @@ from ..common import *
 
 
 def get_mobile_room_url(room_id):
-    return 'http://www.huomao.com/mobile/mob_live?cid=%s' % room_id
+    return 'http://www.huomao.com/mobile/mob_live/%s' % room_id
 
 
 def get_m3u8_url(stream_id):

From 0f3fe97e9caedf976286193aff5dddf430d80962 Mon Sep 17 00:00:00 2001
From: Cheng Gu <guchengf@gmail.com>
Date: Thu, 27 Oct 2016 17:44:54 +0800
Subject: [PATCH 141/149] update: add huomao.com

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index b994ebd1..182fc12a 100644
--- a/README.md
+++ b/README.md
@@ -407,6 +407,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 花瓣     | <http://huaban.com/>           | |✓| |
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 | 芒果TV   | <http://www.mgtv.com/>         |✓| | |
+| 火猫TV   | <http://www.huomao.com/>         |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 

From feffcb656ad2c33b17fb2e20598f8137fc69789c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 30 Oct 2016 00:24:31 +0200
Subject: [PATCH 142/149] [processor.ffmpeg] fix params in
 ffmpeg_download_stream

---
 src/you_get/processor/ffmpeg.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 1c0ba1a3..c6da97f7 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -212,15 +212,6 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     if not (output_dir == '.'):
         output = output_dir + '/' + output
 
-    ffmpeg_params = []
-    #should these exist...
-    if params is not None:
-        if len(params) > 0:
-            for k, v in params:
-                ffmpeg_params.append(k)
-                ffmpeg_params.append(v)
-
-
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
     ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
@@ -230,6 +221,12 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     else:
         ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 
+    if params is not None:
+        if len(params) > 0:
+            for k, v in params:
+                ffmpeg_params.append(k)
+                ffmpeg_params.append(v)
+
     ffmpeg_params.append(output)
 
     print(' '.join(ffmpeg_params))

From 4b55884e86df68c56ae9fce85293f9b757e97576 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 30 Oct 2016 00:26:25 +0200
Subject: [PATCH 143/149] [dailymotion] use ffmpeg_download_stream, fix #1466

---
 src/you_get/extractors/dailymotion.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/dailymotion.py b/src/you_get/extractors/dailymotion.py
index 8b701cd1..2e96c160 100644
--- a/src/you_get/extractors/dailymotion.py
+++ b/src/you_get/extractors/dailymotion.py
@@ -4,6 +4,11 @@ __all__ = ['dailymotion_download']
 
 from ..common import *
 
+def extract_m3u(url):
+    content = get_content(url)
+    m3u_url = re.findall(r'http://.*', content)[0]
+    return match1(m3u_url, r'([^#]+)')
+
 def dailymotion_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     """Downloads Dailymotion videos by URL.
     """
@@ -13,7 +18,7 @@ def dailymotion_download(url, output_dir = '.', merge = True, info_only = False,
     title = match1(html, r'"video_title"\s*:\s*"([^"]+)"') or \
             match1(html, r'"title"\s*:\s*"([^"]+)"')
 
-    for quality in ['720','480','380','240','auto']:
+    for quality in ['1080','720','480','380','240','auto']:
         try:
             real_url = info[quality][0]["url"]
             if real_url:
@@ -21,11 +26,12 @@ def dailymotion_download(url, output_dir = '.', merge = True, info_only = False,
         except KeyError:
             pass
 
-    type, ext, size = url_info(real_url)
+    m3u_url = extract_m3u(real_url)
+    mime, ext, size = 'video/mp4', 'mp4', 0
 
-    print_info(site_info, title, type, size)
+    print_info(site_info, title, mime, size)
     if not info_only:
-        download_urls([real_url], title, ext, size, output_dir, merge = merge)
+        download_url_ffmpeg(m3u_url, title, ext, output_dir=output_dir, merge=merge)
 
 site_info = "Dailymotion.com"
 download = dailymotion_download

From a4f4fb362616862cc283b05122e74be346f1a309 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 30 Oct 2016 16:16:04 +0100
Subject: [PATCH 144/149] Revert "fix for #1405" (fix #1485)

This reverts commit 38ba0dbe48ecac4b7a354e4cf5766cf9415fb3c9.
---
 src/you_get/extractors/youku.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 1fb09e8c..853a75ba 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -314,9 +314,6 @@ class Youku(VideoExtractor):
                                 q         = q
                             )
                         ksegs += [i['server'] for i in json.loads(get_content(u))]
-
-                        if (parse_host(ksegs[len(ksegs)-1])[0] == "vali.cp31.ott.cibntv.net"):
-                            ksegs.pop(len(ksegs)-1)
             except error.HTTPError as e:
                 # Use fallback stream data in case of HTTP 404
                 log.e('[Error] ' + str(e))

From e8514d1370bc748946940c7c2f757db5c9cf42c8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 01:44:04 +0100
Subject: [PATCH 145/149] version 0.4.575

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 6d91656c..6d4f6c4f 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.555'
+__version__ = '0.4.575'

From 391ca5643a355c310db786e467c6929fd5dde53f Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Wed, 2 Nov 2016 20:44:40 -0400
Subject: [PATCH 146/149] [embed] correct tudou pattern

Hyphen-minus (-) is a valid character in Tudou's video ID. It's even
present in the second pattern of tudou_embed_patterns, just not the
first.
---
 src/you_get/extractors/embed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index a177e663..fc4015c4 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -25,7 +25,7 @@ youku_embed_patterns = [ 'youku\.com/v_show/id_([a-zA-Z0-9=]+)',
 """
 http://www.tudou.com/programs/view/html5embed.action?type=0&amp;code=3LS_URGvl54&amp;lcode=&amp;resourceId=0_06_05_99
 """
-tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([a-zA-Z0-9_]+)\&',
+tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([a-zA-Z0-9_-]+)\&',
                          'www\.tudou\.com/v/([a-zA-Z0-9_-]+)/[^"]*v\.swf'
                        ]
 

From 2b0fe3443f844690305caa0a468d1b744c72ced5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 17:03:01 +0100
Subject: [PATCH 147/149] [test] remove test_vimeo

---
 tests/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 638206af..0fa2979a 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -21,9 +21,6 @@ class YouGetTests(unittest.TestCase):
     def test_mixcloud(self):
         mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)
 
-    def test_vimeo(self):
-        vimeo.download("http://vimeo.com/56810854", info_only=True)
-
     def test_youtube(self):
         youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
         youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)

From bc590cbd62ca4350598551e41910c719864f0c36 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 21:32:13 +0100
Subject: [PATCH 148/149] [douban] add support: movie.douban.com

---
 README.md                        |  4 ++--
 src/you_get/extractors/douban.py | 23 +++++++++++++++++------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 182fc12a..40a26803 100644
--- a/README.md
+++ b/README.md
@@ -128,7 +128,7 @@ $ you-get https://github.com/soimort/you-get/archive/master.zip
 or use [chocolatey package manager](https://chocolatey.org):
 
 ```
-> choco upgrade you-get 
+> choco upgrade you-get
 ```
 
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
@@ -373,7 +373,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 爆米花网 | <http://www.baomihua.com/>     |✓| | |
 | **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓| | |
 | Dilidili | <http://www.dilidili.com/>     |✓| | |
-| 豆瓣     | <http://www.douban.com/>       | | |✓|
+| 豆瓣     | <http://www.douban.com/>       |✓| |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
 | Panda<br/>熊猫 | <http://www.panda.tv/>      |✓| | |
 | 凤凰视频 | <http://v.ifeng.com/>          |✓| | |
diff --git a/src/you_get/extractors/douban.py b/src/you_get/extractors/douban.py
index 187e99c0..1a4a67d1 100644
--- a/src/you_get/extractors/douban.py
+++ b/src/you_get/extractors/douban.py
@@ -7,12 +7,23 @@ from ..common import *
 
 def douban_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_html(url)
-    if 'subject' in url:
+
+    if re.match(r'https?://movie', url):
+        title = match1(html, 'name="description" content="([^"]+)')
+        tid = match1(url, 'trailer/(\d+)')
+        real_url = 'https://movie.douban.com/trailer/video_url?tid=%s' % tid
+        type, ext, size = url_info(real_url)
+
+        print_info(site_info, title, type, size)
+        if not info_only:
+            download_urls([real_url], title, ext, size, output_dir, merge = merge)
+
+    elif 'subject' in url:
         titles = re.findall(r'data-title="([^"]*)">', html)
         song_id = re.findall(r'<li class="song-item" id="([^"]*)"', html)
         song_ssid = re.findall(r'data-ssid="([^"]*)"', html)
         get_song_url = 'http://music.douban.com/j/songlist/get_song_url'
-        
+
         for i in range(len(titles)):
             title = titles[i]
             datas = {
@@ -35,16 +46,16 @@ def douban_download(url, output_dir = '.', merge = True, info_only = False, **kw
                 except:
                     pass
 
-    else: 
+    else:
         titles = re.findall(r'"name":"([^"]*)"', html)
         real_urls = [re.sub('\\\\/', '/', i) for i in re.findall(r'"rawUrl":"([^"]*)"', html)]
-        
+
         for i in range(len(titles)):
             title = titles[i]
             real_url = real_urls[i]
-            
+
             type, ext, size = url_info(real_url)
-            
+
             print_info(site_info, title, type, size)
             if not info_only:
                 download_urls([real_url], title, ext, size, output_dir, merge = merge)

From 5601e1fe30316d02ff6f51b4d77689d004ba2f13 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 22:03:56 +0100
Subject: [PATCH 149/149] [bilibili] fix support for bangumi

---
 src/you_get/extractors/bilibili.py | 108 +++++++++++++++--------------
 1 file changed, 56 insertions(+), 52 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index c18290b8..122dea0b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -119,66 +119,70 @@ def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_o
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
-    if re.match(r'https?://bangumi\.bilibili\.com/', url):
-        # quick hack for bangumi URLs
-        url = r1(r'"([^"]+)" class="v-av-link"', html)
-        html = get_content(url)
-
     title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
                    r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
     if title:
         title = unescape_html(title)
         title = escape_file_path(title)
 
-    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
-                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-    assert flashvars
-    flashvars = flashvars.replace(': ', '=')
-    t, cid = flashvars.split('=', 1)
-    cid = cid.split('&')[0]
-    if t == 'cid':
-        if re.match(r'https?://live\.bilibili\.com/', url):
-            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
-            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+    if re.match(r'https?://bangumi\.bilibili\.com/', url):
+        # quick hack for bangumi URLs
+        episode_id = r1(r'data-current-episode-id="(\d+)"', html)
+        cont = post_content('http://bangumi.bilibili.com/web_api/get_source',
+                            post_data={'episode_id': episode_id})
+        cid = json.loads(cont)['result']['cid']
+        bilibili_download_by_cid(str(cid), title, output_dir=output_dir, merge=merge, info_only=info_only)
 
-        else:
-            # multi-P
-            cids = []
-            pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-            for i, page in enumerate(pages):
-                html = get_html("http://www.bilibili.com%s" % page)
-                flashvars = r1_of([r'(cid=\d+)',
-                                   r'flashvars="([^"]+)"',
-                                   r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-                if flashvars:
-                    t, cid = flashvars.split('=', 1)
-                    cids.append(cid.split('&')[0])
-                if url.endswith(page):
-                    cids = [cid.split('&')[0]]
-                    titles = [titles[i]]
-                    break
-
-            # no multi-P
-            if not pages:
-                cids = [cid]
-                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
-
-            for i in range(len(cids)):
-                bilibili_download_by_cid(cids[i],
-                                         titles[i],
-                                         output_dir=output_dir,
-                                         merge=merge,
-                                         info_only=info_only)
-
-    elif t == 'vid':
-        sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'ykid':
-        youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'uid':
-        tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
-        raise NotImplementedError(flashvars)
+        flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
+                           r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+        assert flashvars
+        flashvars = flashvars.replace(': ', '=')
+        t, cid = flashvars.split('=', 1)
+        cid = cid.split('&')[0]
+        if t == 'cid':
+            if re.match(r'https?://live\.bilibili\.com/', url):
+                title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
+                bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+
+            else:
+                # multi-P
+                cids = []
+                pages = re.findall('<option value=\'([^\']*)\'', html)
+                titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
+                for i, page in enumerate(pages):
+                    html = get_html("http://www.bilibili.com%s" % page)
+                    flashvars = r1_of([r'(cid=\d+)',
+                                       r'flashvars="([^"]+)"',
+                                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+                    if flashvars:
+                        t, cid = flashvars.split('=', 1)
+                        cids.append(cid.split('&')[0])
+                    if url.endswith(page):
+                        cids = [cid.split('&')[0]]
+                        titles = [titles[i]]
+                        break
+
+                # no multi-P
+                if not pages:
+                    cids = [cid]
+                    titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
+
+                for i in range(len(cids)):
+                    bilibili_download_by_cid(cids[i],
+                                             titles[i],
+                                             output_dir=output_dir,
+                                             merge=merge,
+                                             info_only=info_only)
+
+        elif t == 'vid':
+            sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        elif t == 'ykid':
+            youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        elif t == 'uid':
+            tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+        else:
+            raise NotImplementedError(flashvars)
 
     if not info_only and not dry_run:
         if not kwargs['caption']: