merge

CHANGELOG.rst

@@ -1,6 +1,21 @@
 Changelog
 =========
 
+0.3.32
+------
+
+*Date: 2014-12-10*
+
+* New site support:
+    - baomihua.com
+    - zhanqi.tv
+* Bug fixes:
+    - DouyuTV
+    - Tudou
+    - Tumblr
+    - Vine
+    - Youku
+
 0.3.31
 ------
 

Makefile

@@ -40,7 +40,7 @@ bdist_egg:
 bdist_wheel:
 	$(SETUP) bdist_wheel
 
-install: bdist_wheel
+install:
 	$(SETUP) install
 
 release: rst

README.md

@@ -75,6 +75,7 @@ Others:
 * 56 (56网) <http://www.56.com>
 * Xiami (虾米) <http://www.xiami.com>
 * YinYueTai (音悦台) <http://www.yinyuetai.com>
+* Zhanqi (战旗TV) <http://www.zhanqi.tv/lives>
 
 ## Prerequisites
 

src/you_get/common.py

@@ -52,7 +52,8 @@ def tr(s):
     if default_encoding == 'utf-8':
         return s
     else:
-        return str(s.encode('utf-8'))[2:-1]
+        return s
+        #return str(s.encode('utf-8'))[2:-1]
 
 # DEPRECATED in favor of match1()
 def r1(pattern, text):
@@ -919,7 +920,12 @@ def script_main(script_name, download, download_playlist = None):
             sys.exit(1)
 
 def url_to_module(url):
+<<<<<<< HEAD
     from .extractors import netease, w56, acfun, baidu, bilibili, blip, catfun, cntv, cbs, coursera, dailymotion, dongting, douban, douyutv, ehow, facebook, freesound, google, sina, ifeng, alive, instagram, iqiyi, joy, jpopsuki, khan, ku6, kugou, kuwo, letv, magisto, miomio, mixcloud, mtv81, nicovideo, pptv, qq, sohu, songtaste, soundcloud, ted, theplatform, tudou, tucao, tumblr, vid48, videobam, vimeo, vine, vk, xiami, yinyuetai, youku, youtube
+=======
+    from .extractors import netease, w56, acfun, baidu, baomihua, bilibili, blip, catfun, cntv, cbs, coursera, dailymotion, dongting, douban, douyutv, ehow, facebook, freesound, google, sina, ifeng, alive, instagram, iqiyi, joy, jpopsuki, khan, ku6, kugou, kuwo, letv, magisto, miomio, mixcloud, mtv81, nicovideo, pptv, qq, sohu, songtaste, soundcloud, ted, theplatform, tudou, tucao, tumblr, vid48, videobam, vimeo, vine, vk, xiami, yinyuetai, youku, youtube, zhanqi
+
+>>>>>>> 1b55b01b047824312c2eba342eed47d1d0503a97
     video_host = r1(r'https?://([^/]+)/', url)
     video_url = r1(r'https?://[^/]+(.*)', url)
     assert video_host and video_url, 'invalid url: ' + url
@@ -989,6 +995,7 @@ def url_to_module(url):
         'youku': youku,
         'youtu': youtube,
         'youtube': youtube,
+        'zhanqi': zhanqi,
     }
     if k in downloads:
         return downloads[k], url

src/you_get/extractors/douyutv.py

@@ -3,26 +3,26 @@
 __all__ = ['douyutv_download']
 
 from ..common import *
-import re
 import json
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False):
+<<<<<<< HEAD
     html = get_html(url)
     room_id_patt = r'"room_id":(\d{1,99}),'
     title_patt = r'<div class="headline clearfix">\s*<h1>([^<]{1,9999})</h1>\s*</div>'
     
     roomid = re.findall(room_id_patt,html)[0]
     title = unescape_html(re.findall(title_patt,html)[0])
+=======
+    room_id = url[url.rfind('/')+1:]
+>>>>>>> 1b55b01b047824312c2eba342eed47d1d0503a97
 
-    conf = get_html("http://www.douyutv.com/api/client/room/"+roomid)
+    content = get_html("http://www.douyutv.com/api/client/room/"+room_id)
-    metadata = json.loads(conf)
+    data = json.loads(content)['data']
-    
+
-    rtmp_live= metadata.get('data').get('rtmp_live')
+    title = data.get('room_name')
-    rtmp_url= metadata.get('data').get('rtmp_url')
+    real_url = data.get('rtmp_url')+'/'+data.get('rtmp_live')
-    real_url = rtmp_url+'/'+rtmp_live
 
-    type, _, _ = url_info(real_url)
-    
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
         download_urls([real_url], title, 'flv', None, output_dir, merge = merge)

src/you_get/extractors/tudou.py

@@ -26,10 +26,10 @@ def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only =
     html = get_html('http://www.tudou.com/programs/view/%s/' % id, faker=True)
 
     iid = r1(r'iid\s*[:=]\s*(\S+)', html)
-    title = r1(r'kw\s*[:=]\s*[\'\"]([^\']+?)[\'\"]', html)
+    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
     tudou_download_by_iid(iid, title, output_dir = output_dir, merge = merge, info_only = info_only)
 
-def tudou_download(url, output_dir = '.', merge = True, info_only = False):
+def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     # Embedded player
     id = r1(r'http://www.tudou.com/v/([^/]+)/', url)
     if id:
@@ -37,14 +37,17 @@ def tudou_download(url, output_dir = '.', merge = True, info_only = False):
 
     html = get_decoded_html(url, faker=True)
 
-    title = r1(r'kw\s*[:=]\s*[\'\"]([^\']+?)[\'\"]', html)
+    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
     assert title
     title = unescape_html(title)
 
     vcode = r1(r'vcode\s*[:=]\s*\'([^\']+)\'', html)
     if vcode:
         from .youku import youku_download_by_vid
-        return youku_download_by_vid(vcode, title=title, output_dir = output_dir, merge = merge, info_only = info_only)
+        if 'stream_id' in kwargs:
+            return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only, stream_id=kwargs['stream_id'])
+        else:
+            return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
     iid = r1(r'iid\s*[:=]\s*(\d+)', html)
     if not iid:

src/you_get/extractors/vine.py

@@ -11,7 +11,7 @@ def vine_download(url, output_dir='.', merge=True, info_only=False):
     title1 = r1(r'<meta property="twitter:title" content="([^"]*)"', html)
     title2 = r1(r'<meta property="twitter:description" content="([^"]*)"', html)
     title = "%s - %s" % (title1, title2) + " [" + vid + "]"
-    url = r1(r'<source src="([^"]*)"', html) or r1(r'<meta itemprop="contentURL" content="([^"]*)"', html)
+    url = r1(r'<source src="([^"]*)"', html) or r1(r'<meta itemprop="contentUrl" content="([^"]*)"', html)
     if url[0:2] == "//":
         url = "http:" + url
     type, ext, size = url_info(url)

src/you_get/extractors/zhanqi.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+__all__ = ['zhanqi_download']
+
+from ..common import *
+import re
+
+def zhanqi_download(url, output_dir = '.', merge = True, info_only = False):
+    html = get_content(url)
+    rtmp_base_patt = r'VideoUrl":"([^"]+)"'
+    rtmp_id_patt = r'VideoID":"([^"]+)"'
+    title_patt = r'<p class="title-name" title="[^"]+">([^<]+)</p>'
+    title_patt_backup = r'<title>([^<]{1,9999})</title>'
+    
+    rtmp_base = match1(html, rtmp_base_patt).replace('\\/','/')
+    rtmp_id = match1(html, rtmp_id_patt).replace('\\/','/')
+    title = match1(html, title_patt) or match1(html, title_patt_backup)
+    title = unescape_html(title)
+
+    real_url = rtmp_base+'/'+rtmp_id
+    
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)
+
+site_info = "zhanqi.tv"
+download = zhanqi_download
+download_playlist = playlist_not_supported('zhanqi')

src/you_get/processor/ffmpeg.py

@@ -26,7 +26,7 @@ def has_ffmpeg_installed():
 def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
     for file in files:
         if os.path.isfile(file):
-            params = [FFMPEG, '-i']
+            params = [FFMPEG, '-y', '-i']
             params.append(file)
             params.append(output)
             subprocess.call(params)
@@ -36,13 +36,13 @@ def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
 def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
-        concat_list = open(output + '.txt', 'w')
+        concat_list = open(output + '.txt', 'w', encoding="utf-8")
         for file in files:
             if os.path.isfile(file):
                 concat_list.write("file '%s'\n" % file)
         concat_list.close()
 
-        params = [FFMPEG, '-f', 'concat', '-i']
+        params = [FFMPEG, '-f', 'concat', '-y', '-i']
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 
@@ -54,7 +54,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
 
     for file in files:
         if os.path.isfile(file):
-            params = [FFMPEG, '-i']
+            params = [FFMPEG, '-y', '-i']
             params.append(file)
             params.append(file + '.mpg')
             subprocess.call(params)
@@ -64,7 +64,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
         for input in inputs:
             o.write(input.read())
 
-    params = [FFMPEG, '-i']
+    params = [FFMPEG, '-y', '-i']
     params.append(output + '.mpg')
     params += ['-vcodec', 'copy', '-acodec', 'copy']
     params.append(output)
@@ -79,7 +79,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
         raise
 
 def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
-    params = [FFMPEG, '-isync', '-i']
+    params = [FFMPEG, '-isync', '-y', '-i']
     params.append('concat:')
     for file in files:
         if os.path.isfile(file):
@@ -97,13 +97,13 @@ def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
 def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
-        concat_list = open(output + '.txt', 'w')
+        concat_list = open(output + '.txt', 'w', encoding="utf-8")
         for file in files:
             if os.path.isfile(file):
                 concat_list.write("file '%s'\n" % file)
         concat_list.close()
 
-        params = [FFMPEG, '-f', 'concat', '-i']
+        params = [FFMPEG, '-f', 'concat', '-y', '-i']
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 
@@ -115,14 +115,14 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
 
     for file in files:
         if os.path.isfile(file):
-            params = [FFMPEG, '-i']
+            params = [FFMPEG, '-y', '-i']
             params.append(file)
             params += ['-map', '0', '-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
             params.append(file + '.ts')
 
             subprocess.call(params)
 
-    params = [FFMPEG, '-i']
+    params = [FFMPEG, '-y', '-i']
     params.append('concat:')
     for file in files:
         f = file + '.ts'
@@ -143,13 +143,13 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
 def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
-        concat_list = open(output + '.txt', 'w')
+        concat_list = open(output + '.txt', 'w', encoding="utf-8")
         for file in files:
             if os.path.isfile(file):
                 concat_list.write("file '%s'\n" % file)
         concat_list.close()
 
-        params = [FFMPEG, '-f', 'concat', '-i']
+        params = [FFMPEG, '-f', 'concat', '-y', '-i']
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 
@@ -161,14 +161,14 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
 
     for file in files:
         if os.path.isfile(file):
-            params = [FFMPEG, '-i']
+            params = [FFMPEG, '-y', '-i']
             params.append(file)
             params += ['-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
             params.append(file + '.ts')
 
             subprocess.call(params)
 
-    params = [FFMPEG, '-i']
+    params = [FFMPEG, '-y', '-i']
     params.append('concat:')
     for file in files:
         f = file + '.ts'

src/you_get/version.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.3.31'
+__version__ = '0.3.32'