From 8bc81d9a70448e5078af86e07c35b12f3f8b1b6a Mon Sep 17 00:00:00 2001 From: Zhang Ning Date: Wed, 29 Jun 2016 20:18:18 +0800 Subject: [PATCH 1/7] iqiyi: support more stream quality algorism form @ERioK thank you @ERioK Signed-off-by: Zhang Ning --- src/you_get/extractors/iqiyi.py | 45 +++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py index 320520fa..bda2c2e8 100644 --- a/src/you_get/extractors/iqiyi.py +++ b/src/you_get/extractors/iqiyi.py @@ -11,8 +11,6 @@ import hashlib import time -from .iqiyi_sc import gen_sc - ''' Changelog: -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf @@ -86,26 +84,31 @@ class Iqiyi(VideoExtractor): name = "爱奇艺 (Iqiyi)" stream_types = [ - {'id': 'high', 'container': 'mp4', 'video_profile': '高清'}, - {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'}, + {'id': 'BD', 'container': 'm3u8', 'video_profile': '全高清'}, + {'id': 'FD', 'container': 'm3u8', 'video_profile': '超高清'}, + {'id': 'TD', 'container': 'm3u8', 'video_profile': '超清'}, + {'id': 'HD', 'container': 'm3u8', 'video_profile': '高清'}, + {'id': 'SD', 'container': 'm3u8', 'video_profile': '标清'}, + {'id': 'LD', 'container': 'm3u8', 'video_profile': '流畅'}, ] - + ''' supported_stream_types = [ 'high', 'standard'] stream_to_bid = { '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96} + ''' + ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD'] + vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'} + vd_2_profile = {21: u'超清', 2: u'高清', 4: u'超高清', 17: u'全高清', 96: u'流畅', 1: u'标清'} - def getVMS(self,rate): - #tm ->the flash run time for md5 usage - #um -> vip 1 normal 0 - #authkey -> for password protected video ,replace '' with your password - #puid user.passportid may empty? - #TODO: support password protected video + def getVMS(self): tvid, vid = self.vid t = int(time.time() * 1000) - sc = gen_sc(tvid, t).decode('utf-8') - vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7) - return json.loads(get_content(vmsreq)[13:]) + src = '76f90cbd92f94a2e925d83e8ccd22cb7' + key = 'd5fb4bd9d50c4be6948c97edd7254b0e' + sc = hashlib.new('md5', bytes(str(t) + key + vid, 'utf-8')).hexdigest() + vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src) + return json.loads(get_content(vmsreq)) def download_playlist_by_url(self, url, **kwargs): self.url = url @@ -128,12 +131,16 @@ class Iqiyi(VideoExtractor): r1(r'vid=([^&]+)', self.url) or \ r1(r'data-player-videoid="([^"]+)"', html) self.vid = (tvid, videoid) + self.title = match1(html, '([^<]+)').split('-')[0] + + info = self.getVMS() + assert info['code'] == 'A00000', 'can\'t play this video' + + for stream in info['data']['vidl']: + stream_id = self.vd_2_id[stream['vd']] + stream_profile = self.vd_2_profile[stream['vd']] + self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0} - for stream in self.supported_stream_types: - info = self.getVMS(self.stream_to_bid[stream]) - if info["code"] == "A00000": - self.title = info['data']['playInfo']['vn'] - self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])} ''' if info["code"] != "A000000": log.e("[error] outdated iQIYI key") From 5966a090f478b876b31632f2b654f27c382ebdba Mon Sep 17 00:00:00 2001 From: David Zhuang <david.zhuang@mail.utoronto.ca> Date: Wed, 29 Jun 2016 15:56:39 -0400 Subject: [PATCH 2/7] [Common, ffmpeg]Exp: Add a ffmpeg downloader and player for any URL --- src/you_get/common.py | 17 +++++++++ src/you_get/processor/ffmpeg.py | 61 +++++++++++++++++++++++++++++++ src/you_get/processor/rtmpdump.py | 1 + 3 files changed, 79 insertions(+) diff --git a/src/you_get/common.py b/src/you_get/common.py index 119640d5..c0097a4e 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -898,6 +898,23 @@ def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', re assert has_rtmpdump_installed(), "RTMPDump not installed." download_rtmpdump_stream(url, title, ext,params, output_dir) +def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False): + assert url + if dry_run: + print('Real URL:\n%s\n' % [url]) + if params.get("-y",False): #None or unset ->False + print('Real Playpath:\n%s\n' % [params.get("-y")]) + return + + if player: + from .processor.ffmpeg import ffmpeg_play_stream + ffmpeg_play_stream(player, url, params) + return + + from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_streaming + assert has_ffmpeg_installed(), "FFmpeg not installed." + ffmpeg_download_stream(url, title, ext, params, output_dir) + def playlist_not_supported(name): def f(*args, **kwargs): raise NotImplementedError('Playlist is not supported for ' + name) diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py index e7ee35d6..1d5d850c 100644 --- a/src/you_get/processor/ffmpeg.py +++ b/src/you_get/processor/ffmpeg.py @@ -199,3 +199,64 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'): for file in files: os.remove(file + '.ts') return True + +def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'): + """str, str->True + WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!! + You can basicly download anything with this function + but better leave it alone with + """ + output = title + '.' + ext + + if not (output_dir == '.'): + output = output_dir + output + + ffmpeg_params = [] + #should these exist... + if len(params) > 0: + for k, v in params: + ffmpeg_params.append(k) + ffmpeg_params.append(v) + + print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...') + ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i'] + ffmpeg_params.append(files) #not the same here!!!! + + if FFMPEG == 'avconv': #who cares? + ffmpeg_params += ['-c', 'copy', output] + else: + ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', output] + + ffmpeg_params.append(output) + + subprocess.call(ffmpeg_params) + + return True + +# +#To be refactor +#Direct copy of rtmpdump.py +# +def ffmpeg_play_stream(player, url, params={}): + ffmpeg_params = [] + #should these exist... + if len(params) > 0: + for k, v in params: + ffmpeg_params.append(k) + ffmpeg_params.append(v) + + print('Playing streaming content with FFmpeg, press Ctrl+C to stop recording...') + ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i'] + ffmpeg_params.append(url) #not the same here!!!! + + if FFMPEG == 'avconv': #who cares? + ffmpeg_params += ['-c', 'copy', '|'] + else: + ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', '|'] + + ffmpeg_params += [player, '-'] + + print(' '.join(ffmpeg_params)) + + subprocess.call(ffmpeg_params) + return diff --git a/src/you_get/processor/rtmpdump.py b/src/you_get/processor/rtmpdump.py index aadb6887..cf5f822c 100644 --- a/src/you_get/processor/rtmpdump.py +++ b/src/you_get/processor/rtmpdump.py @@ -43,6 +43,7 @@ def download_rtmpdump_stream(url, title, ext,params={},output_dir='.'): # #To be refactor +#To the future myself: Remember to refactor the same function in ffmpeg.py # def play_rtmpdump_stream(player, url, params={}): cmdline="rtmpdump -r '%s' "%url From 0d06c260c0ca535d7652e50c60f676f0bdceba14 Mon Sep 17 00:00:00 2001 From: David Zhuang <david.zhuang@mail.utoronto.ca> Date: Wed, 29 Jun 2016 21:28:49 -0400 Subject: [PATCH 3/7] [iQiyi]try to fix CI complaint --- src/you_get/extractors/iqiyi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py index bda2c2e8..4544552d 100644 --- a/src/you_get/extractors/iqiyi.py +++ b/src/you_get/extractors/iqiyi.py @@ -137,9 +137,9 @@ class Iqiyi(VideoExtractor): assert info['code'] == 'A00000', 'can\'t play this video' for stream in info['data']['vidl']: - stream_id = self.vd_2_id[stream['vd']] - stream_profile = self.vd_2_profile[stream['vd']] - self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0} + stream_id = self.vd_2_id[stream['vd']] + stream_profile = self.vd_2_profile[stream['vd']] + self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0} ''' if info["code"] != "A000000": From 2d542c2cd3b2b6a56886342f53ceb476a08818eb Mon Sep 17 00:00:00 2001 From: David Zhuang <david.zhuang@mail.utoronto.ca> Date: Wed, 29 Jun 2016 21:54:46 -0400 Subject: [PATCH 4/7] [iQiyi]CI complaint CI complaint go away --- src/you_get/extractors/iqiyi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py index 4544552d..de8756bf 100644 --- a/src/you_get/extractors/iqiyi.py +++ b/src/you_get/extractors/iqiyi.py @@ -99,7 +99,7 @@ class Iqiyi(VideoExtractor): ''' ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD'] vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'} - vd_2_profile = {21: u'超清', 2: u'高清', 4: u'超高清', 17: u'全高清', 96: u'流畅', 1: u'标清'} + vd_2_profile = {21: '超清', 2: '高清', 4: '超高清', 17: '全高清', 96: '流畅', 1: '标清'} def getVMS(self): tvid, vid = self.vid From fd93b0380d0c04c4766e520e53a6256437e39067 Mon Sep 17 00:00:00 2001 From: David Zhuang <david.zhuang@mail.utoronto.ca> Date: Wed, 29 Jun 2016 23:15:28 -0400 Subject: [PATCH 5/7] [FFmpeg] Change arguments to record M3U --- src/you_get/processor/ffmpeg.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py index 1d5d850c..a387be0a 100644 --- a/src/you_get/processor/ffmpeg.py +++ b/src/you_get/processor/ffmpeg.py @@ -219,16 +219,18 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'): ffmpeg_params.append(v) print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...') - ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i'] + ffmpeg_params = [FFMPEG] + ['-y', '-i'] ffmpeg_params.append(files) #not the same here!!!! if FFMPEG == 'avconv': #who cares? ffmpeg_params += ['-c', 'copy', output] else: - ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', output] + ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc'] ffmpeg_params.append(output) + print(' '.join(ffmpeg_params)) + subprocess.call(ffmpeg_params) return True @@ -252,7 +254,7 @@ def ffmpeg_play_stream(player, url, params={}): if FFMPEG == 'avconv': #who cares? ffmpeg_params += ['-c', 'copy', '|'] else: - ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', '|'] + ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|'] ffmpeg_params += [player, '-'] From 18bc44ea4155d979f8e5fb864e2233d9b3f36eda Mon Sep 17 00:00:00 2001 From: David Zhuang <david.zhuang@mail.utoronto.ca> Date: Wed, 29 Jun 2016 23:42:19 -0400 Subject: [PATCH 6/7] [FFmpeg] Fix stream corrupted if Ctrl+C Use q instead. --- src/you_get/processor/ffmpeg.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py index a387be0a..c23b0eaf 100644 --- a/src/you_get/processor/ffmpeg.py +++ b/src/you_get/processor/ffmpeg.py @@ -218,8 +218,8 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'): ffmpeg_params.append(k) ffmpeg_params.append(v) - print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...') - ffmpeg_params = [FFMPEG] + ['-y', '-i'] + print('Downloading streaming content with FFmpeg, press q to stop recording...') + ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i'] ffmpeg_params.append(files) #not the same here!!!! if FFMPEG == 'avconv': #who cares? @@ -231,7 +231,14 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'): print(' '.join(ffmpeg_params)) - subprocess.call(ffmpeg_params) + try: + a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE) + a.communicate() + except KeyboardInterrupt: + try: + a.stdin.write('q'.encode('utf-8')) + except: + pass return True @@ -247,8 +254,8 @@ def ffmpeg_play_stream(player, url, params={}): ffmpeg_params.append(k) ffmpeg_params.append(v) - print('Playing streaming content with FFmpeg, press Ctrl+C to stop recording...') - ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i'] + print('Playing streaming content with FFmpeg, press 1 to stop recording...') + ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i'] ffmpeg_params.append(url) #not the same here!!!! if FFMPEG == 'avconv': #who cares? @@ -260,5 +267,13 @@ def ffmpeg_play_stream(player, url, params={}): print(' '.join(ffmpeg_params)) - subprocess.call(ffmpeg_params) - return + try: + a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE) + a.communicate() + except KeyboardInterrupt: + try: + a.stdin.write('q'.encode('utf-8')) + except: + pass + + return True \ No newline at end of file From 7452a4bb0e7aac32db89d871e7636741fddf641a Mon Sep 17 00:00:00 2001 From: David Zhuang <david.zhuang@mail.utoronto.ca> Date: Fri, 1 Jul 2016 00:38:26 -0400 Subject: [PATCH 7/7] [iQiyi] fix for PR conflict --- src/you_get/extractors/iqiyi.py | 78 +++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 23 deletions(-) mode change 100644 => 100755 src/you_get/extractors/iqiyi.py diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py old mode 100644 new mode 100755 index de8756bf..a1552fe4 --- a/src/you_get/extractors/iqiyi.py +++ b/src/you_get/extractors/iqiyi.py @@ -8,6 +8,7 @@ import json from math import floor from zlib import decompress import hashlib +from ..util import log import time @@ -79,17 +80,24 @@ def getDispathKey(rid): t=str(int(floor(int(time)/(10*60.0)))) return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest() ''' +def getVMS(tvid, vid): + t = int(time.time() * 1000) + src = '76f90cbd92f94a2e925d83e8ccd22cb7' + key = 'd5fb4bd9d50c4be6948c97edd7254b0e' + sc = hashlib.new('md5', bytes(str(t) + key + vid, 'utf-8')).hexdigest() + vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src) + return json.loads(get_content(vmsreq)) class Iqiyi(VideoExtractor): name = "爱奇艺 (Iqiyi)" stream_types = [ - {'id': 'BD', 'container': 'm3u8', 'video_profile': '全高清'}, - {'id': 'FD', 'container': 'm3u8', 'video_profile': '超高清'}, - {'id': 'TD', 'container': 'm3u8', 'video_profile': '超清'}, - {'id': 'HD', 'container': 'm3u8', 'video_profile': '高清'}, - {'id': 'SD', 'container': 'm3u8', 'video_profile': '标清'}, - {'id': 'LD', 'container': 'm3u8', 'video_profile': '流畅'}, + {'id': '4k', 'container': 'm3u8', 'video_profile': '4k'}, + {'id': 'BD', 'container': 'm3u8', 'video_profile': '1080p'}, + {'id': 'TD', 'container': 'm3u8', 'video_profile': '720p'}, + {'id': 'HD', 'container': 'm3u8', 'video_profile': '540p'}, + {'id': 'SD', 'container': 'm3u8', 'video_profile': '360p'}, + {'id': 'LD', 'container': 'm3u8', 'video_profile': '210p'}, ] ''' supported_stream_types = [ 'high', 'standard'] @@ -97,18 +105,11 @@ class Iqiyi(VideoExtractor): stream_to_bid = { '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96} ''' - ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD'] - vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'} - vd_2_profile = {21: '超清', 2: '高清', 4: '超高清', 17: '全高清', 96: '流畅', 1: '标清'} + ids = ['4k','BD', 'TD', 'HD', 'SD', 'LD'] + vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD', 2: 'HD', 4: 'TD', 17: 'TD', 96: 'LD', 1: 'SD'} + id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p'} + - def getVMS(self): - tvid, vid = self.vid - t = int(time.time() * 1000) - src = '76f90cbd92f94a2e925d83e8ccd22cb7' - key = 'd5fb4bd9d50c4be6948c97edd7254b0e' - sc = hashlib.new('md5', bytes(str(t) + key + vid, 'utf-8')).hexdigest() - vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src) - return json.loads(get_content(vmsreq)) def download_playlist_by_url(self, url, **kwargs): self.url = url @@ -132,15 +133,46 @@ class Iqiyi(VideoExtractor): r1(r'data-player-videoid="([^"]+)"', html) self.vid = (tvid, videoid) self.title = match1(html, '<title>([^<]+)').split('-')[0] - - info = self.getVMS() + tvid, videoid = self.vid + info = getVMS(tvid, videoid) assert info['code'] == 'A00000', 'can\'t play this video' for stream in info['data']['vidl']: - stream_id = self.vd_2_id[stream['vd']] - stream_profile = self.vd_2_profile[stream['vd']] - self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0} + try: + stream_id = self.vd_2_id[stream['vd']] + if stream_id in self.stream_types: + continue + stream_profile = self.id_2_profile[stream_id] + self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0} + except: + log.i("vd: {} is not handled".format(stream['vd'])) + log.i("info is {}".format(stream)) + # why I need do below??? + if not 'BD' in self.stream_types: + p1080_vids = [] + if 18 in info['data']['ctl']['vip']['bids']: + p1080_vids.append(info['data']['ctl']['configs']['18']['vid']) + if 5 in info['data']['ctl']['vip']['bids']: + p1080_vids.append(info['data']['ctl']['configs']['5']['vid']) + for v in p1080_vids: + p1080_info = getVMS(tvid, v) + if info['code'] == 'A00000': + p1080_url = p1080_info['data']['m3u'] + self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0} + break + if not '4k' in self.stream_types: + k4_vids = [] + if 19 in info['data']['ctl']['vip']['bids']: + k4_vids.append(info['data']['ctl']['configs']['19']['vid']) + if 10 in info['data']['ctl']['vip']['bids']: + k4_vids.append(info['data']['ctl']['configs']['10']['vid']) + for v in k4_vids: + k4_info = getVMS(tvid, v) + if info['code'] == 'A00000': + k4_url = k4_info['data']['m3u'] + self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0} + break ''' if info["code"] != "A000000": log.e("[error] outdated iQIYI key") @@ -214,4 +246,4 @@ class Iqiyi(VideoExtractor): site = Iqiyi() download = site.download_by_url iqiyi_download_by_vid = site.download_by_vid -download_playlist = site.download_playlist_by_url +download_playlist = site.download_playlist_by_url \ No newline at end of file