mirror of
https://github.com/soimort/you-get.git
synced 2025-01-23 13:35:16 +03:00
purge dead sites
This commit is contained in:
Mort Yao
parent
96a25273a1
commit
c55a1ea489
@ -37,13 +37,11 @@ SITES = {
|
||||
'cbs' : 'cbs',
|
||||
'coub' : 'coub',
|
||||
'dailymotion' : 'dailymotion',
|
||||
'dilidili' : 'dilidili',
|
||||
'douban' : 'douban',
|
||||
'douyin' : 'douyin',
|
||||
'douyu' : 'douyutv',
|
||||
'ehow' : 'ehow',
|
||||
'facebook' : 'facebook',
|
||||
'fantasy' : 'fantasy',
|
||||
'fc2' : 'fc2video',
|
||||
'flickr' : 'flickr',
|
||||
'freesound' : 'freesound',
|
||||
@ -51,7 +49,6 @@ SITES = {
|
||||
'google' : 'google',
|
||||
'giphy' : 'giphy',
|
||||
'heavy-music' : 'heavymusic',
|
||||
'huaban' : 'huaban',
|
||||
'huomao' : 'huomaotv',
|
||||
'iask' : 'sina',
|
||||
'icourses' : 'icourses',
|
||||
@ -94,7 +91,6 @@ SITES = {
|
||||
'pptv' : 'pptv',
|
||||
'qingting' : 'qingting',
|
||||
'qq' : 'qq',
|
||||
'quanmin' : 'quanmin',
|
||||
'showroom-live' : 'showroom',
|
||||
'sina' : 'sina',
|
||||
'smgbb' : 'bilibili',
|
||||
|
||||
@ -13,20 +13,17 @@ from .ckplayer import *
|
||||
from .cntv import *
|
||||
from .coub import *
|
||||
from .dailymotion import *
|
||||
from .dilidili import *
|
||||
from .douban import *
|
||||
from .douyin import *
|
||||
from .douyutv import *
|
||||
from .ehow import *
|
||||
from .facebook import *
|
||||
from .fantasy import *
|
||||
from .fc2video import *
|
||||
from .flickr import *
|
||||
from .freesound import *
|
||||
from .funshion import *
|
||||
from .google import *
|
||||
from .heavymusic import *
|
||||
from .huaban import *
|
||||
from .icourses import *
|
||||
from .ifeng import *
|
||||
from .imgur import *
|
||||
|
||||
@ -1,89 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__all__ = ['dilidili_download']
|
||||
|
||||
from ..common import *
|
||||
from .ckplayer import ckplayer_download
|
||||
|
||||
headers = {
|
||||
'DNT': '1',
|
||||
'Accept-Encoding': 'gzip, deflate, sdch, br',
|
||||
'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Cache-Control': 'max-age=0',
|
||||
'Referer': 'http://www.dilidili.com/',
|
||||
'Connection': 'keep-alive',
|
||||
'Save-Data': 'on',
|
||||
}
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
|
||||
"""->list"""
|
||||
another_url = 'https://newplayer.jfrft.com/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
|
||||
parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
|
||||
html = get_content(another_url, headers=headers)
|
||||
|
||||
info = re.search(r'(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})', html).groups()
|
||||
info = [i.strip('{}').split('->') for i in info]
|
||||
info = {i[0]: i [1] for i in info}
|
||||
|
||||
stream_types = []
|
||||
for i in zip(info['deft'].split('|'), info['defa'].split('|')):
|
||||
stream_types.append({'id': str(i[1][-1]), 'container': 'mp4', 'video_profile': i[0]})
|
||||
return stream_types
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
|
||||
global headers
|
||||
re_str = r'http://www.dilidili.com/watch\S+'
|
||||
if re.match(r'http://www.dilidili.wang', url):
|
||||
re_str = r'http://www.dilidili.wang/watch\S+'
|
||||
headers['Referer'] = 'http://www.dilidili.wang/'
|
||||
elif re.match(r'http://www.dilidili.mobi', url):
|
||||
re_str = r'http://www.dilidili.mobi/watch\S+'
|
||||
headers['Referer'] = 'http://www.dilidili.mobi/'
|
||||
|
||||
if re.match(re_str, url):
|
||||
html = get_content(url)
|
||||
title = match1(html, r'<title>(.+)丨(.+)</title>') #title
|
||||
|
||||
# player loaded via internal iframe
|
||||
frame_url = re.search(r'<iframe src=\"(.+?)\"', html).group(1)
|
||||
logging.debug('dilidili_download: %s' % frame_url)
|
||||
|
||||
#https://player.005.tv:60000/?vid=a8760f03fd:a04808d307&v=yun&sign=a68f8110cacd892bc5b094c8e5348432
|
||||
html = get_content(frame_url, headers=headers, decoded=False).decode('utf-8')
|
||||
|
||||
match = re.search(r'(.+?)var video =(.+?);', html)
|
||||
vid = match1(html, r'var vid="(.+)"')
|
||||
hd2 = match1(html, r'var hd2="(.+)"')
|
||||
typ = match1(html, r'var typ="(.+)"')
|
||||
sign = match1(html, r'var sign="(.+)"')
|
||||
tmsign = match1(html, r'tmsign=([A-Za-z0-9]+)')
|
||||
ulk = match1(html, r'var ulk="(.+)"')
|
||||
|
||||
# here s the parser...
|
||||
stream_types = dilidili_parser_data_to_stream_types(typ, vid, hd2, sign, tmsign, ulk)
|
||||
|
||||
#get best
|
||||
best_id = max([i['id'] for i in stream_types])
|
||||
|
||||
parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = best_id, sign = sign, tmsign = tmsign, ulk = ulk)
|
||||
|
||||
another_url = 'https://newplayer.jfrft.com/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
|
||||
|
||||
ckplayer_download(another_url, output_dir, merge, info_only, is_xml = True, title = title, headers = headers)
|
||||
|
||||
#type_ = ''
|
||||
#size = 0
|
||||
|
||||
#type_, ext, size = url_info(url)
|
||||
#print_info(site_info, title, type_, size)
|
||||
#if not info_only:
|
||||
#download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
|
||||
|
||||
site_info = "dilidili"
|
||||
download = dilidili_download
|
||||
download_playlist = playlist_not_supported('dilidili')
|
||||
@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__all__ = ['fantasy_download']
|
||||
|
||||
from ..common import *
|
||||
import json
|
||||
import random
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
|
||||
def fantasy_download_by_id_channelId(id = 0, channelId = 0, output_dir = '.', merge = True, info_only = False,
|
||||
**kwargs):
|
||||
api_url = 'http://www.fantasy.tv/tv/playDetails.action?' \
|
||||
'myChannelId=1&id={id}&channelId={channelId}&t={t}'.format(id = id,
|
||||
channelId = channelId,
|
||||
t = str(random.random())
|
||||
)
|
||||
html = get_content(api_url)
|
||||
html = json.loads(html)
|
||||
|
||||
if int(html['status']) != 100000:
|
||||
raise Exception('API error!')
|
||||
|
||||
title = html['data']['tv']['title']
|
||||
|
||||
video_url = html['data']['tv']['videoPath']
|
||||
headers = fake_headers.copy()
|
||||
headers['Referer'] = api_url
|
||||
type, ext, size = url_info(video_url, headers=headers)
|
||||
|
||||
print_info(site_info, title, type, size)
|
||||
if not info_only:
|
||||
download_urls([video_url], title, ext, size, output_dir, merge = merge, headers = headers)
|
||||
|
||||
|
||||
def fantasy_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
|
||||
if 'fantasy.tv' not in url:
|
||||
raise Exception('Wrong place!')
|
||||
|
||||
q = parse_qs(urlparse(url).query)
|
||||
|
||||
if 'tvId' not in q or 'channelId' not in q:
|
||||
raise Exception('No enough arguments!')
|
||||
|
||||
tvId = q['tvId'][0]
|
||||
channelId = q['channelId'][0]
|
||||
|
||||
fantasy_download_by_id_channelId(id = tvId, channelId = channelId, output_dir = output_dir, merge = merge,
|
||||
info_only = info_only, **kwargs)
|
||||
|
||||
|
||||
site_info = "fantasy.tv"
|
||||
download = fantasy_download
|
||||
download_playlist = playlist_not_supported('fantasy.tv')
|
||||
@ -1,85 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import math
|
||||
import traceback
|
||||
import urllib.parse as urlparse
|
||||
|
||||
from ..common import *
|
||||
|
||||
__all__ = ['huaban_download']
|
||||
|
||||
site_info = '花瓣 (Huaban)'
|
||||
|
||||
LIMIT = 100
|
||||
|
||||
|
||||
class Board:
|
||||
def __init__(self, title, pins):
|
||||
self.title = title
|
||||
self.pins = pins
|
||||
self.pin_count = len(pins)
|
||||
|
||||
|
||||
class Pin:
|
||||
host = 'http://img.hb.aicdn.com/'
|
||||
|
||||
def __init__(self, pin_json):
|
||||
img_file = pin_json['file']
|
||||
self.id = str(pin_json['pin_id'])
|
||||
self.url = urlparse.urljoin(self.host, img_file['key'])
|
||||
self.ext = img_file['type'].split('/')[-1]
|
||||
|
||||
|
||||
def construct_url(url, **params):
|
||||
param_str = urlparse.urlencode(params)
|
||||
return url + '?' + param_str
|
||||
|
||||
|
||||
def extract_json_data(url, **params):
|
||||
url = construct_url(url, **params)
|
||||
html = get_content(url, headers=fake_headers)
|
||||
json_string = match1(html, r'app.page\["board"\] = (.*?});')
|
||||
json_data = json.loads(json_string)
|
||||
return json_data
|
||||
|
||||
|
||||
def extract_board_data(url):
|
||||
json_data = extract_json_data(url, limit=LIMIT)
|
||||
pin_list = json_data['pins']
|
||||
title = json_data['title']
|
||||
pin_count = json_data['pin_count']
|
||||
pin_count -= len(pin_list)
|
||||
|
||||
while pin_count > 0:
|
||||
json_data = extract_json_data(url, max=pin_list[-1]['pin_id'],
|
||||
limit=LIMIT)
|
||||
pins = json_data['pins']
|
||||
pin_list += pins
|
||||
pin_count -= len(pins)
|
||||
|
||||
return Board(title, list(map(Pin, pin_list)))
|
||||
|
||||
|
||||
def huaban_download_board(url, output_dir, **kwargs):
|
||||
kwargs['merge'] = False
|
||||
board = extract_board_data(url)
|
||||
output_dir = os.path.join(output_dir, board.title)
|
||||
print_info(site_info, board.title, 'jpg', float('Inf'))
|
||||
for pin in board.pins:
|
||||
download_urls([pin.url], pin.id, pin.ext, float('Inf'),
|
||||
output_dir=output_dir, faker=True, **kwargs)
|
||||
|
||||
|
||||
def huaban_download(url, output_dir='.', **kwargs):
|
||||
if re.match(r'http://huaban\.com/boards/\d+/', url):
|
||||
huaban_download_board(url, output_dir, **kwargs)
|
||||
else:
|
||||
print('Only board (画板) pages are supported currently')
|
||||
print('ex: http://huaban.com/boards/12345678/')
|
||||
|
||||
|
||||
download = huaban_download
|
||||
download_playlist = playlist_not_supported("huaban")
|
||||
@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
__all__ = ['quanmin_download']
|
||||
|
||||
from ..common import *
|
||||
import json
|
||||
|
||||
def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
|
||||
roomid = url.split('/')[3].split('?')[0]
|
||||
|
||||
json_request_url = 'http://m.quanmin.tv/json/rooms/{}/noinfo6.json'.format(roomid)
|
||||
content = get_html(json_request_url)
|
||||
data = json.loads(content)
|
||||
|
||||
title = data["title"]
|
||||
|
||||
if not data["play_status"]:
|
||||
raise ValueError("The live stream is not online!")
|
||||
|
||||
real_url = data["live"]["ws"]["flv"]["5"]["src"]
|
||||
|
||||
print_info(site_info, title, 'flv', float('inf'))
|
||||
if not info_only:
|
||||
download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
|
||||
|
||||
site_info = "quanmin.tv"
|
||||
download = quanmin_download
|
||||
download_playlist = playlist_not_supported('quanmin')
|
||||
Loading…
Reference in New Issue
Block a user