PublicMirrors/you-get
1
0
Fork 0
mirror of https://github.com/soimort/you-get.git synced 2025-01-24 22:15:03 +03:00
Code Issues Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'add-dyn-chunked' of https://github.com/liushuyu/you-get into liushuyu-add-dyn-chunked

Browse Source
This commit is contained in:
Mort Yao 2016-11-03 16:56:57 +01:00
commit 68e8be7af3
No known key found for this signature in database
GPG Key ID: 07DA00CB78203251
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/you_get/common.py
View File

@ -547,7 +547,11 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
os.remove(filepath) # on Windows rename could fail if destination filepath exists os.remove(filepath) # on Windows rename could fail if destination filepath exists
os.rename(temp_filepath, filepath) os.rename(temp_filepath, filepath)
def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}): def url_save_chunked(url, filepath, bar, dyn_callback=None, chunk_size=0, ignore_range=False, refer=None, is_part=False, faker=False, headers={}):
def dyn_update_url(received):
if callable(dyn_callback):
logging.debug('Calling callback %s for new URL from %s' % (dyn_callback.__name__, received))
return dyn_callback(received)
if os.path.exists(filepath): if os.path.exists(filepath):
if not force: if not force:
if not is_part: if not is_part:
@ -585,19 +589,26 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
else: else:
headers = {} headers = {}
if received: if received:
url = dyn_update_url(received)
if not ignore_range:
headers['Range'] = 'bytes=' + str(received) + '-' headers['Range'] = 'bytes=' + str(received) + '-'
if refer: if refer:
headers['Referer'] = refer headers['Referer'] = refer
response = request.urlopen(request.Request(url, headers = headers), None) response = request.urlopen(request.Request(url, headers=headers), None)
with open(temp_filepath, open_mode) as output: with open(temp_filepath, open_mode) as output:
this_chunk = received
while True: while True:
buffer = response.read(1024 * 256) buffer = response.read(1024 * 256)
if not buffer: if not buffer:
break break
output.write(buffer) output.write(buffer)
received += len(buffer) received += len(buffer)
if chunk_size and (received - this_chunk) >= chunk_size:
url = dyn_callback(received)
this_chunk = received
response = request.urlopen(request.Request(url, headers=headers), None)
if bar: if bar:
bar.update_received(len(buffer)) bar.update_received(len(buffer))
@ -846,7 +857,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
print() print()
def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}): def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
assert urls assert urls
if dry_run: if dry_run:
print('Real URLs:\n%s\n' % urls) print('Real URLs:\n%s\n' % urls)
@ -860,7 +871,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
filename = '%s.%s' % (title, ext) filename = '%s.%s' % (title, ext)
filepath = os.path.join(output_dir, filename) filepath = os.path.join(output_dir, filename)
if total_size and ext in ('ts'): if total_size:
if not force and os.path.exists(filepath[:-3] + '.mkv'): if not force and os.path.exists(filepath[:-3] + '.mkv'):
print('Skipping %s: file already exists' % filepath[:-3] + '.mkv') print('Skipping %s: file already exists' % filepath[:-3] + '.mkv')
print() print()
@ -875,7 +886,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
print('Downloading %s ...' % tr(filename)) print('Downloading %s ...' % tr(filename))
filepath = os.path.join(output_dir, filename) filepath = os.path.join(output_dir, filename)
parts.append(filepath) parts.append(filepath)
url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers) url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers, **kwargs)
bar.done() bar.done()
if not merge: if not merge: