PublicMirrors/you-get
1
0
Fork 0
mirror of https://github.com/soimort/you-get.git synced 2025-02-10 12:12:26 +03:00
Code Issues Actions Packages Projects Releases 1 Wiki Activity

feat: parallel run support for url_locations

Browse Source
This commit is contained in:
QingQiz 2020-07-21 14:50:36 +08:00
parent e0c7476338
commit 2275b0ae66
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/you_get/common.py
View File

@ -508,7 +508,6 @@ def post_content(url, headers={}, post_data={}, decoded=True, **kwargs):
def parallel_in_process(target, params_list, **kwargs): def parallel_in_process(target, params_list, **kwargs):
import functools
from multiprocessing import Pool from multiprocessing import Pool
global lambda_to_function global lambda_to_function
@ -520,7 +519,6 @@ def parallel_in_process(target, params_list, **kwargs):
def parallel_in_thread(target, params_list, sort, **kwargs): def parallel_in_thread(target, params_list, sort, **kwargs):
import queue import queue
from queue import Queue
from threading import Thread from threading import Thread
tasks = queue.Queue() tasks = queue.Queue()
@ -666,8 +664,9 @@ def url_info(url, faker=False, headers={}):
def url_locations(urls, faker=False, headers={}): def url_locations(urls, faker=False, headers={}):
locations = [] params = [[url, faker, headers] for url in urls]
for url in urls:
def action(url, faker, headers):
logging.debug('url_locations: %s' % url) logging.debug('url_locations: %s' % url)
if faker: if faker:
@ -681,8 +680,9 @@ def url_locations(urls, faker=False, headers={}):
else: else:
response = urlopen_with_retry(request.Request(url)) response = urlopen_with_retry(request.Request(url))
locations.append(response.url) return response.url
return locations
return parallel_run(action, paras, True)
def url_save( def url_save(