From 2275b0ae6637b457de2c5d3926d604490b584756 Mon Sep 17 00:00:00 2001 From: QingQiz Date: Tue, 21 Jul 2020 14:50:36 +0800 Subject: [PATCH] feat: parallel run support for url_locations --- src/you_get/common.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index cb76e159..712035d4 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -508,7 +508,6 @@ def post_content(url, headers={}, post_data={}, decoded=True, **kwargs): def parallel_in_process(target, params_list, **kwargs): - import functools from multiprocessing import Pool global lambda_to_function @@ -520,7 +519,6 @@ def parallel_in_process(target, params_list, **kwargs): def parallel_in_thread(target, params_list, sort, **kwargs): import queue - from queue import Queue from threading import Thread tasks = queue.Queue() @@ -666,8 +664,9 @@ def url_info(url, faker=False, headers={}): def url_locations(urls, faker=False, headers={}): - locations = [] - for url in urls: + params = [[url, faker, headers] for url in urls] + + def action(url, faker, headers): logging.debug('url_locations: %s' % url) if faker: @@ -681,8 +680,9 @@ def url_locations(urls, faker=False, headers={}): else: response = urlopen_with_retry(request.Request(url)) - locations.append(response.url) - return locations + return response.url + + return parallel_run(action, paras, True) def url_save(