From 69f92e9fc67a180640316d908b133059fc8f4fec Mon Sep 17 00:00:00 2001 From: QingQiz Date: Tue, 21 Jul 2020 14:51:24 +0800 Subject: [PATCH] [soundcloud] get resource information in parallel --- src/you_get/extractors/soundcloud.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py index ecd3fc8d..5f3978d0 100644 --- a/src/you_get/extractors/soundcloud.py +++ b/src/you_get/extractors/soundcloud.py @@ -5,6 +5,8 @@ __all__ = ['sndcd_download'] from ..common import * import re import json +import operator +import functools import urllib.error @@ -28,19 +30,19 @@ def get_resource_info(resource_url, client_id): ids = [i['id'] for i in info if i.get('comment_count') is None] ids = list(map(str, ids)) - ids_split = ['%2C'.join(ids[i:i+10]) for i in range(0, len(ids), 10)] + ids_split = [['%2C'.join(ids[i:i+10])] for i in range(0, len(ids), 10)] api_url = 'https://api-v2.soundcloud.com/tracks?ids={ids}&client_id={client_id}&%5Bobject%20Object%5D=&app_version=1584348206&app_locale=en' - res = [] - for ids in ids_split: + def action(ids): uri = api_url.format(ids=ids, client_id=client_id) cont = get_content(uri, decoded=True) - res += json.loads(cont) + return json.loads(cont) - res = iter(res) - info = [next(res) if i.get('comment_count') is None else i for i in info] + res = parallel_run(action, ids_split, True) + res = iter(functools.reduce(operator.iconcat, res)) + + return [next(res) if i.get('comment_count') is None else i for i in info] - return info def sndcd_download(url, output_dir='.', merge=True, info_only=False, **kwargs):