From fa451eb0595b1abdc7d6407c6578aba084168289 Mon Sep 17 00:00:00 2001 From: chrisww Date: Tue, 14 Jul 2020 13:14:36 -0400 Subject: [PATCH] Handle socket timeout in response.read() response.read() itself could also throw excpetions, and this cannot be covered by urlopen_with_retry. --- src/you_get/common.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index c0dde42b..fa23f10f 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -435,8 +435,15 @@ def get_content(url, headers={}, decoded=True): cookies.add_cookie_header(req) req.headers.update(req.unredirected_hdrs) - response = urlopen_with_retry(req) - data = response.read() + for i in range(retry_time): + response = urlopen_with_retry(req) + try: + data = response.read() + break + except socket.timeout as e: + logging.debug('get_content attempt {} failed: {}'.format(i + 1, e)) + if i + 1 == retry_time: + raise e # Handle HTTP compression for gzip and deflate (zlib) content_encoding = response.getheader('Content-Encoding')