From 4e05b55394d659d7f7d8782972aa67634c38857a Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 10 Feb 2016 16:41:52 +0800 Subject: [PATCH 1/8] Revert "Revert pull request #898" This reverts commit 966840e22a4e48c69a306dcd2d807c2a8a106327. --- src/you_get/common.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index 8bece7c5..6244c579 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -541,8 +541,7 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker = os.rename(temp_filepath, filepath) class SimpleProgressBar: - bar_size = term.get_terminal_size()[1] - 42 - bar = '{0:>5}% ({1:>5}/{2:<5}MB) ├{3:─<' + str(bar_size) + '}┤[{4}/{5}] {6}' + term_size = term.get_terminal_size()[1] def __init__(self, total_size, total_pieces = 1): self.displayed = False @@ -553,6 +552,12 @@ class SimpleProgressBar: self.speed = '' self.last_updated = time.time() + total_pieces_len = len(str(total_pieces)) + # 38 is the size of all statically known size in self.bar + self.bar_size = self.term_size - 38 - 2*total_pieces_len + self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % ( + self.bar_size, total_pieces_len, total_pieces_len) + def update(self): self.displayed = True bar_size = self.bar_size @@ -688,11 +693,13 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg if len(urls) == 1: url = urls[0] print('Downloading %s ...' % tr(output_filename)) + bar.update() url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers) bar.done() else: parts = [] print('Downloading %s.%s ...' % (tr(title), ext)) + bar.update() for i, url in enumerate(urls): filename = '%s[%02d].%s' % (title, i, ext) filepath = os.path.join(output_dir, filename) From 9fa6ef1f6f248f6aaa24f801bdee14a76e22eb19 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 10 Feb 2016 16:21:39 +0800 Subject: [PATCH 2/8] [bar] don't overflow on large files (>1G) https://github.com/soimort/you-get/pull/898#issuecomment-180015169 --- src/you_get/common.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index 6244c579..83cd3271 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -554,9 +554,11 @@ class SimpleProgressBar: total_pieces_len = len(str(total_pieces)) # 38 is the size of all statically known size in self.bar - self.bar_size = self.term_size - 38 - 2*total_pieces_len - self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % ( - self.bar_size, total_pieces_len, total_pieces_len) + total_str = '%5s' % round(self.total_size / 1048576, 1) + total_str_width = max(len(total_str), 5) + self.bar_size = self.term_size - 28 - 2*total_pieces_len - 2*total_str_width + self.bar = '{:>5}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % ( + total_str_width, total_str, self.bar_size, total_pieces_len, total_pieces_len) def update(self): self.displayed = True @@ -573,7 +575,7 @@ class SimpleProgressBar: else: plus = '' bar = '█' * dots + plus - bar = self.bar.format(percent, round(self.received / 1048576, 1), round(self.total_size / 1048576, 1), bar, self.current_piece, self.total_pieces, self.speed) + bar = self.bar.format(percent, round(self.received / 1048576, 1), bar, self.current_piece, self.total_pieces, self.speed) sys.stdout.write('\r' + bar) sys.stdout.flush() From af42e7cedcbe67dcb14e41ff11480c1572575d20 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 10 Feb 2016 16:23:13 +0800 Subject: [PATCH 3/8] [bar] show "100%" instead of "100.0%" --- src/you_get/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index 83cd3271..aaa9457b 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -556,15 +556,15 @@ class SimpleProgressBar: # 38 is the size of all statically known size in self.bar total_str = '%5s' % round(self.total_size / 1048576, 1) total_str_width = max(len(total_str), 5) - self.bar_size = self.term_size - 28 - 2*total_pieces_len - 2*total_str_width - self.bar = '{:>5}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % ( + self.bar_size = self.term_size - 27 - 2*total_pieces_len - 2*total_str_width + self.bar = '{:>4}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % ( total_str_width, total_str, self.bar_size, total_pieces_len, total_pieces_len) def update(self): self.displayed = True bar_size = self.bar_size percent = round(self.received * 100 / self.total_size, 1) - if percent > 100: + if percent >= 100: percent = 100 dots = bar_size * int(percent) // 100 plus = int(percent) - dots // bar_size * 100 From bf59adcd8b6991ae873005eb15b16fec91d60792 Mon Sep 17 00:00:00 2001 From: lilydjwg Date: Wed, 10 Feb 2016 16:26:10 +0800 Subject: [PATCH 4/8] [bar] handle speed >= 1GiB/s nicely I suppose there won't be >= 1TiB/s speed in the near future. --- src/you_get/common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/you_get/common.py b/src/you_get/common.py index aaa9457b..08c9b45a 100755 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -583,8 +583,10 @@ class SimpleProgressBar: self.received += n time_diff = time.time() - self.last_updated bytes_ps = n / time_diff if time_diff else 0 - if bytes_ps >= 1048576: - self.speed = '{:4.0f} MB/s'.format(bytes_ps / 1048576) + if bytes_ps >= 1024 ** 3: + self.speed = '{:4.0f} GB/s'.format(bytes_ps / 1024 ** 3) + elif bytes_ps >= 1024 ** 2: + self.speed = '{:4.0f} MB/s'.format(bytes_ps / 1024 ** 2) elif bytes_ps >= 1024: self.speed = '{:4.0f} kB/s'.format(bytes_ps / 1024) else: From 94fbbf4d0e6d3d4d8abbb1f08b84940793a1e3c1 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Thu, 18 Feb 2016 04:23:43 +0100 Subject: [PATCH 5/8] add .github/ISSUE_TEMPLATE.md - See --- .github/ISSUE_TEMPLATE.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..d6a52e59 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,17 @@ +In case of any encountered problem, always check your network status first. That is, please ensure the video you want to download can be streamed properly in your web browser. + +* Keep in mind that some videos on some hosting sites may have a region restriction, e.g., Youku is blocking access to some videos from IP addresses outside mainland China, and YouTube is also blocking some videos in Germany. + +Please include: + +* Your exact command line, like `you-get -i "www.youtube.com/watch?v=sGwy8DsUJ4M"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem. + +* Your full console output. + +* If you executed the command and got no response, please re-run the command with `--debug`, kill the process with keyboard shortcut `Ctrl-C` and include the full console output. + +* The output of `you-get --version`, or `git rev-parse HEAD` -- if you are using a Git version (but always remember to keep up-to-date!) + +* The output of `python --version`. + +* If possible, you may include your IP address and proxy setting information as well. From 26ed950f16b6754b3816140421037d51502dec76 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Thu, 18 Feb 2016 05:25:38 +0100 Subject: [PATCH 6/8] update .github/ISSUE_TEMPLATE.md --- .github/ISSUE_TEMPLATE.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index d6a52e59..5ec3a5e1 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,17 +1,17 @@ -In case of any encountered problem, always check your network status first. That is, please ensure the video you want to download can be streamed properly in your web browser. +Please make sure these boxes are checked before submitting your issue – thank you! -* Keep in mind that some videos on some hosting sites may have a region restriction, e.g., Youku is blocking access to some videos from IP addresses outside mainland China, and YouTube is also blocking some videos in Germany. +- [ ] You can actually watch the video in your browser, but not download them with `you-get`. +- [ ] Your `you-get` is up-to-date. +- [ ] The issue is not yet reported on or . +- [ ] The issue (or question) is really about `you-get`, not about some other code or project. -Please include: +Run the command with the `--debug` option, and paste the full output inside the fences: -* Your exact command line, like `you-get -i "www.youtube.com/watch?v=sGwy8DsUJ4M"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem. +``` +[PASTE IN ME] +``` -* Your full console output. +If there's anything else you would like to say (e.g. in case your issue is not about downloading a specific video; it might as well be a general discussion or proposal for a new feature), fill in the box below; otherwise, you may want to post an emoji or meme instead: -* If you executed the command and got no response, please re-run the command with `--debug`, kill the process with keyboard shortcut `Ctrl-C` and include the full console output. - -* The output of `you-get --version`, or `git rev-parse HEAD` -- if you are using a Git version (but always remember to keep up-to-date!) - -* The output of `python --version`. - -* If possible, you may include your IP address and proxy setting information as well. +> [WRITE SOMETHING] +> [OR HAVE SOME :icecream:!] From c192708fbbef237c4012a9776946cfdbd35e6150 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Thu, 18 Feb 2016 06:37:21 +0100 Subject: [PATCH 7/8] .github/PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..c57b2816 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ +**(PLEASE DELETE ALL THESE AFTER READING)** + +Thank you for the pull request! `you-get` is a growing open source project, which would not have been possible without contributors like you. + +Here are some simple rules to follow, please recheck them before sending the pull request: + +- [ ] If you want to propose two or more unrelated patches, please open separate pull requests for them, instead of one; +- [ ] All pull requests should be based upon the latest `develop` branch; +- [ ] Name your branch (from which you will send the pull request) properly; use a meaningful name like `add-this-shining-feature` rather than just `develop`; +- [ ] All commit messages, as well as comments in code, should be written in understandable English. + +As a contributor, you must be aware that + +- [ ] You agree to contribute your code to this project, under the terms of the MIT license, so that any person may freely use or redistribute them; of course, you will still reserve the copyright for your own authorship. +- [ ] You may not contribute any code not authored by yourself, unless they are licensed under either public domain or the MIT license, literally. + +Not all pull requests can eventually be merged. I consider merged / unmerged patches as equally important for the community: as long as you think a patch would be helpful, someone else might find it helpful, too, therefore they could take your fork and benefit in some way. In any case, I would like to thank you in advance for taking your time to contribute to this project. + +Cheers, +Mort + +**(PLEASE REPLACE ALL ABOVE WITH A DETAILED DESCRIPTION OF YOUR PULL REQUEST)** From aabf30641bbb5bba535a98a7e5cfe4d0aea29976 Mon Sep 17 00:00:00 2001 From: Mort Yao Date: Thu, 18 Feb 2016 15:50:14 +0100 Subject: [PATCH 8/8] [tudou] fix #925 --- src/you_get/extractors/tudou.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py index 9f4f1245..d6a7f815 100644 --- a/src/you_get/extractors/tudou.py +++ b/src/you_get/extractors/tudou.py @@ -9,12 +9,11 @@ def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid)) temp = max([data[i] for i in data if 'size' in data[i][0]], key=lambda x:sum([part['size'] for part in x])) vids, size = [t["k"] for t in temp], sum([t["size"] for t in temp]) - urls = [[n.firstChild.nodeValue.strip() - for n in - parseString( - get_html('http://ct.v2.tudou.com/f?id=%s' % vid)) - .getElementsByTagName('f')][0] - for vid in vids] + + urls = [] + for vid in vids: + for i in parseString(get_html('http://ct.v2.tudou.com/f?id=%s' % vid)).getElementsByTagName('f'): + urls.append(i.firstChild.nodeValue.strip()) ext = r1(r'http://[\w.]*/(\w+)/[\w.]*', urls[0])