From 439354e730d8b864de9401536c93220467ccb355 Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Wed, 24 Mar 2021 17:48:59 +0000
Subject: [PATCH 01/26] add HDR support for bilibili

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a812d72d..a696b398 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -12,6 +12,8 @@ class Bilibili(VideoExtractor):
 
     # Bilibili media encoding options, in descending quality order.
     stream_types = [
+        {'id': 'hdflv2', 'quality': 125, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '3840p', 'desc': '真彩 HDR'},
         {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '2160p', 'desc': '超清 4K'},
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,

From 25204d8841cdbbcad4f5df357c80853f34286025 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 29 Mar 2021 16:38:54 +0200
Subject: [PATCH 02/26] [test] remove test_bilibili (videos deleted)

---
 tests/test.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 0f7595b3..5a86ee8f 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,14 +40,6 @@ class YouGetTests(unittest.TestCase):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    def test_bilibili(self):
-        bilibili.download(
-            "https://www.bilibili.com/watchlater/#/BV1PE411q7mZ/p6", info_only=True
-        )
-        bilibili.download(
-            "https://www.bilibili.com/watchlater/#/av74906671/p6", info_only=True
-        )
-
     def test_soundcloud(self):
         ## single song
         soundcloud.download(

From 17eff492fe1db0c378f2447b54c5f09ed98b2626 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 29 Mar 2021 16:44:07 +0200
Subject: [PATCH 03/26] version 0.4.1520

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 95505814..60bff607 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1500'
+__version__ = '0.4.1520'

From ef9ff72183acd93b1b10b2b836d145447cceb016 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 2 Apr 2021 01:44:36 +0200
Subject: [PATCH 04/26] [bilibili] fix a bug for non-interative multi-part
 videos
 (https://github.com/soimort/you-get/pull/2746#pullrequestreview-626492105)

---
 src/you_get/extractors/bilibili.py | 32 ++++++++++--------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a696b398..644c5af4 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -181,7 +181,7 @@ class Bilibili(VideoExtractor):
             self.download_playlist_by_url(self.url, **kwargs)
             return
 
-        # regular av video
+        # regular video
         if sort == 'video':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
@@ -601,13 +601,21 @@ class Bilibili(VideoExtractor):
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
 
-        # regular av video
+        # regular video
         if sort == 'video':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
             aid = initial_state['videoData']['aid']
             pn = initial_state['videoData']['videos']
-            if pn!= len(initial_state['videoData']['pages']):#interaction video 互动视频
+
+            if pn == len(initial_state['videoData']['pages']):
+                # non-interative video
+                for pi in range(1, pn + 1):
+                     purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
+                     self.__class__().download_by_url(purl, **kwargs)
+
+            else:
+                # interative video
                 search_node_list = []
                 download_cid_set = set([initial_state['videoData']['cid']])
                 params = {
@@ -658,24 +666,6 @@ class Bilibili(VideoExtractor):
                                     self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
                                 self.extract(**kwargs)
                                 self.download(**kwargs)
-            else:
-                playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-                playinfo = json.loads(playinfo_text) if playinfo_text else None
-
-                html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-                playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-                playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-                p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or '1')-1
-                for pi in range(p,pn):
-                    self.prepare_by_cid(aid,initial_state['videoData']['pages'][pi]['cid'],'%s (P%s. %s)' % (initial_state['videoData']['title'], pi+1, initial_state['videoData']['pages'][pi]['part']),html_content,playinfo,playinfo_,url)
-                    try:
-                        self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
-                    except:
-                        self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
-                    self.extract(**kwargs)
-                    self.download(**kwargs)
-                    # purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi+1)
-                    # self.__class__().download_by_url(purl, **kwargs)
 
         elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME

From 1c841f7e8ce60130572a8f03fb038eda99deff6a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 4 Apr 2021 02:59:00 +0200
Subject: [PATCH 05/26] [bilibili] redirect /s/ URLs

---
 src/you_get/extractors/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 644c5af4..edb656c7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -162,6 +162,11 @@ class Bilibili(VideoExtractor):
             self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
             html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url))
 
+        # redirect: s
+        elif re.match(r'https?://(www\.)?bilibili\.com/s/(.+)', self.url):
+            self.url = 'https://www.bilibili.com/%s' % match1(self.url, r'/s/(.+)')
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
         # sort it out
         if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
             sort = 'audio'

From 144886840212d5d0ee059858e6493dd265927376 Mon Sep 17 00:00:00 2001
From: flewsea <w48325832@gmail.com>
Date: Fri, 7 May 2021 00:20:30 +0800
Subject: [PATCH 06/26] skip private video

---
 src/you_get/extractors/iwara.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index 67a41d41..37cd712a 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -27,6 +27,9 @@ def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     api_url = video_url + '/api/video/' + video_hash
     content = get_content(api_url, headers=headers)
     data = json.loads(content)
+    if len(data)<1 :
+        print('Maybe is Private Video?'+'['+title+']')
+        return True;
     down_urls = 'https:' + data[0]['uri']
     type, ext, size = url_info(down_urls, headers=headers)
     print_info(site_info, title+data[0]['resolution'], type, size)
@@ -35,10 +38,8 @@ def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         download_urls([down_urls], title, ext, size, output_dir, merge=merge, headers=headers)
 
 def download_playlist_by_url( url, **kwargs):
-    video_page = get_content(url)
-    # url_first=re.findall(r"(http[s]?://[^/]+)",url)
+    video_page = get_html(url)
     url_first=match1(url, r"(http[s]?://[^/]+)")
-    # print (url_first)
     videos = set(re.findall(r'<a href="(/videos/[^"]+)"', video_page))
     if(len(videos)>0):
         for video in videos:

From 5104dd3058c342998b32dabdd47230c5f3af75c6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 May 2021 15:38:02 +0200
Subject: [PATCH 07/26] [youtube] close #2890

---
 src/you_get/extractors/youtube.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 7e4c74f0..487869cf 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -195,8 +195,9 @@ class YouTube(VideoExtractor):
         # Get video info
         # 'eurl' is a magic parameter that can bypass age restriction
         # full form: 'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'
-        video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
-        logging.debug('STATUS: %s' % video_info['status'][0])
+        #video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
+        #logging.debug('STATUS: %s' % video_info['status'][0])
+        video_info = {'status': ['ok'], 'use_cipher_signature': 'True'}
 
         ytplayer_config = None
         if 'status' not in video_info:
@@ -253,11 +254,16 @@ class YouTube(VideoExtractor):
             else:
                 # Parse video page instead
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-                ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
 
-                self.title = json.loads(ytplayer_config["args"]["player_response"])["videoDetails"]["title"]
-                self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
-                stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
+                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
+
+                self.title = ytInitialPlayerResponse["videoDetails"]["title"]
+                if re.search('([^"]*/base\.js)"', video_page):
+                    self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
+                else:
+                    self.html5player = None
+
+                stream_list = ytInitialPlayerResponse['streamingData']['formats']
 
         elif video_info['status'] == ['fail']:
             logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])

From e1db00a8fdc228f695b7c643823303ab7e3fe0f0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 May 2021 15:38:53 +0200
Subject: [PATCH 08/26] test on Python 3.9

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index b3d50ff7..daae6668 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, pypy3]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, pypy3]
 
     steps:
     - uses: actions/checkout@v2

From f9fbe8fae02a6ada9f64e0dcfbb960c856701ad6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 May 2021 15:43:24 +0200
Subject: [PATCH 09/26] version 0.4.1525

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 60bff607..63b4af56 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1520'
+__version__ = '0.4.1525'

From 0fcbe3c5a79142f141a55cdd6705f562384b0eed Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 1 Jun 2021 18:14:56 +0200
Subject: [PATCH 10/26] [youtube] s_to_sig: add prefix to prevent namespace
 pollution

---
 src/you_get/extractors/youtube.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 487869cf..58614c5f 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -76,11 +76,13 @@ class YouTube(VideoExtractor):
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl_RGK2l/en_US/base.js
         # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
+        # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
-            code = re.sub(r'(\W)(as|if|in|is|or)\(', r'\1_\2(', code)
+            # add prefix '_sig_' to prevent namespace pollution
+            code = re.sub(r'(\W)([$\w][$\w])\(', r'\1_sig_\2(', code)
             code = re.sub(r'\$', '_dollar', code)
-            code = re.sub(r'\{', r':\n\t', code)
+            code = re.sub(r'\{', r': ', code)
             code = re.sub(r'\}', r'\n', code)
             code = re.sub(r'var\s+', r'', code)
             code = re.sub(r'(\w+).join\(""\)', r'"".join(\1)', code)
@@ -99,7 +101,7 @@ class YouTube(VideoExtractor):
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
-        f1def = 'function main_%s%s' % (f1, f1def)  # prefix to avoid potential namespace conflict
+        f1def = 'function %s%s' % (f1, f1def)
         code = tr_js(f1def)
         f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
         for f2 in f2s:
@@ -112,13 +114,13 @@ class YouTube(VideoExtractor):
                 f2def = 'function {}({},b){}'.format(f2e, f2def.group(1), f2def.group(2))
             f2 = re.sub(r'(as|if|in|is|or)', r'_\1', f2)
             f2 = re.sub(r'\$', '_dollar', f2)
-            code = code + 'global %s\n' % f2 + tr_js(f2def)
+            code = code + 'global _sig_%s\n' % f2 + tr_js(f2def)
 
         f1 = re.sub(r'(as|if|in|is|or)', r'_\1', f1)
         f1 = re.sub(r'\$', '_dollar', f1)
-        code = code + 'sig=main_%s(s)' % f1  # prefix to avoid potential namespace conflict
+        code = code + '_sig=_sig_%s(s)' % f1
         exec(code, globals(), locals())
-        return locals()['sig']
+        return locals()['_sig']
 
     def chunk_by_range(url, size):
         urls = []

From edb358d0b878ba12824b4a39dd5f28cab41a051f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 1 Jun 2021 18:33:26 +0200
Subject: [PATCH 11/26] version 0.4.1527

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 63b4af56..70ca2ef5 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1525'
+__version__ = '0.4.1527'

From d40a15de5c15babe7daa4eb939e5c2f50a90f68c Mon Sep 17 00:00:00 2001
From: nuomi1 <nuomi1@qq.com>
Date: Fri, 4 Jun 2021 22:35:20 +0800
Subject: [PATCH 12/26] fix: url

---
 src/you_get/extractors/missevan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index c2b25a90..5018852f 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -353,7 +353,7 @@ class MissEvan(VideoExtractor):
 
     @staticmethod
     def url_resource(uri):
-        return 'https://static.missevan.com/' + uri
+        return uri if re.match(r'^https?:/{2}\w.+$', uri) else 'https://static.missevan.com/' + uri
 
 site = MissEvan()
 site_info = 'MissEvan.com'

From 3881ed3f949ee63d53fb8f7f7b1e28c4f654ab25 Mon Sep 17 00:00:00 2001
From: nuomi1 <nuomi1@qq.com>
Date: Fri, 4 Jun 2021 22:35:50 +0800
Subject: [PATCH 13/26] feat: new missevan_stream_types

---
 src/you_get/extractors/missevan.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 5018852f..b7f413f2 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -75,17 +75,13 @@ class _Dispatcher(object):
         raise _NoMatchException()
 
 missevan_stream_types = [
-    {'id': 'source', 'quality': '源文件', 'url_json_key': 'soundurl',
-     'resource_url_fmt': 'sound/{resource_url}'},
-    {'id': '320', 'quality': '320 Kbps', 'url_json_key': 'soundurl_64'},
+    {'id': 'source', 'quality': '源文件', 'url_json_key': 'soundurl'},
     {'id': '128', 'quality': '128 Kbps', 'url_json_key': 'soundurl_128'},
-    {'id': '32', 'quality': '32 Kbps', 'url_json_key': 'soundurl_32'},
     {'id': 'covers', 'desc': '封面图', 'url_json_key': 'cover_image',
      'default_src': 'covers/nocover.png',
      'resource_url_fmt': 'covers/{resource_url}'},
-    {'id': 'coversmini', 'desc': '封面缩略图', 'url_json_key': 'cover_image',
-     'default_src': 'coversmini/nocover.png',
-     'resource_url_fmt': 'coversmini/{resource_url}'}
+    {'id': 'coversmini', 'desc': '封面缩略图', 'url_json_key': 'front_cover',
+     'default_src': 'coversmini/nocover.png'}
 ]
 
 def _get_resource_uri(data, stream_type):

From 5445f5ecde1823a200ec24ea0b74328b95299b57 Mon Sep 17 00:00:00 2001
From: Philip Xu <pyx@xrefactor.com>
Date: Tue, 22 Jun 2021 17:51:17 -0400
Subject: [PATCH 14/26] Update douyin.py

The site douyin.com changed to a more user friendly website, this is the updated extractor for that.
---
 src/you_get/extractors/douyin.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index e39775f4..5de1fdd9 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -1,7 +1,7 @@
 # coding=utf-8
 
 import re
-import json
+from urllib.parse import unquote
 
 from ..common import (
     url_size,
@@ -18,17 +18,17 @@ __all__ = ['douyin_download_by_url']
 
 def douyin_download_by_url(url, **kwargs):
     page_content = get_content(url, headers=fake_headers)
-    match_rule = re.compile(r'var data = \[(.*?)\];')
-    video_info = json.loads(match_rule.findall(page_content)[0])
-    video_url = video_info['video']['play_addr']['url_list'][0]
-    # fix: https://www.douyin.com/share/video/6553248251821165832
-    # if there is no title, use desc
-    cha_list = video_info['cha_list']
-    if cha_list:
-        title = cha_list[0]['cha_name']
-    else:
-        title = video_info['desc']
+    # The easiest way to get the title is, obviously, from <title>
+    title = re.findall(r'<title.*>(.*)</title>', page_content)[0].strip()
+    # Remove the site name from title
+    site_name = ' - 抖音'
+    if title.endswith(site_name):
+        title = title[:-len(site_name)]
     video_format = 'mp4'
+    # The video url is url escaped, as of today, there are 4 working CDN video
+    # urls for the same video, I chose the shortest one.
+    cdn_pattern = r'(api\.amemv\.com.*PackSourceEnum_AWEME_DETAIL)'
+    video_url = 'https://' + unquote(re.findall(cdn_pattern, page_content)[0])
     size = url_size(video_url, faker=True)
     print_info(
         site_info='douyin.com', title=title,

From b9dbae8b40588bc134af6bb7406492ec68fffded Mon Sep 17 00:00:00 2001
From: zhancat200801 <zhancat200801@sina.com>
Date: Tue, 29 Jun 2021 13:18:09 +0800
Subject: [PATCH 15/26] modify qq.py

---
 src/you_get/extractors/qq.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 6411b195..e38770e9 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -35,6 +35,7 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
 
     part_urls= []
     total_size = 0
+    ext = None
     for part in range(1, seg_cnt+1):
         if fc_cnt == 0:
             # fix json parsing error

From 71780ae4aa4a47fd817dc4c8485ce4aac96a8633 Mon Sep 17 00:00:00 2001
From: Philip Xu <pyx@xrefactor.com>
Date: Tue, 6 Jul 2021 06:24:52 -0400
Subject: [PATCH 16/26] Update douyin.py

Updated the extractors for the latest www.douyin.com
---
 src/you_get/extractors/douyin.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index 5de1fdd9..8067b1b5 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -1,6 +1,7 @@
 # coding=utf-8
 
 import re
+import json
 from urllib.parse import unquote
 
 from ..common import (
@@ -18,17 +19,17 @@ __all__ = ['douyin_download_by_url']
 
 def douyin_download_by_url(url, **kwargs):
     page_content = get_content(url, headers=fake_headers)
-    # The easiest way to get the title is, obviously, from <title>
-    title = re.findall(r'<title.*>(.*)</title>', page_content)[0].strip()
-    # Remove the site name from title
-    site_name = ' - 抖音'
-    if title.endswith(site_name):
-        title = title[:-len(site_name)]
+    # The video player and video source are rendered client-side, the data
+    # contains in a <script id="RENDER_DATA" type="application/json"> tag
+    # quoted, unquote the whole page content then search using regex with
+    # regular string.
+    page_content = unquote(page_content)
+    title = re.findall(r'"desc":"([^"]*)"', page_content)[0].strip()
     video_format = 'mp4'
-    # The video url is url escaped, as of today, there are 4 working CDN video
-    # urls for the same video, I chose the shortest one.
-    cdn_pattern = r'(api\.amemv\.com.*PackSourceEnum_AWEME_DETAIL)'
-    video_url = 'https://' + unquote(re.findall(cdn_pattern, page_content)[0])
+    # video URLs are in this pattern {"src":"THE_URL"}, in json format
+    urls_pattern = r'"playAddr":(\[.*?\])'
+    urls = json.loads(re.findall(urls_pattern, page_content)[0])
+    video_url = 'https:' + urls[0]['src']
     size = url_size(video_url, faker=True)
     print_info(
         site_info='douyin.com', title=title,

From 69c61b11760e1650b071b8273349f07df482edee Mon Sep 17 00:00:00 2001
From: kagamia <amethyst50504724@msn.com>
Date: Sun, 11 Jul 2021 21:49:31 +0800
Subject: [PATCH 17/26] miaopai: use weibo h5 api

---
 src/you_get/extractors/miaopai.py | 51 ++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 0ddcadba..05c1e650 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -19,7 +19,7 @@ fake_headers_mobile = {
 
 def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
-    page_url = 'http://video.weibo.com/show?fid=' + fid + '&type=mp4'
+    page_url = 'https://video.weibo.com/show?fid=' + fid + '&type=mp4'
 
     mobile_page = get_content(page_url, headers=fake_headers_mobile)
     url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
@@ -78,6 +78,51 @@ def miaopai_download_story(url, output_dir='.', merge=False, info_only=False, **
         download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
 
 
+def miaopai_download_h5api(url, output_dir='.', merge=False, info_only=False, **kwargs):
+    oid = match1(url, r'/show/(\d{4}:\w+)')
+    page = "/show/%s" % oid
+    data_url = 'https://h5.video.weibo.com/api/component?%s' % parse.urlencode({
+        'page': page
+    })
+    headers = {}
+    headers.update(fake_headers_mobile)
+    headers['origin'] = 'https://h5.video.weibo.com'
+    headers['page-referer'] = page
+    headers['referer'] = 'https://h5.video.weibo.com/show/%s' % oid
+    post_data = {
+        "data": json.dumps({
+            "Component_Play_Playinfo": {"oid": oid}
+        })
+    }
+    data_content = post_content(data_url, headers=headers, post_data=post_data)
+    data = json.loads(data_content)
+    if data['msg'] != 'succ':
+        raise Exception('Weibo api returns non-success: (%s)%s'.format(data['code'], data['msg']))
+
+    play_info = data['data']['Component_Play_Playinfo']
+    title = play_info['title']
+
+    # get video formats and sort by size desc
+    video_formats = []
+    for fmt, relative_uri in play_info['urls'].items():
+        url = "https:%s" % relative_uri
+        type, ext, size = url_info(url, headers=headers)
+        video_formats.append({
+            'fmt': fmt,
+            'url': url,
+            'type': type,
+            'ext': ext,
+            'size': size,
+        })
+    video_formats.sort(key=lambda v:v['size'], reverse=True)
+    selected_video = video_formats[0]
+    video_url, ext, size = selected_video['url'], selected_video['ext'], selected_video['size']
+
+    print_info(site_info, title, ext, size)
+    if not info_only:
+        download_urls([video_url], fs.legitimize(title), ext, total_size=size, headers=headers, **kwargs)
+
+
 def miaopai_download_direct(url, output_dir='.', merge=False, info_only=False, **kwargs):
     mobile_page = get_content(url, headers=fake_headers_mobile)
     try:
@@ -108,12 +153,16 @@ def miaopai_download(url, output_dir='.', merge=False, info_only=False, **kwargs
     if re.match(r'^http[s]://.*\.weibo\.com/tv/v/(\w+)', url):
         return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
+    if re.match(r'^http[s]://(.+\.)?weibo\.com/(tv/)?show/(\d{4}:\w+)', url):
+        return miaopai_download_h5api(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
     fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
         miaopai_download_by_fid(fid, output_dir, merge, info_only)
     elif '/p/230444' in url:
         fid = match1(url, r'/p/230444(\w+)')
         miaopai_download_by_fid('1034:'+fid, output_dir, merge, info_only)
+        pass
     else:
         mobile_page = get_content(url, headers = fake_headers_mobile)
         hit = re.search(r'"page_url"\s*:\s*"([^"]+)"', mobile_page)

From 3044f5c16aa4742bd71127bff5211273d638156a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Jul 2021 18:34:33 +0200
Subject: [PATCH 18/26] [common] manually add cookies in get_content and
 post_content (due to python's lack of support for #HttpOnly_)

---
 src/you_get/common.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 67ef581b..597ed45a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -433,8 +433,17 @@ def get_content(url, headers={}, decoded=True):
 
     req = request.Request(url, headers=headers)
     if cookies:
-        cookies.add_cookie_header(req)
-        req.headers.update(req.unredirected_hdrs)
+        # NOTE: Do not use cookies.add_cookie_header(req)
+        # #HttpOnly_ cookies were not supported by CookieJar and MozillaCookieJar properly until python 3.10
+        # See also:
+        # - https://github.com/python/cpython/pull/17471
+        # - https://bugs.python.org/issue2190
+        # Here we add cookies to the request headers manually
+        cookie_strings = []
+        for cookie in list(cookies):
+            cookie_strings.append(cookie.name + '=' + cookie.value)
+        cookie_headers = {'Cookie': '; '.join(cookie_strings)}
+        req.headers.update(cookie_headers)
 
     response = urlopen_with_retry(req)
     data = response.read()
@@ -477,8 +486,17 @@ def post_content(url, headers={}, post_data={}, decoded=True, **kwargs):
 
     req = request.Request(url, headers=headers)
     if cookies:
-        cookies.add_cookie_header(req)
-        req.headers.update(req.unredirected_hdrs)
+        # NOTE: Do not use cookies.add_cookie_header(req)
+        # #HttpOnly_ cookies were not supported by CookieJar and MozillaCookieJar properly until python 3.10
+        # See also:
+        # - https://github.com/python/cpython/pull/17471
+        # - https://bugs.python.org/issue2190
+        # Here we add cookies to the request headers manually
+        cookie_strings = []
+        for cookie in list(cookies):
+            cookie_strings.append(cookie.name + '=' + cookie.value)
+        cookie_headers = {'Cookie': '; '.join(cookie_strings)}
+        req.headers.update(cookie_headers)
     if kwargs.get('post_data_raw'):
         post_data_enc = bytes(kwargs['post_data_raw'], 'utf-8')
     else:

From 47bd107686802c2ee0c3faa083c39d07b7db59dd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Jul 2021 18:41:05 +0200
Subject: [PATCH 19/26] [instagram] use get_content

---
 src/you_get/extractors/instagram.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 0a9da345..e7c79717 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -6,14 +6,14 @@ from ..common import *
 
 def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = r1(r'([^?]*)', url)
-    html = get_html(url, faker=True)
+    cont = get_content(url, headers=fake_headers)
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
-    description = r1(r'<meta property="og:title" content="([^"]*)"', html) or \
-        r1(r'<title>\s([^<]*)</title>', html) # with logged-in cookies
+    description = r1(r'<meta property="og:title" content="([^"]*)"', cont) or \
+        r1(r'<title>\s([^<]*)</title>', cont) # with logged-in cookies
     title = "{} [{}]".format(description.replace("\n", " "), vid)
 
-    stream = r1(r'<meta property="og:video" content="([^"]*)"', html)
+    stream = r1(r'<meta property="og:video" content="([^"]*)"', cont)
     if stream:
         _, ext, size = url_info(stream)
 
@@ -21,14 +21,14 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         if not info_only:
             download_urls([stream], title, ext, size, output_dir, merge=merge)
     else:
-        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
+        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', cont)
         try:
             info = json.loads(data.group(1))
             post = info['entry_data']['PostPage'][0]
             assert post
         except:
             # with logged-in cookies
-            data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', html)
+            data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', cont)
             if data is not None:
                 log.e('[Warning] Cookies needed.')
             post = json.loads(data.group(1))

From ad24e68baa5106522c6d4a37395896e9776ea88f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Jul 2021 18:46:41 +0200
Subject: [PATCH 20/26] version 0.4.1536

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 70ca2ef5..dd603918 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1527'
+__version__ = '0.4.1536'

From f4ec55b00e5b8743727b138d06f6ffadadd49579 Mon Sep 17 00:00:00 2001
From: Tim Gates <tim.gates@iress.com>
Date: Sat, 31 Jul 2021 09:34:46 +1000
Subject: [PATCH 21/26] docs: Fix a few typos

There are small typos in:
- src/you_get/extractors/flickr.py
- src/you_get/extractors/mtv81.py
- src/you_get/extractors/qingting.py

Fixes:
- Should read `several` rather than `serveral`.
- Should read `channel` rather than `chaanel`.
- Should read `approach` rather than `approch`.
---
 src/you_get/extractors/flickr.py   | 2 +-
 src/you_get/extractors/mtv81.py    | 2 +-
 src/you_get/extractors/qingting.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/flickr.py b/src/you_get/extractors/flickr.py
index 2535dd1c..79fca4ff 100644
--- a/src/you_get/extractors/flickr.py
+++ b/src/you_get/extractors/flickr.py
@@ -73,7 +73,7 @@ def get_api_key(page):
     match = match1(page, pattern_inline_api_key)
     # this happens only when the url points to a gallery page
     # that contains no inline api_key(and never makes xhr api calls)
-    # in fact this might be a better approch for getting a temporary api key
+    # in fact this might be a better approach for getting a temporary api key
     # since there's no place for a user to add custom information that may
     # misguide the regex in the homepage
     if not match:
diff --git a/src/you_get/extractors/mtv81.py b/src/you_get/extractors/mtv81.py
index b92f74bc..ef432159 100644
--- a/src/you_get/extractors/mtv81.py
+++ b/src/you_get/extractors/mtv81.py
@@ -28,7 +28,7 @@ def mtv81_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     #
     # rtmpdump  -r 'rtmpe://cp30865.edgefcs.net/ondemand/mtviestor/_!/intlod/MTVInternational/MBUS/GeoLocals/00JP/VIAMTVI/PYC/201304/7122HVAQ4/00JPVIAMTVIPYC7122HVAQ4_640x_360_1200_m30.mp4' -o "title.mp4" --swfVfy http://media.mtvnservices.com/player/prime/mediaplayerprime.1.10.8.swf
     #
-    # because rtmpdump is unstable,may try serveral times
+    # because rtmpdump is unstable,may try several times
     #
     if not info_only:
         # import pdb
diff --git a/src/you_get/extractors/qingting.py b/src/you_get/extractors/qingting.py
index 9859d4be..8dd1b14f 100644
--- a/src/you_get/extractors/qingting.py
+++ b/src/you_get/extractors/qingting.py
@@ -10,7 +10,7 @@ __all__ = ['qingting_download_by_url']
 
 class Qingting(VideoExtractor):
     # every resource is described by its channel id and program id
-    # so vid is tuple (chaanel_id, program_id)
+    # so vid is tuple (channel_id, program_id)
 
     name = 'Qingting'
     stream_types = [

From b97e9484430e2344d3c332d39880ba3ae68890e8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 05:54:36 +0200
Subject: [PATCH 22/26] [youtube] tr_js: support 3-char main function names

---
 src/you_get/extractors/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 58614c5f..81b45ac5 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -77,10 +77,11 @@ class YouTube(VideoExtractor):
         # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
         # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
+        # - https://www.youtube.com/s/player/50e823fc/player_ias.vflset/sv_SE/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             # add prefix '_sig_' to prevent namespace pollution
-            code = re.sub(r'(\W)([$\w][$\w])\(', r'\1_sig_\2(', code)
+            code = re.sub(r'(\W)([$\w][$\w][$\w]?)\(', r'\1_sig_\2(', code)
             code = re.sub(r'\$', '_dollar', code)
             code = re.sub(r'\{', r': ', code)
             code = re.sub(r'\}', r'\n', code)

From 09cd505311af7dff87d344436a7f4a87b1763cd1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 06:39:48 +0200
Subject: [PATCH 23/26] [test] disable test_soundcloud temporarily

---
 tests/test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 5a86ee8f..1989455f 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -42,9 +42,9 @@ class YouGetTests(unittest.TestCase):
 
     def test_soundcloud(self):
         ## single song
-        soundcloud.download(
-            'https://soundcloud.com/keiny-pham/impure-bird', info_only=True
-        )
+        #soundcloud.download(
+        #    'https://soundcloud.com/keiny-pham/impure-bird', info_only=True
+        #)
         ## playlist
         #soundcloud.download(
         #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True

From e1e1503b08c037ec0eb06c6c951240d004cf342b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 06:44:15 +0200
Subject: [PATCH 24/26] [test] disable test_soundcloud temporarily

---
 tests/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index 1989455f..4a2a117c 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,7 +40,7 @@ class YouGetTests(unittest.TestCase):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    def test_soundcloud(self):
+    #def test_soundcloud(self):
         ## single song
         #soundcloud.download(
         #    'https://soundcloud.com/keiny-pham/impure-bird', info_only=True

From 5498c377ff3e8c03831ecef9defe18bea8b4937d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 06:47:09 +0200
Subject: [PATCH 25/26] version 0.4.1545

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index dd603918..f7daa7f8 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1536'
+__version__ = '0.4.1545'

From 63fd9716a8740fc6862b70a474e398ca6e9f26bd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 27 Aug 2021 05:14:00 +0200
Subject: [PATCH 26/26] [universal] fix blogger

---
 src/you_get/extractors/universal.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index abc69475..fdc7426d 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -70,12 +70,13 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                       '[-_][6-9]\d\dx1\d\d\d\.jpe?g',
                       '[-_][6-9]\d\dx[6-9]\d\d\.jpe?g',
                       's1600/[\w%]+\.jpe?g', # blogger
+                      'blogger\.googleusercontent\.com/img/a/\w*', # blogger
                       'img[6-9]\d\d/[\w%]+\.jpe?g' # oricon?
         ]
 
         urls = []
         for i in media_exts:
-            urls += re.findall(r'(https?://[^ ;&"\'\\<>]+' + i + r'[^ ;&"\'\\<>]*)', page)
+            urls += re.findall(r'(https?://[^ ;&"\'\\<>]*' + i + r'[^ ;&"\'\\<>]*)', page)
 
             p_urls = re.findall(r'(https?%3A%2F%2F[^;&"]+' + i + r'[^;&"]*)', page)
             urls += [parse.unquote(url) for url in p_urls]