diff --git a/README.md b/README.md index fe312c72..8a9ac7cc 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,6 @@ For a complete list of all available options, see: -x | --http-proxy Use specific HTTP proxy for downloading. -y | --extractor-proxy Use specific HTTP proxy for extracting stream data. --no-proxy Don't use any proxy. (ignore $http_proxy) - -S | --sogou Use a Sogou proxy server for downloading. - --sogou-proxy Run a standalone Sogou proxy server. --debug Show traceback on KeyboardInterrupt. ## License diff --git a/README.txt b/README.txt index f6106aed..f8ffd878 100644 --- a/README.txt +++ b/README.txt @@ -194,8 +194,6 @@ For a complete list of all available options, see:: -x | --http-proxy Use specific HTTP proxy for downloading. -y | --extractor-proxy Use specific HTTP proxy for extracting stream data. --no-proxy Don't use any proxy. (ignore $http_proxy) - -S | --sogou Use a Sogou proxy server for downloading. - --sogou-proxy Run a standalone Sogou proxy server. --debug Show traceback on KeyboardInterrupt. License diff --git a/src/you_get/common.py b/src/you_get/common.py index d2ab5c7b..253e0a18 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -11,14 +11,13 @@ import platform import threading from .version import __version__ -from .util import log, sogou_proxy_server, get_filename, unescape_html +from .util import log +from .util.strings import get_filename, unescape_html dry_run = False force = False player = None extractor_proxy = None -sogou_proxy = None -sogou_env = None cookies_txt = None fake_headers = { @@ -764,9 +763,6 @@ def parse_host(host): port = o.port or 0 return (hostname, port) -def get_sogou_proxy(): - return sogou_proxy - def set_proxy(proxy): proxy_handler = request.ProxyHandler({ 'http': '%s:%s' % proxy, @@ -803,16 +799,6 @@ def download_main(download, download_playlist, urls, playlist, **kwargs): else: download(url, **kwargs) -def get_version(): - try: - import subprocess - real_dir = os.path.dirname(os.path.realpath(__file__)) - git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=real_dir, stderr=subprocess.DEVNULL).decode('utf-8').strip() - assert git_hash - return '%s-%s' % (__version__, git_hash) - except: - return __version__ - def script_main(script_name, download, download_playlist = None): version = 'You-Get %s, a video downloader.' % get_version() help = 'Usage: %s [OPTION]... [URL]...\n' % script_name @@ -832,13 +818,11 @@ def script_main(script_name, download, download_playlist = None): -x | --http-proxy Use specific HTTP proxy for downloading. -y | --extractor-proxy Use specific HTTP proxy for extracting stream data. --no-proxy Don't use any proxy. (ignore $http_proxy) - -S | --sogou Use a Sogou proxy server for downloading. - --sogou-proxy Run a standalone Sogou proxy server. --debug Show traceback on KeyboardInterrupt. ''' - short_opts = 'Vhfiuc:nSF:o:p:x:y:' - opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'sogou', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'sogou-proxy=', 'sogou-env=', 'lang='] + short_opts = 'Vhfiuc:nF:o:p:x:y:' + opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang='] if download_playlist: short_opts = 'l' + short_opts opts = ['playlist'] + opts @@ -854,8 +838,6 @@ def script_main(script_name, download, download_playlist = None): global dry_run global player global extractor_proxy - global sogou_proxy - global sogou_env global cookies_txt cookies_txt = None @@ -904,33 +886,14 @@ def script_main(script_name, download, download_playlist = None): proxy = a elif o in ('-y', '--extractor-proxy'): extractor_proxy = a - elif o in ('-S', '--sogou'): - sogou_proxy = ("0.0.0.0", 0) - elif o in ('--sogou-proxy',): - sogou_proxy = parse_host(a) - elif o in ('--sogou-env',): - sogou_env = a elif o in ('--lang',): lang = a else: log.e("try 'you-get --help' for more options") sys.exit(2) if not args: - if sogou_proxy is not None: - try: - if sogou_env is not None: - server = sogou_proxy_server(sogou_proxy, network_env=sogou_env) - else: - server = sogou_proxy_server(sogou_proxy) - server.serve_forever() - except KeyboardInterrupt: - if traceback: - raise - else: - sys.exit() - else: - print(help) - sys.exit() + print(help) + sys.exit() set_http_proxy(proxy) diff --git a/src/you_get/extractor/sohu.py b/src/you_get/extractor/sohu.py index 9a1e109b..6ee472e0 100644 --- a/src/you_get/extractor/sohu.py +++ b/src/you_get/extractor/sohu.py @@ -19,14 +19,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) assert vid - # Open Sogou proxy if required - if get_sogou_proxy() is not None: - server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w')) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.daemon = True - server_thread.start() - set_proxy(server.server_address) - if re.match(r'http://tv.sohu.com/', url): data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid)) for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]: @@ -58,11 +50,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): urls.append(real_url(host, prot, file, new)) assert data['clipsURL'][0].endswith('.mp4') - # Close Sogou proxy if required - if get_sogou_proxy() is not None: - server.shutdown() - unset_proxy() - print_info(site_info, title, 'mp4', size) if not info_only: download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge) diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py index 947ea465..5345f0ac 100644 --- a/src/you_get/util/__init__.py +++ b/src/you_get/util/__init__.py @@ -2,5 +2,4 @@ from .fs import * from .log import * -from .sogou_proxy import * from .strings import * diff --git a/src/you_get/util/sogou_proxy.py b/src/you_get/util/sogou_proxy.py deleted file mode 100644 index 01ffb572..00000000 --- a/src/you_get/util/sogou_proxy.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python - -# Original code from: -# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/ - -from . import log - -from http.client import HTTPResponse -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn -from threading import Thread -import random, socket, struct, sys, time - -def sogou_proxy_server( - host=("0.0.0.0", 0), - network_env='CERNET', - ostream=sys.stderr): - """Returns a Sogou proxy server object. - """ - - x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27' - proxy_host = 'h0.cnc.bj.ie.sogou.com' - proxy_port = 80 - - def sogou_hash(t, host): - s = (t + host + 'SogouExplorerProxy').encode('ascii') - code = len(s) - dwords = int(len(s) / 4) - rest = len(s) % 4 - v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s) - for vv in v: - if type(vv) != bytes: - a = (vv & 0xFFFF) - b = (vv >> 16) - code += a - code = code ^ (((code << 5) ^ b) << 0xb) - # To avoid overflows - code &= 0xffffffff - code += code >> 0xb - if rest == 3: - code += s[len(s) - 2] * 256 + s[len(s) - 3] - code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10) - code &= 0xffffffff - code += code >> 0xb - elif rest == 2: - code += (s[len(s) - 1]) * 256 + (s[len(s) - 2]) - code ^= code << 0xb - code &= 0xffffffff - code += code >> 0x11 - elif rest == 1: - code += s[len(s) - 1] - code ^= code << 0xa - code &= 0xffffffff - code += code >> 0x1 - code ^= code * 8 - code &= 0xffffffff - code += code >> 5 - code ^= code << 4 - code = code & 0xffffffff - code += code >> 0x11 - code ^= code << 0x19 - code = code & 0xffffffff - code += code >> 6 - code = code & 0xffffffff - return hex(code)[2:].rstrip('L').zfill(8) - - class Handler(BaseHTTPRequestHandler): - _socket = None - def do_proxy(self): - try: - if self._socket is None: - self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._socket.connect((proxy_host, proxy_port)) - self._socket.send(self.requestline.encode('ascii') + b'\r\n') - log.d(self.requestline, ostream) - - # Add Sogou Verification Tags - self.headers['X-Sogou-Auth'] = x_sogou_auth - t = hex(int(time.time()))[2:].rstrip('L').zfill(8) - self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host']) - self.headers['X-Sogou-Timestamp'] = t - self._socket.send(str(self.headers).encode('ascii') + b'\r\n') - - # Send POST data - if self.command == 'POST': - self._socket.send(self.rfile.read(int(self.headers['Content-Length']))) - response = HTTPResponse(self._socket, method=self.command) - response.begin() - - # Response - status = 'HTTP/1.1 %s %s' % (response.status, response.reason) - self.wfile.write(status.encode('ascii') + b'\r\n') - h = '' - for hh, vv in response.getheaders(): - if hh.upper() != 'TRANSFER-ENCODING': - h += hh + ': ' + vv + '\r\n' - self.wfile.write(h.encode('ascii') + b'\r\n') - while True: - response_data = response.read(8192) - if len(response_data) == 0: - break - self.wfile.write(response_data) - - except socket.error: - log.e('Socket error for ' + self.requestline, ostream) - - def do_POST(self): - self.do_proxy() - - def do_GET(self): - self.do_proxy() - - class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - pass - - # Server starts - log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream) - - try: - server = ThreadingHTTPServer(host, Handler) - except Exception as ex: - log.wtf("Socket error: %s" % ex, ostream) - exit(1) - host = server.server_address - - if network_env.upper() == 'CERNET': - proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10) - elif network_env.upper() == 'CTCNET': - proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3) - elif network_env.upper() == 'CNCNET': - proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3) - elif network_env.upper() == 'DXT': - proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10) - else: - proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10) - - log.i('Remote host: %s' % log.underlined(proxy_host), ostream) - log.i('Proxy server running on %s' % - log.underlined("%s:%s" % host), ostream) - - return server