remove sogou_proxy

README.md

@@ -186,8 +186,6 @@ For a complete list of all available options, see:
         -x | --http-proxy <HOST:PORT>            Use specific HTTP proxy for downloading.
         -y | --extractor-proxy <HOST:PORT>       Use specific HTTP proxy for extracting stream data.
              --no-proxy                          Don't use any proxy. (ignore $http_proxy)
-        -S | --sogou                             Use a Sogou proxy server for downloading.
-             --sogou-proxy <HOST:PORT>           Run a standalone Sogou proxy server.
              --debug                             Show traceback on KeyboardInterrupt.
 
 ## License

README.txt

@@ -194,8 +194,6 @@ For a complete list of all available options, see::
         -x | --http-proxy <HOST:PORT>            Use specific HTTP proxy for downloading.
         -y | --extractor-proxy <HOST:PORT>       Use specific HTTP proxy for extracting stream data.
              --no-proxy                          Don't use any proxy. (ignore $http_proxy)
-        -S | --sogou                             Use a Sogou proxy server for downloading.
-             --sogou-proxy <HOST:PORT>           Run a standalone Sogou proxy server.
              --debug                             Show traceback on KeyboardInterrupt.
 
 License

src/you_get/common.py

@@ -11,14 +11,13 @@ import platform
 import threading
 
 from .version import __version__
-from .util import log, sogou_proxy_server, get_filename, unescape_html
+from .util import log
+from .util.strings import get_filename, unescape_html
 
 dry_run = False
 force = False
 player = None
 extractor_proxy = None
-sogou_proxy = None
-sogou_env = None
 cookies_txt = None
 
 fake_headers = {
@@ -764,9 +763,6 @@ def parse_host(host):
     port = o.port or 0
     return (hostname, port)
 
-def get_sogou_proxy():
-    return sogou_proxy
-
 def set_proxy(proxy):
     proxy_handler = request.ProxyHandler({
         'http': '%s:%s' % proxy,
@@ -803,16 +799,6 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         else:
             download(url, **kwargs)
 
-def get_version():
-    try:
-        import subprocess
-        real_dir = os.path.dirname(os.path.realpath(__file__))
-        git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=real_dir, stderr=subprocess.DEVNULL).decode('utf-8').strip()
-        assert git_hash
-        return '%s-%s' % (__version__, git_hash)
-    except:
-        return __version__
-
 def script_main(script_name, download, download_playlist = None):
     version = 'You-Get %s, a video downloader.' % get_version()
     help = 'Usage: %s [OPTION]... [URL]...\n' % script_name
@@ -832,13 +818,11 @@ def script_main(script_name, download, download_playlist = None):
     -x | --http-proxy <HOST:PORT>            Use specific HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>       Use specific HTTP proxy for extracting stream data.
          --no-proxy                          Don't use any proxy. (ignore $http_proxy)
-    -S | --sogou                             Use a Sogou proxy server for downloading.
-         --sogou-proxy <HOST:PORT>           Run a standalone Sogou proxy server.
          --debug                             Show traceback on KeyboardInterrupt.
     '''
 
-    short_opts = 'Vhfiuc:nSF:o:p:x:y:'
+    short_opts = 'Vhfiuc:nF:o:p:x:y:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'sogou', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'sogou-proxy=', 'sogou-env=', 'lang=']
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
     if download_playlist:
         short_opts = 'l' + short_opts
         opts = ['playlist'] + opts
@@ -854,8 +838,6 @@ def script_main(script_name, download, download_playlist = None):
     global dry_run
     global player
     global extractor_proxy
-    global sogou_proxy
-    global sogou_env
     global cookies_txt
     cookies_txt = None
 
@@ -904,33 +886,14 @@ def script_main(script_name, download, download_playlist = None):
             proxy = a
         elif o in ('-y', '--extractor-proxy'):
             extractor_proxy = a
-        elif o in ('-S', '--sogou'):
-            sogou_proxy = ("0.0.0.0", 0)
-        elif o in ('--sogou-proxy',):
-            sogou_proxy = parse_host(a)
-        elif o in ('--sogou-env',):
-            sogou_env = a
         elif o in ('--lang',):
             lang = a
         else:
             log.e("try 'you-get --help' for more options")
             sys.exit(2)
     if not args:
-        if sogou_proxy is not None:
+        print(help)
-            try:
+        sys.exit()
-                if sogou_env is not None:
-                    server = sogou_proxy_server(sogou_proxy, network_env=sogou_env)
-                else:
-                    server = sogou_proxy_server(sogou_proxy)
-                server.serve_forever()
-            except KeyboardInterrupt:
-                if traceback:
-                    raise
-                else:
-                    sys.exit()
-        else:
-            print(help)
-            sys.exit()
 
     set_http_proxy(proxy)
 

src/you_get/extractor/sohu.py

@@ -19,14 +19,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
         vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
     assert vid
 
-    # Open Sogou proxy if required
-    if get_sogou_proxy() is not None:
-        server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w'))
-        server_thread = threading.Thread(target=server.serve_forever)
-        server_thread.daemon = True
-        server_thread.start()
-        set_proxy(server.server_address)
-
     if re.match(r'http://tv.sohu.com/', url):
         data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
         for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
@@ -58,11 +50,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
             urls.append(real_url(host, prot, file, new))
         assert data['clipsURL'][0].endswith('.mp4')
 
-    # Close Sogou proxy if required
-    if get_sogou_proxy() is not None:
-        server.shutdown()
-        unset_proxy()
-
     print_info(site_info, title, 'mp4', size)
     if not info_only:
         download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)

src/you_get/util/__init__.py

@@ -2,5 +2,4 @@
 
 from .fs import *
 from .log import *
-from .sogou_proxy import *
 from .strings import *

src/you_get/util/sogou_proxy.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-
-# Original code from:
-# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/
-
-from . import log
-
-from http.client import HTTPResponse
-from http.server import BaseHTTPRequestHandler, HTTPServer
-from socketserver import ThreadingMixIn
-from threading import Thread
-import random, socket, struct, sys, time
-
-def sogou_proxy_server(
-    host=("0.0.0.0", 0),
-    network_env='CERNET',
-    ostream=sys.stderr):
-    """Returns a Sogou proxy server object.
-    """
-
-    x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27'
-    proxy_host = 'h0.cnc.bj.ie.sogou.com'
-    proxy_port = 80
-
-    def sogou_hash(t, host):
-        s = (t + host + 'SogouExplorerProxy').encode('ascii')
-        code = len(s)
-        dwords = int(len(s) / 4)
-        rest = len(s) % 4
-        v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s)
-        for vv in v:
-            if type(vv) != bytes:
-                a = (vv & 0xFFFF)
-                b = (vv >> 16)
-                code += a
-                code = code ^ (((code << 5) ^ b) << 0xb)
-                # To avoid overflows
-                code &= 0xffffffff
-                code += code >> 0xb
-        if rest == 3:
-            code += s[len(s) - 2] * 256 + s[len(s) - 3]
-            code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10)
-            code &= 0xffffffff
-            code += code >> 0xb
-        elif rest == 2:
-            code += (s[len(s) - 1]) * 256 + (s[len(s) - 2])
-            code ^= code << 0xb
-            code &= 0xffffffff
-            code += code >> 0x11
-        elif rest == 1:
-            code += s[len(s) - 1]
-            code ^= code << 0xa
-            code &= 0xffffffff
-            code += code >> 0x1
-        code ^= code * 8
-        code &= 0xffffffff
-        code += code >> 5
-        code ^= code << 4
-        code = code & 0xffffffff
-        code += code >> 0x11
-        code ^= code << 0x19
-        code = code & 0xffffffff
-        code += code >> 6
-        code = code & 0xffffffff
-        return hex(code)[2:].rstrip('L').zfill(8)
-
-    class Handler(BaseHTTPRequestHandler):
-        _socket = None
-        def do_proxy(self):
-            try:
-                if self._socket is None:
-                    self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                    self._socket.connect((proxy_host, proxy_port))
-                self._socket.send(self.requestline.encode('ascii') + b'\r\n')
-                log.d(self.requestline, ostream)
-
-                # Add Sogou Verification Tags
-                self.headers['X-Sogou-Auth'] = x_sogou_auth
-                t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
-                self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host'])
-                self.headers['X-Sogou-Timestamp'] = t
-                self._socket.send(str(self.headers).encode('ascii') + b'\r\n')
-
-                # Send POST data
-                if self.command == 'POST':
-                    self._socket.send(self.rfile.read(int(self.headers['Content-Length'])))
-                response = HTTPResponse(self._socket, method=self.command)
-                response.begin()
-
-                # Response
-                status = 'HTTP/1.1 %s %s' % (response.status, response.reason)
-                self.wfile.write(status.encode('ascii') + b'\r\n')
-                h = ''
-                for hh, vv in response.getheaders():
-                    if hh.upper() != 'TRANSFER-ENCODING':
-                        h += hh + ': ' + vv + '\r\n'
-                self.wfile.write(h.encode('ascii') + b'\r\n')
-                while True:
-                    response_data = response.read(8192)
-                    if len(response_data) == 0:
-                        break
-                    self.wfile.write(response_data)
-
-            except socket.error:
-                log.e('Socket error for ' + self.requestline, ostream)
-
-        def do_POST(self):
-            self.do_proxy()
-
-        def do_GET(self):
-            self.do_proxy()
-
-    class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
-        pass
-
-    # Server starts
-    log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream)
-
-    try:
-        server = ThreadingHTTPServer(host, Handler)
-    except Exception as ex:
-        log.wtf("Socket error: %s" % ex, ostream)
-        exit(1)
-    host = server.server_address
-
-    if network_env.upper() == 'CERNET':
-        proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
-    elif network_env.upper() == 'CTCNET':
-        proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3)
-    elif network_env.upper() == 'CNCNET':
-        proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3)
-    elif network_env.upper() == 'DXT':
-        proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10)
-    else:
-        proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
-
-    log.i('Remote host: %s' % log.underlined(proxy_host), ostream)
-    log.i('Proxy server running on %s' %
-        log.underlined("%s:%s" % host), ostream)
-
-    return server