PublicMirrors/you-get
1
0
Fork 0
mirror of https://github.com/soimort/you-get.git synced 2025-02-02 16:24:00 +03:00
Code Issues Actions Packages Projects Releases 1 Wiki Activity

remove sogou_proxy

Browse Source
This commit is contained in:
Mort Yao 2014-07-20 21:26:47 +02:00
parent 2adc337093
commit c3b56def6d
6 changed files with 6 additions and 202 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -186,8 +186,6 @@ For a complete list of all available options, see:
-x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
-y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
--no-proxy Don't use any proxy. (ignore $http_proxy)
-S | --sogou Use a Sogou proxy server for downloading.
--sogou-proxy <HOST:PORT> Run a standalone Sogou proxy server.
--debug Show traceback on KeyboardInterrupt.
## License

2
README.txt
View File

@ -194,8 +194,6 @@ For a complete list of all available options, see::
-x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
-y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
--no-proxy Don't use any proxy. (ignore $http_proxy)
-S | --sogou Use a Sogou proxy server for downloading.
--sogou-proxy <HOST:PORT> Run a standalone Sogou proxy server.
--debug Show traceback on KeyboardInterrupt.
License

49
src/you_get/common.py
View File

@ -11,14 +11,13 @@ import platform
import threading
from .version import __version__
from .util import log, sogou_proxy_server, get_filename, unescape_html
from .util import log
from .util.strings import get_filename, unescape_html
dry_run = False
force = False
player = None
extractor_proxy = None
sogou_proxy = None
sogou_env = None
cookies_txt = None
fake_headers = {
@ -764,9 +763,6 @@ def parse_host(host):
port = o.port or 0
return (hostname, port)
def get_sogou_proxy():
return sogou_proxy
def set_proxy(proxy):
proxy_handler = request.ProxyHandler({
'http': '%s:%s' % proxy,
@ -803,16 +799,6 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
else:
download(url, **kwargs)
def get_version():
try:
import subprocess
real_dir = os.path.dirname(os.path.realpath(__file__))
git_hash = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=real_dir, stderr=subprocess.DEVNULL).decode('utf-8').strip()
assert git_hash
return '%s-%s' % (__version__, git_hash)
except:
return __version__
def script_main(script_name, download, download_playlist = None):
version = 'You-Get %s, a video downloader.' % get_version()
help = 'Usage: %s [OPTION]... [URL]...\n' % script_name
@ -832,13 +818,11 @@ def script_main(script_name, download, download_playlist = None):
-x | --http-proxy <HOST:PORT> Use specific HTTP proxy for downloading.
-y | --extractor-proxy <HOST:PORT> Use specific HTTP proxy for extracting stream data.
--no-proxy Don't use any proxy. (ignore $http_proxy)
-S | --sogou Use a Sogou proxy server for downloading.
--sogou-proxy <HOST:PORT> Run a standalone Sogou proxy server.
--debug Show traceback on KeyboardInterrupt.
'''
short_opts = 'Vhfiuc:nSF:o:p:x:y:'
opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'sogou', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'sogou-proxy=', 'sogou-env=', 'lang=']
short_opts = 'Vhfiuc:nF:o:p:x:y:'
opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-merge', 'no-proxy', 'debug', 'format=', 'stream=', 'itag=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
if download_playlist:
short_opts = 'l' + short_opts
opts = ['playlist'] + opts
@ -854,8 +838,6 @@ def script_main(script_name, download, download_playlist = None):
global dry_run
global player
global extractor_proxy
global sogou_proxy
global sogou_env
global cookies_txt
cookies_txt = None
@ -904,33 +886,14 @@ def script_main(script_name, download, download_playlist = None):
proxy = a
elif o in ('-y', '--extractor-proxy'):
extractor_proxy = a
elif o in ('-S', '--sogou'):
sogou_proxy = ("0.0.0.0", 0)
elif o in ('--sogou-proxy',):
sogou_proxy = parse_host(a)
elif o in ('--sogou-env',):
sogou_env = a
elif o in ('--lang',):
lang = a
else:
log.e("try 'you-get --help' for more options")
sys.exit(2)
if not args:
if sogou_proxy is not None:
try:
if sogou_env is not None:
server = sogou_proxy_server(sogou_proxy, network_env=sogou_env)
else:
server = sogou_proxy_server(sogou_proxy)
server.serve_forever()
except KeyboardInterrupt:
if traceback:
raise
else:
sys.exit()
else:
print(help)
sys.exit()
print(help)
sys.exit()
set_http_proxy(proxy)

13
src/you_get/extractor/sohu.py
View File

@ -19,14 +19,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
assert vid
# Open Sogou proxy if required
if get_sogou_proxy() is not None:
server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w'))
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
set_proxy(server.server_address)
if re.match(r'http://tv.sohu.com/', url):
data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
@ -58,11 +50,6 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False):
urls.append(real_url(host, prot, file, new))
assert data['clipsURL'][0].endswith('.mp4')
# Close Sogou proxy if required
if get_sogou_proxy() is not None:
server.shutdown()
unset_proxy()
print_info(site_info, title, 'mp4', size)
if not info_only:
download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)

1
src/you_get/util/__init__.py
View File

@ -2,5 +2,4 @@
from .fs import *
from .log import *
from .sogou_proxy import *
from .strings import *

141
src/you_get/util/sogou_proxy.py
View File

@ -1,141 +0,0 @@
#!/usr/bin/env python
# Original code from:
# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/
from . import log
from http.client import HTTPResponse
from http.server import BaseHTTPRequestHandler, HTTPServer
from socketserver import ThreadingMixIn
from threading import Thread
import random, socket, struct, sys, time
def sogou_proxy_server(
host=("0.0.0.0", 0),
network_env='CERNET',
ostream=sys.stderr):
"""Returns a Sogou proxy server object.
"""
x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27'
proxy_host = 'h0.cnc.bj.ie.sogou.com'
proxy_port = 80
def sogou_hash(t, host):
s = (t + host + 'SogouExplorerProxy').encode('ascii')
code = len(s)
dwords = int(len(s) / 4)
rest = len(s) % 4
v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s)
for vv in v:
if type(vv) != bytes:
a = (vv & 0xFFFF)
b = (vv >> 16)
code += a
code = code ^ (((code << 5) ^ b) << 0xb)
# To avoid overflows
code &= 0xffffffff
code += code >> 0xb
if rest == 3:
code += s[len(s) - 2] * 256 + s[len(s) - 3]
code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10)
code &= 0xffffffff
code += code >> 0xb
elif rest == 2:
code += (s[len(s) - 1]) * 256 + (s[len(s) - 2])
code ^= code << 0xb
code &= 0xffffffff
code += code >> 0x11
elif rest == 1:
code += s[len(s) - 1]
code ^= code << 0xa
code &= 0xffffffff
code += code >> 0x1
code ^= code * 8
code &= 0xffffffff
code += code >> 5
code ^= code << 4
code = code & 0xffffffff
code += code >> 0x11
code ^= code << 0x19
code = code & 0xffffffff
code += code >> 6
code = code & 0xffffffff
return hex(code)[2:].rstrip('L').zfill(8)
class Handler(BaseHTTPRequestHandler):
_socket = None
def do_proxy(self):
try:
if self._socket is None:
self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._socket.connect((proxy_host, proxy_port))
self._socket.send(self.requestline.encode('ascii') + b'\r\n')
log.d(self.requestline, ostream)
# Add Sogou Verification Tags
self.headers['X-Sogou-Auth'] = x_sogou_auth
t = hex(int(time.time()))[2:].rstrip('L').zfill(8)
self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host'])
self.headers['X-Sogou-Timestamp'] = t
self._socket.send(str(self.headers).encode('ascii') + b'\r\n')
# Send POST data
if self.command == 'POST':
self._socket.send(self.rfile.read(int(self.headers['Content-Length'])))
response = HTTPResponse(self._socket, method=self.command)
response.begin()
# Response
status = 'HTTP/1.1 %s %s' % (response.status, response.reason)
self.wfile.write(status.encode('ascii') + b'\r\n')
h = ''
for hh, vv in response.getheaders():
if hh.upper() != 'TRANSFER-ENCODING':
h += hh + ': ' + vv + '\r\n'
self.wfile.write(h.encode('ascii') + b'\r\n')
while True:
response_data = response.read(8192)
if len(response_data) == 0:
break
self.wfile.write(response_data)
except socket.error:
log.e('Socket error for ' + self.requestline, ostream)
def do_POST(self):
self.do_proxy()
def do_GET(self):
self.do_proxy()
class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
pass
# Server starts
log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream)
try:
server = ThreadingHTTPServer(host, Handler)
except Exception as ex:
log.wtf("Socket error: %s" % ex, ostream)
exit(1)
host = server.server_address
if network_env.upper() == 'CERNET':
proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
elif network_env.upper() == 'CTCNET':
proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3)
elif network_env.upper() == 'CNCNET':
proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3)
elif network_env.upper() == 'DXT':
proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10)
else:
proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10)
log.i('Remote host: %s' % log.underlined(proxy_host), ostream)
log.i('Proxy server running on %s' %
log.underlined("%s:%s" % host), ostream)
return server