Merge pull request #1 from soimort/master

sync the newest

CHANGELOG.txt

@@ -1,6 +1,14 @@
 Changelog
 =========
 
+0.3.8
+-----
+
+*Date: 2013-04-05*
+
+* Add support for:
+    - Coursera
+
 0.3.7
 -----
 

README.md

@@ -14,6 +14,7 @@ Fork me on GitHub: <https://github.com/soimort/you-get>
 
 * YouTube <http://www.youtube.com>
 * Vimeo <http://vimeo.com>
+* Coursera <https://www.coursera.org>
 * Blip <http://blip.tv>
 * Dailymotion <http://dailymotion.com>
 * Facebook <http://facebook.com>
@@ -22,6 +23,7 @@ Fork me on GitHub: <https://github.com/soimort/you-get>
 * Vine <http://vine.co>
 * SoundCloud <http://soundcloud.com>
 * Mixcloud <http://www.mixcloud.com>
+* Freesound <http://www.freesound.org>
 * JPopsuki <http://jpopsuki.tv>
 * VID48 <http://vid48.com>
 * Niconico (ニコニコ動画) <http://www.nicovideo.jp>
@@ -226,6 +228,7 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y
 
 * YouTube <http://www.youtube.com>
 * Vimeo <http://vimeo.com>
+* Coursera <https://www.coursera.org>
 * Blip <http://blip.tv>
 * Dailymotion <http://dailymotion.com>
 * Facebook <http://facebook.com>
@@ -234,6 +237,7 @@ You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/y
 * Vine <http://vine.co>
 * SoundCloud <http://soundcloud.com>
 * Mixcloud <http://www.mixcloud.com>
+* Freesound <http://www.freesound.org>
 * JPopsuki <http://jpopsuki.tv>
 * VID48 <http://vid48.com>
 * NICONICO动画 <http://www.nicovideo.jp>

README.txt

@@ -17,6 +17,7 @@ Supported Sites (As of Now)
 
 * YouTube http://www.youtube.com
 * Vimeo http://vimeo.com
+* Coursera https://www.coursera.org
 * Blip http://blip.tv
 * Dailymotion http://dailymotion.com
 * Facebook http://facebook.com
@@ -25,6 +26,7 @@ Supported Sites (As of Now)
 * Vine http://vine.co
 * SoundCloud http://soundcloud.com
 * Mixcloud http://www.mixcloud.com
+* Freesound http://www.freesound.org
 * JPopsuki http://jpopsuki.tv
 * VID48 http://vid48.com
 * Niconico (ニコニコ動画) http://www.nicovideo.jp
@@ -47,7 +49,7 @@ Supported Sites (As of Now)
 * Sohu (搜狐视频) http://tv.sohu.com
 * 56 (56网) http://www.56.com
 * Xiami (虾米) http://www.xiami.com
-* Baidu (百度音乐) <http://music.baidu.com>
+* Baidu (百度音乐) http://music.baidu.com
 
 Dependencies
 ------------

src/you_get/__main__.py

@@ -24,9 +24,11 @@ def url_to_module(url):
         'bilibili': bilibili,
         'blip': blip,
         'cntv': cntv,
+        'coursera': coursera,
         'dailymotion': dailymotion,
         'douban': douban,
         'facebook': facebook,
+        'freesound': freesound,
         'google': googleplus,
         'iask': sina,
         'ifeng': ifeng,

src/you_get/downloader/__init__.py

@@ -4,9 +4,11 @@ from .acfun import *
 from .bilibili import *
 from .blip import *
 from .cntv import *
+from .coursera import *
 from .dailymotion import *
 from .douban import *
 from .facebook import *
+from .freesound import *
 from .googleplus import *
 from .ifeng import *
 from .iqiyi import *

src/you_get/downloader/coursera.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+__all__ = ['coursera_download']
+
+from ..common import *
+
+def coursera_login(user, password, csrf_token):
+    url = 'https://www.coursera.org/maestro/api/user/login'
+    my_headers = {
+        'Cookie': ('csrftoken=%s' % csrf_token),
+        'Referer': 'https://www.coursera.org',
+        'X-CSRFToken': csrf_token,
+    }
+    
+    values = {
+        'email_address': user,
+        'password': password,
+    }
+    form_data = parse.urlencode(values).encode('utf-8')
+    
+    response = request.urlopen(request.Request(url, headers = my_headers, data = form_data))
+    
+    return response.headers
+
+def coursera_download(url, output_dir = '.', merge = True, info_only = False):
+    course_code = r1(r'coursera.org/([^/]+)', url)
+    
+    request.install_opener(request.build_opener(request.HTTPCookieProcessor()))
+    
+    response = request.urlopen(request.Request(url))
+    csrf_token = r1(r'csrf_token=([^;]+);', response.headers['Set-Cookie'])
+    
+    import netrc, getpass
+    info = netrc.netrc().authenticators('coursera.org')
+    if info is None:
+        user = input("User:     ")
+        password = getpass.getpass("Password: ")
+    else:
+        user, password = info[0], info[2]
+    print("Logging in...")
+    
+    coursera_login(user, password, csrf_token)
+    
+    request.urlopen("https://class.coursera.org/%s/auth/auth_redirector?type=login&subtype=normal" % course_code) # necessary!
+    
+    html = get_html(url)
+    
+    course_name = "%s (%s)" % (r1(r'course_strings_name = "([^"]+)"', html), course_code)
+    output_dir = os.path.join(output_dir, course_name)
+    
+    materials = re.findall(r'<a target="_new" href="([^"]+)"', html)
+    num_of_slides = len(re.findall(r'title="[Ss]lides', html))
+    num_of_srts = len(re.findall(r'title="Subtitles \(srt\)"', html))
+    num_of_texts = len(re.findall(r'title="Subtitles \(text\)"', html))
+    num_of_mp4s = len(re.findall(r'title="Video \(MP4\)"', html))
+    num_of_others = len(materials) - num_of_slides - num_of_srts - num_of_texts - num_of_mp4s
+    
+    print("MOOC Site:               ", site_info)
+    print("Course Name:             ", course_name)
+    print("Num of Videos (MP4):     ", num_of_mp4s)
+    print("Num of Subtitles (srt):  ", num_of_srts)
+    print("Num of Subtitles (text): ", num_of_texts)
+    print("Num of Slides:           ", num_of_slides)
+    print("Num of other resources:  ", num_of_others)
+    print()
+    
+    if info_only:
+        return
+    
+    # Process downloading
+    
+    names = re.findall(r'<div class="hidden">([^<]+)</div>', html)
+    assert len(names) == len(materials)
+    
+    for i in range(len(materials)):
+        title = names[i]
+        resource_url = materials[i]
+        ext = r1(r'format=(.+)', resource_url) or r1(r'\.(\w\w\w\w|\w\w\w|\w\w|\w)$', resource_url) or r1(r'download.(mp4)', resource_url)
+        _, _, size = url_info(resource_url)
+        
+        if ext == 'mp4':
+            download_urls([resource_url], title, ext, size, output_dir, merge = merge)
+        else:
+            download_url_chunked(resource_url, title, ext, size, output_dir, merge = merge)
+    
+    return
+
+def download_url_chunked(url, title, ext, size, output_dir = '.', refer = None, merge = True, faker = False):
+    if dry_run:
+        print('Real URL:\n', [url], '\n')
+        return
+    
+    title = escape_file_path(title)
+    if ext:
+        filename = '%s.%s' % (title, ext)
+    else:
+        filename = title
+    filepath = os.path.join(output_dir, filename)
+    
+    if not force and os.path.exists(filepath):
+        print('Skipping %s: file already exists' % tr(filepath))
+        print()
+        return
+    
+    bar = DummyProgressBar()
+    print('Downloading %s ...' % tr(filename))
+    url_save_chunked(url, filepath, bar, refer = refer, faker = faker)
+    bar.done()
+    
+    print()
+    return
+
+site_info = "Coursera"
+download = coursera_download
+download_playlist = playlist_not_supported('coursera')

src/you_get/downloader/freesound.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+__all__ = ['freesound_download']
+
+from ..common import *
+
+def freesound_download(url, output_dir = '.', merge = True, info_only = False):
+    page = get_html(url)
+    
+    title = r1(r'<meta property="og:title" content="([^"]*)"', page)
+    preview_url = r1(r'<meta property="og:audio" content="([^"]*)"', page)
+    
+    type, ext, size = url_info(preview_url)
+    
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls([preview_url], title, ext, size, output_dir, merge = merge)
+
+site_info = "Freesound.org"
+download = freesound_download
+download_playlist = playlist_not_supported('freesound')

src/you_get/version.py

@@ -2,5 +2,5 @@
 
 __all__ = ['__version__', '__date__']
 
-__version__ = '0.3.7'
+__version__ = '0.3.8'
-__date__ = '2013-03-29'
+__date__ = '2013-04-05'

tests/test.py

@@ -12,9 +12,9 @@ def test_urls(urls):
 
 class YouGetTests(unittest.TestCase):
     
-    def test_blip(self):
+    def test_freesound(self):
         test_urls([
-            "http://blip.tv/clojure/sam-aaron-programming-music-with-overtone-5970273",
+            "http://www.freesound.org/people/Corsica_S/sounds/184419/",
         ])
         
     def test_googleplus(self):