release 2014.06.26

The webpage only contains shortened links for the videos, since the server
doesn't support HEAD requests, we use an specific extractor for them.

test/test_playlists.py

@@ -30,6 +30,7 @@ from youtube_dl.extractor import (
     SoundcloudPlaylistIE,
     TeacherTubeClassroomIE,
     LivestreamIE,
+    LivestreamOriginalIE,
     NHLVideocenterIE,
     BambuserChannelIE,
     BandcampAlbumIE,
@@ -40,6 +41,7 @@ from youtube_dl.extractor import (
     KhanAcademyIE,
     EveryonesMixtapeIE,
     RutubeChannelIE,
+    RutubePersonIE,
     GoogleSearchIE,
     GenericIE,
     TEDIE,
@@ -154,6 +156,14 @@ class TestPlaylists(unittest.TestCase):
         self.assertEqual(result['title'], 'TEDCity2.0 (English)')
         self.assertTrue(len(result['entries']) >= 4)
 
+    def test_livestreamoriginal_folder(self):
+        dl = FakeYDL()
+        ie = LivestreamOriginalIE(dl)
+        result = ie.extract('https://www.livestream.com/newplay/folder?dirId=a07bf706-d0e4-4e75-a747-b021d84f2fd3')
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['id'], 'a07bf706-d0e4-4e75-a747-b021d84f2fd3')
+        self.assertTrue(len(result['entries']) >= 28)
+
     def test_nhl_videocenter(self):
         dl = FakeYDL()
         ie = NHLVideocenterIE(dl)
@@ -256,10 +266,18 @@ class TestPlaylists(unittest.TestCase):
     def test_rutube_channel(self):
         dl = FakeYDL()
         ie = RutubeChannelIE(dl)
-        result = ie.extract('http://rutube.ru/tags/video/1409')
+        result = ie.extract('http://rutube.ru/tags/video/1800/')
         self.assertIsPlaylist(result)
-        self.assertEqual(result['id'], '1409')
-        self.assertTrue(len(result['entries']) >= 34)
+        self.assertEqual(result['id'], '1800')
+        self.assertTrue(len(result['entries']) >= 68)
+
+    def test_rutube_person(self):
+        dl = FakeYDL()
+        ie = RutubePersonIE(dl)
+        result = ie.extract('http://rutube.ru/video/person/313878/')
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['id'], '313878')
+        self.assertTrue(len(result['entries']) >= 37)
 
     def test_multiple_brightcove_videos(self):
         # https://github.com/rg3/youtube-dl/issues/2283

youtube_dl/extractor/__init__.py

@@ -147,7 +147,11 @@ from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
-from .livestream import LivestreamIE, LivestreamOriginalIE
+from .livestream import (
+    LivestreamIE,
+    LivestreamOriginalIE,
+    LivestreamShortenerIE,
+)
 from .lynda import (
     LyndaIE,
     LyndaCourseIE
@@ -255,6 +259,7 @@ from .soundcloud import (
     SoundcloudUserIE,
     SoundcloudPlaylistIE
 )
+from .soundgasm import SoundgasmIE
 from .southparkstudios import (
     SouthParkStudiosIE,
     SouthparkDeIE,

youtube_dl/extractor/bliptv.py

@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class BlipTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z]+)))'
+    _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+]+)))'
 
     _TESTS = [
         {

youtube_dl/extractor/comedycentral.py

@@ -130,7 +130,7 @@ class ComedyCentralShowsIE(InfoExtractor):
                 raise ExtractorError('Invalid redirected URL: ' + url)
             if mobj.group('episode') == '':
                 raise ExtractorError('Redirected URL is still not specific: ' + url)
-            epTitle = mobj.group('episode').rpartition('/')[-1]
+            epTitle = (mobj.group('episode') or mobj.group('videotitle')).rpartition('/')[-1]
 
         mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', webpage)
         if len(mMovieParams) == 0:

youtube_dl/extractor/common.py

@@ -459,6 +459,9 @@ class InfoExtractor(object):
         if secure: regexes = self._og_regexes('video:secure_url') + regexes
         return self._html_search_regex(regexes, html, name, **kargs)
 
+    def _og_search_url(self, html, **kargs):
+        return self._og_search_property('url', html, **kargs)
+
     def _html_search_meta(self, name, html, display_name=None, fatal=False):
         if display_name is None:
             display_name = name

youtube_dl/extractor/dailymotion.py

@@ -150,7 +150,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
         return {
             'id':       video_id,
             'formats': formats,
-            'uploader': info['owner_screenname'],
+            'uploader': info['owner.screenname'],
             'upload_date':  video_upload_date,
             'title':    self._og_search_title(webpage),
             'subtitles':    video_subtitles,

youtube_dl/extractor/discovery.py

@@ -7,9 +7,9 @@ from .common import InfoExtractor
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'http://dsc\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
+    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
     _TEST = {
-        'url': 'http://dsc.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
+        'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
         'md5': 'e12614f9ee303a6ccef415cb0793eba2',
         'info_dict': {
             'id': '614784',

youtube_dl/extractor/livestream.py

@@ -9,6 +9,7 @@ from ..utils import (
     compat_urlparse,
     xpath_with_ns,
     compat_str,
+    orderedSet,
 )
 
 
@@ -64,7 +65,10 @@ class LivestreamIE(InfoExtractor):
 # The original version of Livestream uses a different system
 class LivestreamOriginalIE(InfoExtractor):
     IE_NAME = 'livestream:original'
-    _VALID_URL = r'https?://www\.livestream\.com/(?P<user>[^/]+)/video\?.*?clipId=(?P<id>.*?)(&|$)'
+    _VALID_URL = r'''(?x)https?://www\.livestream\.com/
+        (?P<user>[^/]+)/(?P<type>video|folder)
+        (?:\?.*?Id=|/)(?P<id>.*?)(&|$)
+        '''
     _TEST = {
         'url': 'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
         'info_dict': {
@@ -78,10 +82,7 @@ class LivestreamOriginalIE(InfoExtractor):
         },
     }
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        user = mobj.group('user')
+    def _extract_video(self, user, video_id):
         api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
 
         info = self._download_xml(api_url, video_id)
@@ -99,3 +100,44 @@ class LivestreamOriginalIE(InfoExtractor):
             'ext': 'flv',
             'thumbnail': thumbnail_url,
         }
+
+    def _extract_folder(self, url, folder_id):
+        webpage = self._download_webpage(url, folder_id)
+        urls = orderedSet(re.findall(r'<a href="(https?://livestre\.am/.*?)"', webpage))
+
+        return {
+            '_type': 'playlist',
+            'id': folder_id,
+            'entries': [{
+                '_type': 'url',
+                'url': video_url,
+            } for video_url in urls],
+        }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        id = mobj.group('id')
+        user = mobj.group('user')
+        url_type = mobj.group('type')
+        if url_type == 'folder':
+            return self._extract_folder(url, id)
+        else:
+            return self._extract_video(user, id)
+
+
+# The server doesn't support HEAD request, the generic extractor can't detect
+# the redirection
+class LivestreamShortenerIE(InfoExtractor):
+    IE_NAME = 'livestream:shortener'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://livestre\.am/(?P<id>.+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        id = mobj.group('id')
+        webpage = self._download_webpage(url, id)
+
+        return {
+            '_type': 'url',
+            'url': self._og_search_url(webpage),
+        }

youtube_dl/extractor/soundgasm.py

@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SoundgasmIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
+    _TEST = {
+        'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
+        'md5': '010082a2c802c5275bb00030743e75ad',
+        'info_dict': {
+            'id': '88abd86ea000cafe98f96321b23cc1206cbcbcc9',
+            'ext': 'm4a',
+            'title': 'ytdl_Piano-sample',
+            'description': 'Royalty Free Sample Music'
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('title')
+        audio_title = mobj.group('user') + '_' + mobj.group('title')
+        webpage = self._download_webpage(url, display_id)
+        audio_url = self._html_search_regex(
+            r'(?s)m4a\:\s"([^"]+)"', webpage, 'audio URL')
+        audio_id = re.split('\/|\.', audio_url)[-2]
+        description = self._html_search_regex(
+            r'(?s)<li>Description:\s(.*?)<\/li>', webpage, 'description',
+            fatal=False)
+
+        return {
+            'id': audio_id,
+            'display_id': display_id,
+            'url': audio_url,
+            'title': audio_title,
+            'description': description
+        }

youtube_dl/extractor/teachertube.py

@@ -14,7 +14,7 @@ class TeacherTubeIE(InfoExtractor):
     IE_NAME = 'teachertube'
     IE_DESC = 'teachertube.com videos'
 
-    _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(viewVideo\.php\?video_id=|music\.php\?music_id=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(viewVideo\.php\?video_id=|music\.php\?music_id=|video/|audio/)(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.teachertube.com/viewVideo.php?video_id=339997',
@@ -66,6 +66,7 @@ class TeacherTubeIE(InfoExtractor):
 
         media_urls = re.findall(r'data-contenturl="([^"]+)"', webpage)
         media_urls.extend(re.findall(r'var\s+filePath\s*=\s*"([^"]+)"', webpage))
+        media_urls.extend(re.findall(r'\'file\'\s*:\s*["\']([^"\']+)["\'],', webpage))
 
         formats = [
             {
@@ -79,7 +80,7 @@ class TeacherTubeIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._html_search_regex(r'var\s+thumbUrl\s*=\s*"([^"]+)"', webpage, 'thumbnail'),
+            'thumbnail': self._html_search_regex(r'\'image\'\s*:\s*["\']([^"\']+)["\']', webpage, 'thumbnail'),
             'formats': formats,
             'description': description,
         }

youtube_dl/extractor/videott.py

@@ -4,7 +4,10 @@ import re
 import base64
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    unified_strdate,
+    int_or_none,
+)
 
 
 class VideoTtIE(InfoExtractor):
@@ -50,9 +53,9 @@ class VideoTtIE(InfoExtractor):
             'thumbnail': settings['config']['thumbnail'],
             'upload_date': unified_strdate(video['added']),
             'uploader': video['owner'],
-            'view_count': int(video['view_count']),
-            'comment_count': int(video['comment_count']),
-            'like_count': int(video['liked']),
-            'dislike_count': int(video['disliked']),
+            'view_count': int_or_none(video['view_count']),
+            'comment_count': None if video.get('comment_count') == '--' else int_or_none(video['comment_count']),
+            'like_count': int_or_none(video['liked']),
+            'dislike_count': int_or_none(video['disliked']),
             'formats': formats,
         }

youtube_dl/extractor/wistia.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import json
 import re
 
@@ -5,14 +7,16 @@ from .common import InfoExtractor
 
 
 class WistiaIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:fast\.)?wistia\.net/embed/iframe/(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'https?://(?:fast\.)?wistia\.net/embed/iframe/(?P<id>[a-z0-9]+)'
 
     _TEST = {
-        u"url": u"http://fast.wistia.net/embed/iframe/sh7fpupwlt",
-        u"file": u"sh7fpupwlt.mov",
-        u"md5": u"cafeb56ec0c53c18c97405eecb3133df",
-        u"info_dict": {
-            u"title": u"cfh_resourceful_zdkh_final_1"
+        'url': 'http://fast.wistia.net/embed/iframe/sh7fpupwlt',
+        'md5': 'cafeb56ec0c53c18c97405eecb3133df',
+        'info_dict': {
+            'id': 'sh7fpupwlt',
+            'ext': 'mov',
+            'title': 'Being Resourceful',
+            'duration': 117,
         },
     }
 
@@ -22,7 +26,7 @@ class WistiaIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         data_json = self._html_search_regex(
-            r'Wistia.iframeInit\((.*?), {}\);', webpage, u'video data')
+            r'Wistia\.iframeInit\((.*?), {}\);', webpage, 'video data')
 
         data = json.loads(data_json)
 
@@ -54,4 +58,5 @@ class WistiaIE(InfoExtractor):
             'title': data['name'],
             'formats': formats,
             'thumbnails': thumbnails,
+            'duration': data.get('duration'),
         }

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.06.24.1'
+__version__ = '2014.06.26'