release 2014.04.03

It fails to find the element in python 2.6 and it's not required, the
element is a direct child of the item node.

test/test_all_urls.py

@@ -144,7 +144,15 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('http://video.pbs.org/widget/partnerplayer/980042464/', ['PBS'])
 
     def test_ComedyCentralShows(self):
-        self.assertMatch('http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview', ['ComedyCentralShows'])
+        self.assertMatch(
+            'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
+            ['ComedyCentralShows'])
+        self.assertMatch(
+            'http://thecolbertreport.cc.com/videos/29w6fx/-realhumanpraise-for-fox-news',
+            ['ComedyCentralShows'])
+        self.assertMatch(
+            'http://thecolbertreport.cc.com/videos/gh6urb/neil-degrasse-tyson-pt--1?xrs=eml_col_031114',
+            ['ComedyCentralShows'])
 
 if __name__ == '__main__':
     unittest.main()

test/test_playlists.py

@@ -42,6 +42,7 @@ from youtube_dl.extractor import (
     ToypicsUserIE,
     XTubeUserIE,
     InstagramUserIE,
+    CSpanIE,
 )
 
 
@@ -314,6 +315,19 @@ class TestPlaylists(unittest.TestCase):
         }
         expect_info_dict(self, EXPECTED, test_video)
 
+    def test_CSpan_playlist(self):
+        dl = FakeYDL()
+        ie = CSpanIE(dl)
+        result = ie.extract(
+            'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall')
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['id'], '342759')
+        self.assertEqual(
+            result['title'], 'General Motors Ignition Switch Recall')
+        self.assertEqual(len(result['entries']), 9)
+        whole_duration = sum(e['duration'] for e in result['entries'])
+        self.assertEqual(whole_duration, 14855)
+
 
 if __name__ == '__main__':
     unittest.main()

youtube_dl/extractor/__init__.py

@@ -156,6 +156,7 @@ from .mtv import (
     MTVIE,
     MTVIggyIE,
 )
+from .musicplayon import MusicPlayOnIE
 from .muzu import MuzuTVIE
 from .myspace import MySpaceIE
 from .myspass import MySpassIE

youtube_dl/extractor/brightcove.py

@@ -87,7 +87,7 @@ class BrightcoveIE(InfoExtractor):
         object_str = object_str.replace('<--', '<!--')
         object_str = fix_xml_ampersands(object_str)
 
-        object_doc = xml.etree.ElementTree.fromstring(object_str)
+        object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
 
         fv_el = find_xpath_attr(object_doc, './param', 'name', 'flashVars')
         if fv_el is not None:

youtube_dl/extractor/comedycentral.py

@@ -43,11 +43,13 @@ class ComedyCentralShowsIE(InfoExtractor):
                           (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
                          (full-episodes/(?P<episode>.*)|
                           (?P<clip>
-                              (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
+                              (?:videos/[^/]+/(?P<videotitle>[^/?#]+))
-                              |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))|
+                              |(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
+                              |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
+                          )|
                           (?P<interview>
                               extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
-                     $'''
+                     (?:[?#].*|$)'''
     _TEST = {
         'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
         'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
@@ -57,7 +59,7 @@ class ComedyCentralShowsIE(InfoExtractor):
             'upload_date': '20121213',
             'description': 'Kristen Stewart learns to let loose in "On the Road."',
             'uploader': 'thedailyshow',
-            'title': 'thedailyshow-kristen-stewart part 1',
+            'title': 'thedailyshow kristen-stewart part 1',
         }
     }
 
@@ -102,7 +104,9 @@ class ComedyCentralShowsIE(InfoExtractor):
             assert mobj is not None
 
         if mobj.group('clip'):
-            if mobj.group('showname') == 'thedailyshow':
+            if mobj.group('videotitle'):
+                epTitle = mobj.group('videotitle')
+            elif mobj.group('showname') == 'thedailyshow':
                 epTitle = mobj.group('tdstitle')
             else:
                 epTitle = mobj.group('cntitle')
@@ -161,7 +165,7 @@ class ComedyCentralShowsIE(InfoExtractor):
             content = itemEl.find('.//{http://search.yahoo.com/mrss/}content')
             duration = float_or_none(content.attrib.get('duration'))
             mediagen_url = content.attrib['url']
-            guid = itemEl.find('.//guid').text.rpartition(':')[-1]
+            guid = itemEl.find('./guid').text.rpartition(':')[-1]
 
             cdoc = self._download_xml(
                 mediagen_url, epTitle,

youtube_dl/extractor/common.py

@@ -252,6 +252,17 @@ class InfoExtractor(object):
                 outf.write(webpage_bytes)
 
         content = webpage_bytes.decode(encoding, 'replace')
+
+        if (u'<title>Access to this site is blocked</title>' in content and
+                u'Websense' in content[:512]):
+            msg = u'Access to URL %s has been blocked by Websense filtering software in your network.' % urlh.geturl()
+            blocked_iframe = self._html_search_regex(
+                r'<iframe src="([^"]+)"', content,
+                u'Websense information URL', default=None)
+            if blocked_iframe:
+                msg += u' Visit %s for more details' % blocked_iframe
+            raise ExtractorError(msg, expected=True)
+
         return (content, urlh)
 
     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):

youtube_dl/extractor/cspan.py

@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     unescapeHTML,
     find_xpath_attr,
 )
@@ -54,18 +55,29 @@ class CSpanIE(InfoExtractor):
         info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
         data = self._download_json(info_url, video_id)
 
-        url = unescapeHTML(data['video']['files'][0]['path']['#text'])
+        doc = self._download_xml(
-
+            'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
-        doc = self._download_xml('http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
             video_id)
 
-        def find_string(s):
+        title = find_xpath_attr(doc, './/string', 'name', 'title').text
-            return find_xpath_attr(doc, './/string', 'name', s).text
+        thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
+
+        files = data['video']['files']
+
+        entries = [{
+            'id': '%s_%d' % (video_id, partnum + 1),
+            'title': (
+                title if len(files) == 1 else
+                '%s part %d' % (title, partnum + 1)),
+            'url': unescapeHTML(f['path']['#text']),
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': int_or_none(f.get('length', {}).get('#text')),
+        } for partnum, f in enumerate(files)]
 
         return {
+            '_type': 'playlist',
+            'entries': entries,
+            'title': title,
             'id': video_id,
-            'title': find_string('title'),
-            'url': url,
-            'description': description,
-            'thumbnail': find_string('poster'),
         }

youtube_dl/extractor/generic.py

@@ -82,6 +82,17 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Brightcove'],
         },
+        {
+            'url': 'http://www.championat.com/video/football/v/87/87499.html',
+            'md5': 'fb973ecf6e4a78a67453647444222983',
+            'info_dict': {
+                'id': '3414141473001',
+                'ext': 'mp4',
+                'title': 'Видео. Удаление Дзагоева (ЦСКА)',
+                'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
+                'uploader': 'Championat',
+            },
+        },
         # Direct link to a video
         {
             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',

youtube_dl/extractor/musicplayon.py

@@ -0,0 +1,75 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class MusicPlayOnIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?musicplayon\.com/play(?:-touch)?\?(?:v|pl=100&play)=(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://en.musicplayon.com/play?v=433377',
+        'info_dict': {
+            'id': '433377',
+            'ext': 'mp4',
+            'title': 'Rick Ross - Interview On Chelsea Lately (2014)',
+            'description': 'Rick Ross Interview On Chelsea Lately',
+            'duration': 342,
+            'uploader': 'ultrafish',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        page = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(page)
+        description = self._og_search_description(page)
+        thumbnail = self._og_search_thumbnail(page)
+        duration = self._html_search_meta('video:duration', page, 'duration', fatal=False)
+        view_count = self._og_search_property('count', page, fatal=False)
+        uploader = self._html_search_regex(
+            r'<div>by&nbsp;<a href="[^"]+" class="purple">([^<]+)</a></div>', page, 'uploader', fatal=False)
+
+        formats = [
+            {
+                'url': 'http://media0-eu-nl.musicplayon.com/stream-mobile?id=%s&type=.mp4' % video_id,
+                'ext': 'mp4',
+            }
+        ]
+
+        manifest = self._download_webpage(
+            'http://en.musicplayon.com/manifest.m3u8?v=%s' % video_id, video_id, 'Downloading manifest')
+
+        for entry in manifest.split('#')[1:]:
+            if entry.startswith('EXT-X-STREAM-INF:'):
+                meta, url, _ = entry.split('\n')
+                params = dict(param.split('=') for param in meta.split(',')[1:])
+                formats.append({
+                    'url': url,
+                    'ext': 'mp4',
+                    'tbr': int(params['BANDWIDTH']),
+                    'width': int(params['RESOLUTION'].split('x')[1]),
+                    'height': int(params['RESOLUTION'].split('x')[-1]),
+                    'format_note': params['NAME'].replace('"', '').strip(),
+                })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': int_or_none(duration),
+            'view_count': int_or_none(view_count),
+            'formats': formats,
+        }

youtube_dl/extractor/pyvideo.py

@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import os
 
@@ -5,45 +7,50 @@ from .common import InfoExtractor
 
 
 class PyvideoIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
+    _VALID_URL = r'http://(?:www\.)?pyvideo\.org/video/(?P<id>\d+)/(.*)'
-    _TESTS = [{
+
-        u'url': u'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
+    _TESTS = [
-        u'file': u'24_4WWkSmNo.mp4',
+        {
-        u'md5': u'de317418c8bc76b1fd8633e4f32acbc6',
+            'url': 'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes',
-        u'info_dict': {
+            'md5': 'de317418c8bc76b1fd8633e4f32acbc6',
-            u"title": u"Become a logging expert in 30 minutes",
+            'info_dict': {
-            u"description": u"md5:9665350d466c67fb5b1598de379021f7",
+                'id': '24_4WWkSmNo',
-            u"upload_date": u"20130320",
+                'ext': 'mp4',
-            u"uploader": u"NextDayVideo",
+                'title': 'Become a logging expert in 30 minutes',
-            u"uploader_id": u"NextDayVideo",
+                'description': 'md5:9665350d466c67fb5b1598de379021f7',
+                'upload_date': '20130320',
+                'uploader': 'NextDayVideo',
+                'uploader_id': 'NextDayVideo',
+            },
+            'add_ie': ['Youtube'],
         },
-        u'add_ie': ['Youtube'],
+        {
-    },
+            'url': 'http://pyvideo.org/video/2542/gloriajw-spotifywitherikbernhardsson182m4v',
-    {
+            'md5': '5fe1c7e0a8aa5570330784c847ff6d12',
-        u'url': u'http://pyvideo.org/video/2542/gloriajw-spotifywitherikbernhardsson182m4v',
+            'info_dict': {
-        u'md5': u'5fe1c7e0a8aa5570330784c847ff6d12',
+                'id': '2542',
-        u'info_dict': {
+                'ext': 'm4v',
-            u'id': u'2542',
+                'title': 'Gloriajw-SpotifyWithErikBernhardsson182',
-            u'ext': u'm4v',
+            },
-            u'title': u'Gloriajw-SpotifyWithErikBernhardsson182',
         },
-    },
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage)
 
+        webpage = self._download_webpage(url, video_id)
+
+        m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage)
         if m_youtube is not None:
             return self.url_result(m_youtube.group(1), 'Youtube')
 
-        title = self._html_search_regex(r'<div class="section">.*?<h3>([^>]+?)</h3>',
+        title = self._html_search_regex(
-            webpage, u'title', flags=re.DOTALL)
+            r'<div class="section">.*?<h3>([^>]+?)</h3>', webpage, 'title', flags=re.DOTALL)
-        video_url = self._search_regex([r'<source src="(.*?)"',
+        video_url = self._search_regex(
-            r'<dt>Download</dt>.*?<a href="(.+?)"'],
+            [r'<source src="(.*?)"', r'<dt>Download</dt>.*?<a href="(.+?)"'],
-            webpage, u'video url', flags=re.DOTALL)
+            webpage, 'video url', flags=re.DOTALL)
+
         return {
             'id': video_id,
             'title': os.path.splitext(title)[0],

youtube_dl/extractor/rutube.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 import itertools
 
 from .common import InfoExtractor
@@ -40,15 +39,13 @@ class RutubeIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        api_response = self._download_webpage(
+        video = self._download_json(
             'http://rutube.ru/api/video/%s/?format=json' % video_id,
             video_id, 'Downloading video JSON')
-        video = json.loads(api_response)
 
-        api_response = self._download_webpage(
+        trackinfo = self._download_json(
             'http://rutube.ru/api/play/trackinfo/%s/?format=json' % video_id,
             video_id, 'Downloading trackinfo JSON')
-        trackinfo = json.loads(api_response)
 
         # Some videos don't have the author field
         author = trackinfo.get('author') or {}
@@ -82,10 +79,9 @@ class RutubeChannelIE(InfoExtractor):
     def _extract_videos(self, channel_id, channel_title=None):
         entries = []
         for pagenum in itertools.count(1):
-            api_response = self._download_webpage(
+            page = self._download_json(
                 self._PAGE_TEMPLATE % (channel_id, pagenum),
                 channel_id, 'Downloading page %s' % pagenum)
-            page = json.loads(api_response)
             results = page['results']
             if not results:
                 break
@@ -111,10 +107,9 @@ class RutubeMovieIE(RutubeChannelIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         movie_id = mobj.group('id')
-        api_response = self._download_webpage(
+        movie = self._download_json(
             self._MOVIE_TEMPLATE % movie_id, movie_id,
             'Downloading movie JSON')
-        movie = json.loads(api_response)
         movie_name = movie['name']
         return self._extract_videos(movie_id, movie_name)
 

youtube_dl/extractor/vk.py

@@ -16,7 +16,7 @@ from ..utils import (
 
 class VKIE(InfoExtractor):
     IE_NAME = 'vk.com'
-    _VALID_URL = r'https?://vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>\d+).*?\bid=(?P<id>\d+)|(?:videos.*?\?.*?z=)?video(?P<videoid>.*?)(?:\?|%2F|$))'
+    _VALID_URL = r'https?://vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|(?:videos.*?\?.*?z=)?video(?P<videoid>.*?)(?:\?|%2F|$))'
     _NETRC_MACHINE = 'vk'
 
     _TESTS = [

youtube_dl/extractor/youtube.py

@@ -1446,7 +1446,9 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 break
 
             more = self._download_json(
-                'https://youtube.com/%s' % mobj.group('more'), playlist_id, 'Downloading page #%s' % page_num)
+                'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+                'Downloading page #%s' % page_num,
+                transform_source=uppercase_escape)
             content_html = more['content_html']
             more_widget_html = more['load_more_widget_html']
 
@@ -1736,11 +1738,10 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         feed_entries = []
         paging = 0
         for i in itertools.count(1):
-            info = self._download_webpage(self._FEED_TEMPLATE % paging,
+            info = self._download_json(self._FEED_TEMPLATE % paging,
                                           u'%s feed' % self._FEED_NAME,
                                           u'Downloading page %s' % i)
-            info = json.loads(info)
+            feed_html = info.get('feed_html') or info.get('content_html')
-            feed_html = info['feed_html']
             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
             ids = orderedSet(m.group(1) for m in m_ids)
             feed_entries.extend(

youtube_dl/utils.py

@@ -1264,8 +1264,8 @@ class PagedList(object):
 
 def uppercase_escape(s):
     return re.sub(
-        r'\\U([0-9a-fA-F]{8})',
+        r'\\U[0-9a-fA-F]{8}',
-        lambda m: compat_chr(int(m.group(1), base=16)), s)
+        lambda m: m.group(0).decode('unicode-escape'), s)
 
 try:
     struct.pack(u'!I', 0)

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.03.30.1'
+__version__ = '2014.04.03'