release 2014.02.27.1

#2346 #2469)

test/test_download.py

@@ -73,9 +73,7 @@ def generator(test_case):
         if 'playlist' not in test_case:
             info_dict = test_case.get('info_dict', {})
             if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
-                print_skipping('The output file cannot be know, the "file" '
-                    'key is missing or the info_dict is incomplete')
-                return
+                raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
             return

test/test_youtube_lists.py

@@ -118,6 +118,8 @@ class TestYoutubeLists(unittest.TestCase):
         self.assertEqual(original_video['id'], 'rjFaenf1T-Y')
 
     def test_youtube_toptracks(self):
+        print('Skipping: The playlist page gives error 500')
+        return
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)
         result = ie.extract('https://www.youtube.com/playlist?list=MCUS')

youtube_dl/__init__.py

@@ -47,12 +47,14 @@ __authors__  = (
     'Michael Kaiser',
     'Niklas Laxström',
     'David Triendl',
+    'Anthony Weems',
 )
 
 __license__ = 'Public Domain'
 
 import codecs
 import getpass
+import io
 import locale
 import optparse
 import os

youtube_dl/extractor/__init__.py

@@ -137,7 +137,7 @@ from .malemotion import MalemotionIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
-from .mit import TechTVMITIE, MITIE
+from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mixcloud import MixcloudIE
 from .mpora import MporaIE
 from .mofosex import MofosexIE
@@ -152,7 +152,10 @@ from .myspass import MySpassIE
 from .myvideo import MyVideoIE
 from .naver import NaverIE
 from .nba import NBAIE
-from .nbc import NBCNewsIE
+from .nbc import (
+    NBCIE,
+    NBCNewsIE,
+)
 from .ndr import NDRIE
 from .ndtv import NDTVIE
 from .newgrounds import NewgroundsIE
@@ -172,6 +175,7 @@ from .podomatic import PodomaticIE
 from .pornhd import PornHdIE
 from .pornhub import PornHubIE
 from .pornotube import PornotubeIE
+from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .radiofrance import RadioFranceIE
 from .rbmaradio import RBMARadioIE

youtube_dl/extractor/crunchyroll.py

@@ -1,7 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re, base64, zlib
+import re
+import json
+import base64
+import zlib
+
 from hashlib import sha1
 from math import pow, sqrt, floor
 from .common import InfoExtractor
@@ -19,13 +23,15 @@ from ..aes import (
     inc,
 )
 
+
 class CrunchyrollIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
-    _TESTS = [{
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _TEST = {
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
-        'file': '645513.flv',
         #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412',
         'info_dict': {
+            'id': '645513',
+            'ext': 'flv',
             'title': 'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!',
             'description': 'md5:2d17137920c64f2f49981a7797d275ef',
             'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
@@ -36,7 +42,7 @@ class CrunchyrollIE(InfoExtractor):
             # rtmp
             'skip_download': True,
         },
-    }]
+    }
 
     _FORMAT_IDS = {
         '360': ('60', '106'),
@@ -68,7 +74,7 @@ class CrunchyrollIE(InfoExtractor):
             shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode('ascii')).digest())
             # Extend 160 Bit hash to 256 Bit
             return shaHash + [0] * 12
-        
+
         key = obfuscate_key(id)
         class Counter:
             __value = iv
@@ -80,9 +86,8 @@ class CrunchyrollIE(InfoExtractor):
         return zlib.decompress(decrypted_data)
 
     def _convert_subtitles_to_srt(self, subtitles):
-        i=1
         output = ''
-        for start, end, text in re.findall(r'<event [^>]*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles):
+        for i, (start, end, text) in enumerate(re.findall(r'<event [^>]*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles), 1):
             start = start.replace('.', ',')
             end = end.replace('.', ',')
             text = clean_html(text)
@@ -90,7 +95,6 @@ class CrunchyrollIE(InfoExtractor):
             if not text:
                 continue
             output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
-            i+=1
         return output
 
     def _real_extract(self,url):
@@ -108,6 +112,12 @@ class CrunchyrollIE(InfoExtractor):
         if note_m:
             raise ExtractorError(note_m)
 
+        mobj = re.search(r'Page\.messaging_box_controller\.addItems\(\[(?P<msg>{.+?})\]\)', webpage)
+        if mobj:
+            msg = json.loads(mobj.group('msg'))
+            if msg.get('type') == 'error':
+                raise ExtractorError('crunchyroll returned error: %s' % msg['message_body'], expected=True)
+
         video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
         video_title = re.sub(r' {2,}', ' ', video_title)
         video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
@@ -123,7 +133,7 @@ class CrunchyrollIE(InfoExtractor):
         playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
         playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
         playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
-        
+
         stream_id = self._search_regex(r'<media_id>([^<]+)', playerdata, 'stream_id')
         video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
 
@@ -161,7 +171,7 @@ class CrunchyrollIE(InfoExtractor):
             data = base64.b64decode(data)
 
             subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
-            lang_code = self._search_regex(r'lang_code=\'([^\']+)', subtitle, 'subtitle_lang_code', fatal=False)
+            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
             subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)

youtube_dl/extractor/generic.py

@@ -83,10 +83,10 @@ class GenericIE(InfoExtractor):
         # Direct link to a video
         {
             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
-            'file': 'trailer.mp4',
             'md5': '67d406c2bcb6af27fa886f31aa934bbe',
             'info_dict': {
                 'id': 'trailer',
+                'ext': 'mp4',
                 'title': 'trailer',
                 'upload_date': '20100513',
             }
@@ -94,7 +94,6 @@ class GenericIE(InfoExtractor):
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            'file': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ.mp4',
             'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
             'info_dict': {
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
@@ -102,6 +101,22 @@ class GenericIE(InfoExtractor):
                 'title': '2cc213299525360.mov',  # that's what we get
             },
         },
+        # google redirect
+        {
+            'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
+            'info_dict': {
+                'id': 'cmQHVoWB5FY',
+                'ext': 'mp4',
+                'upload_date': '20130224',
+                'uploader_id': 'TheVerge',
+                'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.',
+                'uploader': 'The Verge',
+                'title': 'First Firefox OS phones side-by-side',
+            },
+            'params': {
+                'skip_download': False,
+            }
+        }
     ]
 
     def report_download_webpage(self, video_id):
@@ -411,6 +426,18 @@ class GenericIE(InfoExtractor):
         if mobj is None:
             # HTML5 video
             mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
+        if mobj is None:
+            mobj = re.search(
+                r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'([^\']+)\'"',
+                webpage)
+            if mobj:
+                new_url = mobj.group(1)
+                self.report_following_redirect(new_url)
+                return {
+                    '_type': 'url',
+                    'url': new_url,
+                }
         if mobj is None:
             raise ExtractorError('Unsupported URL: %s' % url)
 

youtube_dl/extractor/lifenews.py

@@ -32,7 +32,7 @@ class LifeNewsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        webpage = self._download_webpage('http://lifenews.ru/mobile/news/%s' % video_id, video_id, 'Downloading page')
+        webpage = self._download_webpage('http://lifenews.ru/news/%s' % video_id, video_id, 'Downloading page')
 
         video_url = self._html_search_regex(
             r'<video.*?src="([^"]+)".*?></video>', webpage, 'video URL')
@@ -50,7 +50,7 @@ class LifeNewsIE(InfoExtractor):
         view_count = self._html_search_regex(
             r'<div class=\'views\'>(\d+)</div>', webpage, 'view count', fatal=False)
         comment_count = self._html_search_regex(
-            r'<div class=\'comments\'>(\d+)</div>', webpage, 'comment count', fatal=False)
+            r'<div class=\'comments\'>\s*<span class=\'counter\'>(\d+)</span>', webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
             r'<time datetime=\'([^\']+)\'>', webpage, 'upload date',fatal=False)

youtube_dl/extractor/metacafe.py

@@ -166,6 +166,7 @@ class MetacafeIE(InfoExtractor):
 
         video_title = self._html_search_regex(r'(?im)<title>(.*) - Video</title>', webpage, u'title')
         description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
         video_uploader = self._html_search_regex(
                 r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
                 webpage, u'uploader nickname', fatal=False)
@@ -183,6 +184,7 @@ class MetacafeIE(InfoExtractor):
             'uploader': video_uploader,
             'upload_date':  None,
             'title':    video_title,
+            'thumbnail':thumbnail,
             'ext':      video_ext,
             'age_limit': age_limit,
         }

youtube_dl/extractor/mit.py

@@ -1,24 +1,30 @@
+from __future__ import unicode_literals
+
 import re
 import json
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
+    compat_urlparse,
     clean_html,
+    ExtractorError,
     get_element_by_id,
 )
 
 
 class TechTVMITIE(InfoExtractor):
-    IE_NAME = u'techtv.mit.edu'
+    IE_NAME = 'techtv.mit.edu'
     _VALID_URL = r'https?://techtv\.mit\.edu/(videos|embeds)/(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
-        u'file': u'25418.mp4',
-        u'md5': u'1f8cb3e170d41fd74add04d3c9330e5f',
-        u'info_dict': {
-            u'title': u'MIT DNA Learning Center Set',
-            u'description': u'md5:82313335e8a8a3f243351ba55bc1b474',
+        'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
+        'md5': '1f8cb3e170d41fd74add04d3c9330e5f',
+        'info_dict': {
+            'id': '25418',
+            'ext': 'mp4',
+            'title': 'MIT DNA Learning Center Set',
+            'description': 'md5:82313335e8a8a3f243351ba55bc1b474',
         },
     }
 
@@ -27,12 +33,12 @@ class TechTVMITIE(InfoExtractor):
         video_id = mobj.group('id')
         raw_page = self._download_webpage(
             'http://techtv.mit.edu/videos/%s' % video_id, video_id)
-        clean_page = re.compile(u'<!--.*?-->', re.S).sub(u'', raw_page)
+        clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page)
 
-        base_url = self._search_regex(r'ipadUrl: \'(.+?cloudfront.net/)',
-            raw_page, u'base url')
-        formats_json = self._search_regex(r'bitrates: (\[.+?\])', raw_page,
-            u'video formats')
+        base_url = self._search_regex(
+            r'ipadUrl: \'(.+?cloudfront.net/)', raw_page, 'base url')
+        formats_json = self._search_regex(
+            r'bitrates: (\[.+?\])', raw_page, 'video formats')
         formats_mit = json.loads(formats_json)
         formats = [
             {
@@ -48,28 +54,31 @@ class TechTVMITIE(InfoExtractor):
 
         title = get_element_by_id('edit-title', clean_page)
         description = clean_html(get_element_by_id('edit-description', clean_page))
-        thumbnail = self._search_regex(r'playlist:.*?url: \'(.+?)\'',
-            raw_page, u'thumbnail', flags=re.DOTALL)
+        thumbnail = self._search_regex(
+            r'playlist:.*?url: \'(.+?)\'',
+            raw_page, 'thumbnail', flags=re.DOTALL)
 
-        return {'id': video_id,
-                'title': title,
-                'formats': formats,
-                'description': description,
-                'thumbnail': thumbnail,
-                }
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+        }
 
 
 class MITIE(TechTVMITIE):
-    IE_NAME = u'video.mit.edu'
+    IE_NAME = 'video.mit.edu'
     _VALID_URL = r'https?://video\.mit\.edu/watch/(?P<title>[^/]+)'
 
     _TEST = {
-        u'url': u'http://video.mit.edu/watch/the-government-is-profiling-you-13222/',
-        u'file': u'21783.mp4',
-        u'md5': u'7db01d5ccc1895fc5010e9c9e13648da',
-        u'info_dict': {
-            u'title': u'The Government is Profiling You',
-            u'description': u'md5:ad5795fe1e1623b73620dbfd47df9afd',
+        'url': 'http://video.mit.edu/watch/the-government-is-profiling-you-13222/',
+        'md5': '7db01d5ccc1895fc5010e9c9e13648da',
+        'info_dict': {
+            'id': '21783',
+            'ext': 'mp4',
+            'title': 'The Government is Profiling You',
+            'description': 'md5:ad5795fe1e1623b73620dbfd47df9afd',
         },
     }
 
@@ -77,7 +86,73 @@ class MITIE(TechTVMITIE):
         mobj = re.match(self._VALID_URL, url)
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
-        self.to_screen('%s: Extracting %s url' % (page_title, TechTVMITIE.IE_NAME))
-        embed_url = self._search_regex(r'<iframe .*?src="(.+?)"', webpage,
-            u'embed url')
+        embed_url = self._search_regex(
+            r'<iframe .*?src="(.+?)"', webpage, 'embed url')
         return self.url_result(embed_url, ie='TechTVMIT')
+
+
+class OCWMITIE(InfoExtractor):
+    IE_NAME = 'ocw.mit.edu'
+    _VALID_URL = r'^http://ocw\.mit\.edu/courses/(?P<topic>[a-z0-9\-]+)'
+    _BASE_URL = 'http://ocw.mit.edu/'
+
+    _TESTS = [
+        {
+            'url': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/',
+            'info_dict': {
+                'id': 'EObHWIEKGjA',
+                'ext': 'mp4',
+                'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
+                'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
+                #'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
+            }
+        },
+        {
+            'url': 'http://ocw.mit.edu/courses/mathematics/18-01sc-single-variable-calculus-fall-2010/1.-differentiation/part-a-definition-and-basic-rules/session-1-introduction-to-derivatives/',
+            'info_dict': {
+                'id': '7K1sB05pE0A',
+                'ext': 'mp4',
+                'title': 'Session 1: Introduction to Derivatives',
+                'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
+                #'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        topic = mobj.group('topic')
+
+        webpage = self._download_webpage(url, topic)
+        title = self._html_search_meta('WT.cg_s', webpage)
+        description = self._html_search_meta('Description', webpage)
+
+        # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, start, stop, captions_file)
+        embed_chapter_media = re.search(r'ocw_embed_chapter_media\((.+?)\)', webpage)
+        if embed_chapter_media:
+            metadata = re.sub(r'[\'"]', '', embed_chapter_media.group(1))
+            metadata = re.split(r', ?', metadata)
+            yt = metadata[1]
+            subs = compat_urlparse.urljoin(self._BASE_URL, metadata[7])
+        else:
+            # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, captions_file)
+            embed_media = re.search(r'ocw_embed_media\((.+?)\)', webpage)
+            if embed_media:
+                metadata = re.sub(r'[\'"]', '', embed_media.group(1))
+                metadata = re.split(r', ?', metadata)
+                yt = metadata[1]
+                subs = compat_urlparse.urljoin(self._BASE_URL, metadata[5])
+            else:
+                raise ExtractorError('Unable to find embedded YouTube video.')
+        video_id = YoutubeIE.extract_id(yt)
+
+        return {
+            '_type': 'url_transparent',
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'url': yt,
+            'url_transparent'
+            'subtitles': subs,
+            'ie_key': 'Youtube',
+        }

youtube_dl/extractor/mixcloud.py

@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MixcloudIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
+    _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/([^/]+)'
     IE_NAME = 'mixcloud'
 
     _TEST = {

youtube_dl/extractor/nbc.py

@@ -6,6 +6,30 @@ from .common import InfoExtractor
 from ..utils import find_xpath_attr, compat_str
 
 
+class NBCIE(InfoExtractor):
+    _VALID_URL = r'http://www\.nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
+
+    _TEST = {
+        'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+        'md5': '54d0fbc33e0b853a65d7b4de5c06d64e',
+        'info_dict': {
+            'id': 'u1RInQZRN7QJ',
+            'ext': 'flv',
+            'title': 'I Am a Firefighter',
+            'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id)
+        theplatform_url = self._search_regex('class="video-player video-player-full" data-mpx-url="(.*?)"', webpage, 'theplatform url')
+        if theplatform_url.startswith('//'):
+            theplatform_url = 'http:' + theplatform_url
+        return self.url_result(theplatform_url)
+
+
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'https?://www\.nbcnews\.com/video/.+?/(?P<id>\d+)'
 

youtube_dl/extractor/prosiebensat1.py

@@ -0,0 +1,297 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from hashlib import sha1
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse,
+    unified_strdate,
+    clean_html,
+    RegexNotFoundError,
+)
+
+
+class ProSiebenSat1IE(InfoExtractor):
+    IE_NAME = 'prosiebensat1'
+    IE_DESC = 'ProSiebenSat.1 Digital'
+    _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|ran|the-voice-of-germany)\.de|fem\.com)/(?P<id>.+)'
+
+    _TESTS = [
+        {
+            'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
+            'info_dict': {
+                'id': '2104602',
+                'ext': 'mp4',
+                'title': 'Staffel 2, Episode 18 - Jahresrückblick',
+                'description': 'md5:8733c81b702ea472e069bc48bb658fc1',
+                'upload_date': '20131231',
+                'duration': 5845.04,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.prosieben.de/videokatalog/Gesellschaft/Leben/Trends/video-Lady-Umstyling-f%C3%BCr-Audrina-Rebekka-Audrina-Fergen-billig-aussehen-Battal-Modica-700544.html',
+            'info_dict': {
+                'id': '2570327',
+                'ext': 'mp4',
+                'title': 'Lady-Umstyling für Audrina',
+                'description': 'md5:4c16d0c17a3461a0d43ea4084e96319d',
+                'upload_date': '20131014',
+                'duration': 606.76,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'Seems to be broken',
+        },
+        {
+            'url': 'http://www.prosiebenmaxx.de/yep/one-piece/video/148-folge-48-gold-rogers-heimat-ganze-folge',
+            'info_dict': {
+                'id': '2437108',
+                'ext': 'mp4',
+                'title': 'Folge 48: Gold Rogers Heimat',
+                'description': 'Ruffy erreicht die Insel, auf der der berühmte Gold Roger lebte und hingerichtet wurde.',
+                'upload_date': '20140226',
+                'duration': 1401.48,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
+            'info_dict': {
+                'id': '2904997',
+                'ext': 'mp4',
+                'title': 'Sexy laufen in Ugg Boots',
+                'description': 'md5:edf42b8bd5bc4e5da4db4222c5acb7d6',
+                'upload_date': '20140122',
+                'duration': 245.32,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
+            'info_dict': {
+                'id': '2906572',
+                'ext': 'mp4',
+                'title': 'Im Interview: Kai Wiesinger',
+                'description': 'md5:e4e5370652ec63b95023e914190b4eb9',
+                'upload_date': '20140225',
+                'duration': 522.56,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
+            'info_dict': {
+                'id': '2992323',
+                'ext': 'mp4',
+                'title': 'Jagd auf Fertigkost im Elsthal - Teil 2',
+                'description': 'md5:2669cde3febe9bce13904f701e774eb6',
+                'upload_date': '20140225',
+                'duration': 2410.44,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
+            'info_dict': {
+                'id': '3004256',
+                'ext': 'mp4',
+                'title': 'Schalke: Tönnies möchte Raul zurück',
+                'description': 'md5:4b5b271d9bcde223b54390754c8ece3f',
+                'upload_date': '20140226',
+                'duration': 228.96,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
+            'info_dict': {
+                'id': '2572814',
+                'ext': 'mp4',
+                'title': 'Andreas Kümmert: Rocket Man',
+                'description': 'md5:6ddb02b0781c6adf778afea606652e38',
+                'upload_date': '20131017',
+                'duration': 469.88,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.fem.com/wellness/videos/wellness-video-clip-kurztripps-zum-valentinstag.html',
+            'info_dict': {
+                'id': '2156342',
+                'ext': 'mp4',
+                'title': 'Kurztrips zum Valentinstag',
+                'description': 'md5:8ba6301e70351ae0bedf8da00f7ba528',
+                'upload_date': '20130206',
+                'duration': 307.24,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+    ]
+
+    _CLIPID_REGEXES = [
+        r'"clip_id"\s*:\s+"(\d+)"',
+        r'clipid: "(\d+)"',
+    ]
+    _TITLE_REGEXES = [
+        r'<h2 class="subtitle" itemprop="name">\s*(.+?)</h2>',
+        r'<header class="clearfix">\s*<h3>(.+?)</h3>',
+        r'<!-- start video -->\s*<h1>(.+?)</h1>',
+        r'<div class="ep-femvideos-pi4-video-txt">\s*<h2>(.+?)</h2>',
+    ]
+    _DESCRIPTION_REGEXES = [
+        r'<p itemprop="description">\s*(.+?)</p>',
+        r'<div class="videoDecription">\s*<p><strong>Beschreibung</strong>: (.+?)</p>',
+        r'<div class="g-plusone" data-size="medium"></div>\s*</div>\s*</header>\s*(.+?)\s*<footer>',
+        r'<p>(.+?)</p>\s*<div class="ep-femvideos-pi4-video-footer">',
+    ]
+    _UPLOAD_DATE_REGEXES = [
+        r'<meta property="og:published_time" content="(.+?)">',
+        r'<span>\s*(\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}) \|\s*<span itemprop="duration"',
+        r'<footer>\s*(\d{2}\.\d{2}\.\d{4}) \d{2}:\d{2} Uhr',
+        r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
+        r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        page = self._download_webpage(url, video_id, 'Downloading page')
+
+        def extract(patterns, name, page, fatal=False):
+            for pattern in patterns:
+                mobj = re.search(pattern, page)
+                if mobj:
+                    return clean_html(mobj.group(1))
+            if fatal:
+                raise RegexNotFoundError(u'Unable to extract %s' % name)
+            return None
+
+        clip_id = extract(self._CLIPID_REGEXES, 'clip id', page, fatal=True)
+
+        access_token = 'testclient'
+        client_name = 'kolibri-1.2.5'
+        client_location = url
+
+        videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse.urlencode({
+            'access_token': access_token,
+            'client_location': client_location,
+            'client_name': client_name,
+            'ids': clip_id,
+        })
+
+        videos = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')
+
+        duration = float(videos[0]['duration'])
+        source_ids = [source['id'] for source in videos[0]['sources']]
+        source_ids_str = ','.join(map(str, source_ids))
+
+        g = '01!8d8F_)r9]4s[qeuXfP%'
+
+        client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name])
+                                 .encode('utf-8')).hexdigest()
+
+        sources_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources?%s' % (clip_id, compat_urllib_parse.urlencode({
+            'access_token': access_token,
+            'client_id': client_id,
+            'client_location': client_location,
+            'client_name': client_name,
+        }))
+
+        sources = self._download_json(sources_api_url, clip_id, 'Downloading sources JSON')
+        server_id = sources['server_id']
+
+        client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id,
+                                          client_location, source_ids_str, g, client_name])
+                                 .encode('utf-8')).hexdigest()
+
+        url_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url?%s' % (clip_id, compat_urllib_parse.urlencode({
+            'access_token': access_token,
+            'client_id': client_id,
+            'client_location': client_location,
+            'client_name': client_name,
+            'server_id': server_id,
+            'source_ids': source_ids_str,
+        }))
+
+        urls = self._download_json(url_api_url, clip_id, 'Downloading urls JSON')
+
+        title = extract(self._TITLE_REGEXES, 'title', page, fatal=True)
+        description = extract(self._DESCRIPTION_REGEXES, 'description', page)
+        thumbnail = self._og_search_thumbnail(page)
+
+        upload_date = extract(self._UPLOAD_DATE_REGEXES, 'upload date', page)
+        if upload_date:
+            upload_date = unified_strdate(upload_date)
+
+        formats = []
+
+        urls_sources = urls['sources']
+        if isinstance(urls_sources, dict):
+            urls_sources = urls_sources.values()
+
+        def fix_bitrate(bitrate):
+            return bitrate / 1000 if bitrate % 1000 == 0 else bitrate
+
+        for source in urls_sources:
+            protocol = source['protocol']
+            if protocol == 'rtmp' or protocol == 'rtmpe':
+                mobj = re.search(r'^(?P<url>rtmpe?://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', source['url'])
+                if not mobj:
+                    continue
+                formats.append({
+                    'url': mobj.group('url'),
+                    'app': mobj.group('app'),
+                    'play_path': mobj.group('playpath'),
+                    'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
+                    'page_url': 'http://www.prosieben.de',
+                    'vbr': fix_bitrate(source['bitrate']),
+                    'ext': 'mp4',
+                    'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
+                })
+            else:
+                formats.append({
+                    'url': source['url'],
+                    'vbr': fix_bitrate(source['bitrate']),
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': clip_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+        }

youtube_dl/extractor/rtlnow.py

@@ -1,148 +1,165 @@
 # encoding: utf-8
-
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    clean_html,
     ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
 )
 
 
 class RTLnowIE(InfoExtractor):
     """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'(?:http://)?(?P<url>(?P<domain>rtl-now\.rtl\.de|rtl2now\.rtl2\.de|(?:www\.)?voxnow\.de|(?:www\.)?rtlnitronow\.de|(?:www\.)?superrtlnow\.de|(?:www\.)?n-tvnow\.de)/+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
-    _TESTS = [{
-        'url': 'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
-        'file': '90419.flv',
-        'info_dict': {
-            'upload_date': '20070416',
-            'title': 'Ahornallee - Folge 1 - Der Einzug',
-            'description': 'Folge 1 - Der Einzug',
+    _VALID_URL = r'''(?x)
+                        (?:https?://)?
+                        (?P<url>
+                            (?P<domain>
+                                rtl-now\.rtl\.de|
+                                rtl2now\.rtl2\.de|
+                                (?:www\.)?voxnow\.de|
+                                (?:www\.)?rtlnitronow\.de|
+                                (?:www\.)?superrtlnow\.de|
+                                (?:www\.)?n-tvnow\.de)
+                            /+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?
+                            (?:container_id|film_id)=(?P<video_id>[0-9]+)&
+                            player=1(?:&season=[0-9]+)?(?:&.*)?
+                        )'''
+
+    _TESTS = [
+        {
+            'url': 'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
+            'info_dict': {
+                'id': '90419',
+                'ext': 'flv',
+                'title': 'Ahornallee - Folge 1 - Der Einzug',
+                'description': 'md5:ce843b6b5901d9a7f7d04d1bbcdb12de',
+                'upload_date': '20070416',
+                'duration': 1685,
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'Only works from Germany',
         },
-        'params': {
-            'skip_download': True,
+        {
+            'url': 'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
+            'info_dict': {
+                'id': '69756',
+                'ext': 'flv',
+                'title': 'Ärger im Revier - Ein junger Ladendieb, ein handfester Streit u.a.',
+                'description': 'md5:3fb247005ed21a935ffc82b7dfa70cf0',
+                'thumbnail': 'http://autoimg.static-fra.de/rtl2now/219850/1500x1500/image2.jpg',
+                'upload_date': '20120519',
+                'duration': 1245,
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'skip': 'Only works from Germany',
         },
-        'skip': 'Only works from Germany',
-    },
-    {
-        'url': 'http://rtl2now.rtl2.de/aerger-im-revier/episode-15-teil-1.php?film_id=69756&player=1&season=2&index=5',
-        'file': '69756.flv',
-        'info_dict': {
-            'upload_date': '20120519',
-            'title': 'Ärger im Revier - Ein junger Ladendieb, ein handfester Streit...',
-            'description': 'Ärger im Revier - Ein junger Ladendieb, ein handfester Streit u.a.',
-            'thumbnail': 'http://autoimg.static-fra.de/rtl2now/219850/1500x1500/image2.jpg',
+        {
+            'url': 'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
+            'info_dict': {
+                'id': '13883',
+                'ext': 'flv',
+                'title': 'Voxtours - Südafrika-Reporter II',
+                'description': 'md5:de7f8d56be6fd4fed10f10f57786db00',
+                'upload_date': '20090627',
+                'duration': 1800,
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
-        'params': {
-            'skip_download': True,
+        {
+            'url': 'http://superrtlnow.de/medicopter-117/angst.php?film_id=99205&player=1',
+            'info_dict': {
+                'id': '99205',
+                'ext': 'flv',
+                'title': 'Medicopter 117 - Angst!',
+                'description': 'md5:895b1df01639b5f61a04fc305a5cb94d',
+                'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
+                'upload_date': '20080928',
+                'duration': 2691,
+            },
+            'params': {
+                'skip_download': True,
+            },
         },
-        'skip': 'Only works from Germany',
-    },
-    {
-        'url': 'http://www.voxnow.de/voxtours/suedafrika-reporter-ii.php?film_id=13883&player=1&season=17',
-        'file': '13883.flv',
-        'info_dict': {
-            'upload_date': '20090627',
-            'title': 'Voxtours - Südafrika-Reporter II',
-            'description': 'Südafrika-Reporter II',
+        {
+            'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
+            'info_dict': {
+                'id': '153819',
+                'ext': 'flv',
+                'title': 'Deluxe - Alles was Spaß macht - Thema u.a.: Luxushotel für Vierbeiner',
+                'description': 'md5:c3705e1bb32e1a5b2bcd634fc065c631',
+                'thumbnail': 'http://autoimg.static-fra.de/ntvnow/383157/1500x1500/image2.jpg',
+                'upload_date': '20140221',
+                'duration': 2429,
+            },
+            'skip': 'Only works from Germany',
         },
-        'params': {
-            'skip_download': True,
-        },
-    },
-    {
-        'url': 'http://superrtlnow.de/medicopter-117/angst.php?film_id=99205&player=1',
-        'file': '99205.flv',
-        'info_dict': {
-            'upload_date': '20080928', 
-            'title': 'Medicopter 117 - Angst!',
-            'description': 'Angst!',
-            'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg'
-        },
-        'params': {
-            'skip_download': True,
-        },
-    },
-    {
-        'url': 'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10',
-        'file': '124903.flv',
-        'info_dict': {
-            'upload_date': '20130101',
-            'title': 'Top Gear vom 01.01.2013',
-            'description': 'Episode 1',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'Only works from Germany',
-    }]
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-
-        webpage_url = 'http://' + mobj.group('url')
-        video_page_url = 'http://' + mobj.group('domain') + '/'
+        video_page_url = 'http://%s/' % mobj.group('domain')
         video_id = mobj.group('video_id')
 
-        webpage = self._download_webpage(webpage_url, video_id)
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
 
-        note_m = re.search(r'''(?sx)
-            <div[ ]style="margin-left:[ ]20px;[ ]font-size:[ ]13px;">(.*?)
-            <div[ ]id="playerteaser">''', webpage)
-        if note_m:
-            msg = clean_html(note_m.group(1))
-            raise ExtractorError(msg)
+        mobj = re.search(r'(?s)<div style="margin-left: 20px; font-size: 13px;">(.*?)<div id="playerteaser">', webpage)
+        if mobj:
+            raise ExtractorError(clean_html(mobj.group(1)), expected=True)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
+
+        upload_date = unified_strdate(self._html_search_meta('uploadDate', webpage, 'upload date'))
+
+        mobj = re.search(r'<meta itemprop="duration" content="PT(?P<seconds>\d+)S" />', webpage)
+        duration = int(mobj.group('seconds')) if mobj else None
 
-        video_title = self._html_search_regex(
-            r'<title>(?P<title>[^<]+?)( \| [^<]*)?</title>',
-            webpage, 'title')
         playerdata_url = self._html_search_regex(
-            r'\'playerdata\': \'(?P<playerdata_url>[^\']+)\'',
-            webpage, 'playerdata_url')
+            r"'playerdata': '(?P<playerdata_url>[^']+)'", webpage, 'playerdata_url')
 
-        playerdata = self._download_webpage(playerdata_url, video_id)
-        mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)(?:\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr)?\]\]></title>', playerdata)
-        if mobj:
-            video_description = mobj.group('description')
-            if mobj.group('upload_date_Y'):
-                video_upload_date = mobj.group('upload_date_Y')
-            elif mobj.group('upload_date_y'):
-                video_upload_date = '20' + mobj.group('upload_date_y')
+        playerdata = self._download_xml(playerdata_url, video_id, 'Downloading player data XML')
+
+        videoinfo = playerdata.find('./playlist/videoinfo')
+        
+        formats = []
+        for filename in videoinfo.findall('filename'):
+            mobj = re.search(r'(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>.+)', filename.text)
+            if mobj:
+                fmt = {
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:' + mobj.group('play_path'),
+                    'page_url': video_page_url,
+                    'player_url': video_page_url + 'includes/vodplayer.swf',
+                }
             else:
-                video_upload_date = None
-            if video_upload_date:
-                video_upload_date += mobj.group('upload_date_m') + mobj.group('upload_date_d')
-        else:
-            video_description = None
-            video_upload_date = None
-            self._downloader.report_warning('Unable to extract description and upload date')
-
-        # Thumbnail: not every video has an thumbnail
-        mobj = re.search(r'<meta property="og:image" content="(?P<thumbnail>[^"]+)">', webpage)
-        if mobj:
-            video_thumbnail = mobj.group('thumbnail')
-        else:
-            video_thumbnail = None
-
-        mobj = re.search(r'<filename [^>]+><!\[CDATA\[(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>[^\]]+)\]\]></filename>', playerdata)
-        if mobj is None:
-            raise ExtractorError('Unable to extract media URL')
-        video_url = mobj.group('url')
-        video_play_path = 'mp4:' + mobj.group('play_path')
-        video_player_url = video_page_url + 'includes/vodplayer.swf'
+                fmt = {
+                    'url': filename.text,
+                }
+            fmt.update({
+                'width': int_or_none(filename.get('width')),
+                'height': int_or_none(filename.get('height')),
+                'vbr': int_or_none(filename.get('bitrate')),
+                'ext': 'flv',
+            })
+            formats.append(fmt)
 
         return {
             'id': video_id,
-            'url': video_url,
-            'play_path': video_play_path,
-            'page_url': video_page_url,
-            'player_url': video_player_url,
-            'ext': 'flv',
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_upload_date,
-            'thumbnail': video_thumbnail,
-        }
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'formats': formats,
+        }

youtube_dl/extractor/testurl.py

@@ -39,6 +39,8 @@ class TestURLIE(InfoExtractor):
                     ('Found multiple matching extractors: %s' %
                         ' '.join(ie.IE_NAME for ie in matching_extractors)),
                     expected=True)
+        else:
+            extractor = matching_extractors[0]
 
         num_str = mobj.group('num')
         num = int(num_str) if num_str else 0

youtube_dl/extractor/theplatform.py

@@ -13,7 +13,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/[^/]+/
-           (?P<config>[^/\?]+/(?:swf|config)/select/)?
+           (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
     _TEST = {
@@ -54,10 +54,15 @@ class ThePlatformIE(InfoExtractor):
 
         f4m_node = body.find(_x('smil:seq/smil:video'))
         if f4m_node is not None:
+            f4m_url = f4m_node.attrib['src']
+            if 'manifest.f4m?' not in f4m_url:
+                f4m_url += '?'
+            # the parameters are from syfy.com, other sites may use others,
+            # they also work for nbc.com
+            f4m_url += '&g=UXWGVKRWHFSP&hdcore=3.0.3'
             formats = [{
                 'ext': 'flv',
-                # the parameters are from syfy.com, other sites may use others
-                'url': f4m_node.attrib['src'] + '?g=UXWGVKRWHFSP&hdcore=3.0.3',
+                'url': f4m_url,
             }]
         else:
             base_url = head.find(_x('smil:meta')).attrib['base']
@@ -95,9 +100,10 @@ class ThePlatformIE(InfoExtractor):
         if mobj.group('config'):
             config_url = url+ '&form=json'
             config_url = config_url.replace('swf/', 'config/')
+            config_url = config_url.replace('onsite/', 'onsite/config/')
             config_json = self._download_webpage(config_url, video_id, u'Downloading config')
             config = json.loads(config_json)
-            smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4'
+            smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
             smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
                 'format=smil&mbr=true'.format(video_id))

youtube_dl/extractor/vevo.py

@@ -24,9 +24,10 @@ class VevoIE(InfoExtractor):
         (?P<id>[^&?#]+)'''
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
-        'file': 'GB1101300280.mp4',
         "md5": "06bea460acb744eab74a9d7dcb4bfd61",
         'info_dict': {
+            'id': 'GB1101300280',
+            'ext': 'mp4',
             "upload_date": "20130624",
             "uploader": "Hurts",
             "title": "Somebody to Die For",
@@ -34,6 +35,33 @@ class VevoIE(InfoExtractor):
             "width": 1920,
             "height": 1080,
         }
+    }, {
+        'note': 'v3 SMIL format',
+        'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
+        'md5': '893ec0e0d4426a1d96c01de8f2bdff58',
+        'info_dict': {
+            'id': 'USUV71302923',
+            'ext': 'mp4',
+            'upload_date': '20140219',
+            'uploader': 'Cassadee Pope',
+            'title': 'I Wish I Could Break Your Heart',
+            'duration': 226.101,
+            'age_limit': 0,
+        }
+    }, {
+        'note': 'Age-limited video',
+        'url': 'https://www.vevo.com/watch/justin-timberlake/tunnel-vision-explicit/USRV81300282',
+        'info_dict': {
+            'id': 'USRV81300282',
+            'ext': 'mp4',
+            'age_limit': 18,
+            'title': 'Tunnel Vision (Explicit)',
+            'uploader': 'Justin Timberlake',
+            'upload_date': '20130704',
+        },
+        'params': {
+            'skip_download': 'true',
+        }
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
@@ -105,9 +133,31 @@ class VevoIE(InfoExtractor):
         video_info = self._download_json(json_url, video_id)['video']
 
         formats = self._formats_from_json(video_info)
+
+        is_explicit = video_info.get('isExplicit')
+        if is_explicit is True:
+            age_limit = 18
+        elif is_explicit is False:
+            age_limit = 0
+        else:
+            age_limit = None
+
+        # Download SMIL
+        smil_blocks = sorted((
+            f for f in video_info['videoVersions']
+            if f['sourceType'] == 13),
+            key=lambda f: f['version'])
+
+        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
+            self._SMIL_BASE_URL, video_id, video_id.lower())
+        if smil_blocks:
+            smil_url_m = self._search_regex(
+                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
+                fatal=False)
+            if smil_url_m is not None:
+                smil_url = smil_url_m
+
         try:
-            smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
-                self._SMIL_BASE_URL, video_id, video_id.lower())
             smil_xml = self._download_webpage(smil_url, video_id,
                                               'Downloading SMIL info')
             formats.extend(self._formats_from_smil(smil_xml))
@@ -128,4 +178,5 @@ class VevoIE(InfoExtractor):
             'upload_date': upload_date.strftime('%Y%m%d'),
             'uploader': video_info['mainArtists'][0]['artistName'],
             'duration': video_info['duration'],
+            'age_limit': age_limit,
         }

youtube_dl/extractor/xhamster.py

@@ -103,6 +103,7 @@ class XHamsterIE(InfoExtractor):
         }]
 
         if not hd:
+            mrss_url = self._search_regex(r'<link rel="canonical" href="([^"]+)', webpage, 'mrss_url')
             webpage = self._download_webpage(mrss_url + '?hd', video_id, note='Downloading HD webpage')
             if is_hd(webpage):
                 video_url = extract_video_url(webpage)

youtube_dl/extractor/youtube.py

@@ -199,9 +199,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '135': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'DASH video', 'preference': -40},
         '136': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'DASH video', 'preference': -40},
         '137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
-        '138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40},
+        '138': {'ext': 'mp4', 'height': 2160, 'resolution': '2160p', 'format_note': 'DASH video', 'preference': -40},
         '160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40},
-        '264': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
+        '264': {'ext': 'mp4', 'height': 1440, 'resolution': '1440p', 'format_note': 'DASH video', 'preference': -40},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},

youtube_dl/utils.py

@@ -772,6 +772,7 @@ def unified_strdate(date_str):
         '%B %d %Y',
         '%b %d %Y',
         '%Y-%m-%d',
+        '%d.%m.%Y',
         '%d/%m/%Y',
         '%Y/%m/%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.02.25'
+__version__ = '2014.02.27.1'