release 2013.10.09

[youtube] Do not recognize attribution link as user (Fixes #1573 )
Merge pull request #1584 from wingsuit/master
2013-10-09 23:50:47 +02:00 · 2013-10-09 23:50:38 +02:00 · 2013-10-09 07:44:06 -07:00 · 2013-10-09 16:41:36 +02:00 · 2013-10-09 21:56:09 +08:00 · 2013-10-08 21:25:38 +02:00
41 changed files with 1020 additions and 587 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@ updates_key.pem
 *.mp4
 *.part
 test/testdata
+.tox
--- a/README.md
+++ b/README.md
@@ -31,8 +31,9 @@ which means you can modify it, redistribute it or use it however you like.
    --proxy URL                Use the specified HTTP/HTTPS proxy
    --no-check-certificate     Suppress HTTPS certificate validation.
    --cache-dir None           Location in the filesystem where youtube-dl can
-                               store downloaded information permanently.
-                               ~/.youtube-dl/cache by default
+                               store downloaded information permanently. By
+                               default $XDG_CACHE_HOME/youtube-dl or ~/.cache
+                               /youtube-dl .
    --no-cache-dir             Disable filesystem caching

 ## Video Selection:
@@ -50,6 +51,10 @@ which means you can modify it, redistribute it or use it however you like.
    --date DATE                download only videos uploaded in this date
    --datebefore DATE          download only videos uploaded before this date
    --dateafter DATE           download only videos uploaded after this date
+    --no-playlist              download only the currently playing video
+    --age-limit YEARS          download only videos suitable for the given age
+    --download-archive FILE    Download only videos not present in the archive
+                               file. Record all downloaded videos in it.

 ## Download Options:
    -r, --rate-limit LIMIT     maximum download rate (e.g. 50k or 44.6m)
--- a/devscripts/youtube_genalgo.py
+++ b/devscripts/youtube_genalgo.py
@@ -1,116 +0,0 @@
-#!/usr/bin/env python
-# encoding: utf-8
-
-# Generate youtube signature algorithm from test cases
-
-import sys
-
-tests = [
-    # 93 - vfl79wBKW 2013/07/20
-    (u"qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`~\"€",
-     u".>/?;:|}][{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWQ098765'321mnbvcxzasdfghjklpoiu"),
-    # 92 - vflQw-fB4 2013/07/17
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`~\"",
-     "mrtyuioplkjhgfdsazxcvbnq1234567890QWERTY}IOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]\"|:;"),
-    # 91 - vfl79wBKW 2013/07/20 (sporadic)
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`~",
-     "/?;:|}][{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWQ09876543.1mnbvcxzasdfghjklpoiu"),
-    # 90
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`",
-     "mrtyuioplkjhgfdsazxcvbne1234567890QWER[YUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={`]}|"),
-    # 89 
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'",
-     "/?;:|}<[{=+-_)(*&^%$#@!MqBVCXZASDFGHJKLPOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuyt"),
-    # 88 - vflapUV9V 2013/08/28
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<",
-     "ioplkjhgfdsazxcvbnm12<4567890QWERTYUIOZLKJHGFDSAeXCVBNM!@#$%^&*()_-+={[]}|:;?/>.3"),
-    # 87
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$^&*()_-+={[]}|:;?/>.<",
-     "uioplkjhgfdsazxcvbnm1t34567890QWE2TYUIOPLKJHGFDSAZXCVeNM!@#$^&*()_-+={[]}|:;?/>.<"),
-    # 86 - vflHql6Pr 2013/09/24
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
-     ";}|[{=+-d)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYT_EWQ0987654321mnbvcxzas/fghjklpoiuytrewq"),
-    # 85 - vflkuzxcs 2013/09/11
-    ('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[',
-     '3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@'),
-    # 84 - vflHql6Pr 2013/09/24 (sporadic)
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?>.<",
-     "}[{=+-_)g*&^%$#@!MNBVCXZASDFGHJKLPOIUYTRE(Q0987654321mnbvcxzasdf?hjklpoiuytrewq"),
-    # 83
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!#$%^&*()_+={[};?/>.<",
-     ".>/?;}[{=+_)(*&^%<#!MNBVCXZASPFGHJKLwOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuytreq"),
-    # 82 - vflGNjMhJ 2013/09/12
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.<",
-     ".>/?;}[<=+-(*&^%$#@!MNBVCXeASDFGHKLPOqUYTREWQ0987654321mnbvcxzasdfghjklpoiuytrIwZ"),
-    # 81 - vflLC8JvQ 2013/07/25
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.",
-     "C>/?;}[{=+-(*&^%$#@!MNBVYXZASDFGHKLPOIU.TREWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
-    # 80 - vflZK4ZYR 2013/08/23 (sporadic)
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>",
-     "wertyuioplkjhgfdsaqxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&z(-+={[};?/>"),
-    # 79 - vflLC8JvQ 2013/07/25 (sporadic)
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/",
-     "Z?;}[{=+-(*&^%$#@!MNBVCXRASDFGHKLPOIUYT/EWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
-]
-
-tests_age_gate = [
-    # 86 - vflqinMWD
-    ("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
-     "ertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!/#$%^&*()_-+={[|};?@"),
-]
-
-def find_matching(wrong, right):
-    idxs = [wrong.index(c) for c in right]
-    return compress(idxs)
-    return ('s[%d]' % i for i in idxs)
-
-def compress(idxs):
-    def _genslice(start, end, step):
-        starts = '' if start == 0 else str(start)
-        ends = ':%d' % (end+step)
-        steps = '' if step == 1 else (':%d' % step)
-        return 's[%s%s%s]' % (starts, ends, steps)
-
-    step = None
-    for i, prev in zip(idxs[1:], idxs[:-1]):
-        if step is not None:
-            if i - prev == step:
-                continue
-            yield _genslice(start, prev, step)
-            step = None
-            continue
-        if i - prev in [-1, 1]:
-            step = i - prev
-            start = prev
-            continue
-        else:
-            yield 's[%d]' % prev
-    if step is None:
-        yield 's[%d]' % i
-    else:
-        yield _genslice(start, i, step)
-
-def _assert_compress(inp, exp):
-    res = list(compress(inp))
-    if res != exp:
-        print('Got %r, expected %r' % (res, exp))
-        assert res == exp
-_assert_compress([0,2,4,6], ['s[0]', 's[2]', 's[4]', 's[6]'])
-_assert_compress([0,1,2,4,6,7], ['s[:3]', 's[4]', 's[6:8]'])
-_assert_compress([8,0,1,2,4,7,6,9], ['s[8]', 's[:3]', 's[4]', 's[7:5:-1]', 's[9]'])
-
-def gen(wrong, right, indent):
-    code = ' + '.join(find_matching(wrong, right))
-    return 'if len(s) == %d:\n%s    return %s\n' % (len(wrong), indent, code)
-
-def genall(tests):
-    indent = ' ' * 8
-    return indent + (indent + 'el').join(gen(wrong, right, indent) for wrong,right in tests)
-
-def main():
-    print(genall(tests))
-    print(u'    Age gate:')
-    print(genall(tests_age_gate))
-
-if __name__ == '__main__':
-    main()
--- a/test/helper.py
+++ b/test/helper.py
@@ -1,6 +1,9 @@
+import errno
 import io
 import json
 import os.path
+import re
+import types

 import youtube_dl.extractor
 from youtube_dl import YoutubeDL, YoutubeDLHandler
@@ -9,30 +12,47 @@ from youtube_dl.utils import (
    compat_urllib_request,
 )

-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
+youtube_dl._setup_opener(timeout=10)

 PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
 with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
    parameters = json.load(pf)

+
+def try_rm(filename):
+    """ Remove a file if it exists """
+    try:
+        os.remove(filename)
+    except OSError as ose:
+        if ose.errno != errno.ENOENT:
+            raise
+
+
 class FakeYDL(YoutubeDL):
    def __init__(self):
-        self.result = []
        # Different instances of the downloader can't share the same dictionary
        # some test set the "sublang" parameter, which would break the md5 checks.
-        self.params = dict(parameters)
-    def to_screen(self, s):
+        params = dict(parameters)
+        super(FakeYDL, self).__init__(params)
+        self.result = []
+        
+    def to_screen(self, s, skip_eol=None):
        print(s)
+
    def trouble(self, s, tb=None):
        raise Exception(s)
+
    def download(self, x):
        self.result.append(x)

+    def expect_warning(self, regex):
+        # Silence an expected warning matching a regex
+        old_report_warning = self.report_warning
+        def report_warning(self, message):
+            if re.match(regex, message): return
+            old_report_warning(message)
+        self.report_warning = types.MethodType(report_warning, self)
+
 def get_testcases():
    for ie in youtube_dl.extractor.gen_extractors():
        t = getattr(ie, '_TEST', None)
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl import YoutubeDL
+from helper import try_rm
+
+
+def _download_restricted(url, filename, age):
+    """ Returns true iff the file has been downloaded """
+
+    params = {
+        'age_limit': age,
+        'skip_download': True,
+        'writeinfojson': True,
+        "outtmpl": "%(id)s.%(ext)s",
+    }
+    ydl = YoutubeDL(params)
+    ydl.add_default_info_extractors()
+    json_filename = filename + '.info.json'
+    try_rm(json_filename)
+    ydl.download([url])
+    res = os.path.exists(json_filename)
+    try_rm(json_filename)
+    return res
+
+
+class TestAgeRestriction(unittest.TestCase):
+    def _assert_restricted(self, url, filename, age, old_age=None):
+        self.assertTrue(_download_restricted(url, filename, old_age))
+        self.assertFalse(_download_restricted(url, filename, age))
+
+    def test_youtube(self):
+        self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)
+
+    def test_youporn(self):
+        self._assert_restricted(
+            'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
+            '505835.mp4', 2, old_age=25)
+
+    def test_pornotube(self):
+        self._assert_restricted(
+            'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
+            '1689755.flv', 13)
+
+
+if __name__ == '__main__':
+    unittest.main()
--- a/test/test_dailymotion_subtitles.py
+++ b/test/test_dailymotion_subtitles.py
@@ -2,8 +2,6 @@

 import sys
 import unittest
-import json
-import io
 import hashlib

 # Allow direct execution
@@ -45,15 +43,18 @@ class TestDailymotionSubtitles(unittest.TestCase):
        subtitles = self.getSubtitles()
        self.assertEqual(len(subtitles.keys()), 5)
    def test_list_subtitles(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
        self.DL.params['listsubtitles'] = True
        info_dict = self.getInfoDict()
        self.assertEqual(info_dict, None)
    def test_automatic_captions(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
        self.DL.params['writeautomaticsub'] = True
        self.DL.params['subtitleslang'] = ['en']
        subtitles = self.getSubtitles()
        self.assertTrue(len(subtitles.keys()) == 0)
    def test_nosubtitles(self):
+        self.DL.expect_warning(u'video doesn\'t have subtitles')
        self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python

-import errno
 import hashlib
 import io
 import os
@@ -20,22 +19,6 @@ PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "para

 RETRIES = 3

-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
-socket.setdefaulttimeout(10)
-
-def _try_rm(filename):
-    """ Remove a file if it exists """
-    try:
-        os.remove(filename)
-    except OSError as ose:
-        if ose.errno != errno.ENOENT:
-            raise
-
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()

 class YoutubeDL(youtube_dl.YoutubeDL):
@@ -54,7 +37,8 @@ def _file_md5(fn):
    with open(fn, 'rb') as f:
        return hashlib.md5(f.read()).hexdigest()

-from helper import get_testcases
+import helper  # Set up remaining global configuration
+from helper import get_testcases, try_rm
 defs = get_testcases()

 with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
@@ -97,9 +81,9 @@ def generator(test_case):

        test_cases = test_case.get('playlist', [test_case])
        for tc in test_cases:
-            _try_rm(tc['file'])
-            _try_rm(tc['file'] + '.part')
-            _try_rm(tc['file'] + '.info.json')
+            try_rm(tc['file'])
+            try_rm(tc['file'] + '.part')
+            try_rm(tc['file'] + '.info.json')
        try:
            for retry in range(1, RETRIES + 1):
                try:
@@ -145,9 +129,9 @@ def generator(test_case):
                    self.assertTrue(key in info_dict.keys() and info_dict[key])
        finally:
            for tc in test_cases:
-                _try_rm(tc['file'])
-                _try_rm(tc['file'] + '.part')
-                _try_rm(tc['file'] + '.info.json')
+                try_rm(tc['file'])
+                try_rm(tc['file'] + '.part')
+                try_rm(tc['file'] + '.info.json')

    return test_template

--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -27,6 +27,14 @@ class TestYoutubeLists(unittest.TestCase):
        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
        self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])

+    def test_youtube_playlist_noplaylist(self):
+        dl = FakeYDL()
+        dl.params['noplaylist'] = True
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        self.assertEqual(result['_type'], 'url')
+        self.assertEqual(YoutubeIE()._extract_id(result['url']), 'FXxLjLQi3Fg')
+
    def test_issue_673(self):
        dl = FakeYDL()
        ie = YoutubePlaylistIE(dl)
--- a/test/test_youtube_subtitles.py
+++ b/test/test_youtube_subtitles.py
@@ -2,8 +2,6 @@

 import sys
 import unittest
-import json
-import io
 import hashlib

 # Allow direct execution
@@ -56,6 +54,7 @@ class TestYoutubeSubtitles(unittest.TestCase):
        subtitles = self.getSubtitles()
        self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
    def test_youtube_list_subtitles(self):
+        self.DL.expect_warning(u'Video doesn\'t have automatic captions')
        self.DL.params['listsubtitles'] = True
        info_dict = self.getInfoDict()
        self.assertEqual(info_dict, None)
@@ -66,6 +65,7 @@ class TestYoutubeSubtitles(unittest.TestCase):
        subtitles = self.getSubtitles()
        self.assertTrue(subtitles['it'] is not None)
    def test_youtube_nosubtitles(self):
+        self.DL.expect_warning(u'video doesn\'t have subtitles')
        self.url = 'sAjKT8FhjI8'
        self.DL.params['writesubtitles'] = True
        self.DL.params['allsubtitles'] = True
--- a/tox.ini
+++ b/tox.ini
@@ -0,0 +1,5 @@
+[tox]
+envlist = py26,py27,py33
+[testenv]
+deps = nose
+commands = nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose test
--- a/youtube_dl/PostProcessor.py
+++ b/youtube_dl/PostProcessor.py
@@ -444,8 +444,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
        if information['ext'] != u'mp4':
            self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
            return True, information
-        sub_langs = [key for key in information['subtitles']]
+        if not information.get('subtitles'):
+            self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed') 
+            return True, information

+        sub_langs = [key for key in information['subtitles']]
        filename = information['filepath']
        input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]

--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -3,6 +3,7 @@

 from __future__ import absolute_import

+import errno
 import io
 import os
 import re
@@ -83,6 +84,12 @@ class YoutubeDL(object):
    skip_download:     Skip the actual download of the video file
    cachedir:          Location of the cache files in the filesystem.
                       None to disable filesystem cache.
+    noplaylist:        Download single video instead of a playlist if in doubt.
+    age_limit:         An integer representing the user's age in years.
+                       Unsuitable videos for the given age are skipped.
+    downloadarchive:   File name of a file where all downloads are recorded.
+                       Videos already present in the file are not downloaded
+                       again.
    
    The following parameters are not used by YoutubeDL itself, they are used by
    the FileDownloader:
@@ -112,7 +119,7 @@ class YoutubeDL(object):
                and not params['restrictfilenames']):
            # On Python 3, the Unicode filesystem API will throw errors (#1474)
            self.report_warning(
-                u'Assuming --restrict-filenames isnce file system encoding '
+                u'Assuming --restrict-filenames since file system encoding '
                u'cannot encode all charactes. '
                u'Set the LC_ALL environment variable to fix this.')
            params['restrictfilenames'] = True
@@ -308,6 +315,13 @@ class YoutubeDL(object):
            dateRange = self.params.get('daterange', DateRange())
            if date not in dateRange:
                return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
+        age_limit = self.params.get('age_limit')
+        if age_limit is not None:
+            if age_limit < info_dict.get('age_limit', 0):
+                return u'Skipping "' + title + '" because it is age restricted'
+        if self.in_download_archive(info_dict):
+            return (u'%(title)s has already been recorded in archive'
+                    % info_dict)
        return None
        
    def extract_info(self, url, download=True, ie_key=None, extra_info={}):
@@ -545,11 +559,15 @@ class YoutubeDL(object):
                thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
                self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
                               (info_dict['extractor'], info_dict['id']))
-                uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
-                with open(thumb_filename, 'wb') as thumbf:
-                    shutil.copyfileobj(uf, thumbf)
-                self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
-                               (info_dict['extractor'], info_dict['id'], thumb_filename))
+                try:
+                    uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
+                    with open(thumb_filename, 'wb') as thumbf:
+                        shutil.copyfileobj(uf, thumbf)
+                    self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
+                        (info_dict['extractor'], info_dict['id'], thumb_filename))
+                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                    self.report_warning(u'Unable to download thumbnail "%s": %s' %
+                        (info_dict['thumbnail'], compat_str(err)))

        if not self.params.get('skip_download', False):
            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
@@ -573,6 +591,8 @@ class YoutubeDL(object):
                    self.report_error(u'postprocessing: %s' % str(err))
                    return

+        self.record_download_archive(info_dict)
+
    def download(self, url_list):
        """Download a given list of URLs."""
        if len(url_list) > 1 and self.fixed_template():
@@ -612,3 +632,26 @@ class YoutubeDL(object):
                os.remove(encodeFilename(filename))
            except (IOError, OSError):
                self.report_warning(u'Unable to remove downloaded video file')
+
+    def in_download_archive(self, info_dict):
+        fn = self.params.get('download_archive')
+        if fn is None:
+            return False
+        vid_id = info_dict['extractor'] + u' ' + info_dict['id']
+        try:
+            with locked_file(fn, 'r', encoding='utf-8') as archive_file:
+                for line in archive_file:
+                    if line.strip() == vid_id:
+                        return True
+        except IOError as ioe:
+            if ioe.errno != errno.ENOENT:
+                raise
+        return False
+
+    def record_download_archive(self, info_dict):
+        fn = self.params.get('download_archive')
+        if fn is None:
+            return
+        vid_id = info_dict['extractor'] + u' ' + info_dict['id']
+        with locked_file(fn, 'a', encoding='utf-8') as archive_file:
+            archive_file.write(vid_id + u'\n')
--- a/youtube_dl/init.py
+++ b/youtube_dl/init.py
@@ -36,6 +36,7 @@ __authors__  = (
 __license__ = 'Public Domain'

 import codecs
+import collections
 import getpass
 import optparse
 import os
@@ -168,8 +169,8 @@ def parseOpts(overrideArguments=None):
    general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
    general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
    general.add_option(
-        '--cache-dir', dest='cachedir', default=u'~/.youtube-dl/cache',
-        help='Location in the filesystem where youtube-dl can store downloaded information permanently. %default by default')
+        '--cache-dir', dest='cachedir', default=get_cachedir(),
+        help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
    general.add_option(
        '--no-cache-dir', action='store_const', const=None, dest='cachedir',
        help='Disable filesystem caching')
@@ -187,6 +188,13 @@ def parseOpts(overrideArguments=None):
    selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
    selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
    selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
+    selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
+    selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
+                         help='download only videos suitable for the given age',
+                         default=None, type=int)
+    selection.add_option('--download-archive', metavar='FILE',
+                         dest='download_archive',
+                         help='Download only videos not present in the archive file. Record all downloaded videos in it.')


    authentication.add_option('-u', '--username',
@@ -369,9 +377,13 @@ def parseOpts(overrideArguments=None):
    else:
        xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
        if xdg_config_home:
-            userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
+            userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
+            if not os.path.isfile(userConfFile):
+                userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
        else:
-            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
+            if not os.path.isfile(userConfFile):
+                userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
        systemConf = _readOptions('/etc/youtube-dl.conf')
        userConf = _readOptions(userConfFile)
        commandLineConf = sys.argv[1:]
@@ -436,27 +448,7 @@ def _real_main(argv=None):
    all_urls = batchurls + args
    all_urls = [url.strip() for url in all_urls]

-    # General configuration
-    cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-    if opts.proxy is not None:
-        if opts.proxy == '':
-            proxies = {}
-        else:
-            proxies = {'http': opts.proxy, 'https': opts.proxy}
-    else:
-        proxies = compat_urllib_request.getproxies()
-        # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
-        if 'http' in proxies and 'https' not in proxies:
-            proxies['https'] = proxies['http']
-    proxy_handler = compat_urllib_request.ProxyHandler(proxies)
-    https_handler = make_HTTPS_handler(opts)
-    opener = compat_urllib_request.build_opener(https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
-    # Delete the default user-agent header, which would otherwise apply in
-    # cases where our custom HTTP handler doesn't come into play
-    # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
-    opener.addheaders =[]
-    compat_urllib_request.install_opener(opener)
-    socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+    opener = _setup_opener(jar=jar, opts=opts)

    extractors = gen_extractors()

@@ -473,6 +465,8 @@ def _real_main(argv=None):
            if not ie._WORKING:
                continue
            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
+            if desc is False:
+                continue
            if hasattr(ie, 'SEARCH_KEY'):
                _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
                _COUNTS = (u'', u'5', u'10', u'all')
@@ -599,6 +593,7 @@ def _real_main(argv=None):
        'progress_with_newline': opts.progress_with_newline,
        'playliststart': opts.playliststart,
        'playlistend': opts.playlistend,
+        'noplaylist': opts.noplaylist,
        'logtostderr': opts.outtmpl == '-',
        'consoletitle': opts.consoletitle,
        'nopart': opts.nopart,
@@ -625,6 +620,8 @@ def _real_main(argv=None):
        'daterange': date,
        'cachedir': opts.cachedir,
        'youtube_print_sig_code': opts.youtube_print_sig_code,
+        'age_limit': opts.age_limit,
+        'download_archive': opts.download_archive,
        })

    if opts.verbose:
@@ -644,7 +641,12 @@ def _real_main(argv=None):
            except:
                pass
        write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
-        write_string(u'[debug] Proxy map: ' + str(proxy_handler.proxies) + u'\n')
+
+        proxy_map = {}
+        for handler in opener.handlers:
+            if hasattr(handler, 'proxies'):
+                proxy_map.update(handler.proxies)
+        write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')

    ydl.add_default_info_extractors()

@@ -658,7 +660,7 @@ def _real_main(argv=None):

    # Update version
    if opts.update_self:
-        update_self(ydl.to_screen, opts.verbose, sys.argv[0])
+        update_self(ydl.to_screen, opts.verbose)

    # Maybe do nothing
    if len(all_urls) < 1:
@@ -682,6 +684,37 @@ def _real_main(argv=None):

    sys.exit(retcode)

+
+def _setup_opener(jar=None, opts=None, timeout=300):
+    if opts is None:
+        FakeOptions = collections.namedtuple(
+            'FakeOptions', ['proxy', 'no_check_certificate'])
+        opts = FakeOptions(proxy=None, no_check_certificate=False)
+
+    cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+    if opts.proxy is not None:
+        if opts.proxy == '':
+            proxies = {}
+        else:
+            proxies = {'http': opts.proxy, 'https': opts.proxy}
+    else:
+        proxies = compat_urllib_request.getproxies()
+        # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
+        if 'http' in proxies and 'https' not in proxies:
+            proxies['https'] = proxies['http']
+    proxy_handler = compat_urllib_request.ProxyHandler(proxies)
+    https_handler = make_HTTPS_handler(opts)
+    opener = compat_urllib_request.build_opener(
+        https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
+    # Delete the default user-agent header, which would otherwise apply in
+    # cases where our custom HTTP handler doesn't come into play
+    # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
+    opener.addheaders = []
+    compat_urllib_request.install_opener(opener)
+    socket.setdefaulttimeout(timeout)
+    return opener
+
+
 def main(argv=None):
    try:
        _real_main(argv)
--- a/youtube_dl/extractor/init.py
+++ b/youtube_dl/extractor/init.py
@@ -117,6 +117,7 @@ from .veehd import VeeHDIE
 from .veoh import VeohIE
 from .vevo import VevoIE
 from .vice import ViceIE
+from .viddler import ViddlerIE
 from .videofyme import VideofyMeIE
 from .vimeo import VimeoIE, VimeoChannelIE
 from .vine import VineIE
@@ -140,6 +141,7 @@ from .youtube import (
    YoutubeShowIE,
    YoutubeSubscriptionsIE,
    YoutubeRecommendedIE,
+    YoutubeTruncatedURLIE,
    YoutubeWatchLaterIE,
    YoutubeFavouritesIE,
 )
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -1,8 +1,10 @@
 import re
 import xml.etree.ElementTree
+import json

 from .common import InfoExtractor
 from ..utils import (
+    compat_urlparse,
    determine_ext,
 )

@@ -14,10 +16,9 @@ class AppleTrailersIE(InfoExtractor):
        u"playlist": [
            {
                u"file": u"manofsteel-trailer4.mov",
-                u"md5": u"11874af099d480cc09e103b189805d5f",
+                u"md5": u"d97a8e575432dbcb81b7c3acb741f8a8",
                u"info_dict": {
                    u"duration": 111,
-                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_11624.jpg",
                    u"title": u"Trailer 4",
                    u"upload_date": u"20130523",
                    u"uploader_id": u"wb",
@@ -25,10 +26,9 @@ class AppleTrailersIE(InfoExtractor):
            },
            {
                u"file": u"manofsteel-trailer3.mov",
-                u"md5": u"07a0a262aae5afe68120eed61137ab34",
+                u"md5": u"b8017b7131b721fb4e8d6f49e1df908c",
                u"info_dict": {
                    u"duration": 182,
-                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_10793.jpg",
                    u"title": u"Trailer 3",
                    u"upload_date": u"20130417",
                    u"uploader_id": u"wb",
@@ -36,10 +36,9 @@ class AppleTrailersIE(InfoExtractor):
            },
            {
                u"file": u"manofsteel-trailer.mov",
-                u"md5": u"e401fde0813008e3307e54b6f384cff1",
+                u"md5": u"d0f1e1150989b9924679b441f3404d48",
                u"info_dict": {
                    u"duration": 148,
-                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_8703.jpg",
                    u"title": u"Trailer",
                    u"upload_date": u"20121212",
                    u"uploader_id": u"wb",
@@ -47,10 +46,9 @@ class AppleTrailersIE(InfoExtractor):
            },
            {
                u"file": u"manofsteel-teaser.mov",
-                u"md5": u"76b392f2ae9e7c98b22913c10a639c97",
+                u"md5": u"5fe08795b943eb2e757fa95cb6def1cb",
                u"info_dict": {
                    u"duration": 93,
-                    u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_6899.jpg",
                    u"title": u"Teaser",
                    u"upload_date": u"20120721",
                    u"uploader_id": u"wb",
@@ -59,87 +57,61 @@ class AppleTrailersIE(InfoExtractor):
        ]
    }

+    _JSON_RE = r'iTunes.playURL\((.*?)\);'
+
    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        movie = mobj.group('movie')
        uploader_id = mobj.group('company')

-        playlist_url = url.partition(u'?')[0] + u'/includes/playlists/web.inc'
+        playlist_url = compat_urlparse.urljoin(url, u'includes/playlists/itunes.inc')
        playlist_snippet = self._download_webpage(playlist_url, movie)
-        playlist_cleaned = re.sub(r'(?s)<script>.*?</script>', u'', playlist_snippet)
+        playlist_cleaned = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', playlist_snippet)
+        playlist_cleaned = re.sub(r'<img ([^<]*?)>', r'<img \1/>', playlist_cleaned)
+        # The ' in the onClick attributes are not escaped, it couldn't be parsed
+        # with xml.etree.ElementTree.fromstring
+        # like: http://trailers.apple.com/trailers/wb/gravity/
+        def _clean_json(m):
+            return u'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
+        playlist_cleaned = re.sub(self._JSON_RE, _clean_json, playlist_cleaned)
        playlist_html = u'<html>' + playlist_cleaned + u'</html>'

-        size_cache = {}
-
        doc = xml.etree.ElementTree.fromstring(playlist_html)
        playlist = []
        for li in doc.findall('./div/ul/li'):
-            title = li.find('.//h3').text
+            on_click = li.find('.//a').attrib['onClick']
+            trailer_info_json = self._search_regex(self._JSON_RE,
+                on_click, u'trailer info')
+            trailer_info = json.loads(trailer_info_json)
+            title = trailer_info['title']
            video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
            thumbnail = li.find('.//img').attrib['src']
+            upload_date = trailer_info['posted'].replace('-', '')

-            date_el = li.find('.//p')
-            upload_date = None
-            m = re.search(r':\s?(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<year>[0-9]{2})', date_el.text)
-            if m:
-                upload_date = u'20' + m.group('year') + m.group('month') + m.group('day')
-            runtime_el = date_el.find('./br')
-            m = re.search(r':\s?(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime_el.tail)
+            runtime = trailer_info['runtime']
+            m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
            duration = None
            if m:
                duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))

+            first_url = trailer_info['url']
+            trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
+            settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
+            settings_json = self._download_webpage(settings_json_url, trailer_id, u'Downloading settings json')
+            settings = json.loads(settings_json)
+
            formats = []
-            for formats_el in li.findall('.//a'):
-                if formats_el.attrib['class'] != 'OverlayPanel':
-                    continue
-                target = formats_el.attrib['target']
-
-                format_code = formats_el.text
-                if 'Automatic' in format_code:
-                    continue
-
-                size_q = formats_el.attrib['href']
-                size_id = size_q.rpartition('#videos-')[2]
-                if size_id not in size_cache:
-                    size_url = url + size_q
-                    sizepage_html = self._download_webpage(
-                        size_url, movie,
-                        note=u'Downloading size info %s' % size_id,
-                        errnote=u'Error while downloading size info %s' % size_id,
-                    )
-                    _doc = xml.etree.ElementTree.fromstring(sizepage_html)
-                    size_cache[size_id] = _doc
-
-                sizepage_doc = size_cache[size_id]
-                links = sizepage_doc.findall('.//{http://www.w3.org/1999/xhtml}ul/{http://www.w3.org/1999/xhtml}li/{http://www.w3.org/1999/xhtml}a')
-                for vid_a in links:
-                    href = vid_a.get('href')
-                    if not href.endswith(target):
-                        continue
-                    detail_q = href.partition('#')[0]
-                    detail_url = url + '/' + detail_q
-
-                    m = re.match(r'includes/(?P<detail_id>[^/]+)/', detail_q)
-                    detail_id = m.group('detail_id')
-
-                    detail_html = self._download_webpage(
-                        detail_url, movie,
-                        note=u'Downloading detail %s %s' % (detail_id, size_id),
-                        errnote=u'Error while downloading detail %s %s' % (detail_id, size_id)
-                    )
-                    detail_doc = xml.etree.ElementTree.fromstring(detail_html)
-                    movie_link_el = detail_doc.find('.//{http://www.w3.org/1999/xhtml}a')
-                    assert movie_link_el.get('class') == 'movieLink'
-                    movie_link = movie_link_el.get('href').partition('?')[0].replace('_', '_h')
-                    ext = determine_ext(movie_link)
-                    assert ext == 'mov'
-
-                    formats.append({
-                        'format': format_code,
-                        'ext': ext,
-                        'url': movie_link,
-                    })
+            for format in settings['metadata']['sizes']:
+                # The src is a file pointing to the real video file
+                format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
+                formats.append({
+                    'url': format_url,
+                    'ext': determine_ext(format_url),
+                    'format': format['type'],
+                    'width': format['width'],
+                    'height': int(format['height']),
+                })
+            formats = sorted(formats, key=lambda f: (f['height'], f['width']))

            info = {
                '_type': 'video',
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -115,7 +115,7 @@ class BlipTVIE(InfoExtractor):
                ext = umobj.group(1)

                info = {
-                    'id': data['item_id'],
+                    'id': compat_str(data['item_id']),
                    'url': video_url,
                    'uploader': data['display_name'],
                    'upload_date': upload_date,
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 import re
 import json
 import xml.etree.ElementTree
@@ -7,15 +9,39 @@ from ..utils import (
    compat_urllib_parse,
    find_xpath_attr,
    compat_urlparse,
+
+    ExtractorError,
 )

 class BrightcoveIE(InfoExtractor):
    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
    _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
    _PLAYLIST_URL_TEMPLATE = 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=%s'
-    
-    # There is a test for Brigtcove in GenericIE, that way we test both the download
-    # and the detection of videos, and we don't have to find an URL that is always valid
+
+    _TESTS = [
+        {
+            # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
+            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
+            u'file': u'2371591881001.mp4',
+            u'md5': u'9e80619e0a94663f0bdc849b4566af19',
+            u'note': u'Test Brightcove downloads and detection in GenericIE',
+            u'info_dict': {
+                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
+                u'uploader': u'8TV',
+                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
+            }
+        },
+        {
+            # From http://medianetwork.oracle.com/video/player/1785452137001
+            u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
+            u'file': u'1785452137001.flv',
+            u'info_dict': {
+                u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
+                u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.',
+                u'uploader': u'Oracle',
+            },
+        },
+    ]

    @classmethod
    def _build_brighcove_url(cls, object_str):
@@ -23,6 +49,11 @@ class BrightcoveIE(InfoExtractor):
        Build a Brightcove url from a xml string containing
        <object class="BrightcoveExperience">{params}</object>
        """
+
+        # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
+        object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
+                            lambda m: m.group(1) + '/>', object_str)
+
        object_doc = xml.etree.ElementTree.fromstring(object_str)
        assert u'BrightcoveExperience' in object_doc.attrib['class']
        params = {'flashID': object_doc.attrib['id'],
@@ -72,15 +103,27 @@ class BrightcoveIE(InfoExtractor):
                                    playlist_title=playlist_info['mediaCollectionDTO']['displayName'])

    def _extract_video_info(self, video_info):
-        renditions = video_info['renditions']
-        renditions = sorted(renditions, key=lambda r: r['size'])
-        best_format = renditions[-1]
+        info = {
+            'id': video_info['id'],
+            'title': video_info['displayName'],
+            'description': video_info.get('shortDescription'),
+            'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
+            'uploader': video_info.get('publisherName'),
+        }

-        return {'id': video_info['id'],
-                'title': video_info['displayName'],
-                'url': best_format['defaultURL'], 
+        renditions = video_info.get('renditions')
+        if renditions:
+            renditions = sorted(renditions, key=lambda r: r['size'])
+            best_format = renditions[-1]
+            info.update({
+                'url': best_format['defaultURL'],
                'ext': 'mp4',
-                'description': video_info.get('shortDescription'),
-                'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
-                'uploader': video_info.get('publisherName'),
-                }
+            })
+        elif video_info.get('FLVFullLengthURL') is not None:
+            info.update({
+                'url': video_info['FLVFullLengthURL'],
+                'ext': 'flv',
+            })
+        else:
+            raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
+        return info
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -51,12 +51,12 @@ class ComedyCentralIE(InfoExtractor):
        '400': 'mp4',
    }
    _video_dimensions = {
-        '3500': '1280x720',
-        '2200': '960x540',
-        '1700': '768x432',
-        '1200': '640x360',
-        '750': '512x288',
-        '400': '384x216',
+        '3500': (1280, 720),
+        '2200': (960, 540),
+        '1700': (768, 432),
+        '1200': (640, 360),
+        '750': (512, 288),
+        '400': (384, 216),
    }

    @classmethod
@@ -64,11 +64,13 @@ class ComedyCentralIE(InfoExtractor):
        """Receives a URL and returns True if suitable for this IE."""
        return re.match(cls._VALID_URL, url, re.VERBOSE) is not None

-    def _print_formats(self, formats):
-        print('Available formats:')
-        for x in formats:
-            print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
-
+    @staticmethod
+    def _transform_rtmp_url(rtmp_video_url):
+        m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
+        if not m:
+            raise ExtractorError(u'Cannot transform RTMP url')
+        base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
+        return base + m.group('finalid')

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url, re.VERBOSE)
@@ -155,40 +157,31 @@ class ComedyCentralIE(InfoExtractor):
                self._downloader.report_error(u'unable to download ' + mediaId + ': No videos found')
                continue

-            if self._downloader.params.get('listformats', None):
-                self._print_formats([i[0] for i in turls])
-                return
-
-            # For now, just pick the highest bitrate
-            format,rtmp_video_url = turls[-1]
-
-            # Get the format arg from the arg stream
-            req_format = self._downloader.params.get('format', None)
-
-            # Select format if we can find one
-            for f,v in turls:
-                if f == req_format:
-                    format, rtmp_video_url = f, v
-                    break
-
-            m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
-            if not m:
-                raise ExtractorError(u'Cannot transform RTMP url')
-            base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
-            video_url = base + m.group('finalid')
+            formats = []
+            for format, rtmp_video_url in turls:
+                w, h = self._video_dimensions.get(format, (None, None))
+                formats.append({
+                    'url': self._transform_rtmp_url(rtmp_video_url),
+                    'ext': self._video_extensions.get(format, 'mp4'),
+                    'format_id': format,
+                    'height': h,
+                    'width': w,
+                })

            effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1)
            info = {
                'id': shortMediaId,
-                'url': video_url,
+                'formats': formats,
                'uploader': showId,
                'upload_date': officialDate,
                'title': effTitle,
-                'ext': 'mp4',
-                'format': format,
                'thumbnail': None,
                'description': compat_str(officialTitle),
            }
+
+            # TODO: Remove when #980 has been merged
+            info.update(info['formats'][-1])
+
            results.append(info)

        return results
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -35,6 +35,8 @@ class InfoExtractor(object):
    title:          Video title, unescaped.
    ext:            Video filename extension.

+    Instead of url and ext, formats can also specified.
+
    The following fields are optional:

    format:         The video format, defaults to ext (used for --get-format)
@@ -52,8 +54,20 @@ class InfoExtractor(object):
    view_count:     How many users have watched the video on the platform.
    urlhandle:      [internal] The urlHandle to be used to download the file,
                    like returned by urllib.request.urlopen
+    age_limit:      Age restriction for the video, as an integer (years)
+    formats:        A list of dictionaries for each format available, it must
+                    be ordered from worst to best quality. Potential fields:
+                    * url       Mandatory. The URL of the video file
+                    * ext       Will be calculated from url if missing
+                    * format    A human-readable description of the format
+                                ("mp4 container with h264/opus").
+                                Calculated from width and height if missing.
+                    * format_id A short description of the format
+                                ("mp4_h264_opus" or "19")
+                    * width     Width of the video, if known
+                    * height    Height of the video, if known

-    The fields should all be Unicode strings.
+    Unless mentioned otherwise, the fields should be Unicode strings.

    Subclasses of this one should re-define the _real_initialize() and
    _real_extract() methods and define a _VALID_URL regexp.
@@ -305,6 +319,15 @@ class InfoExtractor(object):
                                        self._og_regex('video')],
                                       html, name, **kargs)

+    def _rta_search(self, html):
+        # See http://www.rtalabel.org/index.php?content=howtofaq#single
+        if re.search(r'(?ix)<meta\s+name="rating"\s+'
+                     r'     content="RTA-5042-1996-1400-1577-RTA"',
+                     html):
+            return 18
+        return 0
+
+
 class SearchInfoExtractor(InfoExtractor):
    """
    Base class for paged search queries extractors.
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -10,25 +10,49 @@ from ..utils import (
    compat_str,
    get_element_by_attribute,
    get_element_by_id,
+    orderedSet,

    ExtractorError,
 )

+class DailymotionBaseInfoExtractor(InfoExtractor):
+    @staticmethod
+    def _build_request(url):
+        """Build a request with the family filter disabled"""
+        request = compat_urllib_request.Request(url)
+        request.add_header('Cookie', 'family_filter=off')
+        return request

-class DailymotionIE(SubtitlesInfoExtractor):
+class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
    """Information Extractor for Dailymotion"""

    _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)'
    IE_NAME = u'dailymotion'
-    _TEST = {
-        u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
-        u'file': u'x33vw9.mp4',
-        u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
-        u'info_dict': {
-            u"uploader": u"Amphora Alex and Van .", 
-            u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
-        }
-    }
+    _TESTS = [
+        {
+            u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
+            u'file': u'x33vw9.mp4',
+            u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
+            u'info_dict': {
+                u"uploader": u"Amphora Alex and Van .", 
+                u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
+            }
+        },
+        # Vevo video
+        {
+            u'url': u'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
+            u'file': u'USUV71301934.mp4',
+            u'info_dict': {
+                u'title': u'Roar (Official)',
+                u'uploader': u'Katy Perry',
+                u'upload_date': u'20130905',
+            },
+            u'params': {
+                u'skip_download': True,
+            },
+            u'skip': u'VEVO is only available in some countries',
+        },
+    ]

    def _real_extract(self, url):
        # Extract id and simplified title from URL
@@ -40,13 +64,21 @@ class DailymotionIE(SubtitlesInfoExtractor):
        url = 'http://www.dailymotion.com/video/%s' % video_id

        # Retrieve video webpage to extract further information
-        request = compat_urllib_request.Request(url)
-        request.add_header('Cookie', 'family_filter=off')
+        request = self._build_request(url)
        webpage = self._download_webpage(request, video_id)

        # Extract URL, uploader and title from webpage
        self.report_extraction(video_id)

+        # It may just embed a vevo video:
+        m_vevo = re.search(
+            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?videoId=(?P<id>[\w]*)',
+            webpage)
+        if m_vevo is not None:
+            vevo_id = m_vevo.group('id')
+            self.to_screen(u'Vevo video detected: %s' % vevo_id)
+            return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
+
        video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
                                             # Looking for official user
                                             r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
@@ -113,7 +145,7 @@ class DailymotionIE(SubtitlesInfoExtractor):
        return {}


-class DailymotionPlaylistIE(InfoExtractor):
+class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
    IE_NAME = u'dailymotion:playlist'
    _VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
    _MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/playlist/.+?".*?>.*?</a>.*?</div>'
@@ -122,16 +154,17 @@ class DailymotionPlaylistIE(InfoExtractor):
    def _extract_entries(self, id):
        video_ids = []
        for pagenum in itertools.count(1):
-            webpage = self._download_webpage(self._PAGE_TEMPLATE % (id, pagenum),
+            request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
+            webpage = self._download_webpage(request,
                                             id, u'Downloading page %s' % pagenum)

            playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
-            video_ids.extend(re.findall(r'data-id="(.+?)" data-ext-id', playlist_el))
+            video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))

            if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                break
        return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
-                   for video_id in video_ids]
+                   for video_id in orderedSet(video_ids)]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -9,7 +9,7 @@ from ..utils import (

 class FlickrIE(InfoExtractor):
    """Information Extractor for Flickr videos"""
-    _VALID_URL = r'(?:https?://)?(?:www\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
+    _VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
    _TEST = {
        u'url': u'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
        u'file': u'5645318632.mp4',
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -70,7 +70,11 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):

 class France2IE(FranceTVBaseInfoExtractor):
    IE_NAME = u'france2.fr'
-    _VALID_URL = r'https?://www\.france2\.fr/emissions/.*?/videos/(?P<id>\d+)'
+    _VALID_URL = r'''(?x)https?://www\.france2\.fr/
+        (?:
+            emissions/.*?/videos/(?P<id>\d+)
+        |   emission/(?P<key>[^/?]+)
+        )'''

    _TEST = {
        u'url': u'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104',
@@ -86,12 +90,20 @@ class France2IE(FranceTVBaseInfoExtractor):

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        if mobj.group('key'):
+            webpage = self._download_webpage(url, mobj.group('key'))
+            video_id = self._html_search_regex(
+                r'''(?x)<div\s+class="video-player">\s*
+                    <a\s+href="http://videos.francetv.fr/video/([0-9]+)"\s+
+                    class="francetv-video-player">''',
+                webpage, u'video ID')
+        else:
+            video_id = mobj.group('id')
        return self._extract_video(video_id)


 class GenerationQuoiIE(InfoExtractor):
-    IE_NAME = u'http://generation-quoi.france2.fr'
+    IE_NAME = u'france2.fr:generation-quoi'
    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<name>.*)(\?|$)'

    _TEST = {
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,17 +29,6 @@ class GenericIE(InfoExtractor):
                u"title": u"R\u00e9gis plante sa Jeep"
            }
        },
-        {
-            u'url': u'http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/',
-            u'file': u'2371591881001.mp4',
-            u'md5': u'9e80619e0a94663f0bdc849b4566af19',
-            u'note': u'Test Brightcove downloads and detection in GenericIE',
-            u'info_dict': {
-                u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
-                u'uploader': u'8TV',
-                u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
-            }
-        },
    ]

    def report_download_webpage(self, video_id):
@@ -128,7 +117,7 @@ class GenericIE(InfoExtractor):
        except ValueError:
            # since this is the last-resort InfoExtractor, if
            # this error is thrown, it'll be thrown here
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError(u'Failed to download URL: %s' % url)

        self.report_extraction(video_id)
        # Look for BrightCove:
@@ -160,12 +149,12 @@ class GenericIE(InfoExtractor):
            # HTML5 video
            mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError(u'Unsupported URL: %s' % url)

        # It's possible that one of the regexes
        # matched, but returned an empty group:
        if mobj.group(1) is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError(u'Did not find a valid video URL at %s' % url)

        video_url = mobj.group(1)
        video_url = compat_urlparse.urljoin(url, video_url)
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -41,7 +41,8 @@ class GooglePlusIE(InfoExtractor):

        # Extract update date
        upload_date = self._html_search_regex(
-            ['title="Timestamp">(.*?)</a>', r'<a.+?class="g-M.+?>(.+?)</a>'],
+            r'''(?x)<a.+?class="o-T-s\s[^"]+"\s+style="display:\s*none"\s*>
+                    ([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
            webpage, u'upload date', fatal=False)
        if upload_date:
            # Convert timestring to a format suitable for filename
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -13,7 +13,7 @@ class IGNIE(InfoExtractor):
    Some videos of it.ign.com are also supported
    """

-    _VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles)(/.+)?/(?P<name_or_id>.+)'
+    _VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles|(?:[^/]*/feature))(/.+)?/(?P<name_or_id>.+)'
    IE_NAME = u'ign.com'

    _CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config'
@@ -21,15 +21,39 @@ class IGNIE(InfoExtractor):
                       r'id="my_show_video">.*?<p>(.*?)</p>',
                       ]

-    _TEST = {
-        u'url': u'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
-        u'file': u'8f862beef863986b2785559b9e1aa599.mp4',
-        u'md5': u'eac8bdc1890980122c3b66f14bdd02e9',
-        u'info_dict': {
-            u'title': u'The Last of Us Review',
-            u'description': u'md5:c8946d4260a4d43a00d5ae8ed998870c',
-        }
-    }
+    _TESTS = [
+        {
+            u'url': u'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
+            u'file': u'8f862beef863986b2785559b9e1aa599.mp4',
+            u'md5': u'eac8bdc1890980122c3b66f14bdd02e9',
+            u'info_dict': {
+                u'title': u'The Last of Us Review',
+                u'description': u'md5:c8946d4260a4d43a00d5ae8ed998870c',
+            }
+        },
+        {
+            u'url': u'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
+            u'playlist': [
+                {
+                    u'file': u'5ebbd138523268b93c9141af17bec937.mp4',
+                    u'info_dict': {
+                        u'title': u'GTA 5 Video Review',
+                        u'description': u'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
+                    },
+                },
+                {
+                    u'file': u'638672ee848ae4ff108df2a296418ee2.mp4',
+                    u'info_dict': {
+                        u'title': u'GTA 5\'s Twisted Beauty in Super Slow Motion',
+                        u'description': u'The twisted beauty of GTA 5 in stunning slow motion.',
+                    },
+                },
+            ],
+            u'params': {
+                u'skip_download': True,
+            },
+        },
+    ]

    def _find_video_id(self, webpage):
        res_id = [r'data-video-id="(.+?)"',
@@ -46,6 +70,13 @@ class IGNIE(InfoExtractor):
        if page_type == 'articles':
            video_url = self._search_regex(r'var videoUrl = "(.+?)"', webpage, u'video url')
            return self.url_result(video_url, ie='IGN')
+        elif page_type != 'video':
+            multiple_urls = re.findall(
+                '<param name="flashvars" value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
+                webpage)
+            if multiple_urls:
+                return [self.url_result(u, ie='IGN') for u in multiple_urls]
+
        video_id = self._find_video_id(webpage)
        result = self._get_video_info(video_id)
        description = self._html_search_regex(self._DESCRIPTION_RE,
@@ -87,6 +118,9 @@ class OneUPIE(IGNIE):
        }
    }

+    # Override IGN tests
+    _TESTS = []
+
    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        id = mobj.group('name_or_id')
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -6,13 +6,14 @@ import xml.etree.ElementTree

 from .common import InfoExtractor

+
 class JeuxVideoIE(InfoExtractor):
    _VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)-\d+\.htm'

    _TEST = {
        u'url': u'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
        u'file': u'5182.mp4',
-        u'md5': u'e0fdb0cd3ce98713ef9c1e1e025779d0',
+        u'md5': u'046e491afb32a8aaac1f44dd4ddd54ee',
        u'info_dict': {
            u'title': u'GC 2013 : Tearaway nous présente ses papiers d\'identité',
            u'description': u'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n',
@@ -23,25 +24,29 @@ class JeuxVideoIE(InfoExtractor):
        mobj = re.match(self._VALID_URL, url)
        title = re.match(self._VALID_URL, url).group(1)
        webpage = self._download_webpage(url, title)
-        m_download = re.search(r'<param name="flashvars" value="config=(.*?)" />', webpage)
-
-        xml_link = m_download.group(1)
+        xml_link = self._html_search_regex(
+            r'<param name="flashvars" value="config=(.*?)" />',
+            webpage, u'config URL')
        
-        id = re.search(r'http://www.jeuxvideo.com/config/\w+/0011/(.*?)/\d+_player\.xml', xml_link).group(1)
+        video_id = self._search_regex(
+            r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml',
+            xml_link, u'video ID')

-        xml_config = self._download_webpage(xml_link, title,
-                                                  'Downloading XML config')
+        xml_config = self._download_webpage(
+            xml_link, title, u'Downloading XML config')
        config = xml.etree.ElementTree.fromstring(xml_config.encode('utf-8'))
-        info = re.search(r'<format\.json>(.*?)</format\.json>',
-                         xml_config, re.MULTILINE|re.DOTALL).group(1)
-        info = json.loads(info)['versions'][0]
+        info_json = self._search_regex(
+            r'(?sm)<format\.json>(.*?)</format\.json>',
+            xml_config, u'JSON information')
+        info = json.loads(info_json)['versions'][0]
        
        video_url = 'http://video720.jeuxvideo.com/' + info['file']

-        return {'id': id,
-                'title' : config.find('titre_video').text,
-                'ext' : 'mp4',
-                'url' : video_url,
-                'description': self._og_search_description(webpage),
-                'thumbnail': config.find('image').text,
-                }
+        return {
+            'id': video_id,
+            'title': config.find('titre_video').text,
+            'ext': 'mp4',
+            'url': video_url,
+            'description': self._og_search_description(webpage),
+            'thumbnail': config.find('image').text,
+        }
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -54,23 +54,26 @@ class MTVIE(InfoExtractor):
    def _get_thumbnail_url(self, uri, itemdoc):
        return 'http://mtv.mtvnimages.com/uri/' + uri

-    def _extract_video_url(self, metadataXml):
+    def _extract_video_formats(self, metadataXml):
        if '/error_country_block.swf' in metadataXml:
            raise ExtractorError(u'This video is not available from your country.', expected=True)
        mdoc = xml.etree.ElementTree.fromstring(metadataXml.encode('utf-8'))
        renditions = mdoc.findall('.//rendition')

-        # For now, always pick the highest quality.
-        rendition = renditions[-1]
-
-        try:
-            _,_,ext = rendition.attrib['type'].partition('/')
-            format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
-            rtmp_video_url = rendition.find('./src').text
-        except KeyError:
-            raise ExtractorError('Invalid rendition field.')
-        video_url = self._transform_rtmp_url(rtmp_video_url)
-        return {'ext': ext, 'url': video_url, 'format': format}
+        formats = []
+        for rendition in mdoc.findall('.//rendition'):
+            try:
+                _, _, ext = rendition.attrib['type'].partition('/')
+                rtmp_video_url = rendition.find('./src').text
+                formats.append({'ext': ext,
+                                'url': self._transform_rtmp_url(rtmp_video_url),
+                                'format_id': rendition.get('bitrate'),
+                                'width': int(rendition.get('width')),
+                                'height': int(rendition.get('height')),
+                                })
+            except (KeyError, TypeError):
+                raise ExtractorError('Invalid rendition field.')
+        return formats

    def _get_video_info(self, itemdoc):
        uri = itemdoc.find('guid').text
@@ -81,19 +84,25 @@ class MTVIE(InfoExtractor):
            mediagen_url += '&acceptMethods=fms'
        mediagen_page = self._download_webpage(mediagen_url, video_id,
                                               u'Downloading video urls')
-        video_info = self._extract_video_url(mediagen_page)

        description_node = itemdoc.find('description')
        if description_node is not None:
            description = description_node.text
        else:
            description = None
-        video_info.update({'title': itemdoc.find('title').text,
-                           'id': video_id,
-                           'thumbnail': self._get_thumbnail_url(uri, itemdoc),
-                           'description': description,
-                           })
-        return video_info
+
+        info = {
+            'title': itemdoc.find('title').text,
+            'formats': self._extract_video_formats(mediagen_page),
+            'id': video_id,
+            'thumbnail': self._get_thumbnail_url(uri, itemdoc),
+            'description': description,
+        }
+
+        # TODO: Remove when #980 has been merged
+        info.update(info['formats'][-1])
+
+        return info

    def _get_videos_info(self, uri):
        video_id = self._id_from_uri(uri)
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -38,6 +38,7 @@ class PornotubeIE(InfoExtractor):
        VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
        upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False)
        if upload_date: upload_date = unified_strdate(upload_date)
+        age_limit = self._rta_search(webpage)

        info = {'id': video_id,
                'url': video_url,
@@ -45,6 +46,7 @@ class PornotubeIE(InfoExtractor):
                'upload_date': upload_date,
                'title': video_title,
                'ext': 'flv',
-                'format': 'flv'}
+                'format': 'flv',
+                'age_limit': age_limit}

        return [info]
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -14,24 +14,30 @@ class RedTubeIE(InfoExtractor):
        }
    }

-    def _real_extract(self,url):
+    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)

        video_id = mobj.group('id')
-        video_extension = 'mp4'        
+        video_extension = 'mp4'
        webpage = self._download_webpage(url, video_id)

        self.report_extraction(video_id)

-        video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">',
-            webpage, u'video URL')
+        video_url = self._html_search_regex(
+            r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')

-        video_title = self._html_search_regex('<h1 class="videoTitle slidePanelMovable">(.+?)</h1>',
+        video_title = self._html_search_regex(
+            r'<h1 class="videoTitle slidePanelMovable">(.+?)</h1>',
            webpage, u'title')

-        return [{
-            'id':       video_id,
-            'url':      video_url,
-            'ext':      video_extension,
-            'title':    video_title,
-        }]
+        # No self-labeling, but they describe themselves as
+        # "Home of Videos Porno"
+        age_limit = 18
+
+        return {
+            'id':        video_id,
+            'url':       video_url,
+            'ext':       video_extension,
+            'title':     video_title,
+            'age_limit': age_limit,
+        }
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -8,8 +8,8 @@ from ..utils import (
 )

 class RTLnowIE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, SUPER RTL NOW and VOX NOW"""
-    _VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?superrtlnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?rtlnitronow\.de/|(?:www\.)?superrtlnow\.de/|(?:www\.)?n-tvnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
    _TESTS = [{
        u'url': u'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
        u'file': u'90419.flv',
@@ -61,8 +61,35 @@ class RTLnowIE(InfoExtractor):
        u'params': {
            u'skip_download': True,
        },
+    },
+    {
+        u'url': u'http://www.rtlnitronow.de/recht-ordnung/lebensmittelkontrolle-erlangenordnungsamt-berlin.php?film_id=127367&player=1&season=1',
+        u'file': u'127367.flv',
+        u'info_dict': {
+            u'upload_date': u'20130926', 
+            u'title': u'Recht & Ordnung - Lebensmittelkontrolle Erlangen/Ordnungsamt...',
+            u'description': u'Lebensmittelkontrolle Erlangen/Ordnungsamt Berlin',
+            u'thumbnail': u'http://autoimg.static-fra.de/nitronow/344787/1500x1500/image2.jpg',
+        },
+        u'params': {
+            u'skip_download': True,
+        },
+    },
+    {
+        u'url': u'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10',
+        u'file': u'124903.flv',
+        u'info_dict': {
+            u'upload_date': u'20130101', 
+            u'title': u'Top Gear vom 01.01.2013',
+            u'description': u'Episode 1',
+        },
+        u'params': {
+            u'skip_download': True,
+        },
+        u'skip': u'Only works from Germany',
    }]

+
    def _real_extract(self,url):
        mobj = re.match(self._VALID_URL, url)

@@ -79,20 +106,23 @@ class RTLnowIE(InfoExtractor):
            msg = clean_html(note_m.group(1))
            raise ExtractorError(msg)

-        video_title = self._html_search_regex(r'<title>(?P<title>[^<]+)</title>',
+        video_title = self._html_search_regex(r'<title>(?P<title>[^<]+?)( \| [^<]*)?</title>',
            webpage, u'title')
        playerdata_url = self._html_search_regex(r'\'playerdata\': \'(?P<playerdata_url>[^\']+)\'',
            webpage, u'playerdata_url')

        playerdata = self._download_webpage(playerdata_url, video_id)
-        mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr\]\]></title>', playerdata)
+        mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)(?:\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr)?\]\]></title>', playerdata)
        if mobj:
            video_description = mobj.group(u'description')
            if mobj.group('upload_date_Y'):
                video_upload_date = mobj.group('upload_date_Y')
-            else:
+            elif mobj.group('upload_date_y'):
                video_upload_date = u'20' + mobj.group('upload_date_y')
-            video_upload_date += mobj.group('upload_date_m')+mobj.group('upload_date_d')
+            else:
+                video_upload_date = None
+            if video_upload_date:
+                video_upload_date += mobj.group('upload_date_m')+mobj.group('upload_date_d')
        else:
            video_description = None
            video_upload_date = None
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -77,12 +77,20 @@ class TEDIE(InfoExtractor):
        
        thumbnail = self._search_regex(r'</span>[\s.]*</div>[\s.]*<img src="(.*?)"',
                                       webpage, 'thumbnail')
+        formats = [{
+            'ext': 'mp4',
+            'url': stream['file'],
+            'format': stream['id']
+            } for stream in info['htmlStreams']]
        info = {
-                'id': info['id'],
-                'url': info['htmlStreams'][-1]['file'],
-                'ext': 'mp4',
-                'title': title,
-                'thumbnail': thumbnail,
-                'description': desc,
-                }
+            'id': info['id'],
+            'title': title,
+            'thumbnail': thumbnail,
+            'description': desc,
+            'formats': formats,
+        }
+
+        # TODO: Remove when #980 has been merged
+        info.update(info['formats'][-1])
+
        return info
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -1,11 +1,15 @@
 import re
 import json
+import xml.etree.ElementTree
+import datetime

 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
    ExtractorError,
 )

+
 class VevoIE(InfoExtractor):
    """
    Accepts urls from vevo.com or in the format 'vevo:{id}'
@@ -15,11 +19,11 @@ class VevoIE(InfoExtractor):
    _TEST = {
        u'url': u'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
        u'file': u'GB1101300280.mp4',
-        u'md5': u'06bea460acb744eab74a9d7dcb4bfd61',
        u'info_dict': {
            u"upload_date": u"20130624",
            u"uploader": u"Hurts",
-            u"title": u"Somebody to Die For"
+            u"title": u"Somebody to Die For",
+            u'duration': 230,
        }
    }

@@ -27,27 +31,47 @@ class VevoIE(InfoExtractor):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

-        json_url = 'http://www.vevo.com/data/video/%s' % video_id
-        base_url = 'http://smil.lvl3.vevo.com'
-        videos_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (base_url, video_id, video_id.lower())
+        json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
        info_json = self._download_webpage(json_url, video_id, u'Downloading json info')
-        links_webpage = self._download_webpage(videos_url, video_id, u'Downloading videos urls')

        self.report_extraction(video_id)
-        video_info = json.loads(info_json)
-        m_urls = list(re.finditer(r'<video src="(?P<ext>.*?):/?(?P<url>.*?)"', links_webpage))
-        if m_urls is None or len(m_urls) == 0:
-            raise ExtractorError(u'Unable to extract video url')
-        # They are sorted from worst to best quality
-        m_url = m_urls[-1]
-        video_url = base_url + '/' + m_url.group('url')
-        ext = m_url.group('ext')
+        video_info = json.loads(info_json)['video']
+        last_version = {'version': -1}
+        for version in video_info['videoVersions']:
+            # These are the HTTP downloads, other types are for different manifests
+            if version['sourceType'] == 2:
+                if version['version'] > last_version['version']:
+                    last_version = version
+        if last_version['version'] == -1:
+            raise ExtractorError(u'Unable to extract last version of the video')

-        return {'url': video_url,
-                'ext': ext,
-                'id': video_id,
-                'title': video_info['title'],
-                'thumbnail': video_info['img'],
-                'upload_date': video_info['launchDate'].replace('/',''),
-                'uploader': video_info['Artists'][0]['title'],
-                }
+        renditions = xml.etree.ElementTree.fromstring(last_version['data'])
+        formats = []
+        # Already sorted from worst to best quality
+        for rend in renditions.findall('rendition'):
+            attr = rend.attrib
+            f_url = attr['url']
+            formats.append({
+                'url': f_url,
+                'ext': determine_ext(f_url),
+                'height': int(attr['frameheight']),
+                'width': int(attr['frameWidth']),
+            })
+
+        date_epoch = int(self._search_regex(
+            r'/Date\((\d+)\)/', video_info['launchDate'], u'launch date'))/1000
+        upload_date = datetime.datetime.fromtimestamp(date_epoch)
+        info = {
+            'id': video_id,
+            'title': video_info['title'],
+            'formats': formats,
+            'thumbnail': video_info['imageUrl'],
+            'upload_date': upload_date.strftime('%Y%m%d'),
+            'uploader': video_info['mainArtists'][0]['artistName'],
+            'duration': video_info['duration'],
+        }
+
+        # TODO: Remove when #980 has been merged
+        info.update(formats[-1])
+
+        return info
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -0,0 +1,64 @@
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class ViddlerIE(InfoExtractor):
+    _VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler.com)/(?:v|embed|player)/(?P<id>[0-9]+)'
+    _TEST = {
+        u"url": u"http://www.viddler.com/v/43903784",
+        u'file': u'43903784.mp4',
+        u'md5': u'fbbaedf7813e514eb7ca30410f439ac9',
+        u'info_dict': {
+            u"title": u"Video Made Easy",
+            u"uploader": u"viddler",
+            u"duration": 100.89,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        embed_url = mobj.group('domain') + u'/embed/' + video_id
+        webpage = self._download_webpage(embed_url, video_id)
+
+        video_sources_code = self._search_regex(
+            r"(?ms)sources\s*:\s*(\{.*?\})", webpage, u'video URLs')
+        video_sources = json.loads(video_sources_code.replace("'", '"'))
+
+        formats = [{
+            'url': video_url,
+            'format': format_id,
+        } for video_url, format_id in video_sources.items()]
+
+        title = self._html_search_regex(
+            r"title\s*:\s*'([^']*)'", webpage, u'title')
+        uploader = self._html_search_regex(
+            r"authorName\s*:\s*'([^']*)'", webpage, u'uploader', fatal=False)
+        duration_s = self._html_search_regex(
+            r"duration\s*:\s*([0-9.]*)", webpage, u'duration', fatal=False)
+        duration = float(duration_s) if duration_s else None
+        thumbnail = self._html_search_regex(
+            r"thumbnail\s*:\s*'([^']*)'",
+            webpage, u'thumbnail', fatal=False)
+
+        info = {
+            '_type': 'video',
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': duration,
+            'formats': formats,
+        }
+
+        # TODO: Remove when #980 has been merged
+        info['formats'][-1]['ext'] = determine_ext(info['formats'][-1]['url'])
+        info.update(info['formats'][-1])
+
+        return info
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -17,7 +17,7 @@ class VimeoIE(InfoExtractor):
    """Information extractor for vimeo.com."""

    # _VALID_URL matches Vimeo URLs
-    _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)(?:[?].*)?$'
+    _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)/?(?:[?].*)?$'
    _NETRC_MACHINE = 'vimeo'
    IE_NAME = u'vimeo'
    _TESTS = [
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -1,4 +1,3 @@
-import datetime
 import itertools
 import json
 import re
@@ -6,86 +5,89 @@ import re
 from .common import InfoExtractor, SearchInfoExtractor
 from ..utils import (
    compat_urllib_parse,
-
-    ExtractorError,
+    compat_urlparse,
+    determine_ext,
+    clean_html,
 )

+
 class YahooIE(InfoExtractor):
    IE_DESC = u'Yahoo screen'
    _VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P<id>\d*?)\.html'
-    _TEST = {
-        u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
-        u'file': u'214727115.flv',
-        u'md5': u'2e717f169c1be93d84d3794a00d4a325',
-        u'info_dict': {
-            u"title": u"Julian Smith & Travis Legg Watch Julian Smith"
+    _TESTS = [
+        {
+            u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
+            u'file': u'214727115.flv',
+            u'info_dict': {
+                u'title': u'Julian Smith & Travis Legg Watch Julian Smith',
+                u'description': u'Julian and Travis watch Julian Smith',
+            },
+            u'params': {
+                # Requires rtmpdump
+                u'skip_download': True,
+            },
        },
-        u'skip': u'Requires rtmpdump'
-    }
+        {
+            u'url': u'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
+            u'file': u'103000935.flv',
+            u'info_dict': {
+                u'title': u'Codefellas - The Cougar Lies with Spanish Moss',
+                u'description': u'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
+            },
+            u'params': {
+                # Requires rtmpdump
+                u'skip_download': True,
+            },
+        },
+    ]

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
        video_id = mobj.group('id')
        webpage = self._download_webpage(url, video_id)
-        m_id = re.search(r'YUI\.namespace\("Media"\)\.CONTENT_ID = "(?P<new_id>.+?)";', webpage)

-        if m_id is None: 
-            # TODO: Check which url parameters are required
-            info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
-            webpage = self._download_webpage(info_url, video_id, u'Downloading info webpage')
-            info_re = r'''<title><!\[CDATA\[(?P<title>.*?)\]\]></title>.*
-                        <description><!\[CDATA\[(?P<description>.*?)\]\]></description>.*
-                        <media:pubStart><!\[CDATA\[(?P<date>.*?)\ .*\]\]></media:pubStart>.*
-                        <media:content\ medium="image"\ url="(?P<thumb>.*?)"\ name="LARGETHUMB"
-                        '''
-            self.report_extraction(video_id)
-            m_info = re.search(info_re, webpage, re.VERBOSE|re.DOTALL)
-            if m_info is None:
-                raise ExtractorError(u'Unable to extract video info')
-            video_title = m_info.group('title')
-            video_description = m_info.group('description')
-            video_thumb = m_info.group('thumb')
-            video_date = m_info.group('date')
-            video_date = datetime.datetime.strptime(video_date, '%m/%d/%Y').strftime('%Y%m%d')
-    
-            # TODO: Find a way to get mp4 videos
-            rest_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;element=stream;outputformat=mrss;id=%s;lmsoverride=1;bw=375;dynamicstream=1;cb=83521105;tech=flv,mp4;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
-            webpage = self._download_webpage(rest_url, video_id, u'Downloading video url webpage')
-            m_rest = re.search(r'<media:content url="(?P<url>.*?)" path="(?P<path>.*?)"', webpage)
-            video_url = m_rest.group('url')
-            video_path = m_rest.group('path')
-            if m_rest is None:
-                raise ExtractorError(u'Unable to extract video url')
+        items_json = self._search_regex(r'YVIDEO_INIT_ITEMS = ({.*?});$',
+            webpage, u'items', flags=re.MULTILINE)
+        items = json.loads(items_json)
+        info = items['mediaItems']['query']['results']['mediaObj'][0]
+        meta = info['meta']

-        else: # We have to use a different method if another id is defined
-            long_id = m_id.group('new_id')
-            info_url = 'http://video.query.yahoo.com/v1/public/yql?q=SELECT%20*%20FROM%20yahoo.media.video.streams%20WHERE%20id%3D%22' + long_id + '%22%20AND%20format%3D%22mp4%2Cflv%22%20AND%20protocol%3D%22rtmp%2Chttp%22%20AND%20plrs%3D%2286Gj0vCaSzV_Iuf6hNylf2%22%20AND%20acctid%3D%22389%22%20AND%20plidl%3D%22%22%20AND%20pspid%3D%22792700001%22%20AND%20offnetwork%3D%22false%22%20AND%20site%3D%22ivy%22%20AND%20lang%3D%22en-US%22%20AND%20region%3D%22US%22%20AND%20override%3D%22none%22%3B&env=prod&format=json&callback=YUI.Env.JSONP.yui_3_8_1_1_1368368376830_335'
-            webpage = self._download_webpage(info_url, video_id, u'Downloading info json')
-            json_str = re.search(r'YUI.Env.JSONP.yui.*?\((.*?)\);', webpage).group(1)
-            info = json.loads(json_str)
-            res = info[u'query'][u'results'][u'mediaObj'][0]
-            stream = res[u'streams'][0]
-            video_path = stream[u'path']
-            video_url = stream[u'host']
-            meta = res[u'meta']
-            video_title = meta[u'title']
-            video_description = meta[u'description']
-            video_thumb = meta[u'thumbnail']
-            video_date = None # I can't find it
+        formats = []
+        for s in info['streams']:
+            format_info = {
+                'width': s.get('width'),
+                'height': s.get('height'),
+                'bitrate': s.get('bitrate'),
+            }
+
+            host = s['host']
+            path = s['path']
+            if host.startswith('rtmp'):
+                format_info.update({
+                    'url': host,
+                    'play_path': path,
+                    'ext': 'flv',
+                })
+            else:
+                format_url = compat_urlparse.urljoin(host, path)
+                format_info['url'] = format_url
+                format_info['ext'] = determine_ext(format_url)
+                
+            formats.append(format_info)
+        formats = sorted(formats, key=lambda f:(f['height'], f['width']))
+
+        info = {
+            'id': video_id,
+            'title': meta['title'],
+            'formats': formats,
+            'description': clean_html(meta['description']),
+            'thumbnail': meta['thumbnail'],
+        }
+        # TODO: Remove when #980 has been merged
+        info.update(formats[-1])
+
+        return info

-        info_dict = {
-                     'id': video_id,
-                     'url': video_url,
-                     'play_path': video_path,
-                     'title':video_title,
-                     'description': video_description,
-                     'thumbnail': video_thumb,
-                     'upload_date': video_date,
-                     'ext': 'flv',
-                     }
-        return info_dict

 class YahooSearchIE(SearchInfoExtractor):
    IE_DESC = u'Yahoo screen search'
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -51,6 +51,7 @@ class YouPornIE(InfoExtractor):
        req = compat_urllib_request.Request(url)
        req.add_header('Cookie', 'age_verified=1')
        webpage = self._download_webpage(req, video_id)
+        age_limit = self._rta_search(webpage)

        # Get JSON parameters
        json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, u'JSON parameters')
@@ -115,7 +116,8 @@ class YouPornIE(InfoExtractor):
                'ext': extension,
                'format': format,
                'thumbnail': thumbnail,
-                'description': video_description
+                'description': video_description,
+                'age_limit': age_limit,
            })

        if self._downloader.params.get('listformats', None):
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -23,9 +23,11 @@ from ..utils import (
    compat_urllib_error,
    compat_urllib_parse,
    compat_urllib_request,
+    compat_urlparse,
    compat_str,

    clean_html,
+    get_cachedir,
    get_element_by_id,
    ExtractorError,
    unescapeHTML,
@@ -361,7 +363,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
            u"info_dict": {
                u"upload_date": u"20120506",
                u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
-                u"description": u"md5:bdac09887d209a4ed54b8f76b2bdaa8b",
+                u"description": u"md5:5b292926389560516e384ac437c0ec07",
                u"uploader": u"Icona Pop",
                u"uploader_id": u"IconaPop"
            }
@@ -378,21 +380,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                u"uploader_id": u"justintimberlakeVEVO"
            }
        },
-        {
-            u'url': u'https://www.youtube.com/watch?v=TGi3HqYrWHE',
-            u'file': u'TGi3HqYrWHE.mp4',
-            u'note': u'm3u8 video',
-            u'info_dict': {
-                u'title': u'Triathlon - Men - London 2012 Olympic Games',
-                u'description': u'- Men -  TR02 - Triathlon - 07 August 2012 - London 2012 Olympic Games',
-                u'uploader': u'olympic',
-                u'upload_date': u'20120807',
-                u'uploader_id': u'olympic',
-            },
-            u'params': {
-                u'skip_download': True,
-            },
-        },
    ]


@@ -435,8 +422,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
        # Read from filesystem cache
        func_id = '%s_%s_%d' % (player_type, player_id, slen)
        assert os.path.basename(func_id) == func_id
-        cache_dir = self._downloader.params.get('cachedir',
-                                                u'~/.youtube-dl/cache')
+        cache_dir = get_cachedir(self._downloader.params)

        cache_enabled = cache_dir is not None
        if cache_enabled:
@@ -1051,12 +1037,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):

        if player_url is not None:
            try:
-                if player_url not in self._player_cache:
+                player_id = (player_url, len(s))
+                if player_id not in self._player_cache:
                    func = self._extract_signature_function(
                        video_id, player_url, len(s)
                    )
-                    self._player_cache[player_url] = func
-                func = self._player_cache[player_url]
+                    self._player_cache[player_id] = func
+                func = self._player_cache[player_id]
                if self._downloader.params.get('youtube_print_sig_code'):
                    self._print_sig_code(func, len(s))
                return func(s)
@@ -1067,6 +1054,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):

            self._downloader.report_warning(
                u'Warning: Falling back to static signature algorithm')
+
        return self._static_decrypt_signature(
            s, video_id, player_url, age_gate)

@@ -1098,9 +1086,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
        elif len(s) == 84:
            return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
        elif len(s) == 83:
-            return s[81:64:-1] + s[82] + s[63:52:-1] + s[45] + s[51:45:-1] + s[1] + s[44:1:-1] + s[0]
+            return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
        elif len(s) == 82:
-            return s[80:73:-1] + s[81] + s[72:54:-1] + s[2] + s[53:43:-1] + s[0] + s[42:2:-1] + s[43] + s[1] + s[54]
+            return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
        elif len(s) == 81:
            return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
        elif len(s) == 80:
@@ -1128,6 +1116,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                'lang': lang,
                'v': video_id,
                'fmt': self._downloader.params.get('subtitlesformat'),
+                'name': l[0],
            })
            url = u'http://www.youtube.com/api/timedtext?' + params
            sub_lang_list[lang] = url
@@ -1262,9 +1251,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
        return url_map

    def _real_extract(self, url):
-        if re.match(r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$', url):
-            self._downloader.report_warning(u'Did you forget to quote the URL? Remember that & is a meta-character in most shells, so you want to put the URL in quotes, like  youtube-dl \'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\' (or simply  youtube-dl BaW_jenozKc  ).')
-
        # Extract original video URL from URL with redirection, like age verification, using next_url parameter
        mobj = re.search(self._NEXT_URL_RE, url)
        if mobj:
@@ -1347,9 +1333,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
            self._downloader.report_warning(u'unable to extract uploader nickname')

        # title
-        if 'title' not in video_info:
-            raise ExtractorError(u'Unable to extract video title')
-        video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
+        if 'title' in video_info:
+            video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
+        else:
+            self._downloader.report_warning(u'Unable to extract video title')
+            video_title = u'_'

        # thumbnail image
        # We try first to get a high quality image:
@@ -1359,7 +1347,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
            video_thumbnail = m_thumb.group(1)
        elif 'thumbnail_url' not in video_info:
            self._downloader.report_warning(u'unable to extract video thumbnail')
-            video_thumbnail = ''
+            video_thumbnail = None
        else:   # don't panic if we can't find it
            video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])

@@ -1404,6 +1392,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
            args = info['args']
            # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
            # this signatures are encrypted
+            if 'url_encoded_fmt_stream_map' not in args:
+                raise ValueError(u'No stream_map present')  # caught below
            m_s = re.search(r'[&,]s=', args['url_encoded_fmt_stream_map'])
            if m_s is not None:
                self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
@@ -1479,7 +1469,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                return

        else:
-            raise ExtractorError(u'no conn or url_encoded_fmt_stream_map information found in video info')
+            raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')

        results = []
        for format_param, video_real_url in video_url_list:
@@ -1503,7 +1493,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                'description':  video_description,
                'player_url':   player_url,
                'subtitles':    video_subtitles,
-                'duration':     video_duration
+                'duration':     video_duration,
+                'age_limit':    18 if age_gate else 0,
            })
        return results

@@ -1537,9 +1528,19 @@ class YoutubePlaylistIE(InfoExtractor):
        mobj = re.match(self._VALID_URL, url, re.VERBOSE)
        if mobj is None:
            raise ExtractorError(u'Invalid URL: %s' % url)
+        playlist_id = mobj.group(1) or mobj.group(2)
+
+        # Check if it's a video-specific URL
+        query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        if 'v' in query_dict:
+            video_id = query_dict['v'][0]
+            if self._downloader.params.get('noplaylist'):
+                self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
+                return self.url_result('https://www.youtube.com/watch?v=' + video_id, 'Youtube')
+            else:
+                self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))

        # Download playlist videos from API
-        playlist_id = mobj.group(1) or mobj.group(2)
        videos = []

        for page_num in itertools.count(1):
@@ -1634,7 +1635,7 @@ class YoutubeChannelIE(InfoExtractor):

 class YoutubeUserIE(InfoExtractor):
    IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?)|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
    _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
    _GDATA_PAGE_SIZE = 50
    _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
@@ -1827,3 +1828,18 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
        webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
        playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
        return self.url_result(playlist_id, 'YoutubePlaylist')
+
+
+class YoutubeTruncatedURLIE(InfoExtractor):
+    IE_NAME = 'youtube:truncated_url'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$'
+
+    def _real_extract(self, url):
+        raise ExtractorError(
+            u'Did you forget to quote the URL? Remember that & is a meta '
+            u'character in most shells, so you want to put the URL in quotes, '
+            u'like  youtube-dl '
+            u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\''
+            u' (or simply  youtube-dl BaW_jenozKc  ).',
+            expected=True)
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -2,16 +2,14 @@ import re

 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
    ExtractorError,
-    unescapeHTML,
 )

+
 class ZDFIE(InfoExtractor):
-    _VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?'
-    _TITLE = r'<h1(?: class="beitragHeadline")?>(?P<title>.*)</h1>'
+    _VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek(?P<hash>#)?\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?'
    _MEDIA_STREAM = r'<a href="(?P<video_url>.+(?P<media_type>.streaming).+/zdf/(?P<quality>[^\/]+)/[^"]*)".+class="play".+>'
-    _MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"'
-    _RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)'

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
@@ -19,6 +17,9 @@ class ZDFIE(InfoExtractor):
            raise ExtractorError(u'Invalid URL: %s' % url)
        video_id = mobj.group('video_id')

+        if mobj.group('hash'):
+            url = url.replace(u'#', u'', 1)
+
        html = self._download_webpage(url, video_id)
        streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)]
        if streams is None:
@@ -27,39 +28,48 @@ class ZDFIE(InfoExtractor):
        # s['media_type'] == 'wstreaming' -> use 'Windows Media Player' and mms url
        # s['media_type'] == 'hstreaming' -> use 'Quicktime' and rtsp url
        # choose first/default media type and highest quality for now
-        for s in streams:        #find 300 - dsl1000mbit
-            if s['quality'] == '300' and s['media_type'] == 'wstreaming':
-                stream_=s
-                break
-        for s in streams:        #find veryhigh - dsl2000mbit
-            if s['quality'] == 'veryhigh' and s['media_type'] == 'wstreaming': # 'hstreaming' - rtsp is not working
-                stream_=s
-                break
-        if stream_ is None:
+        def stream_pref(s):
+            TYPE_ORDER = ['ostreaming', 'hstreaming', 'wstreaming']
+            try:
+                type_pref = TYPE_ORDER.index(s['media_type'])
+            except ValueError:
+                type_pref = 999
+
+            QUALITY_ORDER = ['veryhigh', '300']
+            try:
+                quality_pref = QUALITY_ORDER.index(s['quality'])
+            except ValueError:
+                quality_pref = 999
+
+            return (type_pref, quality_pref)
+
+        sorted_streams = sorted(streams, key=stream_pref)
+        if not sorted_streams:
            raise ExtractorError(u'No stream found.')
+        stream = sorted_streams[0]

-        media_link = self._download_webpage(stream_['video_url'], video_id,'Get stream URL')
+        media_link = self._download_webpage(
+            stream['video_url'],
+            video_id,
+            u'Get stream URL')

-        self.report_extraction(video_id)
-        mobj = re.search(self._TITLE, html)
+        MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"'
+        RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)'
+
+        mobj = re.search(self._MEDIA_STREAM, media_link)
        if mobj is None:
-            raise ExtractorError(u'Cannot extract title')
-        title = unescapeHTML(mobj.group('title'))
-
-        mobj = re.search(self._MMS_STREAM, media_link)
-        if mobj is None:
-            mobj = re.search(self._RTSP_STREAM, media_link)
+            mobj = re.search(RTSP_STREAM, media_link)
            if mobj is None:
                raise ExtractorError(u'Cannot extract mms:// or rtsp:// URL')
-        mms_url = mobj.group('video_url')
+        video_url = mobj.group('video_url')

-        mobj = re.search('(.*)[.](?P<ext>[^.]+)', mms_url)
-        if mobj is None:
-            raise ExtractorError(u'Cannot extract extention')
-        ext = mobj.group('ext')
+        title = self._html_search_regex(
+            r'<h1(?: class="beitragHeadline")?>(.*?)</h1>',
+            html, u'title')

-        return [{'id': video_id,
-                 'url': mms_url,
-                 'title': title,
-                 'ext': ext
-                 }]
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': determine_ext(video_url)
+        }
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -1,6 +1,9 @@
+import io
 import json
 import traceback
 import hashlib
+import subprocess
+import sys
 from zipimport import zipimporter

 from .utils import *
@@ -34,7 +37,7 @@ def rsa_verify(message, signature, key):
    if signature != sha256(message).digest(): return False
    return True

-def update_self(to_screen, verbose, filename):
+def update_self(to_screen, verbose):
    """Update the program file with the latest version from the repository"""

    UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
@@ -42,7 +45,6 @@ def update_self(to_screen, verbose, filename):
    JSON_URL = UPDATE_URL + 'versions.json'
    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)

-
    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
        to_screen(u'It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
        return
@@ -75,11 +77,18 @@ def update_self(to_screen, verbose, filename):
        to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
        return

-    to_screen(u'Updating to version ' + versions_info['latest'] + '...')
-    version = versions_info['versions'][versions_info['latest']]
+    version_id = versions_info['latest']
+    to_screen(u'Updating to version ' + version_id + '...')
+    version = versions_info['versions'][version_id]

    print_notes(to_screen, versions_info['versions'])

+    filename = sys.argv[0]
+    # Py2EXE: Filename could be different
+    if hasattr(sys, "frozen") and not os.path.isfile(filename):
+        if os.path.isfile(filename + u'.exe'):
+            filename += u'.exe'
+
    if not os.access(filename, os.W_OK):
        to_screen(u'ERROR: no write permissions on %s' % filename)
        return
@@ -116,16 +125,18 @@ def update_self(to_screen, verbose, filename):

        try:
            bat = os.path.join(directory, 'youtube-dl-updater.bat')
-            b = open(bat, 'w')
-            b.write("""
-echo Updating youtube-dl...
+            with io.open(bat, 'w') as batfile:
+                batfile.write(u"""
+@echo off
+echo Waiting for file handle to be closed ...
 ping 127.0.0.1 -n 5 -w 1000 > NUL
-move /Y "%s.new" "%s"
-del "%s"
-            \n""" %(exe, exe, bat))
-            b.close()
+move /Y "%s.new" "%s" > NUL
+echo Updated youtube-dl to version %s.
+start /b "" cmd /c del "%%~f0"&exit /b"
+                \n""" % (exe, exe, version_id))

-            os.startfile(bat)
+            subprocess.Popen([bat])  # Continues to run in the background
+            return  # Do not show premature success messages
        except (IOError, OSError) as err:
            if verbose: to_screen(compat_str(traceback.format_exc()))
            to_screen(u'ERROR: unable to overwrite current version')
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -175,7 +175,7 @@ def compat_ord(c):
 compiled_regex_type = type(re.compile(''))

 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
@@ -824,3 +824,105 @@ def intlist_to_bytes(xs):
        return ''.join([chr(x) for x in xs])
    else:
        return bytes(xs)
+
+
+def get_cachedir(params={}):
+    cache_root = os.environ.get('XDG_CACHE_HOME',
+                                os.path.expanduser('~/.cache'))
+    return params.get('cachedir', os.path.join(cache_root, 'youtube-dl'))
+
+
+# Cross-platform file locking
+if sys.platform == 'win32':
+    import ctypes.wintypes
+    import msvcrt
+
+    class OVERLAPPED(ctypes.Structure):
+        _fields_ = [
+            ('Internal', ctypes.wintypes.LPVOID),
+            ('InternalHigh', ctypes.wintypes.LPVOID),
+            ('Offset', ctypes.wintypes.DWORD),
+            ('OffsetHigh', ctypes.wintypes.DWORD),
+            ('hEvent', ctypes.wintypes.HANDLE),
+        ]
+
+    kernel32 = ctypes.windll.kernel32
+    LockFileEx = kernel32.LockFileEx
+    LockFileEx.argtypes = [
+        ctypes.wintypes.HANDLE,     # hFile
+        ctypes.wintypes.DWORD,      # dwFlags
+        ctypes.wintypes.DWORD,      # dwReserved
+        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
+        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
+        ctypes.POINTER(OVERLAPPED)  # Overlapped
+    ]
+    LockFileEx.restype = ctypes.wintypes.BOOL
+    UnlockFileEx = kernel32.UnlockFileEx
+    UnlockFileEx.argtypes = [
+        ctypes.wintypes.HANDLE,     # hFile
+        ctypes.wintypes.DWORD,      # dwReserved
+        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockLow
+        ctypes.wintypes.DWORD,      # nNumberOfBytesToLockHigh
+        ctypes.POINTER(OVERLAPPED)  # Overlapped
+    ]
+    UnlockFileEx.restype = ctypes.wintypes.BOOL
+    whole_low = 0xffffffff
+    whole_high = 0x7fffffff
+
+    def _lock_file(f, exclusive):
+        overlapped = OVERLAPPED()
+        overlapped.Offset = 0
+        overlapped.OffsetHigh = 0
+        overlapped.hEvent = 0
+        f._lock_file_overlapped_p = ctypes.pointer(overlapped)
+        handle = msvcrt.get_osfhandle(f.fileno())
+        if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
+                          whole_low, whole_high, f._lock_file_overlapped_p):
+            raise OSError('Locking file failed: %r' % ctypes.FormatError())
+
+    def _unlock_file(f):
+        assert f._lock_file_overlapped_p
+        handle = msvcrt.get_osfhandle(f.fileno())
+        if not UnlockFileEx(handle, 0,
+                            whole_low, whole_high, f._lock_file_overlapped_p):
+            raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
+
+else:
+    import fcntl
+
+    def _lock_file(f, exclusive):
+        fcntl.lockf(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
+
+    def _unlock_file(f):
+        fcntl.lockf(f, fcntl.LOCK_UN)
+
+
+class locked_file(object):
+    def __init__(self, filename, mode, encoding=None):
+        assert mode in ['r', 'a', 'w']
+        self.f = io.open(filename, mode, encoding=encoding)
+        self.mode = mode
+
+    def __enter__(self):
+        exclusive = self.mode != 'r'
+        try:
+            _lock_file(self.f, exclusive)
+        except IOError:
+            self.f.close()
+            raise
+        return self
+
+    def __exit__(self, etype, value, traceback):
+        try:
+            _unlock_file(self.f)
+        finally:
+            self.f.close()
+
+    def __iter__(self):
+        return iter(self.f)
+
+    def write(self, *args):
+        return self.f.write(*args)
+
+    def read(self, *args):
+        return self.f.read(*args)
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@

-__version__ = '2013.09.24.2'
+__version__ = '2013.10.09'
Author	SHA1	Message	Date
Philipp Hagemeister	8f1ae18a18	release 2013.10.09	2013-10-09 23:50:47 +02:00
Philipp Hagemeister	57da92b7df	[youtube] Do not recognize attribution link as user (Fixes #1573 )	2013-10-09 23:50:38 +02:00
Jaime Marquínez Ferrándiz	df4f632dbc	Merge pull request #1584 from wingsuit/master Tiny tpo	2013-10-09 07:44:06 -07:00
Jaime Marquínez Ferrándiz	a34c2faae4	[youtube] set the 'name' parameter in the subtitles url (fixes #1577 )	2013-10-09 16:41:36 +02:00
Tom	1d368c7589	Tiny tpo	2013-10-09 21:56:09 +08:00
Jaime Marquínez Ferrándiz	88bd97e34c	[vevo] Some improvements (fixes #1580 ) Extract the info from http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc={id} Some videos don't have an smil manifest, extract the video urls directly from the json and use the last version of the video. Extract all the available formats and set the 'formats' field of the result	2013-10-08 21:25:38 +02:00
Jaime Marquínez Ferrándiz	2ae3edb1cf	Fix the printing of the proxy map in debug mode The proxies have to be extracted from the opener.handlers	2013-10-07 21:10:31 +02:00
Philipp Hagemeister	b2ad967e45	Simplify test setup	2013-10-07 19:06:36 +02:00
Philipp Hagemeister	a27b9e8bd5	Move opener setup into a separate helper function	2013-10-07 19:01:47 +02:00
Philipp Hagemeister	4481a754e4	release 2013.10.07	2013-10-07 14:34:19 +02:00
Philipp Hagemeister	faa6ef6bc8	[jeuxvideo] Improve code quality (fixes #1567 )	2013-10-07 14:33:23 +02:00
Philipp Hagemeister	15870e90b0	Restore warning when user forgets to quote URL (#1396 )	2013-10-07 12:21:24 +02:00
Jaime Marquínez Ferrándiz	387ae5f30b	[vimeo] Recognize urls ending in a slash (fixes #1242 )	2013-10-06 21:56:23 +02:00
Philipp Hagemeister	1310bf2474	[redtube] add age_limit	2013-10-06 16:39:35 +02:00
Philipp Hagemeister	b24f347190	Merge branch 'download-archive' Conflicts: youtube_dl/YoutubeDL.py youtube_dl/__init__.py	2013-10-06 16:30:26 +02:00
Philipp Hagemeister	ee6c9f95e1	Remove superfluous parenthesis	2013-10-06 16:28:36 +02:00
Philipp Hagemeister	2a69c6b879	Merge branch 'age_limit'	2013-10-06 16:23:18 +02:00
Philipp Hagemeister	cfadd183c4	Call extracted property age_limit everywhere	2013-10-06 16:23:06 +02:00
Philipp Hagemeister	e484c81f0c	[generic] Clarify error messages	2013-10-06 16:03:18 +02:00
Philipp Hagemeister	7e5e8306fd	release 2013.10.06	2013-10-06 07:13:14 +02:00
Philipp Hagemeister	41e8bca4d0	[viddler] Add basic support (Fixes #1520 )	2013-10-06 07:12:47 +02:00
Philipp Hagemeister	8dbe9899a9	Allow users to specify an age limit (fixes #1545 ) With these changes, users can now restrict what videos are downloaded by the intented audience, by specifying their age with --age-limit YEARS . Add rudimentary support in youtube, pornotube, and youporn.	2013-10-06 06:08:56 +02:00
Philipp Hagemeister	f4aac741d5	Move try_rm to test helpers	2013-10-06 05:47:17 +02:00
Philipp Hagemeister	c1c9a79c49	Add basic --download-archive option Often, users want to be able to download only videos they haven't seen before, despite the video files having been deleted or moved in the mean time. When --download-archive FILE is given, the extractor and ID of every download is recorded in the specified file. If it is already present, the video in question is skipped.	2013-10-06 04:27:10 +02:00
Philipp Hagemeister	226113c880	Merge remote-tracking branch 'origin/tox'	2013-10-05 22:47:44 +02:00
Filippo Valsorda	8932a66e49	[fixup] remove unnecessary commented function	2013-10-05 16:38:37 -04:00
Filippo Valsorda	79cfb46d42	add tox configuration file for easy testing	2013-10-05 16:08:48 -04:00
Filippo Valsorda	00fcc17aee	add capability to suppress expected warnings in tests	2013-10-05 15:55:58 -04:00
Philipp Hagemeister	e94b783c74	[googleplus] Fix upload_date detection	2013-10-05 16:38:33 +02:00
Philipp Hagemeister	97dae9ae07	[bliptv] Make sure video ID is a string	2013-10-05 16:12:29 +02:00
Jaime Marquínez Ferrándiz	c3fef636b5	[dailymotion] Fix playlist extraction The html code has changed, make the video ids extraction more solid.	2013-10-04 14:07:29 +02:00
Philipp Hagemeister	46e28a84ca	[brightcove] Fix up some broken HTML (#1553 )	2013-10-04 11:53:49 +02:00
Philipp Hagemeister	17ad2b3fb1	[yahoo] Switch ext of test	2013-10-04 11:44:56 +02:00
Philipp Hagemeister	5e2a60db4a	[yahoo] Fix test title	2013-10-04 11:44:02 +02:00
Philipp Hagemeister	cd214418f6	[redtube] pep8	2013-10-04 11:41:57 +02:00
Philipp Hagemeister	ba2d9f213e	[jeuxvideo] fix video file md5sum	2013-10-04 11:38:56 +02:00
Philipp Hagemeister	7f8ae73a5d	Include length in player cache ID Some videos use the same player with IDs of multiple lengths. See https://travis-ci.org/rg3/youtube-dl/jobs/12126506#L319 for an example.	2013-10-04 11:36:06 +02:00
Philipp Hagemeister	466880f531	[yahoo] Do not try to run rtmpdump on travis	2013-10-04 11:34:12 +02:00
Philipp Hagemeister	9f1f6d2437	[rtlnow] Skip test on travis	2013-10-04 11:33:14 +02:00
Philipp Hagemeister	9e0f897f6b	[francetv] Use common format for ID of generation-quoi subextractor	2013-10-04 11:30:47 +02:00
Philipp Hagemeister	c0f6aa876f	Merge remote-tracking branch 'origin/master'	2013-10-04 11:14:20 +02:00
Philipp Hagemeister	d93bdee9a6	[comedycentral] Prepare for generic video extraction (#980 )	2013-10-04 11:14:10 +02:00
Philipp Hagemeister	f13d09332d	[mtv] Prepare for #980	2013-10-04 11:10:04 +02:00
Philipp Hagemeister	2f5865cc6d	Clarify that url and ext are optional when formats is given (#980 )	2013-10-04 11:09:43 +02:00
Philipp Hagemeister	deefc05b88	Document formats (for #980 )	2013-10-04 10:40:42 +02:00
Philipp Hagemeister	0d8cb1cc14	[ted] Prepare #980 merge	2013-10-04 10:32:34 +02:00
Jaime Marquínez Ferrándiz	a90b9fd209	Merge pull request #1551 from rzhxeo/flickr [FlickrIE] Fix HTTPS url	2013-10-03 23:14:12 -07:00
rzhxeo	829493439a	[FlickrIE] Fix HTTPS url	2013-10-04 07:47:40 +02:00
Philipp Hagemeister	3cd022f6e6	Merge remote-tracking branch 'rzhxeo/rtl_ntv'	2013-10-04 00:59:11 +02:00
Philipp Hagemeister	abefd1f7c4	Merge remote-tracking branch 'rzhxeo/rtl_upload_date'	2013-10-04 00:58:35 +02:00
Philipp Hagemeister	c21315f273	[youtube] new static 82 signature	2013-10-04 00:43:01 +02:00
Philipp Hagemeister	9ab1018b1a	release 2013.10.04	2013-10-04 00:38:19 +02:00
Philipp Hagemeister	da0a5d2d6e	[france2] Add support for URLs without video IDs (Fixes #1547 )	2013-10-04 00:34:36 +02:00
Jaime Marquínez Ferrándiz	ee6adb166c	[ign] Support more urls and detect multiple videos in articles (fixes #1543 )	2013-10-02 20:59:34 +02:00
Philipp Hagemeister	be8fe32c92	Fix help of --cachedir	2013-10-02 14:37:19 +02:00
Philipp Hagemeister	c38b1e776d	[youtube] Simplify cache_dir code (#1529 )	2013-10-02 08:41:14 +02:00
Philipp Hagemeister	4f8bf17f23	Merge remote-tracking branch 'holomorph/master'	2013-10-02 08:23:53 +02:00
Philipp Hagemeister	ca40186c75	[youtube] Fix static 82 signature (Closes #1539 )	2013-10-02 08:20:00 +02:00
Philipp Hagemeister	a8c6b24155	[youtube] Support videos without a title (Fixes #1391 , Closes #1542 )	2013-10-02 07:25:35 +02:00
Filippo Valsorda	bd8e5c7ca2	Merge pull request #1531 from rg3/no-playlist [youtube] implement --no-playlist to only download current video	2013-10-01 10:08:20 -07:00
Filippo Valsorda	7c61bd36bb	[youtube] correct --no-playlist for python3	2013-10-01 11:58:13 -04:00
Jaime Marquínez Ferrándiz	c54283824c	[dailymotion] Detect vevo videos (fixes #1532 ) All videos from the Vevo user, just embed videos from vevo.com	2013-10-01 15:05:41 +02:00
Philipp Hagemeister	52f15da2ca	release 2013.10.01.1	2013-10-01 14:44:26 +02:00
Philipp Hagemeister	44d466559e	Properly handle stream meap not being present	2013-10-01 14:44:09 +02:00
Philipp Hagemeister	05751eb047	release 2013.10.01	2013-10-01 11:43:54 +02:00
Philipp Hagemeister	f10503db67	Handle videos without url_encoded_fmt_stream_map (Fixes #1535 )	2013-10-01 11:39:11 +02:00
rzhxeo	adfeafe9e1	[RTLnowIE] Allow video description without upload date Some videos (feature films) have no upload date.	2013-10-01 07:22:49 +02:00
rzhxeo	4c62a16f4f	[RTLnowIE] Add support for http://n-tvnow.de	2013-10-01 06:55:30 +02:00
rzhxeo	c0de39e6d4	Merge pull request #2 from rg3/master Update	2013-09-30 21:39:58 -07:00
Mark Oteiza	fa55675593	Support XDG base directory specification	2013-09-30 18:22:38 -04:00
Filippo Valsorda	d4d9920a26	add test for --no-playlist	2013-09-30 18:01:17 -04:00
Filippo Valsorda	47192f92d8	implement --no-playlist to only download current video - closes #755	2013-09-30 16:26:25 -04:00
Jaime Marquínez Ferrándiz	722076a123	[rtlnow] Replace one of the tests The video is no longer available.	2013-09-29 23:07:26 +02:00
Jaime Marquínez Ferrándiz	bb4aa62cf7	[appletrailers] The request for the settings must have the trailer name in lower case (fixes #1329 )	2013-09-29 20:59:19 +02:00
Jaime Marquínez Ferrándiz	843530568f	[appletrailers] Rework extraction (fixes #1387 ) The exraction was broken: * The includes page contains img elements that need to be fixed. * Use the 'itunes.inc' page, it contains a json dictionary for each trailer with information. * Get the formats from 'includes/settings{trailer_name}.json' * Use urljoin to allow urls with a fragment identifier to work Removed the thumbnail urls from the tests, they are different now.	2013-09-29 20:49:58 +02:00
Philipp Hagemeister	138a5454b5	release 2013.09.29	2013-09-29 14:38:37 +02:00
Philipp Hagemeister	d279037036	[update] Prevent cmd window popup on Windows (Fixes #1478 )	2013-09-29 14:37:06 +02:00
Philipp Hagemeister	46353f6783	[update] Look for .exe extension on Windows (Fixes #745 )	2013-09-29 14:37:00 +02:00
Jaime Marquínez Ferrándiz	70922df8b5	[dailymotion] Disable the family filter in the playlists (fixes #1524 )	2013-09-29 12:44:02 +02:00
Jaime Marquínez Ferrándiz	9c15e9de84	[yahoo] Fix video extraction (fixes #1521 ) There's no need to use two different methods. Now we can also download videos over http if possible. Also run the test for rtmp videos, but skip the download.	2013-09-28 21:19:52 +02:00
Philipp Hagemeister	123c10608d	Merge branch 'master' of github.com:rg3/youtube-dl	2013-09-28 15:43:38 +02:00
Philipp Hagemeister	0b7c2485b6	[zdf] Add support for hash URLs and simplify (#1518 )	2013-09-28 15:43:34 +02:00
Jaime Marquínez Ferrándiz	9abb32045a	[youtube] Add hlsvp to the error message if it can't be found and remove the live stream test It's no longer available, other olympics streams have the same problem.	2013-09-27 15:06:27 +02:00
Jaime Marquínez Ferrándiz	f490e77e77	[youtube] Set the thumbnail to None if it can't be extracted	2013-09-27 14:22:36 +02:00
Jaime Marquínez Ferrándiz	2dc592991a	[youtube] update description of test	2013-09-27 14:20:52 +02:00
Jaime Marquínez Ferrándiz	0a60edcfa9	Don't fail if the video thumbnail couldn't be downloaded (fixes #1516 ) Just report a warning	2013-09-27 14:19:19 +02:00
Philipp Hagemeister	c53f9d30c8	Merge branch 'master' of github.com:rg3/youtube-dl	2013-09-27 13:09:58 +02:00
Philipp Hagemeister	509f398292	Remove youtube_genalgo (#1515 ) With the automatic signature extraction, this script has become superfluous now	2013-09-27 13:09:24 +02:00
Jaime Marquínez Ferrándiz	74bab3f0a4	Don't embed subtitles if the list is empty or the field is not set (fixes #1510 )	2013-09-27 08:08:43 +02:00
Philipp Hagemeister	8574862991	Merge remote-tracking branch 'rzhxeo/RTL_T'	2013-09-27 06:25:04 +02:00
Philipp Hagemeister	2de957c7e1	Merge remote-tracking branch 'rzhxeo/RTL'	2013-09-27 06:23:10 +02:00
Philipp Hagemeister	920de7a27d	[youtube] Fix 83 signature (Closes #1511 )	2013-09-27 06:15:21 +02:00
rzhxeo	63efc427cd	[RTLnowIE] Clean video title The title of some videos has the following format: Series - Episode \| Series online schauen bei ... NOW	2013-09-27 06:00:37 +02:00
rzhxeo	ce65fb6c76	[RTLnowIE] Add support for http://rtlnitronow.de	2013-09-27 05:50:16 +02:00
Jaime Marquínez Ferrándiz	4de1994b6e	[brightcove] Use direct url for the tests The test_all_urls.py test failed because BrightcoveIE doesn't match them.	2013-09-26 18:59:56 +02:00
Jaime Marquínez Ferrándiz	592882aa9f	[brightcove] Support videos that only provide flv versions (fixes #1504 ) Moved the test from generic.py to brightcove.py	2013-09-26 13:54:31 +02:00