release 2015.01.11

[drtv] Updated with support for https

AUTHORS

@@ -101,3 +101,4 @@ Thijs Vermeir
 Joel Leclerc
 Christopher Krooss
 Ondřej Caletka
+Dinesh S

CONTRIBUTING.md

@@ -44,7 +44,7 @@ In particular, every site support request issue should only pertain to services
 
 ###  Is anyone going to need the feature?
 
-Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
 
 ###  Is your question about youtube-dl?
 

Makefile

@@ -63,7 +63,7 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	chmod a+x youtube-dl
 
 README.md: youtube_dl/*.py youtube_dl/*/*.py
-	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
+	COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py
 
 CONTRIBUTING.md: README.md
 	python devscripts/make_contributing.py README.md CONTRIBUTING.md

README.md

@@ -60,10 +60,6 @@ which means you can modify it, redistribute it or use it however you like.
                                      they would handle
     --extractor-descriptions         Output descriptions of all supported
                                      extractors
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
-                                     an empty string (--proxy "") for direct
-                                     connection
-    --socket-timeout None            Time to wait before giving up, in seconds
     --default-search PREFIX          Use this prefix for unqualified URLs. For
                                      example "gvsearch2:" downloads two videos
                                      from google videos for  youtube-dl "large
@@ -82,6 +78,18 @@ which means you can modify it, redistribute it or use it however you like.
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
 
+## Network Options:
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
+                                     an empty string (--proxy "") for direct
+                                     connection
+    --socket-timeout SECONDS         Time to wait before giving up, in seconds
+    --source-address IP              Client-side IP address to bind to
+                                     (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4
+                                     (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6
+                                     (experimental)
+
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
@@ -219,6 +227,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      for each command-line argument. If the URL
                                      refers to a playlist, dump the whole
                                      playlist information in a single line.
+    --print-json                     Be quiet and print the video information as
+                                     JSON (video is still being downloaded).
     --newline                        output progress bar as new lines
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
@@ -229,6 +239,10 @@ which means you can modify it, redistribute it or use it however you like.
                                      files in the current directory to debug
                                      problems
     --print-traffic                  Display sent and read HTTP traffic
+    -C, --call-home                  Contact the youtube-dl server for
+                                     debugging.
+    --no-call-home                   Do NOT contact the youtube-dl server for
+                                     debugging.
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
@@ -248,14 +262,15 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
-                                     preference using slashes: -f 22/17/18 .  -f
-                                     mp4 , -f m4a and  -f flv  are also
-                                     supported. You can also use the special
-                                     names "best", "bestvideo", "bestaudio",
-                                     "worst", "worstvideo" and "worstaudio". By
-                                     default, youtube-dl will pick the best
-                                     quality. Use commas to download multiple
-                                     audio formats, such as -f
+                                     preference using slashes, as in -f 22/17/18
+                                     .  Instead of format codes, you can select
+                                     by extension for the extensions aac, m4a,
+                                     mp3, mp4, ogg, wav, webm. You can also use
+                                     the special names "best", "bestvideo",
+                                     "bestaudio", "worst".  By default, youtube-
+                                     dl will pick the best quality. Use commas
+                                     to download multiple audio formats, such as
+                                     -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
@@ -269,6 +284,10 @@ which means you can modify it, redistribute it or use it however you like.
     -F, --list-formats               list all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on
                                      YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g.
+                                     bestvideo+bestaudio), output to given
+                                     container format. One of mkv, mp4, ogg,
+                                     webm, flv.Ignored if no merge is required
 
 ## Subtitle Options:
     --write-sub                      write subtitle file
@@ -315,6 +334,11 @@ which means you can modify it, redistribute it or use it however you like.
     --add-metadata                   write metadata to the video file
     --xattrs                         write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
+    --fixup POLICY                   (experimental) Automatically correct known
+                                     faults of the file. One of never (do
+                                     nothing), warn (only emit a warning),
+                                     detect_or_warn(check whether we can do
+                                     anything about it, warn otherwise
     --prefer-avconv                  Prefer avconv over ffmpeg for running the
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
@@ -326,7 +350,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<Yourname>\youtube-dl.conf`.
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
 
 # OUTPUT TEMPLATE
 
@@ -449,6 +473,14 @@ Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unz
 
 To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
 
+### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
+
+If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+
+To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+
+From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
@@ -597,7 +629,9 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
-Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+**Please include the full output of youtube-dl when run with `-v`**.
+
+The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 

devscripts/make_supportedsites.py

@@ -23,12 +23,12 @@ def main():
 
     def gen_ies_md(ies):
         for ie in ies:
-            ie_md = '**{}**'.format(ie.IE_NAME)
+            ie_md = '**{0}**'.format(ie.IE_NAME)
             ie_desc = getattr(ie, 'IE_DESC', None)
             if ie_desc is False:
                 continue
             if ie_desc is not None:
-                ie_md += ': {}'.format(ie.IE_DESC)
+                ie_md += ': {0}'.format(ie.IE_DESC)
             if not ie.working():
                 ie_md += ' (Currently broken)'
             yield ie_md

test/helper.py

@@ -110,6 +110,20 @@ def expect_info_dict(self, got_dict, expected_dict):
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
+            elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+                got = got_dict.get(info_field)
+                self.assertTrue(
+                    isinstance(got, list),
+                    'Expected field %s to be a list, but it is of type %s' % (
+                        info_field, type(got).__name__))
+                expected_num = int(expected.partition(':')[2])
+                assertGreaterEqual(
+                    self, len(got), expected_num,
+                    'Expected %d items in field %s, but only got %d' % (
+                        expected_num, info_field, len(got)
+                    )
+                )
+                continue
             else:
                 got = got_dict.get(info_field)
             self.assertEqual(expected, got,

test/test_InfoExtractor.py

@@ -40,5 +40,23 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
 
+    def test_html_search_meta(self):
+        ie = self.ie
+        html = '''
+            <meta name="a" content="1" />
+            <meta name='b' content='2'>
+            <meta name="c" content='3'>
+            <meta name=d content='4'>
+            <meta property="e" content='5' >
+            <meta content="6" name="f">
+        '''
+
+        self.assertEqual(ie._html_search_meta('a', html), '1')
+        self.assertEqual(ie._html_search_meta('b', html), '2')
+        self.assertEqual(ie._html_search_meta('c', html), '3')
+        self.assertEqual(ie._html_search_meta('d', html), '4')
+        self.assertEqual(ie._html_search_meta('e', html), '5')
+        self.assertEqual(ie._html_search_meta('f', html), '6')
+
 if __name__ == '__main__':
     unittest.main()

test/test_YoutubeDL.py

@@ -8,6 +8,8 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import copy
+
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
@@ -192,6 +194,37 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'vid-high')
 
+    def test_format_selection_audio_exts(self):
+        formats = [
+            {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
+        ]
+
+        info_dict = _make_result(formats)
+        ydl = YDL({'format': 'best'})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'aac-64')
+
+        ydl = YDL({'format': 'mp3'})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'mp3-64')
+
+        ydl = YDL({'prefer_free_formats': True})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'ogg-64')
+
     def test_format_selection_video(self):
         formats = [
             {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'},

test/test_age_restriction.py

@@ -45,11 +45,6 @@ class TestAgeRestriction(unittest.TestCase):
             'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
             '505835.mp4', 2, old_age=25)
 
-    def test_pornotube(self):
-        self._assert_restricted(
-            'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
-            '1689755.flv', 13)
-
 
 if __name__ == '__main__':
     unittest.main()

test/test_utils.py

@@ -79,6 +79,10 @@ class TestUtil(unittest.TestCase):
         tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430'
         self.assertEqual(sanitize_filename(tests), tests)
 
+        self.assertEqual(
+            sanitize_filename('New World record at 0:12:34'),
+            'New World record at 0_12_34')
+
         forbidden = '"\0\\/'
         for fc in forbidden:
             for fbc in forbidden:

youtube_dl/YoutubeDL.py

@@ -58,6 +58,7 @@ from .utils import (
     takewhile_inclusive,
     UnavailableVideoError,
     url_basename,
+    version_tuple,
     write_json_file,
     write_string,
     YoutubeDLHandler,
@@ -70,6 +71,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
     get_postprocessor,
@@ -203,6 +205,16 @@ class YoutubeDL(object):
 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
+    merge_output_format: Extension to use when merging formats.
+    fixup:             Automatically correct known faults of the file.
+                       One of:
+                       - "never": do nothing
+                       - "warn": only emit a warning
+                       - "detect_or_warn": check whether we can do anything
+                                           about it, warn otherwise
+    source_address:    (Experimental) Client-side IP address to bind to.
+    call_home:         Boolean, true iff we are allowed to contact the
+                       youtube-dl servers for debugging.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -786,7 +798,7 @@ class YoutubeDL(object):
             if video_formats:
                 return video_formats[0]
         else:
-            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a']
+            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
             if format_spec in extensions:
                 filter_f = lambda f: f['ext'] == format_spec
             else:
@@ -909,10 +921,24 @@ class YoutubeDL(object):
                                                   'contain the video, try using '
                                                   '"-f %s+%s"' % (format_2, format_1))
                                 return
+                            output_ext = (
+                                formats_info[0]['ext']
+                                if self.params.get('merge_output_format') is None
+                                else self.params['merge_output_format'])
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
                                 'ext': formats_info[0]['ext'],
+                                'width': formats_info[0].get('width'),
+                                'height': formats_info[0].get('height'),
+                                'resolution': formats_info[0].get('resolution'),
+                                'fps': formats_info[0].get('fps'),
+                                'vcodec': formats_info[0].get('vcodec'),
+                                'vbr': formats_info[0].get('vbr'),
+                                'stretched_ratio': formats_info[0].get('stretched_ratio'),
+                                'acodec': formats_info[1].get('acodec'),
+                                'abr': formats_info[1].get('abr'),
+                                'ext': output_ext,
                             }
                         else:
                             selected_format = None
@@ -1095,51 +1121,69 @@ class YoutubeDL(object):
                                             (info_dict['thumbnail'], compat_str(err)))
 
         if not self.params.get('skip_download', False):
-            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
-                success = True
-            else:
-                try:
-                    def dl(name, info):
-                        fd = get_suitable_downloader(info)(self, self.params)
-                        for ph in self._progress_hooks:
-                            fd.add_progress_hook(ph)
-                        if self.params.get('verbose'):
-                            self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
-                        return fd.download(name, info)
-                    if info_dict.get('requested_formats') is not None:
-                        downloaded = []
-                        success = True
-                        merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                        if not merger._executable:
-                            postprocessors = []
-                            self.report_warning('You have requested multiple '
-                                                'formats but ffmpeg or avconv are not installed.'
-                                                ' The formats won\'t be merged')
-                        else:
-                            postprocessors = [merger]
-                        for f in info_dict['requested_formats']:
-                            new_info = dict(info_dict)
-                            new_info.update(f)
-                            fname = self.prepare_filename(new_info)
-                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                            downloaded.append(fname)
-                            partial_success = dl(fname, new_info)
-                            success = success and partial_success
-                        info_dict['__postprocessors'] = postprocessors
-                        info_dict['__files_to_merge'] = downloaded
+            try:
+                def dl(name, info):
+                    fd = get_suitable_downloader(info)(self, self.params)
+                    for ph in self._progress_hooks:
+                        fd.add_progress_hook(ph)
+                    if self.params.get('verbose'):
+                        self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
+                    return fd.download(name, info)
+                if info_dict.get('requested_formats') is not None:
+                    downloaded = []
+                    success = True
+                    merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
+                    if not merger._executable:
+                        postprocessors = []
+                        self.report_warning('You have requested multiple '
+                                            'formats but ffmpeg or avconv are not installed.'
+                                            ' The formats won\'t be merged')
                     else:
-                        # Just a single file
-                        success = dl(filename, info_dict)
-                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self.report_error('unable to download video data: %s' % str(err))
-                    return
-                except (OSError, IOError) as err:
-                    raise UnavailableVideoError(err)
-                except (ContentTooShortError, ) as err:
-                    self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
-                    return
+                        postprocessors = [merger]
+                    for f in info_dict['requested_formats']:
+                        new_info = dict(info_dict)
+                        new_info.update(f)
+                        fname = self.prepare_filename(new_info)
+                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                        downloaded.append(fname)
+                        partial_success = dl(fname, new_info)
+                        success = success and partial_success
+                    info_dict['__postprocessors'] = postprocessors
+                    info_dict['__files_to_merge'] = downloaded
+                else:
+                    # Just a single file
+                    success = dl(filename, info_dict)
+            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                self.report_error('unable to download video data: %s' % str(err))
+                return
+            except (OSError, IOError) as err:
+                raise UnavailableVideoError(err)
+            except (ContentTooShortError, ) as err:
+                self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                return
 
             if success:
+                # Fixup content
+                stretched_ratio = info_dict.get('stretched_ratio')
+                if stretched_ratio is not None and stretched_ratio != 1:
+                    fixup_policy = self.params.get('fixup')
+                    if fixup_policy is None:
+                        fixup_policy = 'detect_or_warn'
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
+                            info_dict['id'], stretched_ratio))
+                    elif fixup_policy == 'detect_or_warn':
+                        stretched_pp = FFmpegFixupStretchedPP(self)
+                        if stretched_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(stretched_pp)
+                        else:
+                            self.report_warning(
+                                '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id'], stretched_ratio))
+                    else:
+                        assert fixup_policy == 'ignore'
+
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
@@ -1188,14 +1232,15 @@ class YoutubeDL(object):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
-        keep_video = None
         pps_chain = []
         if ie_info.get('__postprocessors') is not None:
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
+            keep_video = None
+            old_filename = info['filepath']
             try:
-                keep_video_wish, new_info = pp.run(info)
+                keep_video_wish, info = pp.run(info)
                 if keep_video_wish is not None:
                     if keep_video_wish:
                         keep_video = keep_video_wish
@@ -1204,12 +1249,12 @@ class YoutubeDL(object):
                         keep_video = keep_video_wish
             except PostProcessingError as e:
                 self.report_error(e.msg)
-        if keep_video is False and not self.params.get('keepvideo', False):
-            try:
-                self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
-                os.remove(encodeFilename(filename))
-            except (IOError, OSError):
-                self.report_warning('Unable to remove downloaded video file')
+            if keep_video is False and not self.params.get('keepvideo', False):
+                try:
+                    self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
+                    os.remove(encodeFilename(old_filename))
+                except (IOError, OSError):
+                    self.report_warning('Unable to remove downloaded video file')
 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case
@@ -1420,6 +1465,17 @@ class YoutubeDL(object):
                 proxy_map.update(handler.proxies)
         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
 
+        if self.params.get('call_home', False):
+            ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
+            self._write_string('[debug] Public IP address: %s\n' % ipaddr)
+            latest_version = self.urlopen(
+                'https://yt-dl.org/latest/version').read().decode('utf-8')
+            if version_tuple(latest_version) > version_tuple(__version__):
+                self.report_warning(
+                    'You are using an outdated version (newest version: %s)! '
+                    'See https://yt-dl.org/update if you need help updating.' %
+                    latest_version)
+
     def _setup_opener(self):
         timeout_val = self.params.get('socket_timeout')
         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
@@ -1450,9 +1506,8 @@ class YoutubeDL(object):
         proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
-        https_handler = make_HTTPS_handler(
-            self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
-        ydlh = YoutubeDLHandler(debuglevel=debuglevel)
+        https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
+        ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             https_handler, proxy_handler, cookie_processor, ydlh)
         # Delete the default user-agent header, which would otherwise apply in

youtube_dl/__init__.py

@@ -166,6 +166,7 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
+
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
@@ -197,7 +198,8 @@ def _real_main(argv=None):
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                      ' template'.format(outtmpl))
 
-    any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_printing = opts.print_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     # PostProcessors
@@ -243,7 +245,7 @@ def _real_main(argv=None):
         'password': opts.password,
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
-        'quiet': (opts.quiet or any_printing),
+        'quiet': (opts.quiet or any_getting or any_printing),
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
         'forcetitle': opts.gettitle,
@@ -253,9 +255,9 @@ def _real_main(argv=None):
         'forceduration': opts.getduration,
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
-        'forcejson': opts.dumpjson,
+        'forcejson': opts.dumpjson or opts.print_json,
         'dump_single_json': opts.dump_single_json,
-        'simulate': opts.simulate or any_printing,
+        'simulate': opts.simulate or any_getting,
         'skip_download': opts.skip_download,
         'format': opts.format,
         'format_limit': opts.format_limit,
@@ -322,7 +324,11 @@ def _real_main(argv=None):
         'encoding': opts.encoding,
         'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
+        'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
+        'fixup': opts.fixup,
+        'source_address': opts.source_address,
+        'call_home': opts.call_home,
     }
 
     with YoutubeDL(ydl_opts) as ydl:

youtube_dl/compat.py

@@ -4,6 +4,7 @@ import getpass
 import optparse
 import os
 import re
+import socket
 import subprocess
 import sys
 
@@ -307,6 +308,32 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
+if sys.version_info < (2, 7):
+    def compat_socket_create_connection(address, timeout, source_address=None):
+        host, port = address
+        err = None
+        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+            af, socktype, proto, canonname, sa = res
+            sock = None
+            try:
+                sock = socket.socket(af, socktype, proto)
+                sock.settimeout(timeout)
+                if source_address:
+                    sock.bind(source_address)
+                sock.connect(sa)
+                return sock
+            except socket.error as _:
+                err = _
+                if sock is not None:
+                    sock.close()
+        if err is not None:
+            raise err
+        else:
+            raise socket.error("getaddrinfo returns an empty list")
+else:
+    compat_socket_create_connection = socket.create_connection
+
+
 # Fix https://github.com/rg3/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -342,6 +369,7 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
+    'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',

youtube_dl/downloader/common.py

@@ -284,8 +284,19 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
+        nooverwrites_and_exists = (
+            self.params.get('nooverwrites', False)
+            and os.path.exists(encodeFilename(filename))
+        )
+
+        continuedl_and_exists = (
+            self.params.get('continuedl', False)
+            and os.path.isfile(encodeFilename(filename))
+            and not self.params.get('nopart', False)
+        )
+
         # Check file already present
-        if filename != '-' and self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

youtube_dl/downloader/f4m.py

@@ -187,24 +187,34 @@ def build_fragments_list(boot_info):
     return res
 
 
-def write_flv_header(stream, metadata):
-    """Writes the FLV header and the metadata to stream"""
+def write_unsigned_int(stream, val):
+    stream.write(struct_pack('!I', val))
+
+
+def write_unsigned_int_24(stream, val):
+    stream.write(struct_pack('!I', val)[1:])
+
+
+def write_flv_header(stream):
+    """Writes the FLV header to stream"""
     # FLV header
     stream.write(b'FLV\x01')
     stream.write(b'\x05')
     stream.write(b'\x00\x00\x00\x09')
-    # FLV File body
     stream.write(b'\x00\x00\x00\x00')
-    # FLVTAG
-    # Script data
-    stream.write(b'\x12')
-    # Size of the metadata with 3 bytes
-    stream.write(struct_pack('!L', len(metadata))[1:])
-    stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
-    stream.write(metadata)
-    # Magic numbers extracted from the output files produced by AdobeHDS.php
-    # (https://github.com/K-S-V/Scripts)
-    stream.write(b'\x00\x00\x01\x73')
+
+
+def write_metadata_tag(stream, metadata):
+    """Writes optional metadata tag to stream"""
+    SCRIPT_TAG = b'\x12'
+    FLV_TAG_HEADER_LEN = 11
+
+    if metadata:
+        stream.write(SCRIPT_TAG)
+        write_unsigned_int_24(stream, len(metadata))
+        stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
+        stream.write(metadata)
+        write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))
 
 
 def _add_ns(prop):
@@ -256,7 +266,11 @@ class F4mFD(FileDownloader):
             bootstrap = self.ydl.urlopen(bootstrap_url).read()
         else:
             bootstrap = base64.b64decode(bootstrap_node.text)
-        metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
+        metadata_node = media.find(_add_ns('metadata'))
+        if metadata_node is not None:
+            metadata = base64.b64decode(metadata_node.text)
+        else:
+            metadata = None
         boot_info = read_bootstrap_info(bootstrap)
 
         fragments_list = build_fragments_list(boot_info)
@@ -269,7 +283,8 @@ class F4mFD(FileDownloader):
 
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
-        write_flv_header(dest_stream, metadata)
+        write_flv_header(dest_stream)
+        write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook

youtube_dl/extractor/__init__.py

@@ -26,7 +26,8 @@ from .arte import (
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
-from .audiomack import AudiomackIE
+from .atttechchannel import ATTTechChannelIE
+from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
@@ -69,6 +70,7 @@ from .cnn import (
     CNNArticleIE,
 )
 from .collegehumor import CollegeHumorIE
+from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE
@@ -206,6 +208,7 @@ from .jove import JoveIE
 from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
 from .kankan import KankanIE
+from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
@@ -274,6 +277,7 @@ from .nbc import (
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE
+from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
@@ -338,6 +342,7 @@ from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
+from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
 from .rtp import RTPIE
@@ -446,7 +451,7 @@ from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tvigle import TvigleIE
-from .tvp import TvpIE
+from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import TwitchIE
@@ -560,8 +565,6 @@ from .zingmp3 import (
     ZingMp3AlbumIE,
 )
 
-from ..utils import age_restricted
-
 _ALL_CLASSES = [
     klass
     for name, klass in globals().items()

youtube_dl/extractor/atttechchannel.py

@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class ATTTechChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://techchannel\.att\.com/play-video\.cfm/([^/]+/)*(?P<id>.+)'
+    _TEST = {
+        'url': 'http://techchannel.att.com/play-video.cfm/2014/1/27/ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
+        'info_dict': {
+            'id': '11316',
+            'display_id': 'ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
+            'ext': 'flv',
+            'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
+            'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20140127',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._search_regex(
+            r"url\s*:\s*'(rtmp://[^']+)'",
+            webpage, 'video URL')
+
+        video_id = self._search_regex(
+            r'mediaid\s*=\s*(\d+)',
+            webpage, 'video id', fatal=False)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'[Rr]elease\s+date:\s*(\d{1,2}/\d{1,2}/\d{4})',
+            webpage, 'upload date', fatal=False), False)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'ext': 'flv',
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+        }

youtube_dl/extractor/audiomack.py

@@ -1,11 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
+import time
+
 from .common import InfoExtractor
 from .soundcloud import SoundcloudIE
-from ..utils import ExtractorError
-
-import time
+from ..utils import (
+    ExtractorError,
+    url_basename,
+)
 
 
 class AudiomackIE(InfoExtractor):
@@ -17,12 +21,13 @@ class AudiomackIE(InfoExtractor):
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
             'info_dict':
             {
-                'id': 'roosh-williams/extraordinary',
+                'id': '310086',
                 'ext': 'mp3',
-                'title': 'Roosh Williams - Extraordinary'
+                'uploader': 'Roosh Williams',
+                'title': 'Extraordinary'
             }
         },
-        # hosted on soundcloud via audiomack
+        # audiomack wrapper around soundcloud song
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
@@ -38,32 +43,97 @@ class AudiomackIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        # URLs end with [uploader name]/[uploader title]
+        # this title is whatever the user types in, and is rarely
+        # the proper song title.  Real metadata is in the api response
+        album_url_tag = self._match_id(url)
 
+        # Request the extended version of the api for extra fields like artist and title
         api_response = self._download_json(
-            "http://www.audiomack.com/api/music/url/song/%s?_=%d" % (
-                video_id, time.time()),
-            video_id)
+            'http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d' % (
+                album_url_tag, time.time()),
+            album_url_tag)
 
-        if "url" not in api_response:
-            raise ExtractorError("Unable to deduce api url of song")
-        realurl = api_response["url"]
+        # API is inconsistent with errors
+        if 'url' not in api_response or not api_response['url'] or 'error' in api_response:
+            raise ExtractorError('Invalid url %s', url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
-        # - if so, pass the work off to the soundcloud extractor
-        if SoundcloudIE.suitable(realurl):
-            return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
-
-        webpage = self._download_webpage(url, video_id)
-        artist = self._html_search_regex(
-            r'<span class="artist">(.*?)</span>', webpage, "artist")
-        songtitle = self._html_search_regex(
-            r'<h1 class="profile-title song-title"><span class="artist">.*?</span>(.*?)</h1>',
-            webpage, "title")
-        title = artist + " - " + songtitle
+        # if so, pass the work off to the soundcloud extractor
+        if SoundcloudIE.suitable(api_response['url']):
+            return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}
 
         return {
-            'id': video_id,
-            'title': title,
-            'url': realurl,
+            'id': api_response.get('id', album_url_tag),
+            'uploader': api_response.get('artist'),
+            'title': api_response.get('title'),
+            'url': api_response['url'],
         }
+
+
+class AudiomackAlbumIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/album/(?P<id>[\w/-]+)'
+    IE_NAME = 'audiomack:album'
+    _TESTS = [
+        # Standard album playlist
+        {
+            'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
+            'playlist_count': 15,
+            'info_dict':
+            {
+                'id': '812251',
+                'title': 'Tha Tour: Part 2 (Official Mixtape)'
+            }
+        },
+        # Album playlist ripped from fakeshoredrive with no metadata
+        {
+            'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
+            'playlist': [{
+                'info_dict': {
+                    'title': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'id': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'ext': 'mp3',
+                }
+            }],
+            'params': {
+                'playliststart': 8,
+                'playlistend': 8,
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        # URLs end with [uploader name]/[uploader title]
+        # this title is whatever the user types in, and is rarely
+        # the proper song title.  Real metadata is in the api response
+        album_url_tag = self._match_id(url)
+        result = {'_type': 'playlist', 'entries': []}
+        # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
+        # Therefore we don't know how many songs the album has and must infi-loop until failure
+        for track_no in itertools.count():
+            # Get song's metadata
+            api_response = self._download_json(
+                'http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
+                % (album_url_tag, track_no, time.time()), album_url_tag,
+                note='Querying song information (%d)' % (track_no + 1))
+
+            # Total failure, only occurs when url is totally wrong
+            # Won't happen in middle of valid playlist (next case)
+            if 'url' not in api_response or 'error' in api_response:
+                raise ExtractorError('Invalid url for track %d of album url %s' % (track_no, url))
+            # URL is good but song id doesn't exist - usually means end of playlist
+            elif not api_response['url']:
+                break
+            else:
+                # Pull out the album metadata and add to result (if it exists)
+                for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
+                    if apikey in api_response and resultkey not in result:
+                        result[resultkey] = api_response[apikey]
+                song_id = url_basename(api_response['url']).rpartition('.')[0]
+                result['entries'].append({
+                    'id': api_response.get('id', song_id),
+                    'uploader': api_response.get('artist'),
+                    'title': api_response.get('title', song_id),
+                    'url': api_response['url'],
+                })
+        return result

youtube_dl/extractor/bet.py

@@ -16,7 +16,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '406429c6-1b8a-463e-83fc-814adb81a9db',
+                'id': '740ab250-bb94-4a8a-8787-fe0de7c74471',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
                 'title': 'BET News Presents: A Conversation With President Obama',
@@ -35,7 +35,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
+                'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
@@ -55,7 +55,6 @@ class BetIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
         media_url = compat_urllib_parse.unquote(self._search_regex(

youtube_dl/extractor/bilibili.py

@@ -4,9 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_parse_qs
 from ..utils import (
-    ExtractorError,
     int_or_none,
     unified_strdate,
 )
@@ -54,45 +52,38 @@ class BiliBiliIE(InfoExtractor):
         thumbnail = self._html_search_meta(
             'thumbnailUrl', video_code, 'thumbnail', fatal=False)
 
-        player_params = compat_parse_qs(self._html_search_regex(
-            r'<iframe .*?class="player" src="https://secure\.bilibili\.(?:tv|com)/secure,([^"]+)"',
-            webpage, 'player params'))
+        cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
 
-        if 'cid' in player_params:
-            cid = player_params['cid'][0]
+        lq_doc = self._download_xml(
+            'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
+            video_id,
+            note='Downloading LQ video info'
+        )
+        lq_durl = lq_doc.find('./durl')
+        formats = [{
+            'format_id': 'lq',
+            'quality': 1,
+            'url': lq_durl.find('./url').text,
+            'filesize': int_or_none(
+                lq_durl.find('./size'), get_attr='text'),
+        }]
 
-            lq_doc = self._download_xml(
-                'http://interface.bilibili.cn/v_cdn_play?cid=%s' % cid,
-                video_id,
-                note='Downloading LQ video info'
-            )
-            lq_durl = lq_doc.find('.//durl')
-            formats = [{
-                'format_id': 'lq',
-                'quality': 1,
-                'url': lq_durl.find('./url').text,
+        hq_doc = self._download_xml(
+            'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
+            video_id,
+            note='Downloading HQ video info',
+            fatal=False,
+        )
+        if hq_doc is not False:
+            hq_durl = hq_doc.find('./durl')
+            formats.append({
+                'format_id': 'hq',
+                'quality': 2,
+                'ext': 'flv',
+                'url': hq_durl.find('./url').text,
                 'filesize': int_or_none(
-                    lq_durl.find('./size'), get_attr='text'),
-            }]
-
-            hq_doc = self._download_xml(
-                'http://interface.bilibili.cn/playurl?cid=%s' % cid,
-                video_id,
-                note='Downloading HQ video info',
-                fatal=False,
-            )
-            if hq_doc is not False:
-                hq_durl = hq_doc.find('.//durl')
-                formats.append({
-                    'format_id': 'hq',
-                    'quality': 2,
-                    'ext': 'flv',
-                    'url': hq_durl.find('./url').text,
-                    'filesize': int_or_none(
-                        hq_durl.find('./size'), get_attr='text'),
-                })
-        else:
-            raise ExtractorError('Unsupported player parameters: %r' % (player_params,))
+                    hq_durl.find('./size'), get_attr='text'),
+            })
 
         self._sort_formats(formats)
         return {

youtube_dl/extractor/buzzfeed.py

@@ -33,7 +33,7 @@ class BuzzFeedIE(InfoExtractor):
             'skip_download': True,  # Got enough YouTube download tests
         },
         'info_dict': {
-            'description': 'Munchkin the Teddy Bear is back !',
+            'description': 're:Munchkin the Teddy Bear is back ?!',
             'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
         },
         'playlist': [{
@@ -42,9 +42,9 @@ class BuzzFeedIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': '© 2014 Munchkin the Shih Tzu\nAll rights reserved\nFacebook: http://facebook.com/MunchkintheShihTzu',
+                'description': 're:© 2014 Munchkin the Shih Tzu',
                 'uploader': 'Munchkin the Shih Tzu',
-                'title': 'Munchkin the Teddy Bear gets her exercise',
+                'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]
     }]

youtube_dl/extractor/collegerama.py

@@ -0,0 +1,92 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
+
+
+class CollegeRamaIE(InfoExtractor):
+    _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
+    _TESTS = [
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
+            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
+            'info_dict': {
+                'id': '585a43626e544bdd97aeb71a0ec907a01d',
+                'ext': 'mp4',
+                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
+                'description': '',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 7713.088,
+                'timestamp': 1413309600,
+                'upload_date': '20141014',
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
+            'md5': 'ef1fdded95bdf19b12c5999949419c92',
+            'info_dict': {
+                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
+                'ext': 'wmv',
+                'title': '64ste Vakantiecursus: Afvalwater',
+                'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'duration': 10853,
+                'timestamp': 1326446400,
+                'upload_date': '20120113',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        player_options_request = {
+            "getPlayerOptionsRequest": {
+                "ResourceId": video_id,
+                "QueryString": "",
+            }
+        }
+
+        request = compat_urllib_request.Request(
+            'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
+            json.dumps(player_options_request))
+        request.add_header('Content-Type', 'application/json')
+
+        player_options = self._download_json(request, video_id)
+
+        presentation = player_options['d']['Presentation']
+        title = presentation['Title']
+        description = presentation.get('Description')
+        thumbnail = None
+        duration = float_or_none(presentation.get('Duration'), 1000)
+        timestamp = int_or_none(presentation.get('UnixTime'), 1000)
+
+        formats = []
+        for stream in presentation['Streams']:
+            for video in stream['VideoUrls']:
+                thumbnail_url = stream.get('ThumbnailUrl')
+                if thumbnail_url:
+                    thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
+                format_id = video['MediaType']
+                if format_id == 'SS':
+                    continue
+                formats.append({
+                    'url': video['Location'],
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+        }

youtube_dl/extractor/common.py

@@ -114,6 +114,9 @@ class InfoExtractor(object):
                                  to add to the request.
                     * http_post_data  Additional data to send with a POST
                                  request.
+                    * stretched_ratio  If given and not 1, indicates that the
+                                       video's pixels are not square.
+                                       width : height ratio as float.
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
@@ -147,6 +150,17 @@ class InfoExtractor(object):
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
     comment_count:  Number of comments on the video
+    comments:       A list of comments, each with one or more of the following
+                    properties (all but one of text or html optional):
+                        * "author" - human-readable name of the comment author
+                        * "author_id" - user ID of the comment author
+                        * "id" - Comment ID
+                        * "html" - Comment as HTML
+                        * "text" - Plain text of the comment
+                        * "timestamp" - UNIX timestamp of comment
+                        * "parent" - ID of the comment this one is replying to.
+                                     Set to "root" to indicate that this is a
+                                     comment to the original video.
     age_limit:      Age restriction for the video, as an integer (years)
     webpage_url:    The url to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
@@ -365,9 +379,19 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
         """ Returns the data of the page as a string """
-        res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+        success = False
+        try_count = 0
+        while success is False:
+            try:
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+                success = True
+            except compat_http_client.IncompleteRead as e:
+                try_count += 1
+                if try_count >= tries:
+                    raise e
+                self._sleep(timeout, video_id)
         if res is False:
             return res
         else:
@@ -594,7 +618,7 @@ class InfoExtractor(object):
         return self._html_search_regex(
             r'''(?isx)<meta
                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
-                    [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
+                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):
@@ -718,8 +742,14 @@ class InfoExtractor(object):
             'Unable to download f4m manifest')
 
         formats = []
+        manifest_version = '1.0'
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
+        if not media_nodes:
+            manifest_version = '2.0'
+            media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
+            if manifest_version == '2.0':
+                manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             format_id = 'f4m-%d' % (i if tbr is None else tbr)
             formats.append({

youtube_dl/extractor/discovery.py

@@ -1,47 +1,45 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
+    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
     _TEST = {
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
-        'md5': 'e12614f9ee303a6ccef415cb0793eba2',
+        'md5': '3c69d77d9b0d82bfd5e5932a60f26504',
         'info_dict': {
-            'id': '614784',
-            'ext': 'mp4',
-            'title': 'MythBusters: Mission Impossible Outtakes',
+            'id': 'mission-impossible-outtakes',
+            'ext': 'flv',
+            'title': 'Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                             ' each other -- to the point of confusing Jamie\'s dog -- and '
                             'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
                             ' back.'),
             'duration': 156,
+            'timestamp': 1303099200,
+            'upload_date': '20110418',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_list_json = self._search_regex(r'var videoListJSON = ({.*?});',
-                                             webpage, 'video list', flags=re.DOTALL)
-        video_list = json.loads(video_list_json)
-        info = video_list['clips'][0]
-        formats = []
-        for f in info['mp4']:
-            formats.append(
-                {'url': f['src'], 'ext': 'mp4', 'tbr': int(f['bitrate'][:-1])})
+        info = self._parse_json(self._search_regex(
+            r'(?s)<script type="application/ld\+json">(.*?)</script>',
+            webpage, 'video info'), video_id)
 
         return {
-            'id': info['contentId'],
-            'title': video_list['name'],
-            'formats': formats,
-            'description': info['videoCaption'],
-            'thumbnail': info.get('videoStillURL') or info.get('thumbnailURL'),
-            'duration': info['duration'],
+            'id': video_id,
+            'title': info['name'],
+            'url': info['contentURL'],
+            'description': info.get('description'),
+            'thumbnail': info.get('thumbnailUrl'),
+            'timestamp': parse_iso8601(info.get('uploadDate')),
+            'duration': int_or_none(info.get('duration')),
         }

youtube_dl/extractor/drtv.py

@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class DRTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
         'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',

youtube_dl/extractor/eighttracks.py

@@ -9,6 +9,9 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class EightTracksIE(InfoExtractor):
@@ -112,14 +115,29 @@ class EightTracksIE(InfoExtractor):
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']
         track_count = data['tracks_count']
+        duration = data['duration']
+        avg_song_duration = float(duration) / track_count
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []
+
         for i in range(track_count):
-            api_json = self._download_webpage(
-                next_url, playlist_id,
-                note='Downloading song information %d/%d' % (i + 1, track_count),
-                errnote='Failed to download song information')
+            api_json = None
+            download_tries = 0
+
+            while api_json is None:
+                try:
+                    api_json = self._download_webpage(
+                        next_url, playlist_id,
+                        note='Downloading song information %d/%d' % (i + 1, track_count),
+                        errnote='Failed to download song information')
+                except ExtractorError:
+                    if download_tries > 3:
+                        raise
+                    else:
+                        download_tries += 1
+                        self._sleep(avg_song_duration, playlist_id)
+
             api_data = json.loads(api_json)
             track_data = api_data['set']['track']
             info = {
@@ -131,6 +149,7 @@ class EightTracksIE(InfoExtractor):
                 'ext': 'm4a',
             }
             entries.append(info)
+
             next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (
                 session, mix_id, track_data['id'])
         return {

youtube_dl/extractor/elpais.py

@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
@@ -24,9 +22,7 @@ class ElPaisIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         prefix = self._html_search_regex(

youtube_dl/extractor/fktv.py

@@ -13,7 +13,7 @@ from ..utils import (
 
 class FKTVIE(InfoExtractor):
     IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
+    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
@@ -26,29 +26,32 @@ class FKTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = int(mobj.group('ep'))
+        episode = int(self._match_id(url))
 
-        server = random.randint(2, 4)
-        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%d.jpg' % episode
-        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
+        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%s.jpg' % episode
+        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/Start' % episode,
                                                episode)
         playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
                                       'playlist', flags=re.DOTALL)
         files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
-        # TODO: return a single multipart video
+
         videos = []
         for i, _ in enumerate(files, 1):
             video_id = '%04d%d' % (episode, i)
-            video_url = 'http://dl%d.fernsehkritik.tv/fernsehkritik%d%s.flv' % (server, episode, '' if i == 1 else '-%d' % i)
+            video_url = 'http://fernsehkritik.tv/js/directme.php?file=%s%s.flv' % (episode, '' if i == 1 else '-%d' % i)
             videos.append({
+                'ext': 'flv',
                 'id': video_id,
                 'url': video_url,
                 'title': clean_html(get_element_by_id('eptitle', start_webpage)),
                 'description': clean_html(get_element_by_id('contentlist', start_webpage)),
                 'thumbnail': video_thumbnail
             })
-        return videos
+        return {
+            '_type': 'multi_video',
+            'entries': videos,
+            'id': 'folge-%s' % episode,
+        }
 
 
 class FKTVPosteckeIE(InfoExtractor):

youtube_dl/extractor/gdcvault.py

@@ -39,7 +39,8 @@ class GDCVaultIE(InfoExtractor):
                 'id': '1015301',
                 'ext': 'flv',
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
-            }
+            },
+            'skip': 'Requires login',
         }
     ]
 

youtube_dl/extractor/huffpost.py

@@ -39,8 +39,9 @@ class HuffPostIE(InfoExtractor):
         data = self._download_json(api_url, video_id)['data']
 
         video_title = data['title']
-        duration = parse_duration(data['running_time'])
-        upload_date = unified_strdate(data['schedule']['starts_at'])
+        duration = parse_duration(data.get('running_time'))
+        upload_date = unified_strdate(
+            data.get('schedule', {}).get('starts_at') or data.get('segment_start_date_time'))
         description = data.get('description')
 
         thumbnails = []
@@ -59,16 +60,11 @@ class HuffPostIE(InfoExtractor):
             'ext': 'mp4',
             'url': url,
             'vcodec': 'none' if key.startswith('audio/') else None,
-        } for key, url in data['sources']['live'].items()]
-        if data.get('fivemin_id'):
-            fid = data['fivemin_id']
-            fcat = str(int(fid) // 100 + 1)
-            furl = 'http://avideos.5min.com/2/' + fcat[-3:] + '/' + fcat + '/' + fid + '.mp4'
-            formats.append({
-                'format': 'fivemin',
-                'url': furl,
-                'preference': 1,
-            })
+        } for key, url in data.get('sources', {}).get('live', {}).items()]
+
+        if not formats and data.get('fivemin_id'):
+            return self.url_result('5min:%s' % data['fivemin_id'])
+
         self._sort_formats(formats)
 
         return {

youtube_dl/extractor/imdb.py

@@ -16,7 +16,6 @@ class ImdbIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
-        'md5': '9f34fa777ade3a6e57a054fdbcb3a068',
         'info_dict': {
             'id': '2524815897',
             'ext': 'mp4',

youtube_dl/extractor/karaoketv.py

@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    js_to_json,
+)
+
+
+class KaraoketvIE(InfoExtractor):
+    _VALID_URL = r'http://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://karaoketv.co.il/?container=songs&id=171568',
+        'info_dict': {
+            'id': '171568',
+            'ext': 'mp4',
+            'title': 'אל העולם שלך - רותם כהן - שרים קריוקי',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        page_video_url = self._og_search_video_url(webpage, video_id)
+        config_json = compat_urllib_parse.unquote_plus(self._search_regex(
+            r'config=(.*)', page_video_url, 'configuration'))
+
+        urls_info_json = self._download_json(
+            config_json, video_id, 'Downloading configuration',
+            transform_source=js_to_json)
+
+        url = urls_info_json['playlist'][0]['url']
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'url': url,
+        }

youtube_dl/extractor/mit.py

@@ -105,6 +105,9 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
+                'upload_date': '20121109',
+                'uploader_id': 'MIT',
+                'uploader': 'MIT OpenCourseWare',
                 # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
@@ -114,6 +117,9 @@ class OCWMITIE(InfoExtractor):
                 'id': '7K1sB05pE0A',
                 'ext': 'mp4',
                 'title': 'Session 1: Introduction to Derivatives',
+                'upload_date': '20090818',
+                'uploader_id': 'MIT',
+                'uploader': 'MIT OpenCourseWare',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
                 # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }

youtube_dl/extractor/netzkino.py

@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    js_to_json,
+    parse_iso8601,
+)
+
+
+class NetzkinoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/(?P<category>[^/]+)/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.netzkino.de/#!/scifikino/rakete-zum-mond',
+        'md5': '92a3f8b76f8d7220acce5377ea5d4873',
+        'info_dict': {
+            'id': 'rakete-zum-mond',
+            'ext': 'mp4',
+            'title': 'Rakete zum Mond (Endstation Mond, Destination Moon)',
+            'comments': 'mincount:3',
+            'description': 'md5:1eddeacc7e62d5a25a2d1a7290c64a28',
+            'upload_date': '20120813',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'timestamp': 1344858571,
+            'age_limit': 12,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        category_id = mobj.group('category')
+        video_id = mobj.group('id')
+
+        api_url = 'http://api.netzkino.de.simplecache.net/capi-2.0a/categories/%s.json?d=www' % category_id
+        api_info = self._download_json(api_url, video_id)
+        info = next(
+            p for p in api_info['posts'] if p['slug'] == video_id)
+        custom_fields = info['custom_fields']
+
+        production_js = self._download_webpage(
+            'http://www.netzkino.de/beta/dist/production.min.js', video_id,
+            note='Downloading player code')
+        avo_js = self._search_regex(
+            r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})',
+            production_js, 'URL templates')
+        templates = self._parse_json(
+            avo_js, video_id, transform_source=js_to_json)
+
+        suffix = {
+            'hds': '.mp4/manifest.f4m',
+            'hls': '.mp4/master.m3u8',
+            'pmd': '.mp4',
+        }
+        film_fn = custom_fields['Streaming'][0]
+        formats = [{
+            'format_id': key,
+            'ext': 'mp4',
+            'url': tpl.replace('{}', film_fn) + suffix[key],
+        } for key, tpl in templates.items()]
+        self._sort_formats(formats)
+
+        comments = [{
+            'timestamp': parse_iso8601(c.get('date'), delimiter=' '),
+            'id': c['id'],
+            'author': c['name'],
+            'html': c['content'],
+            'parent': 'root' if c.get('parent', 0) == 0 else c['parent'],
+        } for c in info.get('comments', [])]
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'comments': comments,
+            'title': info['title'],
+            'age_limit': int_or_none(custom_fields.get('FSK')[0]),
+            'timestamp': parse_iso8601(info.get('date'), delimiter=' '),
+            'description': clean_html(info.get('content')),
+            'thumbnail': info.get('thumbnail'),
+            'playlist_title': api_info.get('title'),
+            'playlist_id': category_id,
+        }

youtube_dl/extractor/nrk.py

@@ -7,8 +7,10 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
+    parse_duration,
     unified_strdate,
 )
+from .subtitles import SubtitlesInfoExtractor
 
 
 class NRKIE(InfoExtractor):
@@ -71,8 +73,8 @@ class NRKIE(InfoExtractor):
         }
 
 
-class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+class NRKTVIE(SubtitlesInfoExtractor):
+    _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -111,9 +113,6 @@ class NRKTVIE(InfoExtractor):
                 'upload_date': '20150106',
             },
             'skip': 'Only works from Norway',
-            'params': {
-                'proxy': '127.0.0.1:8118',
-            },
         },
         {
             'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
@@ -147,12 +146,32 @@ class NRKTVIE(InfoExtractor):
                 'duration': 6947.5199999999995,
             },
             'skip': 'Only works from Norway',
-            'params': {
-                'proxy': '127.0.0.1:8118',
-            },
         }
     ]
 
+    def _seconds2str(self, s):
+        return '%02d:%02d:%02d.%03d' % (s / 3600, (s % 3600) / 60, s % 60, (s % 1) * 1000)
+
+    def _debug_print(self, txt):
+        if self._downloader.params.get('verbose', False):
+            self.to_screen('[debug] %s' % txt)
+
+    def _extract_captions(self, subtitlesurl, video_id, baseurl):
+        url = "%s%s" % (baseurl, subtitlesurl)
+        self._debug_print('%s: Subtitle url: %s' % (video_id, url))
+        captions = self._download_xml(url, video_id, 'Downloading subtitles')
+        lang = captions.get('lang', 'no')
+        ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
+        srt = ''
+        for pos, p in enumerate(ps):
+            begin = parse_duration(p.get('begin'))
+            duration = parse_duration(p.get('dur'))
+            starttime = self._seconds2str(begin)
+            endtime = self._seconds2str(begin + duration)
+            text = '\n'.join(p.itertext())
+            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
+        return {lang: srt}
+
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
 
@@ -160,6 +179,7 @@ class NRKTVIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         part_id = mobj.group('part_id')
+        baseurl = mobj.group('baseurl')
 
         webpage = self._download_webpage(url, video_id)
 
@@ -216,9 +236,18 @@ class NRKTVIE(InfoExtractor):
         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
-
         self._sort_formats(formats)
 
+        subtitles_url = self._html_search_regex(
+            r'data-subtitlesurl[ ]*=[ ]*"([^"]+)"',
+            webpage, 'subtitle URL', default=None)
+        subtitles = None
+        if subtitles_url:
+            subtitles = self._extract_captions(subtitles_url, video_id, baseurl)
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id': video_id,
             'title': title,
@@ -227,4 +256,5 @@ class NRKTVIE(InfoExtractor):
             'upload_date': upload_date,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

youtube_dl/extractor/orf.py

@@ -128,13 +128,16 @@ class ORFTVthekIE(InfoExtractor):
         }
 
 
-# Audios on ORF radio are only available for 7 days, so we can't add tests.
-
-
 class ORFOE1IE(InfoExtractor):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
+    _VALID_URL = r'http://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
+
+    # Audios on ORF radio are only available for 7 days, so we can't add tests.
+    _TEST = {
+        'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
+        'only_matching': True,
+    }
 
     def _real_extract(self, url):
         show_id = self._match_id(url)
@@ -160,7 +163,7 @@ class ORFOE1IE(InfoExtractor):
 
 
 class ORFFM4IE(InfoExtractor):
-    IE_DESC = 'orf:fm4'
+    IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
     _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
 

youtube_dl/extractor/rte.py

@@ -0,0 +1,62 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    float_or_none,
+)
+
+
+class RteIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/(?P<id>[0-9]+)/'
+    _TEST = {
+        'url': 'http://www.rte.ie/player/de/show/10363114/',
+        'info_dict': {
+            'id': '10363114',
+            'ext': 'mp4',
+            'title': 'One News',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'The One O\'Clock News followed by Weather.',
+            'duration': 436.844,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_meta('description', webpage, 'description')
+        duration = float_or_none(self._html_search_meta(
+            'duration', webpage, 'duration', fatal=False), 1000)
+
+        thumbnail_id = self._search_regex(
+            r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
+        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg'
+
+        feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
+        json_string = self._download_json(feeds_url, video_id)
+
+        # f4m_url = server + relative_url
+        f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
+        f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
+        f4m_formats = [{
+            'format_id': f['format_id'],
+            'url': f['url'],
+            'ext': 'mp4',
+            'width': f['width'],
+            'height': f['height'],
+        } for f in f4m_formats]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': f4m_formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

youtube_dl/extractor/sexykarma.py

@@ -24,7 +24,7 @@ class SexyKarmaIE(InfoExtractor):
             'title': 'Taking a quick pee.',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'wildginger7',
-            'upload_date': '20141007',
+            'upload_date': '20141008',
             'duration': 22,
             'view_count': int,
             'comment_count': int,
@@ -45,6 +45,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
@@ -61,6 +62,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }]
 
@@ -114,4 +116,5 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': view_count,
             'comment_count': comment_count,
             'categories': categories,
+            'age_limit': 18,
         }

youtube_dl/extractor/teachertube.py

@@ -57,9 +57,7 @@ class TeacherTubeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_meta('title', webpage, 'title', fatal=True)

youtube_dl/extractor/tudou.py

@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?tudou\.com/(?:listplay|programs|albumplay)/(?:view|(.+?))/(?:([^/]+)|([^/]+))(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',
@@ -27,13 +27,6 @@ class TudouIE(InfoExtractor):
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }, {
-        'url': 'http://www.tudou.com/albumplay/TenTw_JgiPM/PzsAs5usU9A.html',
-        'info_dict': {
-            'title': 'todo.mp4',
-        },
-        'add_ie': ['Youku'],
-        'skip': 'Only works from China'
     }]
 
     def _url_for_id(self, id, quality=None):
@@ -45,8 +38,7 @@ class TudouIE(InfoExtractor):
         return final_url
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(2)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage)
@@ -87,4 +79,9 @@ class TudouIE(InfoExtractor):
             }
             result.append(part_info)
 
-        return result
+        return {
+            '_type': 'multi_video',
+            'entries': result,
+            'id': video_id,
+            'title': title,
+        }

youtube_dl/extractor/tunein.py

@@ -24,7 +24,7 @@ class TuneInIE(InfoExtractor):
     _INFO_DICT = {
         'id': '34682',
         'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
-        'ext': 'AAC',
+        'ext': 'aac',
         'thumbnail': 're:^https?://.*\.png$',
         'location': 'Tacoma, WA',
     }
@@ -78,14 +78,21 @@ class TuneInIE(InfoExtractor):
         for stream in streams:
             if stream.get('Type') == 'Live':
                 is_live = True
+            reliability = stream.get('Reliability')
+            format_note = (
+                'Reliability: %d%%' % reliability
+                if reliability is not None else None)
             formats.append({
+                'preference': (
+                    0 if reliability is None or reliability > 90
+                    else 1),
                 'abr': stream.get('Bandwidth'),
-                'ext': stream.get('MediaType'),
+                'ext': stream.get('MediaType').lower(),
                 'acodec': stream.get('MediaType'),
                 'vcodec': 'none',
                 'url': stream.get('Url'),
-                # Sometimes streams with the highest quality do not exist
-                'preference': stream.get('Reliability'),
+                'source_preference': reliability,
+                'format_note': format_note,
             })
         self._sort_formats(formats)
 

youtube_dl/extractor/tvp.py

@@ -1,37 +1,142 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class TvpIE(InfoExtractor):
     IE_NAME = 'tvp.pl'
-    _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
 
-    _TEST = {
-        'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-        'md5': '148408967a6a468953c0a75cbdaf0d7a',
+    _TESTS = [{
+        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
         'info_dict': {
-            'id': '12878238',
+            'id': '4278035',
             'ext': 'wmv',
-            'title': '31.10.2013 - Odcinek 2',
-            'description': '31.10.2013 - Odcinek 2',
+            'title': 'Ogniem i mieczem, odc. 2',
+            'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
         },
-        'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
-    }
+    }, {
+        'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'info_dict': {
+            'id': '194536',
+            'ext': 'mp4',
+            'title': 'Czas honoru, I seria – odc. 13',
+            #  'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
+        },
+    }, {
+        'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+        'info_dict': {
+            'id': '17916176',
+            'ext': 'mp4',
+            'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': 'true',
+        },
+    }, {
+        'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+        'info_dict': {
+            'id': '17834272',
+            'ext': 'mp4',
+            'title': 'Na sygnale, odc. 39',
+            'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': 'true',
+        },
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id
-        params = self._download_json(
-            json_url, video_id, "Downloading video metadata")
-        video_url = params['video_url']
+        webpage = self._download_webpage(
+            'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
+
+        title = self._og_search_title(webpage)
+        series = self._search_regex(
+            r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
+            webpage, 'series', group='series', default=None)
+        if series is not None and series not in title:
+            title = '%s, %s' % (series, title)
+        description = self._og_search_description(webpage, default=None)
+
+        video_url = self._search_regex(
+            r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
+        if video_url is None:
+            video_url = self._download_json(
+                'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
+                video_id)['video_url']
+
+        ext = video_url.rsplit('.', 1)[-1]
+        if ext != 'ism/manifest':
+            if '/' in ext:
+                ext = 'mp4'
+            formats = [{
+                'format_id': 'direct',
+                'url': video_url,
+                'ext': ext,
+            }]
+        else:
+            m3u8_url = re.sub('([^/]*)\.ism/manifest', r'\1.ism/\1.m3u8', video_url)
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'ext': 'wmv',
-            'url': video_url,
-            'description': self._og_search_description(webpage),
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
+            'description': description,
+            'formats': formats,
+        }
+
+
+class TvpSeriesIE(InfoExtractor):
+    IE_NAME = 'tvp.pl:Series'
+    _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
+
+    _TESTS = [{
+        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
+        'info_dict': {
+            'title': 'Ogniem i mieczem',
+            'id': '4278026',
+        },
+        'playlist_count': 4,
+    }, {
+        'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
+        'info_dict': {
+            'title': 'Boso przez świat',
+            'id': '9329207',
+        },
+        'playlist_count': 86,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id, tries=5)
+
+        title = self._html_search_regex(
+            r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
+        playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
+        playlist = self._download_webpage(
+            'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
+            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5,
+            note='Downloading playlist')
+
+        videos_paths = re.findall(
+            '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
+        entries = [
+            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TvpIE.ie_key())
+            for v_path in videos_paths]
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': display_id,
+            'title': title,
+            'entries': entries,
         }

youtube_dl/extractor/veehd.py

@@ -8,6 +8,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    ExtractorError,
     clean_html,
     get_element_by_id,
 )
@@ -17,13 +18,13 @@ class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://veehd.com/video/4686958',
+        'url': 'http://veehd.com/video/4639434_Solar-Sinter',
         'info_dict': {
-            'id': '4686958',
+            'id': '4639434',
             'ext': 'mp4',
-            'title': 'Time Lapse View from Space ( ISS)',
-            'uploader_id': 'spotted',
-            'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
+            'title': 'Solar Sinter',
+            'uploader_id': 'VideoEyes',
+            'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
         },
     }
 
@@ -34,6 +35,10 @@ class VeeHDIE(InfoExtractor):
         # See https://github.com/rg3/youtube-dl/issues/2102
         self._download_webpage(url, video_id, 'Requesting webpage')
         webpage = self._download_webpage(url, video_id)
+
+        if 'This video has been removed<' in webpage:
+            raise ExtractorError('Video %s has been removed' % video_id, expected=True)
+
         player_path = self._search_regex(
             r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
             webpage, 'player path')
@@ -42,18 +47,35 @@ class VeeHDIE(InfoExtractor):
         self._download_webpage(player_url, video_id, 'Requesting player page')
         player_page = self._download_webpage(
             player_url, video_id, 'Downloading player page')
-        config_json = self._search_regex(
-            r'value=\'config=({.+?})\'', player_page, 'config json')
-        config = json.loads(config_json)
 
-        video_url = compat_urlparse.unquote(config['clip']['url'])
+        config_json = self._search_regex(
+            r'value=\'config=({.+?})\'', player_page, 'config json', default=None)
+
+        if config_json:
+            config = json.loads(config_json)
+            video_url = compat_urlparse.unquote(config['clip']['url'])
+        else:
+            iframe_src = self._search_regex(
+                r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
+            iframe_url = 'http://veehd.com/%s' % iframe_src
+
+            self._download_webpage(iframe_url, video_id, 'Requesting iframe page')
+            iframe_page = self._download_webpage(
+                iframe_url, video_id, 'Downloading iframe page')
+
+            video_url = self._search_regex(
+                r"file\s*:\s*'([^']+)'", iframe_page, 'video url')
+
         title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
-        uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-                                              webpage, 'uploader')
-        thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-                                       webpage, 'thumbnail')
-        description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-                                              webpage, 'description', flags=re.DOTALL)
+        uploader_id = self._html_search_regex(
+            r'<a href="/profile/\d+">(.+?)</a>',
+            webpage, 'uploader')
+        thumbnail = self._search_regex(
+            r'<img id="veehdpreview" src="(.+?)"',
+            webpage, 'thumbnail')
+        description = self._html_search_regex(
+            r'<td class="infodropdown".*?<div>(.*?)<ul',
+            webpage, 'description', flags=re.DOTALL)
 
         return {
             '_type': 'video',

youtube_dl/extractor/vk.py

@@ -166,7 +166,6 @@ class VKIE(InfoExtractor):
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
-        assert m_rutube
         if m_rutube is not None:
             self.to_screen('rutube video detected')
             rutube_url = self._proto_relative_url(

youtube_dl/extractor/washingtonpost.py

@@ -10,14 +10,14 @@ from ..utils import (
 
 
 class WashingtonPostIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
     _TEST = {
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
-            'md5': 'c3f4b4922ffa259243f68e928db2db8c',
+            'md5': '79132cc09ec5309fa590ae46e4cc31bc',
             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
@@ -29,7 +29,7 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
             },
         }, {
-            'md5': 'f645a07652c2950cd9134bb852c5f5eb',
+            'md5': 'e1d5734c06865cc504ad99dc2de0d443',
             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
@@ -44,10 +44,9 @@ class WashingtonPostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('id')
-
+        page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
+
         title = self._og_search_title(webpage)
         uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
         entries = []

youtube_dl/extractor/wdr.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -67,6 +68,10 @@ class WDRIE(InfoExtractor):
                 'upload_date': '20140717',
             },
         },
+        {
+            'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
+            'playlist_mincount': 146,
+        }
     ]
 
     def _real_extract(self, url):
@@ -81,6 +86,27 @@ class WDRIE(InfoExtractor):
                 self.url_result(page_url + href, 'WDR')
                 for href in re.findall(r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX, webpage)
             ]
+
+            if entries:  # Playlist page
+                return self.playlist_result(entries, page_id)
+
+            # Overview page
+            entries = []
+            for page_num in itertools.count(2):
+                hrefs = re.findall(
+                    r'<li class="mediathekvideo"\s*>\s*<img[^>]*>\s*<a href="(/mediathek/video/[^"]+)"',
+                    webpage)
+                entries.extend(
+                    self.url_result(page_url + href, 'WDR')
+                    for href in hrefs)
+                next_url_m = re.search(
+                    r'<li class="nextToLast">\s*<a href="([^"]+)"', webpage)
+                if not next_url_m:
+                    break
+                next_url = page_url + next_url_m.group(1)
+                webpage = self._download_webpage(
+                    next_url, page_id,
+                    note='Downloading playlist page %d' % page_num)
             return self.playlist_result(entries, page_id)
 
         flashvars = compat_parse_qs(
@@ -172,8 +198,7 @@ class WDRMausIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         param_code = self._html_search_regex(
@@ -224,5 +249,3 @@ class WDRMausIE(InfoExtractor):
             'thumbnail': thumbnail,
             'upload_date': upload_date,
         }
-
-# TODO test _1

youtube_dl/extractor/xboxclips.py

@@ -30,7 +30,7 @@ class XboxClipsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
-            r'>(?:Link|Download): <a href="([^"]+)">', webpage, 'video URL')
+            r'>(?:Link|Download): <a[^>]+href="([^"]+)"', webpage, 'video URL')
         title = self._html_search_regex(
             r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
         upload_date = unified_strdate(self._html_search_regex(

youtube_dl/extractor/youtube.py

@@ -264,9 +264,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
@@ -414,7 +414,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'id': 'HtVdAasjOgU',
                 'ext': 'mp4',
                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
-                'description': 'md5:eca57043abae25130f58f655ad9a7771',
+                'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',
@@ -465,6 +465,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # Non-square pixels
+        {
+            'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
+            'info_dict': {
+                'id': '_b-2C3KPAM0',
+                'ext': 'mp4',
+                'stretched_ratio': 16 / 9.,
+                'upload_date': '20110310',
+                'uploader_id': 'AllenMeow',
+                'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
+                'uploader': '孫艾倫',
+                'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1051,6 +1065,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                             f['preference'] = f.get('preference', 0) - 10000
                     formats.extend(dash_formats)
 
+        # Check for malformed aspect ratio
+        stretched_m = re.search(
+            r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
+            video_webpage)
+        if stretched_m:
+            ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+            for f in formats:
+                if f.get('vcodec') != 'none':
+                    f['stretched_ratio'] = ratio
+
         self._sort_formats(formats)
 
         return {

youtube_dl/extractor/zdf.py

@@ -119,7 +119,7 @@ class ZDFChannelIE(InfoExtractor):
         'info_dict': {
             'id': '1586442',
         },
-        'playlist_count': 4,
+        'playlist_count': 3,
     }
     _PAGE_SIZE = 50
 

youtube_dl/options.py

@@ -148,14 +148,6 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
-    general.add_option(
-        '--proxy', dest='proxy',
-        default=None, metavar='URL',
-        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
-    general.add_option(
-        '--socket-timeout',
-        dest='socket_timeout', type=float, default=None,
-        help='Time to wait before giving up, in seconds')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
@@ -173,6 +165,31 @@ def parseOpts(overrideArguments=None):
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
 
+    network = optparse.OptionGroup(parser, 'Network Options')
+    network.add_option(
+        '--proxy', dest='proxy',
+        default=None, metavar='URL',
+        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
+    network.add_option(
+        '--socket-timeout',
+        dest='socket_timeout', type=float, default=None, metavar='SECONDS',
+        help='Time to wait before giving up, in seconds')
+    network.add_option(
+        '--source-address',
+        metavar='IP', dest='source_address', default=None,
+        help='Client-side IP address to bind to (experimental)',
+    )
+    network.add_option(
+        '-4', '--force-ipv4',
+        action='store_const', const='0.0.0.0', dest='source_address',
+        help='Make all connections via IPv4 (experimental)',
+    )
+    network.add_option(
+        '-6', '--force-ipv6',
+        action='store_const', const='::', dest='source_address',
+        help='Make all connections via IPv6 (experimental)',
+    )
+
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
         '--playlist-start',
@@ -267,10 +284,12 @@ def parseOpts(overrideArguments=None):
         action='store', dest='format', metavar='FORMAT', default=None,
         help=(
             'video format code, specify the order of preference using'
-            ' slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also'
-            ' supported. You can also use the special names "best",'
-            ' "bestvideo", "bestaudio", "worst", "worstvideo" and'
-            ' "worstaudio". By default, youtube-dl will pick the best quality.'
+            ' slashes, as in -f 22/17/18 . '
+            ' Instead of format codes, you can select by extension for the '
+            'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
+            'You can also use the special names "best",'
+            ' "bestvideo", "bestaudio", "worst". '
+            ' By default, youtube-dl will pick the best quality.'
             ' Use commas to download multiple audio formats, such as'
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
             ' You can merge the video and audio of two formats into a single'
@@ -300,6 +319,12 @@ def parseOpts(overrideArguments=None):
         '--youtube-skip-dash-manifest',
         action='store_false', dest='youtube_include_dash_manifest',
         help='Do not download the DASH manifest on YouTube videos')
+    video_format.add_option(
+        '--merge-output-format',
+        action='store', dest='merge_output_format', metavar='FORMAT', default=None,
+        help=(
+            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.'
+            'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
     subtitles.add_option(
@@ -443,6 +468,11 @@ def parseOpts(overrideArguments=None):
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,
         help='simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
+    verbosity.add_option(
+        '--print-json',
+        action='store_true', dest='print_json', default=False,
+        help='Be quiet and print the video information as JSON (video is still being downloaded).',
+    )
     verbosity.add_option(
         '--newline',
         action='store_true', dest='progress_with_newline', default=False,
@@ -475,6 +505,14 @@ def parseOpts(overrideArguments=None):
         '--print-traffic',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
+    verbosity.add_option(
+        '-C', '--call-home',
+        dest='call_home', action='store_true', default=False,
+        help='Contact the youtube-dl server for debugging.')
+    verbosity.add_option(
+        '--no-call-home',
+        dest='call_home', action='store_false', default=False,
+        help='Do NOT contact the youtube-dl server for debugging.')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(
@@ -618,6 +656,13 @@ def parseOpts(overrideArguments=None):
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
         help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+    postproc.add_option(
+        '--fixup',
+        metavar='POLICY', dest='fixup', default='detect_or_warn',
+        help='(experimental) Automatically correct known faults of the file. '
+             'One of never (do nothing), warn (only emit a warning), '
+             'detect_or_warn(check whether we can do anything about it, warn '
+             'otherwise')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',
@@ -632,6 +677,7 @@ def parseOpts(overrideArguments=None):
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
 
     parser.add_option_group(general)
+    parser.add_option_group(network)
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)

youtube_dl/postprocessor/__init__.py

@@ -6,6 +6,7 @@ from .ffmpeg import (
     FFmpegAudioFixPP,
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
+    FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -24,6 +25,7 @@ __all__ = [
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',

youtube_dl/postprocessor/ffmpeg.py

@@ -50,6 +50,10 @@ class FFmpegPostProcessor(PostProcessor):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
 
+    @property
+    def available(self):
+        return self._executable is not None
+
     @property
     def _executable(self):
         if self._downloader.params.get('prefer_ffmpeg', False):
@@ -78,11 +82,15 @@ class FFmpegPostProcessor(PostProcessor):
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
 
+        oldest_mtime = min(
+            os.stat(encodeFilename(path)).st_mtime for path in input_paths)
+
         files_cmd = []
         for path in input_paths:
-            files_cmd.extend(['-i', encodeFilename(path, True)])
-        cmd = ([self._executable, '-y'] + files_cmd
-               + [encodeArgument(o) for o in opts] +
+            files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
+        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+               files_cmd +
+               [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
         if self._downloader.params.get('verbose', False):
@@ -93,6 +101,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
+        os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)
@@ -122,8 +131,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                self._probe_executable,
-                '-show_streams',
+                encodeFilename(self._probe_executable, True),
+                encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]
@@ -539,3 +548,22 @@ class FFmpegAudioFixPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegFixupStretchedPP(FFmpegPostProcessor):
+    def run(self, info):
+        stretched_ratio = info.get('stretched_ratio')
+        if stretched_ratio is None or stretched_ratio == 1:
+            return
+
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio]
+        self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return True, info

youtube_dl/update.py

@@ -13,6 +13,7 @@ from .compat import (
     compat_str,
     compat_urllib_request,
 )
+from .utils import make_HTTPS_handler
 from .version import __version__
 
 
@@ -58,9 +59,12 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
+    https_handler = make_HTTPS_handler({})
+    opener = compat_urllib_request.build_opener(https_handler)
+
     # Check if there is a new version
     try:
-        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+        newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
     except:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
@@ -72,7 +76,7 @@ def update_self(to_screen, verbose):
 
     # Download and check versions info
     try:
-        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = opener.open(JSON_URL).read().decode('utf-8')
         versions_info = json.loads(versions_info)
     except:
         if verbose:
@@ -120,7 +124,7 @@ def update_self(to_screen, verbose):
             return
 
         try:
-            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            urlh = opener.open(version['exe'][0])
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):
@@ -166,7 +170,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
     # Zip unix package
     elif isinstance(globals().get('__loader__'), zipimporter):
         try:
-            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            urlh = opener.open(version['bin'][0])
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):

youtube_dl/utils.py

@@ -10,6 +10,7 @@ import ctypes
 import datetime
 import email.utils
 import errno
+import functools
 import gzip
 import itertools
 import io
@@ -34,7 +35,9 @@ from .compat import (
     compat_chr,
     compat_getenv,
     compat_html_entities,
+    compat_http_client,
     compat_parse_qs,
+    compat_socket_create_connection,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
@@ -205,6 +208,10 @@ def get_element_by_attribute(attribute, value, html):
 
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
+
+    if html is None:  # Convenience for sanitizing descriptions etc.
+        return html
+
     # Newline vs <br />
     html = html.replace('\n', ' ')
     html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
@@ -280,6 +287,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
+    # Handle timestamps
+    s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
     result = ''.join(map(replace_insane, s))
     if not is_id:
         while '__' in result:
@@ -387,13 +396,14 @@ def formatSeconds(secs):
         return '%d' % secs
 
 
-def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
+def make_HTTPS_handler(params, **kwargs):
+    opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:
-            return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+            return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
         except TypeError:
             # Python 2.7.8
             # (create_default_context present but HTTPSHandler has no context=)
@@ -416,17 +426,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
                 except ssl.SSLError:
                     self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
 
-        class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler):
-            def https_open(self, req):
-                return self.do_open(HTTPSConnectionV3, req)
-        return HTTPSHandlerV3(**kwargs)
+        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+        return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
 class ExtractorError(Exception):
@@ -540,6 +547,26 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
+def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
+    hc = http_class(*args, **kwargs)
+    source_address = ydl_handler._params.get('source_address')
+    if source_address is not None:
+        sa = (source_address, 0)
+        if hasattr(hc, 'source_address'):  # Python 2.7+
+            hc.source_address = sa
+        else:  # Python 2.6
+            def _hc_connect(self, *args, **kwargs):
+                sock = compat_socket_create_connection(
+                    (self.host, self.port), self.timeout, sa)
+                if is_https:
+                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                else:
+                    self.sock = sock
+            hc.connect = functools.partial(_hc_connect, hc)
+
+    return hc
+
+
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -558,6 +585,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     public domain.
     """
 
+    def __init__(self, params, *args, **kwargs):
+        compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
+        self._params = params
+
+    def http_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, compat_http_client.HTTPConnection, False),
+            req)
+
     @staticmethod
     def deflate(data):
         try:
@@ -627,6 +663,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
+class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
+    def __init__(self, params, https_conn_class=None, *args, **kwargs):
+        compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
+        self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
+        self._params = params
+
+    def https_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, self._https_conn_class, True),
+            req)
+
+
 def parse_iso8601(date_str, delimiter='T'):
     """ Return a UNIX timestamp from the given date """
 
@@ -675,9 +723,6 @@ def unified_strdate(date_str, day_first=True):
         '%b %dth %Y %I:%M%p',
         '%Y-%m-%d',
         '%Y/%m/%d',
-        '%d.%m.%Y',
-        '%d/%m/%Y',
-        '%d/%m/%y',
         '%Y/%m/%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S.%f',
@@ -692,10 +737,16 @@ def unified_strdate(date_str, day_first=True):
     ]
     if day_first:
         format_expressions.extend([
+            '%d.%m.%Y',
+            '%d/%m/%Y',
+            '%d/%m/%y',
             '%d/%m/%Y %H:%M:%S',
         ])
     else:
         format_expressions.extend([
+            '%m.%d.%Y',
+            '%m/%d/%Y',
+            '%m/%d/%y',
             '%m/%d/%Y %H:%M:%S',
         ])
     for expression in format_expressions:

youtube_dl/version.py

@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07.2'
+__version__ = '2015.01.11'