release 2015.02.10.3

This functionality is intended to eventually encompass the current format filtering.

README.md

@@ -77,6 +77,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
+    --no-color                       Do not emit color codes in output.
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
@@ -119,6 +120,23 @@ which means you can modify it, redistribute it or use it however you like.
                                      COUNT views
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
+    --match-filter FILTER            (Experimental) Generic video filter.
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use  --match-filter "like_count > 100 &
+                                     dislike_count <? 50 & description" .
     --no-playlist                    If the URL refers to a video and a
                                      playlist, download only the video.
     --age-limit YEARS                download only videos suitable for the given

docs/supportedsites.md

@@ -225,6 +225,7 @@
  - **mailru**: Видео@Mail.Ru
  - **Malemotion**
  - **MDR**
+ - **media.ccc.de**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**

test/test_utils.py

@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     version_tuple,
     xpath_with_ns,
     render_table,
+    match_str,
 )
 
 
@@ -459,6 +460,37 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             '123  4\n'
             '9999 51')
 
+    def test_match_str(self):
+        self.assertRaises(ValueError, match_str, 'xy>foobar', {})
+        self.assertFalse(match_str('xy', {'x': 1200}))
+        self.assertTrue(match_str('!xy', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 1200}))
+        self.assertFalse(match_str('!x', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 0}))
+        self.assertFalse(match_str('x>0', {'x': 0}))
+        self.assertFalse(match_str('x>0', {}))
+        self.assertTrue(match_str('x>?0', {}))
+        self.assertTrue(match_str('x>1K', {'x': 1200}))
+        self.assertFalse(match_str('x>2K', {'x': 1200}))
+        self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200}))
+        self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200}))
+        self.assertFalse(match_str('y=a212', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'}))
+        self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 90, 'description': 'foo'}))
+        self.assertTrue(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 60, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 10}))
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_youtube_signature.py

@@ -8,11 +8,11 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import io
 import re
 import string
 
+from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
@@ -88,7 +88,8 @@ def make_tfunc(url, stype, sig_input, expected_sig):
         if not os.path.exists(fn):
             compat_urlretrieve(url, fn)
 
-        ie = YoutubeIE()
+        ydl = FakeYDL()
+        ie = YoutubeIE(ydl)
         if stype == 'js':
             with io.open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()

youtube_dl/YoutubeDL.py

@@ -228,6 +228,12 @@ class YoutubeDL(object):
     external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
+    match_filter:      A function that gets called with the info_dict of
+                       every video.
+                       If it returns a message, the video is ignored.
+                       If it returns None, the video is downloaded.
+                       match_filter_func in utils.py is one example for this.
+    no_color:          Do not emit color codes in output.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -485,7 +491,7 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if self._err_file.isatty() and os.name != 'nt':
+            if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
                 _msg_header = '\033[0;33mWARNING:\033[0m'
             else:
                 _msg_header = 'WARNING:'
@@ -497,7 +503,7 @@ class YoutubeDL(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if self._err_file.isatty() and os.name != 'nt':
+        if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
@@ -583,9 +589,16 @@ class YoutubeDL(object):
             if max_views is not None and view_count > max_views:
                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
-            return 'Skipping "%s" because it is age restricted' % title
+            return 'Skipping "%s" because it is age restricted' % video_title
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
+
+        match_filter = self.params.get('match_filter')
+        if match_filter is not None:
+            ret = match_filter(info_dict)
+            if ret is not None:
+                return ret
+
         return None
 
     @staticmethod

youtube_dl/__init__.py

@@ -23,9 +23,10 @@ from .compat import (
 )
 from .utils import (
     DateRange,
-    DEFAULT_OUTTMPL,
     decodeOption,
+    DEFAULT_OUTTMPL,
     DownloadError,
+    match_filter_func,
     MaxDownloadsReached,
     preferredencoding,
     read_batch_urls,
@@ -247,6 +248,9 @@ def _real_main(argv=None):
             xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
+    match_filter = (
+        None if opts.match_filter is None
+        else match_filter_func(opts.match_filter))
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -344,6 +348,8 @@ def _real_main(argv=None):
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
+        'match_filter': match_filter,
+        'no_color': opts.no_color,
     }
 
     with YoutubeDL(ydl_opts) as ydl:

youtube_dl/extractor/__init__.py

@@ -54,6 +54,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE

youtube_dl/extractor/ccc.py

@@ -0,0 +1,99 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    qualities,
+    unified_strdate,
+)
+
+
+class CCCIE(InfoExtractor):
+    IE_NAME = 'media.ccc.de'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+
+    _TEST = {
+        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
+        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'info_dict': {
+            'id': '20131228183',
+            'ext': 'mp4',
+            'title': 'Introduction to Processor Design',
+            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'upload_date': '20131229',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        if self._downloader.params.get('prefer_free_formats'):
+            preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd'])
+        else:
+            preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd'])
+
+        title = self._html_search_regex(
+            r'(?s)<h1>(.*?)</h1>', webpage, 'title')
+        description = self._html_search_regex(
+            r"(?s)<p class='description'>(.*?)</p>",
+            webpage, 'description', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
+            webpage, 'upload date', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
+            webpage, 'view count', fatal=False))
+
+        matches = re.finditer(r'''(?xs)
+            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
+            <a\s+href='(?P<http_url>[^']+)'>\s*
+            (?:
+                .*?
+                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
+            )?''', webpage)
+        formats = []
+        for m in matches:
+            format = m.group('format')
+            format_id = self._search_regex(
+                r'.*/([a-z0-9_-]+)/[^/]*$',
+                m.group('http_url'), 'format id', default=None)
+            vcodec = 'h264' if 'h264' in format_id else (
+                'none' if format_id in ('mp3', 'opus') else None
+            )
+            formats.append({
+                'format_id': format_id,
+                'format': format,
+                'url': m.group('http_url'),
+                'vcodec': vcodec,
+                'preference': preference(format_id),
+            })
+
+            if m.group('torrent_url'):
+                formats.append({
+                    'format_id': 'torrent-%s' % (format if format_id is None else format_id),
+                    'format': '%s (torrent)' % format,
+                    'proto': 'torrent',
+                    'format_note': '(unsupported; will just download the .torrent file)',
+                    'vcodec': vcodec,
+                    'preference': -100 + preference(format_id),
+                    'url': m.group('torrent_url'),
+                })
+        self._sort_formats(formats)
+
+        thumbnail = self._html_search_regex(
+            r"<video.*?poster='([^']+)'", webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'view_count': view_count,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

youtube_dl/extractor/common.py

@@ -514,7 +514,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name

youtube_dl/extractor/firstpost.py

@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,11 +18,10 @@ class FirstpostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
-        title = self._html_search_meta('twitter:title', page, 'title')
+
+        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
         description = self._html_search_meta('twitter:description', page, 'title')
 
         data = self._download_xml(
@@ -42,6 +39,7 @@ class FirstpostIE(InfoExtractor):
                 'height': int(details.find('./height').text.strip()),
             } for details in item.findall('./source/file_details') if details.find('./file').text
         ]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

youtube_dl/extractor/pornhd.py

@@ -46,16 +46,17 @@ class PornHdIE(InfoExtractor):
 
         quality = qualities(['sd', 'hd'])
         sources = json.loads(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
+            webpage, 'sources')))
         formats = []
-        for container, s in sources.items():
-            for qname, video_url in s.items():
-                formats.append({
-                    'url': video_url,
-                    'container': container,
-                    'format_id': '%s-%s' % (container, qname),
-                    'quality': quality(qname),
-                })
+        for qname, video_url in sources.items():
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': qname,
+                'quality': quality(qname),
+            })
         self._sort_formats(formats)
 
         return {

youtube_dl/extractor/vimeo.py

@@ -188,9 +188,9 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         password_request = compat_urllib_request.Request(pass_url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(password_request, video_id,
-                               'Verifying the password',
-                               'Wrong password')
+        return self._download_webpage(
+            password_request, video_id,
+            'Verifying the password', 'Wrong password')
 
     def _verify_player_video_password(self, url, video_id):
         password = self._downloader.params.get('videopassword', None)
@@ -266,7 +266,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
-            if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
+            if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(url)
             else:
@@ -412,12 +412,47 @@ class VimeoChannelIE(InfoExtractor):
     def _extract_list_title(self, webpage):
         return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
+    def _login_list_password(self, page_url, list_id, webpage):
+        login_form = self._search_regex(
+            r'(?s)<form[^>]+?id="pw_form"(.*?)</form>',
+            webpage, 'login form', default=None)
+        if not login_form:
+            return webpage
+
+        password = self._downloader.params.get('videopassword', None)
+        if password is None:
+            raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            value="([^"]*)"
+            ''', login_form))
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        fields['token'] = token
+        fields['password'] = password
+        post = compat_urllib_parse.urlencode(fields)
+        password_path = self._search_regex(
+            r'action="([^"]+)"', login_form, 'password URL')
+        password_url = compat_urlparse.urljoin(page_url, password_path)
+        password_request = compat_urllib_request.Request(password_url, post)
+        password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        self._set_cookie('vimeo.com', 'xsrft', token)
+
+        return self._download_webpage(
+            password_request, list_id,
+            'Verifying the password', 'Wrong password')
+
     def _extract_videos(self, list_id, base_url):
         video_ids = []
         for pagenum in itertools.count(1):
+            page_url = self._page_url(base_url, pagenum)
             webpage = self._download_webpage(
-                self._page_url(base_url, pagenum), list_id,
+                page_url, list_id,
                 'Downloading page %s' % pagenum)
+
+            if pagenum == 1:
+                webpage = self._login_list_password(page_url, list_id, webpage)
+
             video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -464,14 +499,24 @@ class VimeoAlbumIE(VimeoChannelIE):
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
+    }, {
+        'note': 'Password-protected album',
+        'url': 'https://vimeo.com/album/3253534',
+        'info_dict': {
+            'title': 'test',
+            'id': '3253534',
+        },
+        'playlist_count': 1,
+        'params': {
+            'videopassword': 'youtube-dl',
+        }
     }]
 
     def _page_url(self, base_url, pagenum):
         return '%s/page:%d/' % (base_url, pagenum)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        album_id = mobj.group('id')
+        album_id = self._match_id(url)
         return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
 
 

youtube_dl/options.py

@@ -165,6 +165,11 @@ def parseOpts(overrideArguments=None):
         action='store_const', dest='extract_flat', const='in_playlist',
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
+    general.add_option(
+        '--no-color', '--no-colors',
+        action='store_true', dest='no_color',
+        default=False,
+        help='Do not emit color codes in output.')
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(
@@ -244,6 +249,25 @@ def parseOpts(overrideArguments=None):
         '--max-views',
         metavar='COUNT', dest='max_views', default=None, type=int,
         help='Do not download any videos with more than COUNT views')
+    selection.add_option(
+        '--match-filter',
+        metavar='FILTER', dest='match_filter', default=None,
+        help=(
+            '(Experimental) Generic video filter. '
+            'Specify any key (see help for -o for a list of available keys) to'
+            ' match if the key is present, '
+            '!key to check if the key is not present,'
+            'key > NUMBER (like "comment_count > 12", also works with '
+            '>=, <, <=, !=, =) to compare against a number, and '
+            '& to require multiple matches. '
+            'Values which are not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            'For example, to only match videos that have been liked more than '
+            '100 times and disliked less than 50 times (or the dislike '
+            'functionality is not available at the given service), but who '
+            'also have a description, use  --match-filter '
+            '"like_count > 100 & dislike_count <? 50 & description" .'
+        ))
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
@@ -533,7 +557,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='youtube_print_sig_code', default=False,
         help=optparse.SUPPRESS_HELP)
     verbosity.add_option(
-        '--print-traffic',
+        '--print-traffic', '--dump-headers',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
     verbosity.add_option(

youtube_dl/utils.py

@@ -17,6 +17,7 @@ import io
 import json
 import locale
 import math
+import operator
 import os
 import pipes
 import platform
@@ -1678,3 +1679,79 @@ def render_table(header_row, data):
     max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
     format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
     return '\n'.join(format_str % tuple(row) for row in table)
+
+
+def _match_one(filter_part, dct):
+    COMPARISON_OPERATORS = {
+        '<': operator.lt,
+        '<=': operator.le,
+        '>': operator.gt,
+        '>=': operator.ge,
+        '=': operator.eq,
+        '!=': operator.ne,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<key>[a-z_]+)
+        \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+        (?:
+            (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
+            (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
+        )
+        \s*$
+        ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = COMPARISON_OPERATORS[m.group('op')]
+        if m.group('strval') is not None:
+            if m.group('op') not in ('=', '!='):
+                raise ValueError(
+                    'Operator %s does not support string values!' % m.group('op'))
+            comparison_value = m.group('strval')
+        else:
+            try:
+                comparison_value = int(m.group('intval'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('intval'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('intval') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid integer value %r in filter part %r' % (
+                            m.group('intval'), filter_part))
+        actual_value = dct.get(m.group('key'))
+        if actual_value is None:
+            return m.group('none_inclusive')
+        return op(actual_value, comparison_value)
+
+    UNARY_OPERATORS = {
+        '': lambda v: v is not None,
+        '!': lambda v: v is None,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<op>%s)\s*(?P<key>[a-z_]+)
+        \s*$
+        ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = UNARY_OPERATORS[m.group('op')]
+        actual_value = dct.get(m.group('key'))
+        return op(actual_value)
+
+    raise ValueError('Invalid filter part %r' % filter_part)
+
+
+def match_str(filter_str, dct):
+    """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
+
+    return all(
+        _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
+
+
+def match_filter_func(filter_str):
+    def _match_func(info_dict):
+        if match_str(filter_str, info_dict):
+            return None
+        else:
+            video_title = info_dict.get('title', info_dict.get('id', 'video'))
+            return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
+    return _match_func

youtube_dl/version.py

@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.1'
+__version__ = '2015.02.10.3'