release 2013.02.25

MANIFEST.in

@@ -1,3 +1,5 @@
 include README.md
 include test/*.py
 include test/*.json
+include youtube-dl.bash-completion
+include youtube-dl.1

devscripts/release.sh

@@ -22,7 +22,7 @@ if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit
 
 /bin/echo -e "\n### First of all, testing..."
 make cleanall
-nosetests --with-coverage --cover-package=youtube_dl --cover-html test || exit 1
+nosetests --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1
 
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py

test/tests.json

@@ -293,7 +293,8 @@
     "file": "102.mp4",
     "md5": "7bc087e71d16f18f9b8ab9fa62a8a031",
     "info_dict": {
-        "title": "Dan Dennett: The illusion of consciousness"
+        "title": "Dan Dennett: The illusion of consciousness",
+        "thumbnail": "http://images.ted.com/images/ted/488_389x292.jpg"
     }
   },
   {

youtube_dl/FileDownloader.py

@@ -370,12 +370,10 @@ class FileDownloader(object):
         title = info_dict['title']
         matchtitle = self.params.get('matchtitle', False)
         if matchtitle:
-            matchtitle = matchtitle.decode('utf8')
             if not re.search(matchtitle, title, re.IGNORECASE):
                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
         rejecttitle = self.params.get('rejecttitle', False)
         if rejecttitle:
-            rejecttitle = rejecttitle.decode('utf8')
             if re.search(rejecttitle, title, re.IGNORECASE):
                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
         return None

youtube_dl/InfoExtractors.py

@@ -1330,7 +1330,7 @@ class GenericIE(InfoExtractor):
         opener = compat_urllib_request.OpenerDirector()
         for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
                         HTTPMethodFallback, HEADRedirectHandler,
-                        compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
+                        compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
             opener.add_handler(handler())
 
         response = opener.open(HeadRequest(url))
@@ -1366,6 +1366,9 @@ class GenericIE(InfoExtractor):
         if mobj is None:
             # Broaden the search a little bit
             mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
+        if mobj is None:
+            # Broaden the search a little bit: JWPlayer JS loader
+            mobj = re.search(r'[^A-Za-z0-9]?file:\s*["\'](http[^\'"&]*)', webpage)
         if mobj is None:
             self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
             return
@@ -1469,7 +1472,7 @@ class YoutubeSearchIE(InfoExtractor):
             result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
             request = compat_urllib_request.Request(result_url)
             try:
-                data = compat_urllib_request.urlopen(request).read()
+                data = compat_urllib_request.urlopen(request).read().decode('utf-8')
             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                 self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
                 return
@@ -1918,9 +1921,8 @@ class BlipTVUserIE(InfoExtractor):
 
         while True:
             self.report_download_page(username, pagenum)
-
-            request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) )
-
+            url = page_base + "&page=" + str(pagenum)
+            request = compat_urllib_request.Request( url )
             try:
                 page = compat_urllib_request.urlopen(request).read().decode('utf-8')
             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
@@ -3624,18 +3626,22 @@ class SteamIE(InfoExtractor):
         mweb = re.finditer(urlRE, webpage)
         namesRE = r'<span class="title">(?P<videoName>.+?)</span>'
         titles = re.finditer(namesRE, webpage)
+        thumbsRE = r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">'
+        thumbs = re.finditer(thumbsRE, webpage)
         videos = []
-        for vid,vtitle in zip(mweb,titles):
+        for vid,vtitle,thumb in zip(mweb,titles,thumbs):
             video_id = vid.group('videoID')
             title = vtitle.group('videoName')
             video_url = vid.group('videoURL')
+            video_thumb = thumb.group('thumbnail')
             if not video_url:
                 self._downloader.trouble(u'ERROR: Cannot find video url for %s' % video_id)
             info = {
                 'id':video_id,
                 'url':video_url,
                 'ext': 'flv',
-                'title': unescapeHTML(title)
+                'title': unescapeHTML(title),
+                'thumbnail': video_thumb
                   }
             videos.append(info)
         return videos
@@ -4008,31 +4014,30 @@ class TEDIE(InfoExtractor):
                      ([.\s]*?)data-playlist_item_id="(\d+)"
                      ([.\s]*?)data-mediaslug="(?P<mediaSlug>.+?)"
                      '''
-        video_name_RE=r'<p\ class="talk-title"><a href="/talks/(.+).html">(?P<fullname>.+?)</a></p>'
+        video_name_RE=r'<p\ class="talk-title"><a href="(?P<talk_url>/talks/(.+).html)">(?P<fullname>.+?)</a></p>'
         webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage')
         m_videos=re.finditer(video_RE,webpage,re.VERBOSE)
         m_names=re.finditer(video_name_RE,webpage)
         info=[]
         for m_video, m_name in zip(m_videos,m_names):
-            video_dic={
-                       'id': m_video.group('video_id'),
-                       'url': self._talk_video_link(m_video.group('mediaSlug')),
-                       'ext': 'mp4',
-                       'title': m_name.group('fullname')
-                       }
-            info.append(video_dic)
+            video_id=m_video.group('video_id')
+            talk_url='http://www.ted.com%s' % m_name.group('talk_url')
+            info.append(self._talk_info(talk_url,video_id))
         return info
+
     def _talk_info(self, url, video_id=0):
         """Return the video for the talk in the url"""
         m=re.match(self._VALID_URL, url,re.VERBOSE)
         videoName=m.group('name')
         webpage=self._download_webpage(url, video_id, 'Downloading \"%s\" page' % videoName)
         # If the url includes the language we get the title translated
-        title_RE=r'<h1><span id="altHeadline" >(?P<title>[\s\w:/\.\?=\+-\\\']*)</span></h1>'
+        title_RE=r'<h1><span id="altHeadline" >(?P<title>.*)</span></h1>'
         title=re.search(title_RE, webpage).group('title')
         info_RE=r'''<script\ type="text/javascript">var\ talkDetails\ =(.*?)
                         "id":(?P<videoID>[\d]+).*?
                         "mediaSlug":"(?P<mediaSlug>[\w\d]+?)"'''
+        thumb_RE=r'</span>[\s.]*</div>[\s.]*<img src="(?P<thumbnail>.*?)"'
+        thumb_match=re.search(thumb_RE,webpage)
         info_match=re.search(info_RE,webpage,re.VERBOSE)
         video_id=info_match.group('videoID')
         mediaSlug=info_match.group('mediaSlug')
@@ -4041,7 +4046,8 @@ class TEDIE(InfoExtractor):
                 'id': video_id,
                 'url': video_url,
                 'ext': 'mp4',
-                'title': title
+                'title': title,
+                'thumbnail': thumb_match.group('thumbnail')
                 }
         return info
 

youtube_dl/__init__.py

@@ -412,6 +412,7 @@ def _real_main():
             or (opts.useid and u'%(id)s.%(ext)s')
             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
             or u'%(id)s.%(ext)s')
+
     # File downloader
     fd = FileDownloader({
         'usenetrc': opts.usenetrc,
@@ -450,8 +451,8 @@ def _real_main():
         'writeinfojson': opts.writeinfojson,
         'writesubtitles': opts.writesubtitles,
         'subtitleslang': opts.subtitleslang,
-        'matchtitle': opts.matchtitle,
-        'rejecttitle': opts.rejecttitle,
+        'matchtitle': decodeOption(opts.matchtitle),
+        'rejecttitle': decodeOption(opts.rejecttitle),
         'max_downloads': opts.max_downloads,
         'prefer_free_formats': opts.prefer_free_formats,
         'verbose': opts.verbose,

youtube_dl/update.py

@@ -77,10 +77,8 @@ def update_self(to_screen, verbose, filename):
 
     to_screen(u'Updating to version ' + versions_info['latest'] + '...')
     version = versions_info['versions'][versions_info['latest']]
-    if version.get('notes'):
-        to_screen(u'PLEASE NOTE:')
-        for note in version['notes']:
-            to_screen(note)
+
+    print_notes(versions_info['versions'])
 
     if not os.access(filename, os.W_OK):
         to_screen(u'ERROR: no write permissions on %s' % filename)
@@ -158,3 +156,13 @@ del "%s"
             return
 
     to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+
+def print_notes(versions, fromVersion=__version__):
+    notes = []
+    for v,vdata in sorted(versions.items()):
+        if v > fromVersion:
+            notes.extend(vdata.get('notes', []))
+    if notes:
+        to_screen(u'PLEASE NOTE:')
+        for note in notes:
+            to_screen(note)

youtube_dl/utils.py

@@ -420,6 +420,14 @@ def encodeFilename(s):
             encoding = 'utf-8'
         return s.encode(encoding, 'ignore')
 
+def decodeOption(optval):
+    if optval is None:
+        return optval
+    if isinstance(optval, bytes):
+        optval = optval.decode(preferredencoding())
+
+    assert isinstance(optval, compat_str)
+    return optval
 
 class ExtractorError(Exception):
     """Error during info extraction."""

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.02.18'
+__version__ = '2013.02.25'