release 2014.09.14.3

.gitignore

@@ -11,6 +11,7 @@ MANIFEST
 README.txt
 youtube-dl.1
 youtube-dl.bash-completion
+youtube-dl.fish
 youtube-dl
 youtube-dl.exe
 youtube-dl.tar.gz

MANIFEST.in

@@ -2,5 +2,6 @@ include README.md
 include test/*.py
 include test/*.json
 include youtube-dl.bash-completion
+include youtube-dl.fish
 include youtube-dl.1
 recursive-include docs Makefile conf.py *.rst

Makefile

@@ -1,7 +1,7 @@
-all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.fish
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe
@@ -29,6 +29,8 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
 	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
+	install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions
+	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
@@ -36,9 +38,9 @@ test:
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files
+.PHONY: all clean install test tar bash-completion pypi-files fish-completion
 
-pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1
+pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 
 youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py
@@ -64,7 +66,12 @@ youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-co
 
 bash-completion: youtube-dl.bash-completion
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
+youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
+	python devscripts/fish-completion.py
+
+fish-completion: youtube-dl.fish
+
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.fish
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -78,5 +85,6 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		-- \
 		bin devscripts test youtube_dl docs \
 		LICENSE README.md README.txt \
-		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \
+		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
+		youtube-dl.fish setup.py \
 		youtube-dl

devscripts/fish-completion.in

@@ -0,0 +1,5 @@
+
+{{commands}}
+
+
+complete --command youtube-dl --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory"

devscripts/fish-completion.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import optparse
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+from youtube_dl.utils import shell_quote
+
+FISH_COMPLETION_FILE = 'youtube-dl.fish'
+FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'
+
+EXTRA_ARGS = {
+    'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'],
+
+    # Options that need a file parameter
+    'download-archive': ['--require-parameter'],
+    'cookies': ['--require-parameter'],
+    'load-info': ['--require-parameter'],
+    'batch-file': ['--require-parameter'],
+}
+
+def build_completion(opt_parser):
+    commands = []
+
+    for group in opt_parser.option_groups:
+        for option in group.option_list:
+            long_option = option.get_opt_string().strip('-')
+            help_msg = shell_quote([option.help])
+            complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option]
+            if option._short_opts:
+                complete_cmd += ['--short-option', option._short_opts[0].strip('-')]
+            if option.help != optparse.SUPPRESS_HELP:
+                complete_cmd += ['--description', option.help]
+            complete_cmd.extend(EXTRA_ARGS.get(long_option, []))
+            commands.append(shell_quote(complete_cmd))
+
+    with open(FISH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+    filled_template = template.replace('{{commands}}', '\n'.join(commands))
+    with open(FISH_COMPLETION_FILE, 'w') as f:
+        f.write(filled_template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)

setup.py

@@ -48,6 +48,7 @@ if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
 else:
     files_spec = [
         ('etc/bash_completion.d', ['youtube-dl.bash-completion']),
+        ('etc/fish/completions', ['youtube-dl.fish']),
         ('share/doc/youtube_dl', ['README.txt']),
         ('share/man/man1', ['youtube-dl.1'])
     ]

youtube_dl/extractor/cloudy.py

@@ -9,6 +9,8 @@ from ..utils import (
     compat_parse_qs,
     compat_urllib_parse,
     remove_end,
+    HEADRequest,
+    compat_HTTPError,
 )
 
 
@@ -21,6 +23,7 @@ class CloudyIE(InfoExtractor):
         '''
     _EMBED_URL = 'http://www.%s/embed.php?id=%s'
     _API_URL = 'http://www.%s/api/player.api.php?%s'
+    _MAX_TRIES = 2
     _TESTS = [
         {
             'url': 'https://www.cloudy.ec/v/af511e2527aac',
@@ -42,24 +45,30 @@ class CloudyIE(InfoExtractor):
         }
     ]
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_host = mobj.group('host')
-        video_id = mobj.group('id')
+    def _extract_video(self, video_host, video_id, file_key, error_url=None, try_num=0):
 
-        url = self._EMBED_URL % (video_host, video_id)
-        webpage = self._download_webpage(url, video_id)
+        if try_num > self._MAX_TRIES - 1:
+            raise ExtractorError('Unable to extract video URL', expected=True)
 
-        file_key = self._search_regex(
-            r'filekey\s*=\s*"([^"]+)"', webpage, 'file_key')
-        data_url = self._API_URL % (video_host, compat_urllib_parse.urlencode({
+        form = {
             'file': video_id,
             'key': file_key,
-        }))
+        }
+
+        if error_url:
+            form.update({
+                'numOfErrors': try_num,
+                'errorCode': '404',
+                'errorUrl': error_url,
+            })
+
+        data_url = self._API_URL % (video_host, compat_urllib_parse.urlencode(form))
         player_data = self._download_webpage(
             data_url, video_id, 'Downloading player data')
         data = compat_parse_qs(player_data)
 
+        try_num += 1
+
         if 'error' in data:
             raise ExtractorError(
                 '%s error: %s' % (self.IE_NAME, ' '.join(data['error_msg'])),
@@ -69,16 +78,31 @@ class CloudyIE(InfoExtractor):
         if title:
             title = remove_end(title, '&asdasdas').strip()
 
-        formats = []
         video_url = data.get('url', [None])[0]
+
         if video_url:
-            formats.append({
-                'format_id': 'sd',
-                'url': video_url,
-            })
+            try:
+                self._request_webpage(HEADRequest(video_url), video_id, 'Checking video URL')
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code in [404, 410]:
+                    self.report_warning('Invalid video URL, requesting another', video_id)
+                    return self._extract_video(video_host, video_id, file_key, video_url, try_num)
 
         return {
             'id': video_id,
+            'url': video_url,
             'title': title,
-            'formats': formats,
         }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_host = mobj.group('host')
+        video_id = mobj.group('id')
+
+        url = self._EMBED_URL % (video_host, video_id)
+        webpage = self._download_webpage(url, video_id)
+
+        file_key = self._search_regex(
+            r'filekey\s*=\s*"([^"]+)"', webpage, 'file_key')
+
+        return self._extract_video(video_host, video_id, file_key)

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.14.1'
+__version__ = '2014.09.14.3'