release 2013.02.18

enables easier process monitoring when being called with external
scripts.

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 *.pyo
 *~
+*.DS_Store
 wine-py2exe/
 py2exe.log
 *.kate-swp
@@ -13,3 +14,7 @@ youtube-dl.bash-completion
 youtube-dl
 youtube-dl.exe
 youtube-dl.tar.gz
+.coverage
+cover/
+updates_key.pem
+*.egg-info

.travis.yml

@@ -2,12 +2,13 @@ language: python
 python:
   - "2.6"
   - "2.7"
-#  - "3.3"
+  - "3.3"
 script: nosetests test --verbose
 notifications:
   email:
     - filippo.valsorda@gmail.com
-  irc:
-    channels:
-      - "irc.freenode.org#youtube-dl"
-    skip_join: true
+    - phihag@phihag.de
+#  irc:
+#    channels:
+#      - "irc.freenode.org#youtube-dl"
+#    skip_join: true

CHANGELOG

@@ -0,0 +1,14 @@
+2013.01.02  Codename: GIULIA
+
+    * Add support for ComedyCentral clips <nto>
+    * Corrected Vimeo description fetching <Nick Daniels>
+    * Added the --no-post-overwrites argument <Barbu Paul - Gheorghe>
+    * --verbose offers more environment info
+    * New info_dict field: uploader_id
+    * New updates system, with signature checking
+    * New IEs: NBA, JustinTV, FunnyOrDie, TweetReel, Steam, Ustream
+    * Fixed IEs: BlipTv
+    * Fixed for Python 3 IEs: Xvideo, Youku, XNXX, Dailymotion, Vimeo, InfoQ
+    * Simplified IEs and test code
+    * Various (Python 3 and other) fixes
+    * Revamped and expanded tests

LATEST_VERSION

@@ -1 +1 @@
-9999.99.99
+2012.12.99

LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

Makefile

@@ -1,12 +1,16 @@
 all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
 
 clean:
-	rm -rf youtube-dl youtube-dl.exe youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/
+	rm -rf youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz
+
+cleanall: clean
+	rm -f youtube-dl youtube-dl.exe
 
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/man
 SYSCONFDIR=/etc
+PYTHON=/usr/bin/env python
 
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -d $(DESTDIR)$(BINDIR)
@@ -17,14 +21,19 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
 
 test:
-	nosetests2 --nocapture test
+	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
+	nosetests --verbose test
 
-.PHONY: all clean install test
+tar: youtube-dl.tar.gz
+
+.PHONY: all clean install test tar bash-completion pypi-files
+
+pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1
 
 youtube-dl: youtube_dl/*.py
 	zip --quiet youtube-dl youtube_dl/*.py
 	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
-	echo '#!/usr/bin/env python' > youtube-dl
+	echo '#!$(PYTHON)' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
 	chmod a+x youtube-dl
@@ -38,11 +47,22 @@ README.txt: README.md
 youtube-dl.1: README.md
 	pandoc -s -f markdown -t man README.md -o youtube-dl.1
 
-youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.template
+youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.in
 	python devscripts/bash-completion.py
 
-youtube-dl.tar.gz: all
-	tar -czf youtube-dl.tar.gz -s "|^./|./youtube-dl/|" \
-		--exclude="*.pyc" --exclude="*.pyo" --exclude="*~" --exclude="youtube-dl.exe" \
-		--exclude="wine-py2exe/" --exclude="py2exe.log" --exclude="*.kate-swp" \
-		--exclude="build/" --exclude="dist/" --exclude="MANIFEST" --exclude=".git/" .
+bash-completion: youtube-dl.bash-completion
+
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
+	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
+		--exclude '*.DS_Store' \
+		--exclude '*.kate-swp' \
+		--exclude '*.pyc' \
+		--exclude '*.pyo' \
+		--exclude '*~' \
+		--exclude '__pycache' \
+		--exclude '.git' \
+		-- \
+		bin devscripts test youtube_dl \
+		CHANGELOG LICENSE README.md README.txt \
+		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \
+		youtube-dl

README.md

@@ -9,8 +9,8 @@ youtube-dl
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
-2.x (x being at least 6), and it is not platform specific. It should work in
-your Unix box, in Windows or in Mac OS X. It is released to the public domain,
+2.6, 2.7, or 3.3+, and it is not platform specific. It should work on
+your Unix box, on Windows or on Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
 # OPTIONS
@@ -38,6 +38,10 @@ which means you can modify it, redistribute it or use it however you like.
     --reject-title REGEX     skip download for matching titles (regex or
                              caseless sub-string)
     --max-downloads NUMBER   Abort after downloading NUMBER files
+    --min-filesize SIZE      Do not download any videos smaller than SIZE (e.g.
+                             50k or 44.6m)
+    --max-filesize SIZE      Do not download any videos larger than SIZE (e.g.
+                             50k or 44.6m)
 
 ## Filesystem Options:
     -t, --title              use title in file name
@@ -46,15 +50,16 @@ which means you can modify it, redistribute it or use it however you like.
     -A, --auto-number        number downloaded files starting from 00000
     -o, --output TEMPLATE    output filename template. Use %(title)s to get the
                              title, %(uploader)s for the uploader name,
-                             %(autonumber)s to get an automatically incremented
-                             number, %(ext)s for the filename extension,
-                             %(upload_date)s for the upload date (YYYYMMDD),
-                             %(extractor)s for the provider (youtube, metacafe,
-                             etc), %(id)s for the video id and %% for a literal
-                             percent. Use - to output to stdout. Can also be
-                             used to download to a different directory, for
-                             example with -o '/my/downloads/%(uploader)s/%(title
-                             )s-%(id)s.%(ext)s' .
+                             %(uploader_id)s for the uploader nickname if
+                             different, %(autonumber)s to get an automatically
+                             incremented number, %(ext)s for the filename
+                             extension, %(upload_date)s for the upload date
+                             (YYYYMMDD), %(extractor)s for the provider
+                             (youtube, metacafe, etc), %(id)s for the video id
+                             and %% for a literal percent. Use - to output to
+                             stdout. Can also be used to download to a different
+                             directory, for example with -o '/my/downloads/%(upl
+                             oader)s/%(title)s-%(id)s.%(ext)s' .
     --restrict-filenames     Restrict filenames to only ASCII characters, and
                              avoid "&" and spaces in filenames
     -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
@@ -80,6 +85,7 @@ which means you can modify it, redistribute it or use it however you like.
     --get-description        simulate, quiet but print video description
     --get-filename           simulate, quiet but print output filename
     --get-format             simulate, quiet but print output format
+    --newline                output progress bar as new lines
     --no-progress            do not print progress bar
     --console-title          display progress in console titlebar
     -v, --verbose            print various debugging information
@@ -104,17 +110,21 @@ which means you can modify it, redistribute it or use it however you like.
 ## Post-processing Options:
     -x, --extract-audio      convert video files to audio-only files (requires
                              ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
-                             best by default
+    --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", "opus", or
+                             "wav"; best by default
     --audio-quality QUALITY  ffmpeg/avconv audio quality specification, insert a
                              value between 0 (better) and 9 (worse) for VBR or a
                              specific bitrate like 128K (default 5)
+    --recode-video FORMAT    Encode the video to another format if necessary
+                             (currently supported: mp4|flv|ogg|webm)
     -k, --keep-video         keeps the video file on disk after the post-
                              processing; the video is erased by default
+    --no-post-overwrites     do not overwrite post-processed files; the post-
+                             processed files are overwritten by default
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`.
 
 # OUTPUT TEMPLATE
 
@@ -194,6 +204,7 @@ Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/i
 Please include:
 
 * Your exact command line, like `youtube-dl -t "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+* If possible re-run the command with `--verbose`, and include the full output, it is really helpful to us.
 * The output of `youtube-dl --version`
 * The output of `python --version`
 * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).

devscripts/bash-completion.py

@@ -7,7 +7,7 @@ sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
 
 BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
-BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.template"
+BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
 
 def build_completion(opt_parser):
     opts_flag = []

devscripts/gh-pages/add-version.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+import hashlib
+import urllib.request
+
+if len(sys.argv) <= 1:
+	print('Specify the version number as parameter')
+	sys.exit()
+version = sys.argv[1]
+
+with open('update/LATEST_VERSION', 'w') as f:
+	f.write(version)
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+new_version = {}
+
+filenames = {'bin': 'youtube-dl', 'exe': 'youtube-dl.exe', 'tar': 'youtube-dl-%s.tar.gz' % version}
+for key, filename in filenames.items():
+	print('Downloading and checksumming %s...' %filename)
+	url = 'http://youtube-dl.org/downloads/%s/%s' % (version, filename)
+	data = urllib.request.urlopen(url).read()
+	sha256sum = hashlib.sha256(data).hexdigest()
+	new_version[key] = (url, sha256sum)
+
+versions_info['versions'][version] = new_version
+versions_info['latest'] = version
+
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)

devscripts/gh-pages/generate-download.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import hashlib
+import shutil
+import subprocess
+import tempfile
+import urllib.request
+import json
+
+versions_info = json.load(open('update/versions.json'))
+version = versions_info['latest']
+URL = versions_info['versions'][version]['bin'][0]
+
+data = urllib.request.urlopen(URL).read()
+
+# Read template page
+with open('download.html.in', 'r', encoding='utf-8') as tmplf:
+    template = tmplf.read()
+
+md5sum = hashlib.md5(data).hexdigest()
+sha1sum = hashlib.sha1(data).hexdigest()
+sha256sum = hashlib.sha256(data).hexdigest()
+template = template.replace('@PROGRAM_VERSION@', version)
+template = template.replace('@PROGRAM_URL@', URL)
+template = template.replace('@PROGRAM_MD5SUM@', md5sum)
+template = template.replace('@PROGRAM_SHA1SUM@', sha1sum)
+template = template.replace('@PROGRAM_SHA256SUM@', sha256sum)
+template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0])
+template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1])
+template = template.replace('@TAR_URL@', versions_info['versions'][version]['tar'][0])
+template = template.replace('@TAR_SHA256SUM@', versions_info['versions'][version]['tar'][1])
+with open('download.html', 'w', encoding='utf-8') as dlf:
+    dlf.write(template)

devscripts/gh-pages/sign-versions.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import rsa
+import json
+from binascii import hexlify
+
+try:
+    input = raw_input
+except NameError:
+    pass
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+print('Enter the PKCS1 private key, followed by a blank line:')
+privkey = b''
+while True:
+	try:
+		line = input()
+	except EOFError:
+		break
+	if line == '':
+		break
+	privkey += line.encode('ascii') + b'\n'
+privkey = rsa.PrivateKey.load_pkcs1(privkey)
+
+signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
+print('signature: ' + signature)
+
+versions_info['signature'] = signature
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)

devscripts/gh-pages/update-copyright.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import with_statement
+
+import datetime
+import glob
+import io # For Python 2 compatibilty
+import os
+import re
+
+year = str(datetime.datetime.now().year)
+for fn in glob.glob('*.html*'):
+    with io.open(fn, encoding='utf-8') as f:
+        content = f.read()
+    newc = re.sub(u'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', u'Copyright © 2006-' + year, content)
+    if content != newc:
+        tmpFn = fn + '.part'
+        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
+            outf.write(newc)
+        os.rename(tmpFn, fn)

devscripts/release.sh

@@ -1,11 +1,91 @@
-#!/bin/sh
+#!/bin/bash
+
+# IMPORTANT: the following assumptions are made
+# * the GH repo is on the origin remote
+# * the gh-pages branch is named so locally
+# * the git config user.signingkey is properly set
+
+# You will need
+# pip install coverage nose rsa
+
+# TODO
+# release notes
+# make hash on local files
+
+set -e
 
 if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
-if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
-sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
-make all
-git add -A
+if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
+
+/bin/echo -e "\n### First of all, testing..."
+make cleanall
+nosetests --with-coverage --cover-package=youtube_dl --cover-html test || exit 1
+
+/bin/echo -e "\n### Changing version in version.py..."
+sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
+
+/bin/echo -e "\n### Committing CHANGELOG README.md and youtube_dl/version.py..."
+make README.md
+git add CHANGELOG README.md youtube_dl/version.py
 git commit -m "release $version"
-git tag -m "Release $version" "$version"
+
+/bin/echo -e "\n### Now tagging, signing and pushing..."
+git tag -s -m "Release $version" "$version"
+git show "$version"
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+MASTER=$(git rev-parse --abbrev-ref HEAD)
+git push origin $MASTER:master
+git push origin "$version"
+
+/bin/echo -e "\n### OK, now it is time to build the binaries..."
+REV=$(git rev-parse HEAD)
+make youtube-dl youtube-dl.tar.gz
+wget "http://jeromelaheurte.net:8142/download/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe || \
+	wget "http://jeromelaheurte.net:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
+mkdir -p "build/$version"
+mv youtube-dl youtube-dl.exe "build/$version"
+mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"
+RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
+(cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS)
+(cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS)
+(cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
+(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
+git checkout HEAD -- youtube-dl youtube-dl.exe
+
+/bin/echo -e "\n### Signing and uploading the new binaries to youtube-dl.org..."
+for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done
+scp -r "build/$version" ytdl@youtube-dl.org:html/downloads/
+
+/bin/echo -e "\n### Now switching to gh-pages..."
+git clone --branch gh-pages --single-branch . build/gh-pages
+ROOT=$(pwd)
+(
+    set -e
+    ORIGIN_URL=$(git config --get remote.origin.url)
+    cd build/gh-pages
+    "$ROOT/devscripts/gh-pages/add-version.py" $version
+    "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
+    "$ROOT/devscripts/gh-pages/generate-download.py"
+    "$ROOT/devscripts/gh-pages/update-copyright.py"
+    git add *.html *.html.in update
+    git commit -m "release $version"
+    git show HEAD
+    read -p "Is it good, can I push? (y/n) " -n 1
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+    echo
+    git push "$ROOT" gh-pages
+    git push "$ORIGIN_URL" gh-pages
+)
+rm -rf build
+
+make pypi-files
+echo "Uploading to PyPi ..."
+python setup.py sdist upload
+make clean
+
+/bin/echo -e "\n### DONE!"

devscripts/transition_helper_exe/youtube-dl.py

@@ -2,17 +2,48 @@
 
 import sys, os
 import urllib2
+import json, hashlib
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
 
 sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
 sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
-sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
 
 raw_input()
 
 filename = sys.argv[0]
 
-API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
-EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe"
+UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+JSON_URL = UPDATE_URL + 'versions.json'
+UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
 
 if not os.access(filename, os.W_OK):
     sys.exit('ERROR: no write permissions on %s' % filename)
@@ -23,13 +54,35 @@ if not os.access(directory, os.W_OK):
     sys.exit('ERROR: no write permissions on %s' % directory)
 
 try:
-    urlh = urllib2.urlopen(EXE_URL)
+    versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8')
+    versions_info = json.loads(versions_info)
+except:
+    sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
+if not 'signature' in versions_info:
+    sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+signature = versions_info['signature']
+del versions_info['signature']
+if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
+    sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
+
+version = versions_info['versions'][versions_info['latest']]
+
+try:
+    urlh = urllib2.urlopen(version['exe'][0])
     newcontent = urlh.read()
     urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+if newcontent_hash != version['exe'][1]:
+    sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
+
+try:
     with open(exe + '.new', 'wb') as outf:
         outf.write(newcontent)
 except (IOError, OSError) as err:
-    sys.exit('ERROR: unable to download latest version')
+    sys.exit(u'ERROR: unable to write the new version')
 
 try:
     bat = os.path.join(directory, 'youtube-dl-updater.bat')

setup.py

@@ -2,10 +2,14 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import print_function
-from distutils.core import setup
 import pkg_resources
 import sys
 
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
 try:
     import py2exe
     """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""

test/gentests.py

@@ -1,132 +0,0 @@
-#!/usr/bin/env python3
-
-import io  # for python 2
-import json
-import os
-import sys
-import unittest
-
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import youtube_dl.InfoExtractors
-
-HEADER = u'''#!/usr/bin/env python
-
-# DO NOT EDIT THIS FILE BY HAND!
-# It is auto-generated from tests.json and gentests.py.
-
-import hashlib
-import io
-import os
-import json
-import unittest
-import sys
-import socket
-
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-import youtube_dl.FileDownloader
-import youtube_dl.InfoExtractors
-from youtube_dl.utils import *
-
-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
-socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
-
-class FileDownloader(youtube_dl.FileDownloader):
-    def __init__(self, *args, **kwargs):
-        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
-        self.to_stderr = self.to_screen
-
-def _file_md5(fn):
-    with open(fn, 'rb') as f:
-        return hashlib.md5(f.read()).hexdigest()
-try:
-    _skip_unless = unittest.skipUnless
-except AttributeError: # Python 2.6
-    def _skip_unless(cond, reason='No reason given'):
-        def resfunc(f):
-            # Start the function name with test to appease nosetests-2.6
-            def test_wfunc(*args, **kwargs):
-                if cond:
-                    return f(*args, **kwargs)
-                else:
-                    print('Skipped test')
-                    return
-            return test_wfunc
-        return resfunc
-_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
-
-class DownloadTest(unittest.TestCase):
-    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
-
-    def setUp(self):
-        # Clear old files
-        self.tearDown()
-
-        with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf:
-            self.parameters = json.load(pf)
-'''
-
-FOOTER = u'''
-
-if __name__ == '__main__':
-    unittest.main()
-'''
-
-DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
-TEST_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'test_download.py')
-
-def gentests():
-    with io.open(DEF_FILE, encoding='utf-8') as deff:
-        defs = json.load(deff)
-    with io.open(TEST_FILE, 'w', encoding='utf-8') as testf:
-        testf.write(HEADER)
-        spaces = ' ' * 4
-        write = lambda l: testf.write(spaces + l + u'\n')
-
-        for d in defs:
-            name = d['name']
-            ie = getattr(youtube_dl.InfoExtractors, name + 'IE')
-            testf.write(u'\n')
-            write('@_skip_unless(youtube_dl.InfoExtractors.' + name + 'IE._WORKING, "IE marked as not _WORKING")')
-            if not d['file']:
-                write('@_skip("No output file specified")')
-            elif 'skip' in d:
-                write('@_skip(' + repr(d['skip']) + ')')
-            write('def test_' + name + '(self):')
-            write('    filename = ' + repr(d['file']))
-            write('    params = self.parameters')
-            for p in d.get('params', {}):
-                write('    params["' + p + '"] = ' + repr(d['params'][p]))
-            write('    fd = FileDownloader(params)')
-            write('    fd.add_info_extractor(youtube_dl.InfoExtractors.' + name + 'IE())')
-            for ien in d.get('addIEs', []):
-                write('    fd.add_info_extractor(youtube_dl.InfoExtractors.' + ien + 'IE())')
-            write('    fd.download([' + repr(d['url']) + '])')
-            write('    self.assertTrue(os.path.exists(filename))')
-            if 'md5' in d:
-                write('    md5_for_file = _file_md5(filename)')
-                write('    self.assertEqual(md5_for_file, ' + repr(d['md5']) + ')')
-
-        testf.write(u'\n\n')
-        write('def tearDown(self):')
-        for d in defs:
-            if d['file']:
-                write('    if os.path.exists(' + repr(d['file']) + '):')
-                write('        os.remove(' + repr(d['file']) + ')')
-            else:
-                write('    # No file specified for ' + d['name'])
-        testf.write(u'\n')
-        testf.write(FOOTER)
-
-if __name__ == '__main__':
-    gentests()

test/parameters.json

@@ -35,6 +35,6 @@
     "username": null, 
     "verbose": true, 
     "writedescription": false, 
-    "writeinfojson": false, 
+    "writeinfojson": true, 
     "writesubtitles": false
 }

test/test_all_urls.py

@@ -9,8 +9,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
 
-class TestYoutubePlaylistMatching(unittest.TestCase):
-    def test_playlist_matching(self):
+class TestAllURLsMatching(unittest.TestCase):
+    def test_youtube_playlist_matching(self):
         self.assertTrue(YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
         self.assertTrue(YoutubePlaylistIE().suitable(u'PL63F0C78739B09958'))
         self.assertFalse(YoutubePlaylistIE().suitable(u'PLtS2H6bU1M'))
@@ -18,5 +18,10 @@ class TestYoutubePlaylistMatching(unittest.TestCase):
     def test_youtube_matching(self):
         self.assertTrue(YoutubeIE().suitable(u'PLtS2H6bU1M'))
 
+    def test_youtube_extract(self):
+        self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc'), 'BaW_jenozKc')
+
 if __name__ == '__main__':
     unittest.main()

test/test_download.py

@@ -1,197 +1,127 @@
 #!/usr/bin/env python
 
-# DO NOT EDIT THIS FILE BY HAND!
-# It is auto-generated from tests.json and gentests.py.
-
+import errno
 import hashlib
 import io
 import os
 import json
 import unittest
 import sys
+import hashlib
 import socket
 
 # Allow direct execution
-import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import youtube_dl.FileDownloader
 import youtube_dl.InfoExtractors
 from youtube_dl.utils import *
 
+DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
 # General configuration (from __init__, not very elegant...)
 jar = compat_cookiejar.CookieJar()
 cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 proxy_handler = compat_urllib_request.ProxyHandler()
 opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
 compat_urllib_request.install_opener(opener)
-socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
+socket.setdefaulttimeout(10)
+
+def _try_rm(filename):
+    """ Remove a file if it exists """
+    try:
+        os.remove(filename)
+    except OSError as ose:
+        if ose.errno != errno.ENOENT:
+            raise
 
 class FileDownloader(youtube_dl.FileDownloader):
     def __init__(self, *args, **kwargs):
-        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
         self.to_stderr = self.to_screen
+        self.processed_info_dicts = []
+        return youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+    def process_info(self, info_dict):
+        self.processed_info_dicts.append(info_dict)
+        return youtube_dl.FileDownloader.process_info(self, info_dict)
 
 def _file_md5(fn):
     with open(fn, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
-try:
-    _skip_unless = unittest.skipUnless
-except AttributeError: # Python 2.6
-    def _skip_unless(cond, reason='No reason given'):
-        def resfunc(f):
-            # Start the function name with test to appease nosetests-2.6
-            def test_wfunc(*args, **kwargs):
-                if cond:
-                    return f(*args, **kwargs)
-                else:
-                    print('Skipped test')
-                    return
-            return test_wfunc
-        return resfunc
-_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
 
-class DownloadTest(unittest.TestCase):
-    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(DEF_FILE, encoding='utf-8') as deff:
+    defs = json.load(deff)
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
 
+
+class TestDownload(unittest.TestCase):
     def setUp(self):
-        # Clear old files
-        self.tearDown()
+        self.parameters = parameters
+        self.defs = defs
 
-        with io.open(self.PARAMETERS_FILE, encoding='utf-8') as pf:
-            self.parameters = json.load(pf)
+### Dynamically generate tests
+def generator(test_case):
+
+    def test_template(self):
+        ie = getattr(youtube_dl.InfoExtractors, test_case['name'] + 'IE')
+        if not ie._WORKING:
+            print('Skipping: IE marked as not _WORKING')
+            return
+        if 'playlist' not in test_case and not test_case['file']:
+            print('Skipping: No output file specified')
+            return
+        if 'skip' in test_case:
+            print('Skipping: {0}'.format(test_case['skip']))
+            return
+
+        params = self.parameters.copy()
+        params.update(test_case.get('params', {}))
 
-    @_skip_unless(youtube_dl.InfoExtractors.YoutubeIE._WORKING, "IE marked as not _WORKING")
-    def test_Youtube(self):
-        filename = u'BaW_jenozKc.mp4'
-        params = self.parameters
         fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
-        fd.download([u'http://www.youtube.com/watch?v=BaW_jenozKc'])
-        self.assertTrue(os.path.exists(filename))
+        fd.add_info_extractor(ie())
+        for ien in test_case.get('add_ie', []):
+            fd.add_info_extractor(getattr(youtube_dl.InfoExtractors, ien + 'IE')())
+        finished_hook_called = set()
+        def _hook(status):
+            if status['status'] == 'finished':
+                finished_hook_called.add(status['filename'])
+        fd.add_progress_hook(_hook)
 
-    @_skip_unless(youtube_dl.InfoExtractors.DailymotionIE._WORKING, "IE marked as not _WORKING")
-    def test_Dailymotion(self):
-        filename = u'x33vw9.mp4'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.DailymotionIE())
-        fd.download([u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'392c4b85a60a90dc4792da41ce3144eb')
+        test_cases = test_case.get('playlist', [test_case])
+        for tc in test_cases:
+            _try_rm(tc['file'])
+            _try_rm(tc['file'] + '.part')
+            _try_rm(tc['file'] + '.info.json')
+        try:
+            fd.download([test_case['url']])
 
-    @_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
-    def test_Metacafe(self):
-        filename = u'_aUehQsCQtM.flv'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
-        fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
-        fd.download([u'http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/'])
-        self.assertTrue(os.path.exists(filename))
+            for tc in test_cases:
+                if not test_case.get('params', {}).get('skip_download', False):
+                    self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
+                    self.assertTrue(tc['file'] in finished_hook_called)
+                self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
+                if 'md5' in tc:
+                    md5_for_file = _file_md5(tc['file'])
+                    self.assertEqual(md5_for_file, tc['md5'])
+                with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
+                    info_dict = json.load(infof)
+                for (info_field, value) in tc.get('info_dict', {}).items():
+                    self.assertEqual(value, info_dict.get(info_field))
+        finally:
+            for tc in test_cases:
+                _try_rm(tc['file'])
+                _try_rm(tc['file'] + '.part')
+                _try_rm(tc['file'] + '.info.json')
 
-    @_skip_unless(youtube_dl.InfoExtractors.BlipTVIE._WORKING, "IE marked as not _WORKING")
-    def test_BlipTV(self):
-        filename = u'5779306.m4v'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.BlipTVIE())
-        fd.download([u'http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'b2d849efcf7ee18917e4b4d9ff37cafe')
-
-    @_skip_unless(youtube_dl.InfoExtractors.XVideosIE._WORKING, "IE marked as not _WORKING")
-    def test_XVideos(self):
-        filename = u'939581.flv'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.XVideosIE())
-        fd.download([u'http://www.xvideos.com/video939581/funny_porns_by_s_-1'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'1d0c835822f0a71a7bf011855db929d0')
-
-    @_skip_unless(youtube_dl.InfoExtractors.VimeoIE._WORKING, "IE marked as not _WORKING")
-    def test_Vimeo(self):
-        filename = u'14160053.mp4'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.VimeoIE())
-        fd.download([u'http://vimeo.com/14160053'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'60540a4ec7cc378ec84b919c0aed5023')
-
-    @_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
-    def test_Soundcloud(self):
-        filename = u'62986583.mp3'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
-        fd.download([u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'ebef0a451b909710ed1d7787dddbf0d7')
-
-    @_skip_unless(youtube_dl.InfoExtractors.StanfordOpenClassroomIE._WORKING, "IE marked as not _WORKING")
-    def test_StanfordOpenClassroom(self):
-        filename = u'PracticalUnix_intro-environment.mp4'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.StanfordOpenClassroomIE())
-        fd.download([u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'544a9468546059d4e80d76265b0443b8')
-
-    @_skip_unless(youtube_dl.InfoExtractors.XNXXIE._WORKING, "IE marked as not _WORKING")
-    def test_XNXX(self):
-        filename = u'1135332.flv'
-        params = self.parameters
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.XNXXIE())
-        fd.download([u'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'0831677e2b4761795f68d417e0b7b445')
-
-    @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
-    def test_Youku(self):
-        filename = u'XNDgyMDQ2NTQw_part00.flv'
-        params = self.parameters
-        params["test"] = False
-        fd = FileDownloader(params)
-        fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
-        fd.download([u'http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
-        self.assertTrue(os.path.exists(filename))
-        md5_for_file = _file_md5(filename)
-        self.assertEqual(md5_for_file, u'ffe3f2e435663dc2d1eea34faeff5b5b')
-
-
-    def tearDown(self):
-        if os.path.exists(u'BaW_jenozKc.mp4'):
-            os.remove(u'BaW_jenozKc.mp4')
-        if os.path.exists(u'x33vw9.mp4'):
-            os.remove(u'x33vw9.mp4')
-        if os.path.exists(u'_aUehQsCQtM.flv'):
-            os.remove(u'_aUehQsCQtM.flv')
-        if os.path.exists(u'5779306.m4v'):
-            os.remove(u'5779306.m4v')
-        if os.path.exists(u'939581.flv'):
-            os.remove(u'939581.flv')
-        if os.path.exists(u'14160053.mp4'):
-            os.remove(u'14160053.mp4')
-        if os.path.exists(u'62986583.mp3'):
-            os.remove(u'62986583.mp3')
-        if os.path.exists(u'PracticalUnix_intro-environment.mp4'):
-            os.remove(u'PracticalUnix_intro-environment.mp4')
-        if os.path.exists(u'1135332.flv'):
-            os.remove(u'1135332.flv')
-        if os.path.exists(u'XNDgyMDQ2NTQw_part00.flv'):
-            os.remove(u'XNDgyMDQ2NTQw_part00.flv')
+    return test_template
 
+### And add them to TestDownload
+for test_case in defs:
+    test_method = generator(test_case)
+    test_method.__name__ = "test_{0}".format(test_case["name"])
+    setattr(TestDownload, test_method.__name__, test_method)
+    del test_method
 
 
 if __name__ == '__main__':

test/test_utils.py

@@ -82,9 +82,9 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
     def test_sanitize_ids(self):
-        self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
-        self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
-        self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
+        self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+        self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+        self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
 
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])

test/test_write_info_json.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import json
+import os
+import sys
+import unittest
+
+# Allow direct execution
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+        self.to_stderr = self.to_screen
+
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    params = json.load(pf)
+params['writeinfojson'] = True
+params['skip_download'] = True
+params['writedescription'] = True
+
+TEST_ID = 'BaW_jenozKc'
+INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
+DESCRIPTION_FILE = TEST_ID + '.mp4.description'
+EXPECTED_DESCRIPTION = u'''test chars:  "'/\ä↭𝕐
+
+This is a test video for youtube-dl.
+
+For more information, contact phihag@phihag.de .'''
+
+class TestInfoJSON(unittest.TestCase):
+    def setUp(self):
+        # Clear old files
+        self.tearDown()
+
+    def test_info_json(self):
+        ie = youtube_dl.InfoExtractors.YoutubeIE()
+        fd = FileDownloader(params)
+        fd.add_info_extractor(ie)
+        fd.download([TEST_ID])
+        self.assertTrue(os.path.exists(INFO_JSON_FILE))
+        with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
+            jd = json.load(jsonf)
+        self.assertEqual(jd['upload_date'], u'20121002')
+        self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
+        self.assertEqual(jd['id'], TEST_ID)
+        self.assertEqual(jd['extractor'], 'youtube')
+        self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
+        self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
+
+        self.assertTrue(os.path.exists(DESCRIPTION_FILE))
+        with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
+            descr = descf.read()
+        self.assertEqual(descr, EXPECTED_DESCRIPTION)
+
+    def tearDown(self):
+        if os.path.exists(INFO_JSON_FILE):
+            os.remove(INFO_JSON_FILE)
+        if os.path.exists(DESCRIPTION_FILE):
+            os.remove(DESCRIPTION_FILE)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_lists.py

@@ -2,27 +2,30 @@
 
 import sys
 import unittest
-import socket
+import json
 
 # Allow direct execution
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.InfoExtractors import YoutubePlaylistIE
+from youtube_dl.InfoExtractors import YoutubeUserIE,YoutubePlaylistIE
 from youtube_dl.utils import *
 
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
 # General configuration (from __init__, not very elegant...)
 jar = compat_cookiejar.CookieJar()
 cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 proxy_handler = compat_urllib_request.ProxyHandler()
 opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
 compat_urllib_request.install_opener(opener)
-socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
 
 class FakeDownloader(object):
     def __init__(self):
         self.result = []
-        self.params = {}
+        self.params = parameters
     def to_screen(self, s):
         print(s)
     def trouble(self, s):
@@ -57,12 +60,12 @@ class TestYoutubeLists(unittest.TestCase):
         self.assertEqual(DL.result[-1], ['http://www.youtube.com/watch?v=rYefUsYuEp0'])
 
     def test_youtube_channel(self):
-        """I give up, please find a channel that does paginate and test this like test_youtube_playlist_long"""
+        # I give up, please find a channel that does paginate and test this like test_youtube_playlist_long
         pass # TODO
 
     def test_youtube_user(self):
         DL = FakeDownloader()
-        IE = YoutubePlaylistIE(DL)
+        IE = YoutubeUserIE(DL)
         IE.extract('https://www.youtube.com/user/TheLinuxFoundation')
         self.assertTrue(len(DL.result) >= 320)
 

test/test_youtube_subtitles.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+import io
+import hashlib
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(object):
+    def __init__(self):
+        self.result = []
+        self.params = parameters
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class TestYoutubeSubtitles(unittest.TestCase):
+    def test_youtube_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(md5(info_dict[0]['subtitles']), 'c3228550d59116f3c29fba370b55d033')
+
+    def test_youtube_subtitles_it(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitleslang'] = 'it'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(md5(info_dict[0]['subtitles']), '132a88a0daf8e1520f393eb58f1f646a')
+
+if __name__ == '__main__':
+    unittest.main()

test/tests.json

@@ -2,7 +2,14 @@
   {
     "name": "Youtube",
     "url":  "http://www.youtube.com/watch?v=BaW_jenozKc",
-    "file":  "BaW_jenozKc.mp4"
+    "file":  "BaW_jenozKc.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video \"'/\\ä↭𝕐",
+      "uploader": "Philipp Hagemeister",
+      "uploader_id": "phihag",
+      "upload_date": "20121002",
+      "description": "test chars:  \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
+    }
   },
   {
     "name": "Dailymotion",
@@ -12,8 +19,8 @@
   },
   {
     "name": "Metacafe",
-    "addIEs": ["Youtube"],
-    "url":  "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
+    "add_ie": ["Youtube"],
+    "url":  "http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
     "file":  "_aUehQsCQtM.flv"
   },
   {
@@ -28,11 +35,36 @@
     "url":  "http://www.xvideos.com/video939581/funny_porns_by_s_-1",
     "file":  "939581.flv"
   },
+  {
+    "name": "YouPorn",
+    "md5": "c37ddbaaa39058c76a7e86c6813423c1",
+    "url": "http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/",
+    "file": "505835.mp4"
+  },
+  {
+    "name": "Pornotube",
+    "md5": "374dd6dcedd24234453b295209aa69b6",
+    "url": "http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing",
+    "file": "1689755.flv"
+  },
+  {
+    "name": "YouJizz",
+    "md5": "07e15fa469ba384c7693fd246905547c",
+    "url": "http://www.youjizz.com/videos/zeichentrick-1-2189178.html",
+    "file": "2189178.flv"
+  },
   {
     "name": "Vimeo",
-    "md5":  "60540a4ec7cc378ec84b919c0aed5023",
-    "url":  "http://vimeo.com/14160053",
-    "file": "14160053.mp4"
+    "md5":  "8879b6cc097e987f02484baf890129e5",
+    "url":  "http://vimeo.com/56015672",
+    "file": "56015672.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video - ★ \" ' 幸 / \\ ä ↭ 𝕐",
+      "uploader": "Filippo Valsorda",
+      "uploader_id": "user7108434",
+      "upload_date": "20121220",
+      "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: ★ \" ' 幸 / \\ ä ↭ 𝕐"
+    }
   },
   {
     "name": "Soundcloud",
@@ -44,7 +76,8 @@
     "name": "StanfordOpenClassroom",
     "md5":  "544a9468546059d4e80d76265b0443b8",
     "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
-    "file":  "PracticalUnix_intro-environment.mp4"
+    "file":  "PracticalUnix_intro-environment.mp4",
+    "skip": "Currently offline"
   },
   {
     "name": "XNXX",
@@ -58,5 +91,218 @@
     "file": "XNDgyMDQ2NTQw_part00.flv",
     "md5": "ffe3f2e435663dc2d1eea34faeff5b5b",
     "params": { "test": false }
+  },
+  {
+    "name": "NBA",
+    "url": "http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html",
+    "file": "0021200253-okc-bkn-recap.nba.mp4",
+    "md5": "c0edcfc37607344e2ff8f13c378c88a4"
+  },
+  {
+    "name": "JustinTV",
+    "url": "http://www.twitch.tv/thegamedevhub/b/296128360",
+    "file": "296128360.flv",
+    "md5": "ecaa8a790c22a40770901460af191c9a"
+  },
+  {
+    "name": "MyVideo",
+    "url": "http://www.myvideo.de/watch/8229274/bowling_fail_or_win",
+    "file": "8229274.flv",
+    "md5": "2d2753e8130479ba2cb7e0a37002053e"
+  },
+  {
+    "name": "Escapist",
+    "url": "http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate",
+    "file": "6618-Breaking-Down-Baldurs-Gate.flv",
+    "md5": "c6793dbda81388f4264c1ba18684a74d",
+    "skip": "Fails with timeout on Travis"
+  },
+  {
+    "name": "GooglePlus",
+    "url": "https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH",
+    "file": "ZButuJc6CtH.flv"
+  },
+  {
+    "name": "FunnyOrDie",
+    "url": "http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version",
+    "file": "0732f586d7.mp4",
+    "md5": "f647e9e90064b53b6e046e75d0241fbd"
+  },
+  {
+    "name": "TweetReel",
+    "url": "http://tweetreel.com/?77smq",
+    "file": "77smq.mov",
+    "md5": "56b4d9ca9de467920f3f99a6d91255d6",
+    "info_dict": {
+        "uploader": "itszero",
+        "uploader_id": "itszero",
+        "upload_date": "20091225",
+        "description": "Installing Gentoo Linux on Powerbook G4, it turns out the sleep indicator becomes HDD activity indicator :D"
+    }
+  },
+  {
+    "name": "Steam",
+    "url": "http://store.steampowered.com/video/105600/",
+    "playlist": [
+      {
+        "file": "81300.flv",
+        "md5": "f870007cee7065d7c76b88f0a45ecc07",
+        "info_dict": {
+            "title": "Terraria 1.1 Trailer"
+        }
+      },
+      {
+        "file": "80859.flv",
+        "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+        "info_dict": {
+          "title": "Terraria Trailer"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Ustream",
+    "url": "http://www.ustream.tv/recorded/20274954",
+    "file": "20274954.flv",
+    "md5": "088f151799e8f572f84eb62f17d73e5c",
+    "info_dict": {
+        "title": "Young Americans for Liberty February 7, 2012 2:28 AM"
+    }
+  },
+  {
+    "name": "InfoQ",
+    "url": "http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things",
+    "file": "12-jan-pythonthings.mp4",
+    "info_dict": {
+      "title": "A Few of My Favorite [Python] Things"
+    },
+    "params": {
+      "skip_download": true
+    }
+  },
+  {
+    "name": "ComedyCentral",
+    "url": "http://www.thedailyshow.com/watch/thu-december-13-2012/kristen-stewart",
+    "file": "422212.mp4",
+    "md5": "4e2f5cb088a83cd8cdb7756132f9739d",
+    "info_dict": {
+        "title": "thedailyshow-kristen-stewart part 1"
+    }
+  },
+  {
+    "name": "RBMARadio",
+    "url": "http://www.rbmaradio.com/shows/ford-lopatin-live-at-primavera-sound-2011",
+    "file": "ford-lopatin-live-at-primavera-sound-2011.mp3",
+    "md5": "6bc6f9bcb18994b4c983bc3bf4384d95",
+    "info_dict": {
+        "title": "Live at Primavera Sound 2011",
+        "description": "Joel Ford and Daniel \u2019Oneohtrix Point Never\u2019 Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.",
+        "uploader": "Ford & Lopatin",
+        "uploader_id": "ford-lopatin",
+        "location": "Spain"
+    }
+  },
+  {
+    "name": "Facebook",
+    "url": "https://www.facebook.com/photo.php?v=120708114770723",
+    "file": "120708114770723.mp4",
+    "md5": "48975a41ccc4b7a581abd68651c1a5a8",
+    "info_dict": {
+      "title": "PEOPLE ARE AWESOME 2013",
+      "duration": 279
+    }
+  },
+  {
+    "name": "EightTracks",
+    "url": "http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
+    "playlist": [
+      {
+        "file": "11885610.m4a",
+        "md5": "96ce57f24389fc8734ce47f4c1abcc55",
+        "info_dict": {
+          "title": "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885608.m4a",
+        "md5": "4ab26f05c1f7291ea460a3920be8021f",
+        "info_dict": {
+          "title": "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+
+        }
+      },
+      {
+        "file": "11885679.m4a",
+        "md5": "d30b5b5f74217410f4689605c35d1fd7",
+        "info_dict": {
+          "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad"
+        }
+      },
+      {
+        "file": "11885680.m4a",
+        "md5": "4eb0a669317cd725f6bbd336a29f923a",
+        "info_dict": {
+          "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad"
+        }
+      },
+      {
+        "file": "11885682.m4a",
+        "md5": "1893e872e263a2705558d1d319ad19e8",
+        "info_dict": {
+          "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad"
+        }
+      },
+      {
+        "file": "11885683.m4a",
+        "md5": "b673c46f47a216ab1741ae8836af5899",
+        "info_dict": {
+          "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad"
+        }
+      },
+      {
+        "file": "11885684.m4a",
+        "md5": "1d74534e95df54986da7f5abf7d842b7",
+        "info_dict": {
+          "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad"
+        }
+      },
+      {
+        "file": "11885685.m4a",
+        "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
+        "info_dict": {
+          "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Keek",
+    "url": "http://www.keek.com/ytdl/keeks/NODfbab",
+    "file": "NODfbab.mp4",
+    "md5": "9b0636f8c0f7614afa4ea5e4c6e57e83",
+    "info_dict": {
+      "title": "test chars: \"'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de ."
+    }
+
+  },
+  {
+    "name": "TED",
+    "url": "http://www.ted.com/talks/dan_dennett_on_our_consciousness.html",
+    "file": "102.mp4",
+    "md5": "7bc087e71d16f18f9b8ab9fa62a8a031",
+    "info_dict": {
+        "title": "Dan Dennett: The illusion of consciousness"
+    }
+  },
+  {
+    "name": "MySpass",
+    "url": "http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/",
+    "file": "11741.mp4",
+    "md5": "0b49f4844a068f8b33f4b7c88405862b",
+    "info_dict": {
+        "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2"
+    }
   }
 ]

youtube_dl/FileDownloader.py

@@ -4,6 +4,7 @@
 from __future__ import absolute_import
 
 import math
+import io
 import os
 import re
 import socket
@@ -80,6 +81,9 @@ class FileDownloader(object):
     writesubtitles:    Write the video subtitles to a .srt file
     subtitleslang:     Language of the subtitles to download
     test:              Download only first bytes to test the downloader.
+    keepvideo:         Keep the video file after post-processing
+    min_filesize:      Skip files smaller than this size
+    max_filesize:      Skip files larger than this size
     """
 
     params = None
@@ -93,6 +97,7 @@ class FileDownloader(object):
         """Create a FileDownloader object with the given options."""
         self._ies = []
         self._pps = []
+        self._progress_hooks = []
         self._download_retcode = 0
         self._num_downloads = 0
         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
@@ -203,23 +208,28 @@ class FileDownloader(object):
             # already of type unicode()
             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
         elif 'TERM' in os.environ:
-            sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
+            self.to_screen('\033]0;%s\007' % message, skip_eol=True)
 
     def fixed_template(self):
         """Checks if the output template is fixed."""
         return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
 
-    def trouble(self, message=None):
+    def trouble(self, message=None, tb=None):
         """Determine action to take when a download problem appears.
 
         Depending on if the downloader has been configured to ignore
         download errors or not, this method may throw an exception or
         not when errors are found, after printing the message.
+
+        tb, if given, is additional traceback information.
         """
         if message is not None:
             self.to_stderr(message)
         if self.params.get('verbose'):
-            self.to_stderr(u''.join(traceback.format_list(traceback.extract_stack())))
+            if tb is None:
+                tb_data = traceback.format_list(traceback.extract_stack())
+                tb = u''.join(tb_data)
+            self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):
             raise DownloadError(message)
         self._download_retcode = 1
@@ -295,7 +305,11 @@ class FileDownloader(object):
         """Report download progress."""
         if self.params.get('noprogress', False):
             return
-        self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
+        if self.params.get('progress_with_newline', False):
+            self.to_screen(u'[download] %s of %s at %s ETA %s' %
+                (percent_str, data_len_str, speed_str, eta_str))
+        else:
+            self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
         self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
@@ -420,11 +434,8 @@ class FileDownloader(object):
             try:
                 descfn = filename + u'.description'
                 self.report_writedescription(descfn)
-                descfile = open(encodeFilename(descfn), 'wb')
-                try:
-                    descfile.write(info_dict['description'].encode('utf-8'))
-                finally:
-                    descfile.close()
+                with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+                    descfile.write(info_dict['description'])
             except (OSError, IOError):
                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
                 return
@@ -435,11 +446,8 @@ class FileDownloader(object):
             try:
                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
                 self.report_writesubtitles(srtfn)
-                srtfile = open(encodeFilename(srtfn), 'wb')
-                try:
-                    srtfile.write(info_dict['subtitles'].encode('utf-8'))
-                finally:
-                    srtfile.close()
+                with io.open(encodeFilename(srtfn), 'w', encoding='utf-8') as srtfile:
+                    srtfile.write(info_dict['subtitles'])
             except (OSError, IOError):
                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
                 return
@@ -448,17 +456,8 @@ class FileDownloader(object):
             infofn = filename + u'.info.json'
             self.report_writeinfojson(infofn)
             try:
-                json.dump
-            except (NameError,AttributeError):
-                self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
-                return
-            try:
-                infof = open(encodeFilename(infofn), 'wb')
-                try:
-                    json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
-                    json.dump(json_info_dict, infof)
-                finally:
-                    infof.close()
+                json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
+                write_json_file(json_info_dict, encodeFilename(infofn))
             except (OSError, IOError):
                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
                 return
@@ -499,14 +498,28 @@ class FileDownloader(object):
 
                 # Warn if the _WORKING attribute is False
                 if not ie.working():
-                    self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
-                                 u'and will probably not work. If you want to go on, use the -i option.')
+                    self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
+                                   u'and will probably not work. If you want to go on, use the -i option.')
 
                 # Suitable InfoExtractor found
                 suitable_found = True
 
                 # Extract information from URL and process it
-                videos = ie.extract(url)
+                try:
+                    videos = ie.extract(url)
+                except ExtractorError as de: # An error we somewhat expected
+                    self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
+                    break
+                except Exception as e:
+                    if self.params.get('ignoreerrors', False):
+                        self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
+                        break
+                    else:
+                        raise
+
+                if len(videos or []) > 1 and self.fixed_template():
+                    raise SameFileError(self.params['outtmpl'])
+
                 for video in videos or []:
                     video['extractor'] = ie.IE_NAME
                     try:
@@ -524,15 +537,29 @@ class FileDownloader(object):
         return self._download_retcode
 
     def post_process(self, filename, ie_info):
-        """Run the postprocessing chain on the given file."""
+        """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
+        keep_video = None
         for pp in self._pps:
-            info = pp.run(info)
-            if info is None:
-                break
+            try:
+                keep_video_wish,new_info = pp.run(info)
+                if keep_video_wish is not None:
+                    if keep_video_wish:
+                        keep_video = keep_video_wish
+                    elif keep_video is None:
+                        # No clear decision yet, let IE decide
+                        keep_video = keep_video_wish
+            except PostProcessingError as e:
+                self.to_stderr(u'ERROR: ' + e.msg)
+        if keep_video is False and not self.params.get('keepvideo', False):
+            try:
+                self.to_stderr(u'Deleting original file %s (pass -k to keep)' % filename)
+                os.remove(encodeFilename(filename))
+            except (IOError, OSError):
+                self.to_stderr(u'WARNING: Unable to remove downloaded video file')
 
-    def _download_with_rtmpdump(self, filename, url, player_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
@@ -546,7 +573,11 @@ class FileDownloader(object):
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
-        basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
+        basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
+        if player_url is not None:
+            basic_args += ['-W', player_url]
+        if page_url is not None:
+            basic_args += ['--pageUrl', page_url]
         args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
@@ -570,8 +601,15 @@ class FileDownloader(object):
                 retval = 0
                 break
         if retval == 0:
-            self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
             self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
             return True
         else:
             self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
@@ -579,22 +617,29 @@ class FileDownloader(object):
 
     def _do_download(self, filename, info_dict):
         url = info_dict['url']
-        player_url = info_dict.get('player_url', None)
 
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)
+            self._hook_progress({
+                'filename': filename,
+                'status': 'finished',
+            })
             return True
 
         # Attempt to download using rtmpdump
         if url.startswith('rtmp'):
-            return self._download_with_rtmpdump(filename, url, player_url)
+            return self._download_with_rtmpdump(filename, url,
+                                                info_dict.get('player_url', None),
+                                                info_dict.get('page_url', None))
 
         tmpfilename = self.temp_name(filename)
         stream = None
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
+        if 'user_agent' in info_dict:
+            headers['Youtubedl-user-agent'] = info_dict['user_agent']
         basic_request = compat_urllib_request.Request(url, None, headers)
         request = compat_urllib_request.Request(url, None, headers)
 
@@ -651,6 +696,10 @@ class FileDownloader(object):
                             # the one in the hard drive.
                             self.report_file_already_downloaded(filename)
                             self.try_rename(tmpfilename, filename)
+                            self._hook_progress({
+                                'filename': filename,
+                                'status': 'finished',
+                            })
                             return True
                         else:
                             # The length does not match, we start the download over
@@ -669,6 +718,15 @@ class FileDownloader(object):
         data_len = data.info().get('Content-length', None)
         if data_len is not None:
             data_len = int(data_len) + resume_len
+            min_data_len = self.params.get("min_filesize", None)
+            max_data_len =  self.params.get("max_filesize", None)
+            if min_data_len is not None and data_len < min_data_len:
+                self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+                return False
+            if max_data_len is not None and data_len > max_data_len:
+                self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+                return False
+
         data_len_str = self.format_bytes(data_len)
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
@@ -709,6 +767,14 @@ class FileDownloader(object):
                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 
+            self._hook_progress({
+                'downloaded_bytes': byte_counter,
+                'total_bytes': data_len,
+                'tmpfilename': tmpfilename,
+                'filename': filename,
+                'status': 'downloading',
+            })
+
             # Apply rate limit
             self.slow_down(start, byte_counter - resume_len)
 
@@ -725,4 +791,31 @@ class FileDownloader(object):
         if self.params.get('updatetime', True):
             info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 
+        self._hook_progress({
+            'downloaded_bytes': byte_counter,
+            'total_bytes': byte_counter,
+            'filename': filename,
+            'status': 'finished',
+        })
+
         return True
+
+    def _hook_progress(self, status):
+        for ph in self._progress_hooks:
+            ph(status)
+
+    def add_progress_hook(self, ph):
+        """ ph gets called on download progress, with a dictionary with the entries
+        * filename: The final filename
+        * status: One of "downloading" and "finished"
+
+        It can also have some of the following entries:
+
+        * downloaded_bytes: Bytes on disks
+        * total_bytes: Total bytes, None if unknown
+        * tmpfilename: The filename we're currently writing to
+
+        Hooks are guaranteed to be called at least once (with status "finished")
+        if the download is successful.
+        """
+        self._progress_hooks.append(ph)

youtube_dl/PostProcessor.py

@@ -45,30 +45,24 @@ class PostProcessor(object):
         one has an extra field called "filepath" that points to the
         downloaded file.
 
-        When this method returns None, the postprocessing chain is
-        stopped. However, this method may return an information
-        dictionary that will be passed to the next postprocessing
-        object in the chain. It can be the one it received after
-        changing some fields.
+        This method returns a tuple, the first element of which describes
+        whether the original file should be kept (i.e. not deleted - None for
+        no preference), and the second of which is the updated information.
 
         In addition, this method may raise a PostProcessingError
-        exception that will be taken into account by the downloader
-        it was called from.
+        exception if post processing fails.
         """
-        return information # by default, do nothing
+        return None, information # by default, keep file and do nothing
 
-class AudioConversionError(BaseException):
-    def __init__(self, message):
-        self.message = message
+class FFmpegPostProcessorError(PostProcessingError):
+    pass
 
-class FFmpegExtractAudioPP(PostProcessor):
-    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
+class AudioConversionError(PostProcessingError):
+    pass
+
+class FFmpegPostProcessor(PostProcessor):
+    def __init__(self,downloader=None):
         PostProcessor.__init__(self, downloader)
-        if preferredcodec is None:
-            preferredcodec = 'best'
-        self._preferredcodec = preferredcodec
-        self._preferredquality = preferredquality
-        self._keepvideo = keepvideo
         self._exes = self.detect_executables()
 
     @staticmethod
@@ -82,18 +76,45 @@ class FFmpegExtractAudioPP(PostProcessor):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         return dict((program, executable(program)) for program in programs)
 
+    def run_ffmpeg(self, path, out_path, opts):
+        if not self._exes['ffmpeg'] and not self._exes['avconv']:
+            raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
+        cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path)]
+               + opts +
+               [encodeFilename(self._ffmpeg_filename_argument(out_path))])
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout,stderr = p.communicate()
+        if p.returncode != 0:
+            msg = stderr.strip().split('\n')[-1]
+            raise FFmpegPostProcessorError(msg.decode('utf-8', 'replace'))
+
+    def _ffmpeg_filename_argument(self, fn):
+        # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
+        if fn.startswith(u'-'):
+            return u'./' + fn
+        return fn
+
+class FFmpegExtractAudioPP(FFmpegPostProcessor):
+    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
+        FFmpegPostProcessor.__init__(self, downloader)
+        if preferredcodec is None:
+            preferredcodec = 'best'
+        self._preferredcodec = preferredcodec
+        self._preferredquality = preferredquality
+        self._nopostoverwrites = nopostoverwrites
+
     def get_audio_codec(self, path):
         if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
         try:
-            cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
-            handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+            cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', encodeFilename(self._ffmpeg_filename_argument(path))]
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
                 return None
         except (IOError, OSError):
             return None
         audio_codec = None
-        for line in output.split('\n'):
+        for line in output.decode('ascii', 'ignore').split('\n'):
             if line.startswith('codec_name='):
                 audio_codec = line.split('=')[1].strip()
             elif line.strip() == 'codec_type=audio' and audio_codec is not None:
@@ -102,36 +123,32 @@ class FFmpegExtractAudioPP(PostProcessor):
 
     def run_ffmpeg(self, path, out_path, codec, more_opts):
         if not self._exes['ffmpeg'] and not self._exes['avconv']:
-            raise AudioConversionError('ffmpeg or avconv not found. Please install one.')   
+            raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
         if codec is None:
             acodec_opts = []
         else:
             acodec_opts = ['-acodec', codec]
-        cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
-               + acodec_opts + more_opts +
-               ['--', encodeFilename(out_path)])
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout,stderr = p.communicate()
-        if p.returncode != 0:
-            msg = stderr.strip().split('\n')[-1]
-            raise AudioConversionError(msg)
+        opts = ['-vn'] + acodec_opts + more_opts
+        try:
+            FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts)
+        except FFmpegPostProcessorError as err:
+            raise AudioConversionError(err.message)
 
     def run(self, information):
         path = information['filepath']
 
         filecodec = self.get_audio_codec(path)
         if filecodec is None:
-            self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
-            return None
+            raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe')
 
         more_opts = []
         if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
-            if self._preferredcodec == 'm4a' and filecodec == 'aac':
+            if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                 # Lossless, but in another container
                 acodec = 'copy'
-                extension = self._preferredcodec
+                extension = 'm4a'
                 more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
-            elif filecodec in ['aac', 'mp3', 'vorbis']:
+            elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible
                 acodec = 'copy'
                 extension = filecodec
@@ -151,7 +168,7 @@ class FFmpegExtractAudioPP(PostProcessor):
                         more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
         else:
             # We convert the audio (lossy)
-            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
+            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
             extension = self._preferredcodec
             more_opts = []
             if self._preferredquality is not None:
@@ -171,16 +188,19 @@ class FFmpegExtractAudioPP(PostProcessor):
 
         prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
         new_path = prefix + sep + extension
-        self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
         try:
-            self.run_ffmpeg(path, new_path, acodec, more_opts)
+            if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
+                self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
+            else:
+                self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+                self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype,e,tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
-                self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+                msg = u'audio conversion failed: ' + e.message
             else:
-                self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
-            return None
+                msg = u'error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')
+            raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.
         if information.get('filetime') is not None:
@@ -189,12 +209,24 @@ class FFmpegExtractAudioPP(PostProcessor):
             except:
                 self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
 
-        if not self._keepvideo:
-            try:
-                os.remove(encodeFilename(path))
-            except (IOError, OSError):
-                self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
-                return None
-
         information['filepath'] = new_path
-        return information
+        return False,information
+
+class FFmpegVideoConvertor(FFmpegPostProcessor):
+    def __init__(self, downloader=None,preferedformat=None):
+        super(FFmpegVideoConvertor, self).__init__(downloader)
+        self._preferedformat=preferedformat
+
+    def run(self, information):
+        path = information['filepath']
+        prefix, sep, ext = path.rpartition(u'.')
+        outpath = prefix + sep + self._preferedformat
+        if information['ext'] == self._preferedformat:
+            self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
+            return True,information
+        self._downloader.to_screen(u'['+'ffmpeg'+'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) +outpath)
+        self.run_ffmpeg(path, outpath, [])
+        information['filepath'] = outpath
+        information['format'] = self._preferedformat
+        information['ext'] = self._preferedformat
+        return False,information

youtube_dl/__init__.py

@@ -20,6 +20,10 @@ __authors__  = (
     'shizeeg',
     'Filippo Valsorda',
     'Christian Albrecht',
+    'Dave Vasilevsky',
+    'Jaime Marquínez Ferrándiz',
+    'Jeff Crouse',
+    'Osama Khalid',
     )
 
 __license__ = 'Public Domain'
@@ -33,106 +37,15 @@ import socket
 import subprocess
 import sys
 import warnings
+import platform
 
 from .utils import *
+from .update import update_self
 from .version import __version__
 from .FileDownloader import *
-from .InfoExtractors import *
+from .InfoExtractors import gen_extractors
 from .PostProcessor import *
 
-def updateSelf(downloader, filename):
-    """Update the program file with the latest version from the repository"""
-
-    # TODO: at least, check https certificates
-
-    from zipimport import zipimporter
-
-    API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
-    BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
-    EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe"
-
-    if hasattr(sys, "frozen"): # PY2EXE
-        if not os.access(filename, os.W_OK):
-            sys.exit('ERROR: no write permissions on %s' % filename)
-
-        downloader.to_screen(u'Updating to latest version...')
-
-        urla = compat_urllib_request.urlopen(API_URL)
-        download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read()))
-        if not download:
-            downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.')
-            return
-        newversion = download[0]["description"].strip()
-        if newversion == __version__:
-            downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
-            return
-        urla.close()
-
-        exe = os.path.abspath(filename)
-        directory = os.path.dirname(exe)
-        if not os.access(directory, os.W_OK):
-            sys.exit('ERROR: no write permissions on %s' % directory)
-
-        try:
-            urlh = compat_urllib_request.urlopen(EXE_URL)
-            newcontent = urlh.read()
-            urlh.close()
-            with open(exe + '.new', 'wb') as outf:
-                outf.write(newcontent)
-        except (IOError, OSError) as err:
-            sys.exit('ERROR: unable to download latest version')
-
-        try:
-            bat = os.path.join(directory, 'youtube-dl-updater.bat')
-            b = open(bat, 'w')
-            b.write("""
-echo Updating youtube-dl...
-ping 127.0.0.1 -n 5 -w 1000 > NUL
-move /Y "%s.new" "%s"
-del "%s"
-            \n""" %(exe, exe, bat))
-            b.close()
-
-            os.startfile(bat)
-        except (IOError, OSError) as err:
-            sys.exit('ERROR: unable to overwrite current version')
-
-    elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP
-        if not os.access(filename, os.W_OK):
-            sys.exit('ERROR: no write permissions on %s' % filename)
-
-        downloader.to_screen(u'Updating to latest version...')
-
-        urla = compat_urllib_request.urlopen(API_URL)
-        download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"]
-        if not download:
-            downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.')
-            return
-        newversion = download[0]["description"].strip()
-        if newversion == __version__:
-            downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
-            return
-        urla.close()
-
-        try:
-            urlh = compat_urllib_request.urlopen(BIN_URL)
-            newcontent = urlh.read()
-            urlh.close()
-        except (IOError, OSError) as err:
-            sys.exit('ERROR: unable to download latest version')
-
-        try:
-            with open(filename, 'wb') as outf:
-                outf.write(newcontent)
-        except (IOError, OSError) as err:
-            sys.exit('ERROR: unable to overwrite current version')
-
-    else:
-        downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.')
-        return
-
-    downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
-
 def parseOpts():
     def _readOptions(filename_bytes):
         try:
@@ -238,6 +151,9 @@ def parseOpts():
     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
+    selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
+    selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
+
 
     authentication.add_option('-u', '--username',
             dest='username', metavar='USERNAME', help='account username')
@@ -264,7 +180,6 @@ def parseOpts():
             action='store', dest='subtitleslang', metavar='LANG',
             help='language of the closed captions to download (optional) use IETF language tags like \'en\'')
 
-
     verbosity.add_option('-q', '--quiet',
             action='store_true', dest='quiet', help='activates quiet mode', default=False)
     verbosity.add_option('-s', '--simulate',
@@ -287,6 +202,8 @@ def parseOpts():
     verbosity.add_option('--get-format',
             action='store_true', dest='getformat',
             help='simulate, quiet but print output format', default=False)
+    verbosity.add_option('--newline',
+            action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
     verbosity.add_option('--no-progress',
             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
     verbosity.add_option('--console-title',
@@ -295,7 +212,6 @@ def parseOpts():
     verbosity.add_option('-v', '--verbose',
             action='store_true', dest='verbose', help='print various debugging information', default=False)
 
-
     filesystem.add_option('-t', '--title',
             action='store_true', dest='usetitle', help='use title in file name', default=False)
     filesystem.add_option('--id',
@@ -306,7 +222,7 @@ def parseOpts():
             action='store_true', dest='autonumber',
             help='number downloaded files starting from 00000', default=False)
     filesystem.add_option('-o', '--output',
-            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
+            dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
     filesystem.add_option('--restrict-filenames',
             action='store_true', dest='restrictfilenames',
             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
@@ -337,11 +253,15 @@ def parseOpts():
     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-            help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
+            help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
+    postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
+            help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
             help='keeps the video file on disk after the post-processing; the video is erased by default')
+    postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
+            help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 
 
     parser.add_option_group(general)
@@ -362,45 +282,6 @@ def parseOpts():
 
     return parser, opts, args
 
-def gen_extractors():
-    """ Return a list of an instance of every supported extractor.
-    The order does matter; the first extractor matched is the one handling the URL.
-    """
-    return [
-        YoutubePlaylistIE(),
-        YoutubeChannelIE(),
-        YoutubeUserIE(),
-        YoutubeSearchIE(),
-        YoutubeIE(),
-        MetacafeIE(),
-        DailymotionIE(),
-        GoogleIE(),
-        GoogleSearchIE(),
-        PhotobucketIE(),
-        YahooIE(),
-        YahooSearchIE(),
-        DepositFilesIE(),
-        FacebookIE(),
-        BlipTVUserIE(),
-        BlipTVIE(),
-        VimeoIE(),
-        MyVideoIE(),
-        ComedyCentralIE(),
-        EscapistIE(),
-        CollegeHumorIE(),
-        XVideosIE(),
-        SoundcloudIE(),
-        InfoQIE(),
-        MixcloudIE(),
-        StanfordOpenClassroomIE(),
-        MTVIE(),
-        YoukuIE(),
-        XNXXIE(),
-        GooglePlusIE(),
-        ArteTvIE(),
-        GenericIE()
-    ]
-
 def _real_main():
     parser, opts, args = parseOpts()
 
@@ -410,10 +291,13 @@ def _real_main():
     else:
         try:
             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
-            if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
+            if os.access(opts.cookiefile, os.R_OK):
                 jar.load()
         except (IOError, OSError) as err:
-            sys.exit(u'ERROR: unable to open cookie file')
+            if opts.verbose:
+                traceback.print_exc()
+            sys.stderr.write(u'ERROR: unable to open cookie file\n')
+            sys.exit(101)
     # Set user agent
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent
@@ -451,8 +335,8 @@ def _real_main():
     if opts.list_extractors:
         for ie in extractors:
             print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
-            matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
-            all_urls = filter(lambda url: url not in matchedUrls, all_urls)
+            matchedUrls = [url for url in all_urls if ie.suitable(url)]
+            all_urls = [url for url in all_urls if url not in matchedUrls]
             for mu in matchedUrls:
                 print(u'  ' + mu)
         sys.exit(0)
@@ -473,6 +357,16 @@ def _real_main():
         if numeric_limit is None:
             parser.error(u'invalid rate limit specified')
         opts.ratelimit = numeric_limit
+    if opts.min_filesize is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
+        if numeric_limit is None:
+            parser.error(u'invalid min_filesize specified')
+        opts.min_filesize = numeric_limit
+    if opts.max_filesize is not None:
+        numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
+        if numeric_limit is None:
+            parser.error(u'invalid max_filesize specified')
+        opts.max_filesize = numeric_limit
     if opts.retries is not None:
         try:
             opts.retries = int(opts.retries)
@@ -496,13 +390,28 @@ def _real_main():
     except (TypeError, ValueError) as err:
         parser.error(u'invalid playlist end number specified')
     if opts.extractaudio:
-        if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
+        if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
             parser.error(u'invalid audio format specified')
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
         if not opts.audioquality.isdigit():
             parser.error(u'invalid audio quality specified')
+    if opts.recodevideo is not None:
+        if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
+            parser.error(u'invalid video recode format specified')
 
+    if sys.version_info < (3,):
+        # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
+        if opts.outtmpl is not None:
+            opts.outtmpl = opts.outtmpl.decode(preferredencoding())
+    outtmpl =((opts.outtmpl is not None and opts.outtmpl)
+            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
+            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
+            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
+            or (opts.useid and u'%(id)s.%(ext)s')
+            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
+            or u'%(id)s.%(ext)s')
     # File downloader
     fd = FileDownloader({
         'usenetrc': opts.usenetrc,
@@ -520,14 +429,7 @@ def _real_main():
         'format': opts.format,
         'format_limit': opts.format_limit,
         'listformats': opts.listformats,
-        'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
-            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
-            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
-            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
-            or (opts.useid and u'%(id)s.%(ext)s')
-            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
-            or u'%(id)s.%(ext)s'),
+        'outtmpl': outtmpl,
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
@@ -537,6 +439,7 @@ def _real_main():
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
         'noprogress': opts.noprogress,
+        'progress_with_newline': opts.progress_with_newline,
         'playliststart': opts.playliststart,
         'playlistend': opts.playlistend,
         'logtostderr': opts.outtmpl == '-',
@@ -553,9 +456,23 @@ def _real_main():
         'prefer_free_formats': opts.prefer_free_formats,
         'verbose': opts.verbose,
         'test': opts.test,
+        'keepvideo': opts.keepvideo,
+        'min_filesize': opts.min_filesize,
+        'max_filesize': opts.max_filesize
         })
 
     if opts.verbose:
+        fd.to_screen(u'[debug] youtube-dl version ' + __version__)
+        try:
+            sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                  cwd=os.path.dirname(os.path.abspath(__file__)))
+            out, err = sp.communicate()
+            out = out.decode().strip()
+            if re.match('[0-9a-f]+', out):
+                fd.to_screen(u'[debug] Git HEAD: ' + out)
+        except:
+            pass
+        fd.to_screen(u'[debug] Python version %s - %s' %(platform.python_version(), platform.platform()))
         fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies))
 
     for extractor in extractors:
@@ -563,11 +480,13 @@ def _real_main():
 
     # PostProcessors
     if opts.extractaudio:
-        fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
+        fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
+    if opts.recodevideo:
+        fd.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 
     # Update version
     if opts.update_self:
-        updateSelf(fd, sys.argv[0])
+        update_self(fd.to_screen, opts.verbose, sys.argv[0])
 
     # Maybe do nothing
     if len(all_urls) < 1:

youtube_dl/update.py

@@ -0,0 +1,160 @@
+import json
+import traceback
+import hashlib
+from zipimport import zipimporter
+
+from .utils import *
+from .version import __version__
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+def update_self(to_screen, verbose, filename):
+    """Update the program file with the latest version from the repository"""
+
+    UPDATE_URL = "http://rg3.github.com/youtube-dl/update/"
+    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+    JSON_URL = UPDATE_URL + 'versions.json'
+    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+
+    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
+        to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.')
+        return
+
+    # Check if there is a new version
+    try:
+        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+        return
+    if newversion == __version__:
+        to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+        return
+
+    # Download and check versions info
+    try:
+        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = json.loads(versions_info)
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
+        return
+    if not 'signature' in versions_info:
+        to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+        return
+    signature = versions_info['signature']
+    del versions_info['signature']
+    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
+        to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
+        return
+
+    to_screen(u'Updating to version ' + versions_info['latest'] + '...')
+    version = versions_info['versions'][versions_info['latest']]
+    if version.get('notes'):
+        to_screen(u'PLEASE NOTE:')
+        for note in version['notes']:
+            to_screen(note)
+
+    if not os.access(filename, os.W_OK):
+        to_screen(u'ERROR: no write permissions on %s' % filename)
+        return
+
+    # Py2EXE
+    if hasattr(sys, "frozen"):
+        exe = os.path.abspath(filename)
+        directory = os.path.dirname(exe)
+        if not os.access(directory, os.W_OK):
+            to_screen(u'ERROR: no write permissions on %s' % directory)
+            return
+
+        try:
+            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['exe'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(exe + '.new', 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to write the new version')
+            return
+
+        try:
+            bat = os.path.join(directory, 'youtube-dl-updater.bat')
+            b = open(bat, 'w')
+            b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+            \n""" %(exe, exe, bat))
+            b.close()
+
+            os.startfile(bat)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    # Zip unix package
+    elif isinstance(globals().get('__loader__'), zipimporter):
+        try:
+            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['bin'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(filename, 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')

youtube_dl/utils.py

@@ -3,10 +3,12 @@
 
 import gzip
 import io
+import json
 import locale
 import os
 import re
 import sys
+import traceback
 import zlib
 import email.utils
 import json
@@ -51,6 +53,12 @@ try:
 except ImportError: # Python 2
     import httplib as compat_http_client
 
+try:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
+    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
 try:
     from urllib.parse import parse_qs as compat_parse_qs
 except ImportError: # Python 2
@@ -147,6 +155,7 @@ std_headers = {
     'Accept-Encoding': 'gzip, deflate',
     'Accept-Language': 'en-us,en;q=0.5',
 }
+
 def preferredencoding():
     """Get preferred encoding.
 
@@ -169,6 +178,17 @@ else:
         assert type(s) == type(u'')
         print(s)
 
+# In Python 2.x, json.dump expects a bytestream.
+# In Python 3.x, it writes to a character stream
+if sys.version_info < (3,0):
+    def write_json_file(obj, fn):
+        with open(fn, 'wb') as f:
+            json.dump(obj, f)
+else:
+    def write_json_file(obj, fn):
+        with open(fn, 'w', encoding='utf-8') as f:
+            json.dump(obj, f)
+
 def htmlentity_transform(matchobj):
     """Transforms an HTML entity to a character.
 
@@ -195,10 +215,11 @@ def htmlentity_transform(matchobj):
     return (u'&%s;' % entity)
 
 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class IDParser(compat_html_parser.HTMLParser):
-    """Modified HTMLParser that isolates a tag with the specified id"""
-    def __init__(self, id):
-        self.id = id
+class AttrParser(compat_html_parser.HTMLParser):
+    """Modified HTMLParser that isolates a tag with the specified attribute"""
+    def __init__(self, attribute, value):
+        self.attribute = attribute
+        self.value = value
         self.result = None
         self.started = False
         self.depth = {}
@@ -223,7 +244,7 @@ class IDParser(compat_html_parser.HTMLParser):
         attrs = dict(attrs)
         if self.started:
             self.find_startpos(None)
-        if 'id' in attrs and attrs['id'] == self.id:
+        if self.attribute in attrs and attrs[self.attribute] == self.value:
             self.result = [tag]
             self.started = True
             self.watch_startpos = True
@@ -259,10 +280,20 @@ class IDParser(compat_html_parser.HTMLParser):
             lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
         lines[-1] = lines[-1][:self.result[2][1]]
         return '\n'.join(lines).strip()
+# Hack for https://github.com/rg3/youtube-dl/issues/662
+if sys.version_info < (2, 7, 3):
+    AttrParser.parse_endtag = (lambda self, i:
+        i + len("</scr'+'ipt>")
+        if self.rawdata[i:].startswith("</scr'+'ipt>")
+        else compat_html_parser.HTMLParser.parse_endtag(self, i))
 
 def get_element_by_id(id, html):
-    """Return the content of the tag with the specified id in the passed HTML document"""
-    parser = IDParser(id)
+    """Return the content of the tag with the specified ID in the passed HTML document"""
+    return get_element_by_attribute("id", id, html)
+
+def get_element_by_attribute(attribute, value, html):
+    """Return the content of the tag with the specified attribute in the passed HTML document"""
+    parser = AttrParser(attribute, value)
     try:
         parser.loads(html)
     except compat_html_parser.HTMLParseError:
@@ -274,7 +305,8 @@ def clean_html(html):
     """Clean an HTML snippet into a readable string"""
     # Newline vs <br />
     html = html.replace('\n', ' ')
-    html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
     # Strip html tags
     html = re.sub('<.*?>', '', html)
     # Replace html entities
@@ -383,7 +415,24 @@ def encodeFilename(s):
         # match Windows 9x series as well. Besides, NT 4 is obsolete.)
         return s
     else:
-        return s.encode(sys.getfilesystemencoding(), 'ignore')
+        encoding = sys.getfilesystemencoding()
+        if encoding is None:
+            encoding = 'utf-8'
+        return s.encode(encoding, 'ignore')
+
+
+class ExtractorError(Exception):
+    """Error during info extraction."""
+    def __init__(self, msg, tb=None):
+        """ tb, if given, is the original traceback (so that it can be printed out). """
+        super(ExtractorError, self).__init__(msg)
+        self.traceback = tb
+
+    def format_traceback(self):
+        if self.traceback is None:
+            return None
+        return u''.join(traceback.format_tb(self.traceback))
+
 
 class DownloadError(Exception):
     """Download Error exception.
@@ -410,7 +459,8 @@ class PostProcessingError(Exception):
     This exception may be raised by PostProcessor's .run() method to
     indicate an error in the postprocessing task.
     """
-    pass
+    def __init__(self, msg):
+        self.msg = msg
 
 class MaxDownloadsReached(Exception):
     """ --max-downloads limit has been reached. """
@@ -441,14 +491,6 @@ class ContentTooShortError(Exception):
         self.downloaded = downloaded
         self.expected = expected
 
-
-class Trouble(Exception):
-    """Trouble helper exception
-
-    This is an exception to be handled with
-    FileDownloader.trouble
-    """
-
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -483,14 +525,19 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         return ret
 
     def http_request(self, req):
-        for h in std_headers:
+        for h,v in std_headers.items():
             if h in req.headers:
                 del req.headers[h]
-            req.add_header(h, std_headers[h])
+            req.add_header(h, v)
         if 'Youtubedl-no-compression' in req.headers:
             if 'Accept-encoding' in req.headers:
                 del req.headers['Accept-encoding']
             del req.headers['Youtubedl-no-compression']
+        if 'Youtubedl-user-agent' in req.headers:
+            if 'User-agent' in req.headers:
+                del req.headers['User-agent']
+            req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
+            del req.headers['Youtubedl-user-agent']
         return req
 
     def http_response(self, req, resp):

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2012.12.11'
+__version__ = '2013.02.18'