release 2013.06.21

Also extracts the description.

.gitignore

@@ -1,6 +1,20 @@
 *.pyc
 *.pyo
 *~
+*.DS_Store
 wine-py2exe/
 py2exe.log
 *.kate-swp
+build/
+dist/
+MANIFEST
+README.txt
+youtube-dl.1
+youtube-dl.bash-completion
+youtube-dl
+youtube-dl.exe
+youtube-dl.tar.gz
+.coverage
+cover/
+updates_key.pem
+*.egg-info

.travis.yml

@@ -1,9 +1,15 @@
 language: python
-#specify the python version
 python:
   - "2.6"
   - "2.7"
-#command to install the setup
+  - "3.3"
-install:
+script: nosetests test --verbose
-# command to run tests
+notifications:
-script: nosetests test --nocapture
+  email:
+    - filippo.valsorda@gmail.com
+    - phihag@phihag.de
+    - jaime.marquinez.ferrandiz+travis@gmail.com
+#  irc:
+#    channels:
+#      - "irc.freenode.org#youtube-dl"
+#    skip_join: true

CHANGELOG

@@ -0,0 +1,14 @@
+2013.01.02  Codename: GIULIA
+
+    * Add support for ComedyCentral clips <nto>
+    * Corrected Vimeo description fetching <Nick Daniels>
+    * Added the --no-post-overwrites argument <Barbu Paul - Gheorghe>
+    * --verbose offers more environment info
+    * New info_dict field: uploader_id
+    * New updates system, with signature checking
+    * New IEs: NBA, JustinTV, FunnyOrDie, TweetReel, Steam, Ustream
+    * Fixed IEs: BlipTv
+    * Fixed for Python 3 IEs: Xvideo, Youku, XNXX, Dailymotion, Vimeo, InfoQ
+    * Simplified IEs and test code
+    * Various (Python 3 and other) fixes
+    * Revamped and expanded tests

LATEST_VERSION

@@ -1 +1 @@
-2012.11.29
+2012.12.99

LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

MANIFEST.in

@@ -0,0 +1,5 @@
+include README.md
+include test/*.py
+include test/*.json
+include youtube-dl.bash-completion
+include youtube-dl.1

Makefile

@@ -1,13 +1,26 @@
-all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
-# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
 
 clean:
-	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
+	rm -rf youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz
+
+cleanall: clean
+	rm -f youtube-dl youtube-dl.exe
 
 PREFIX=/usr/local
 BINDIR=$(PREFIX)/bin
 MANDIR=$(PREFIX)/man
-SYSCONFDIR=/etc
+PYTHON=/usr/bin/env python
+
+# set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
+ifeq ($(PREFIX),/usr)
+    SYSCONFDIR=/etc
+else
+    ifeq ($(PREFIX),/usr/local)
+        SYSCONFDIR=/etc
+    else
+        SYSCONFDIR=$(PREFIX)/etc
+    endif
+endif
 
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -d $(DESTDIR)$(BINDIR)
@@ -18,40 +31,48 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
 
 test:
-	nosetests2 --nocapture test
+	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
+	nosetests --verbose test
 
-.PHONY: all clean install test README.md youtube-dl.bash-completion
+tar: youtube-dl.tar.gz
-# TODO un-phony README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+
+.PHONY: all clean install test tar bash-completion pypi-files
+
+pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1
 
 youtube-dl: youtube_dl/*.py
-	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
+	zip --quiet youtube-dl youtube_dl/*.py
-	echo '#!/usr/bin/env python' > youtube-dl
+	zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py
+	echo '#!$(PYTHON)' > youtube-dl
 	cat youtube-dl.zip >> youtube-dl
 	rm youtube-dl.zip
 	chmod a+x youtube-dl
 
-youtube-dl.exe: youtube_dl/*.py
-	bash devscripts/wine-py2exe.sh build_exe.py
-
 README.md: youtube_dl/*.py
-	@options=$$(COLUMNS=80 python -m youtube_dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
+	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
-		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
+
-		footer=$$(sed -e '1,/.*# CONFIGURATION/ d' README.md) && \
+README.txt: README.md
-		echo "$${header}" > README.md && \
+	pandoc -f markdown -t plain README.md -o README.txt
-		echo >> README.md && \
-		echo '# OPTIONS' >> README.md && \
-		echo "$${options}" >> README.md&& \
-		echo >> README.md && \
-		echo '# CONFIGURATION' >> README.md && \
-		echo "$${footer}" >> README.md
 
 youtube-dl.1: README.md
-	pandoc -s -w man README.md -o youtube-dl.1
+	pandoc -s -f markdown -t man README.md -o youtube-dl.1
 
-youtube-dl.bash-completion: README.md
+youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.in
-	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
+	python devscripts/bash-completion.py
-		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
-		echo "$${content}" > youtube-dl.bash-completion
 
-LATEST_VERSION: youtube_dl/__init__.py
+bash-completion: youtube-dl.bash-completion
-	python -m youtube_dl --version > LATEST_VERSION
+
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion
+	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
+		--exclude '*.DS_Store' \
+		--exclude '*.kate-swp' \
+		--exclude '*.pyc' \
+		--exclude '*.pyo' \
+		--exclude '*~' \
+		--exclude '__pycache' \
+		--exclude '.git' \
+		-- \
+		bin devscripts test youtube_dl \
+		CHANGELOG LICENSE README.md README.txt \
+		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion setup.py \
+		youtube-dl

README.md

@@ -9,109 +9,146 @@ youtube-dl
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
-2.x (x being at least 6), and it is not platform specific. It should work in
+2.6, 2.7, or 3.3+, and it is not platform specific. It should work on
-your Unix box, in Windows or in Mac OS X. It is released to the public domain,
+your Unix box, on Windows or on Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
 # OPTIONS
-    -h, --help               print this help text and exit
+    -h, --help                 print this help text and exit
-    --version                print program version and exit
+    --version                  print program version and exit
-    -U, --update             update this program to latest version
+    -U, --update               update this program to latest version
-    -i, --ignore-errors      continue on download errors
+    -i, --ignore-errors        continue on download errors
-    -r, --rate-limit LIMIT   download rate limit (e.g. 50k or 44.6m)
+    -r, --rate-limit LIMIT     maximum download rate (e.g. 50k or 44.6m)
-    -R, --retries RETRIES    number of retries (default is 10)
+    -R, --retries RETRIES      number of retries (default is 10)
-    --buffer-size SIZE       size of download buffer (e.g. 1024 or 16k) (default
+    --buffer-size SIZE         size of download buffer (e.g. 1024 or 16k)
-                             is 1024)
+                               (default is 1024)
-    --no-resize-buffer       do not automatically adjust the buffer size. By
+    --no-resize-buffer         do not automatically adjust the buffer size. By
-                             default, the buffer size is automatically resized
+                               default, the buffer size is automatically resized
-                             from an initial value of SIZE.
+                               from an initial value of SIZE.
-    --dump-user-agent        display the current browser identification
+    --dump-user-agent          display the current browser identification
-    --user-agent UA          specify a custom user agent
+    --user-agent UA            specify a custom user agent
-    --list-extractors        List all supported extractors and the URLs they
+    --referer REF              specify a custom referer, use if the video access
-                             would handle
+                               is restricted to one domain
+    --list-extractors          List all supported extractors and the URLs they
+                               would handle
+    --proxy URL                Use the specified HTTP/HTTPS proxy
+    --no-check-certificate     Suppress HTTPS certificate validation.
 
 ## Video Selection:
-    --playlist-start NUMBER  playlist video to start at (default is 1)
+    --playlist-start NUMBER    playlist video to start at (default is 1)
-    --playlist-end NUMBER    playlist video to end at (default is last)
+    --playlist-end NUMBER      playlist video to end at (default is last)
-    --match-title REGEX      download only matching titles (regex or caseless
+    --match-title REGEX        download only matching titles (regex or caseless
-                             sub-string)
+                               sub-string)
-    --reject-title REGEX     skip download for matching titles (regex or
+    --reject-title REGEX       skip download for matching titles (regex or
-                             caseless sub-string)
+                               caseless sub-string)
-    --max-downloads NUMBER   Abort after downloading NUMBER files
+    --max-downloads NUMBER     Abort after downloading NUMBER files
+    --min-filesize SIZE        Do not download any videos smaller than SIZE
+                               (e.g. 50k or 44.6m)
+    --max-filesize SIZE        Do not download any videos larger than SIZE (e.g.
+                               50k or 44.6m)
+    --date DATE                download only videos uploaded in this date
+    --datebefore DATE          download only videos uploaded before this date
+    --dateafter DATE           download only videos uploaded after this date
 
 ## Filesystem Options:
-    -t, --title              use title in file name
+    -t, --title                use title in file name (default)
-    --id                     use video ID in file name
+    --id                       use only video ID in file name
-    -l, --literal            [deprecated] alias of --title
+    -l, --literal              [deprecated] alias of --title
-    -A, --auto-number        number downloaded files starting from 00000
+    -A, --auto-number          number downloaded files starting from 00000
-    -o, --output TEMPLATE    output filename template. Use %(title)s to get the
+    -o, --output TEMPLATE      output filename template. Use %(title)s to get
-                             title, %(uploader)s for the uploader name,
+                               the title, %(uploader)s for the uploader name,
-                             %(autonumber)s to get an automatically incremented
+                               %(uploader_id)s for the uploader nickname if
-                             number, %(ext)s for the filename extension,
+                               different, %(autonumber)s to get an automatically
-                             %(upload_date)s for the upload date (YYYYMMDD),
+                               incremented number, %(ext)s for the filename
-                             %(extractor)s for the provider (youtube, metacafe,
+                               extension, %(upload_date)s for the upload date
-                             etc), %(id)s for the video id and %% for a literal
+                               (YYYYMMDD), %(extractor)s for the provider
-                             percent. Use - to output to stdout.
+                               (youtube, metacafe, etc), %(id)s for the video id
-    --restrict-filenames     Restrict filenames to only ASCII characters, and
+                               , %(playlist)s for the playlist the video is in,
-                             avoid "&" and spaces in filenames
+                               %(playlist_index)s for the position in the
-    -a, --batch-file FILE    file containing URLs to download ('-' for stdin)
+                               playlist and %% for a literal percent. Use - to
-    -w, --no-overwrites      do not overwrite files
+                               output to stdout. Can also be used to download to
-    -c, --continue           resume partially downloaded files
+                               a different directory, for example with -o '/my/d
-    --no-continue            do not resume partially downloaded files (restart
+                               ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
-                             from beginning)
+    --autonumber-size NUMBER   Specifies the number of digits in %(autonumber)s
-    --cookies FILE           file to read cookies from and dump cookie jar in
+                               when it is present in output filename template or
-    --no-part                do not use .part files
+                               --autonumber option is given
-    --no-mtime               do not use the Last-modified header to set the file
+    --restrict-filenames       Restrict filenames to only ASCII characters, and
-                             modification time
+                               avoid "&" and spaces in filenames
-    --write-description      write video description to a .description file
+    -a, --batch-file FILE      file containing URLs to download ('-' for stdin)
-    --write-info-json        write video metadata to a .info.json file
+    -w, --no-overwrites        do not overwrite files
+    -c, --continue             resume partially downloaded files
+    --no-continue              do not resume partially downloaded files (restart
+                               from beginning)
+    --cookies FILE             file to read cookies from and dump cookie jar in
+    --no-part                  do not use .part files
+    --no-mtime                 do not use the Last-modified header to set the
+                               file modification time
+    --write-description        write video description to a .description file
+    --write-info-json          write video metadata to a .info.json file
+    --write-thumbnail          write thumbnail image to disk
 
 ## Verbosity / Simulation Options:
-    -q, --quiet              activates quiet mode
+    -q, --quiet                activates quiet mode
-    -s, --simulate           do not download the video and do not write anything
+    -s, --simulate             do not download the video and do not write
-                             to disk
+                               anything to disk
-    --skip-download          do not download the video
+    --skip-download            do not download the video
-    -g, --get-url            simulate, quiet but print URL
+    -g, --get-url              simulate, quiet but print URL
-    -e, --get-title          simulate, quiet but print title
+    -e, --get-title            simulate, quiet but print title
-    --get-thumbnail          simulate, quiet but print thumbnail URL
+    --get-id                   simulate, quiet but print id
-    --get-description        simulate, quiet but print video description
+    --get-thumbnail            simulate, quiet but print thumbnail URL
-    --get-filename           simulate, quiet but print output filename
+    --get-description          simulate, quiet but print video description
-    --get-format             simulate, quiet but print output format
+    --get-filename             simulate, quiet but print output filename
-    --no-progress            do not print progress bar
+    --get-format               simulate, quiet but print output format
-    --console-title          display progress in console titlebar
+    --newline                  output progress bar as new lines
-    -v, --verbose            print various debugging information
+    --no-progress              do not print progress bar
+    --console-title            display progress in console titlebar
+    -v, --verbose              print various debugging information
+    --dump-intermediate-pages  print downloaded pages to debug problems(very
+                               verbose)
 
 ## Video Format Options:
-    -f, --format FORMAT      video format code
+    -f, --format FORMAT        video format code, specifiy the order of
-    --all-formats            download all available video formats
+                               preference using slashes: "-f 22/17/18"
-    --prefer-free-formats    prefer free video formats unless a specific one is
+    --all-formats              download all available video formats
-                             requested
+    --prefer-free-formats      prefer free video formats unless a specific one
-    --max-quality FORMAT     highest quality format to download
+                               is requested
-    -F, --list-formats       list all available formats (currently youtube only)
+    --max-quality FORMAT       highest quality format to download
-    --write-srt              write video closed captions to a .srt file
+    -F, --list-formats         list all available formats (currently youtube
-                             (currently youtube only)
+                               only)
-    --srt-lang LANG          language of the closed captions to download
+    --write-sub                write subtitle file (currently youtube only)
-                             (optional) use IETF language tags like 'en'
+    --only-sub                 [deprecated] alias of --skip-download
+    --all-subs                 downloads all the available subtitles of the
+                               video (currently youtube only)
+    --list-subs                lists all available subtitles for the video
+                               (currently youtube only)
+    --sub-format LANG          subtitle format [srt/sbv] (default=srt)
+                               (currently youtube only)
+    --sub-lang LANG            language of the subtitles to download (optional)
+                               use IETF language tags like 'en'
 
 ## Authentication Options:
-    -u, --username USERNAME  account username
+    -u, --username USERNAME    account username
-    -p, --password PASSWORD  account password
+    -p, --password PASSWORD    account password
-    -n, --netrc              use .netrc authentication data
+    -n, --netrc                use .netrc authentication data
 
 ## Post-processing Options:
-    -x, --extract-audio      convert video files to audio-only files (requires
+    -x, --extract-audio        convert video files to audio-only files (requires
-                             ffmpeg or avconv and ffprobe or avprobe)
+                               ffmpeg or avconv and ffprobe or avprobe)
-    --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
+    --audio-format FORMAT      "best", "aac", "vorbis", "mp3", "m4a", "opus", or
-                             best by default
+                               "wav"; best by default
-    --audio-quality QUALITY  ffmpeg/avconv audio quality specification, insert a
+    --audio-quality QUALITY    ffmpeg/avconv audio quality specification, insert
-                             value between 0 (better) and 9 (worse) for VBR or a
+                               a value between 0 (better) and 9 (worse) for VBR
-                             specific bitrate like 128K (default 5)
+                               or a specific bitrate like 128K (default 5)
-    -k, --keep-video         keeps the video file on disk after the post-
+    --recode-video FORMAT      Encode the video to another format if necessary
-                             processing; the video is erased by default
+                               (currently supported: mp4|flv|ogg|webm)
+    -k, --keep-video           keeps the video file on disk after the post-
+                               processing; the video is erased by default
+    --no-post-overwrites       do not overwrite post-processed files; the post-
+                               processed files are overwritten by default
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.local/config/youtube-dl.conf`.
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`.
 
 # OUTPUT TEMPLATE
 
@@ -125,6 +162,8 @@ The `-o` option allows users to indicate a template for the output file names. T
  - `ext`: The sequence will be replaced by the appropriate extension (like flv or mp4).
  - `epoch`: The sequence will be replaced by the Unix epoch when creating the file.
  - `autonumber`: The sequence will be replaced by a five-digit number that will be increased with each download, starting at zero.
+ - `playlist`: The name or the id of the playlist that contains the video.
+ - `playlist_index`: The index of the video in the playlist, a five-digit number.
 
 The current default template is `%(id)s.%(ext)s`, but that will be switchted to `%(title)s-%(id)s.%(ext)s` (which can be requested with `-t` at the moment).
 
@@ -135,6 +174,19 @@ In some cases, you don't want special characters such as 中, spaces, or &, such
     $ youtube-dl --get-filename -o "%(title)s.%(ext)s" BaW_jenozKc --restrict-filenames
     youtube-dl_test_video_.mp4          # A simple file name
 
+# VIDEO SELECTION
+
+Videos can be filtered by their upload date using the options `--date`, `--datebefore` or `--dateafter`, they accept dates in two formats:
+
+ - Absolute dates: Dates in the format `YYYYMMDD`.
+ - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
+ 
+Examples:
+
+	$ youtube-dl --dateafter now-6months #will only download the videos uploaded in the last 6 months
+	$ youtube-dl --date 19700101 #will only download the videos uploaded in January 1, 1970
+	$ youtube-dl --dateafter 20000101 --datebefore 20100101 #will only download the videos uploaded between 2000 and 2010
+
 # FAQ
 
 ### Can you please put the -b option back?
@@ -170,14 +222,6 @@ The error
 
 means you're using an outdated version of Python. Please update to Python 2.6 or 2.7.
 
-To run youtube-dl under Python 2.5, you'll have to manually check it out like this:
-
-	git clone git://github.com/rg3/youtube-dl.git
-	cd youtube-dl
-	python -m youtube_dl --help
-
-Please note that Python 2.5 is not supported anymore.
-
 ### What is this binary file? Where has the code gone?
 
 Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unzip it (might need renaming to `youtube-dl.zip` first on some systems) or clone the git repository, as laid out above. If you modify the code, you can run it by executing the `__main__.py` file. To recompile the executable, run `make youtube-dl`.
@@ -199,6 +243,9 @@ Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/i
 Please include:
 
 * Your exact command line, like `youtube-dl -t "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+* If possible re-run the command with `--verbose`, and include the full output, it is really helpful to us.
 * The output of `youtube-dl --version`
 * The output of `python --version`
 * The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.

bin/youtube-dl

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+import youtube_dl
+
+if __name__ == '__main__':
+    youtube_dl.main()

build_exe.py

@@ -1,48 +0,0 @@
-from distutils.core import setup
-import py2exe
-import sys, os
-
-"""This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
-
-# If run without args, build executables
-if len(sys.argv) == 1:
-    sys.argv.append("py2exe")
-
-# os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) # conflict with wine-py2exe.sh
-sys.path.append('./youtube_dl')
-
-options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe']
-}
-
-console = [{
-    "script":"./youtube_dl/__main__.py",
-    "dest_base": "youtube-dl",
-}]
-
-init_file = open('./youtube_dl/__init__.py')
-for line in init_file.readlines():
-    if line.startswith('__version__'):
-        version = line[11:].strip(" ='\n")
-        break
-else:
-    version = ''
-
-setup(name='youtube-dl',
-      version=version,
-      description='Small command-line program to download videos from YouTube.com and other video sites',
-      url='https://github.com/rg3/youtube-dl',
-      packages=['youtube_dl'],
-      
-      console = console,
-      options = {"py2exe": options},
-      zipfile = None,
-)
-
-import shutil
-shutil.rmtree("build")
-

devscripts/bash-completion.in

@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="{{flags}}"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl

devscripts/bash-completion.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+
+BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
+BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
+
+def build_completion(opt_parser):
+    opts_flag = []
+    for group in opt_parser.option_groups:
+        for option in group.option_list:
+            #for every long flag
+            opts_flag.append(option.get_opt_string())
+    with open(BASH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+    with open(BASH_COMPLETION_FILE, "w") as f:
+        #just using the special char
+        filled_template = template.replace("{{flags}}", " ".join(opts_flag))
+        f.write(filled_template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)

devscripts/gh-pages/add-version.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+import hashlib
+import urllib.request
+
+if len(sys.argv) <= 1:
+	print('Specify the version number as parameter')
+	sys.exit()
+version = sys.argv[1]
+
+with open('update/LATEST_VERSION', 'w') as f:
+	f.write(version)
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+new_version = {}
+
+filenames = {'bin': 'youtube-dl', 'exe': 'youtube-dl.exe', 'tar': 'youtube-dl-%s.tar.gz' % version}
+for key, filename in filenames.items():
+	print('Downloading and checksumming %s...' %filename)
+	url = 'http://youtube-dl.org/downloads/%s/%s' % (version, filename)
+	data = urllib.request.urlopen(url).read()
+	sha256sum = hashlib.sha256(data).hexdigest()
+	new_version[key] = (url, sha256sum)
+
+versions_info['versions'][version] = new_version
+versions_info['latest'] = version
+
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)

devscripts/gh-pages/generate-download.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+import hashlib
+import shutil
+import subprocess
+import tempfile
+import urllib.request
+import json
+
+versions_info = json.load(open('update/versions.json'))
+version = versions_info['latest']
+URL = versions_info['versions'][version]['bin'][0]
+
+data = urllib.request.urlopen(URL).read()
+
+# Read template page
+with open('download.html.in', 'r', encoding='utf-8') as tmplf:
+    template = tmplf.read()
+
+md5sum = hashlib.md5(data).hexdigest()
+sha1sum = hashlib.sha1(data).hexdigest()
+sha256sum = hashlib.sha256(data).hexdigest()
+template = template.replace('@PROGRAM_VERSION@', version)
+template = template.replace('@PROGRAM_URL@', URL)
+template = template.replace('@PROGRAM_MD5SUM@', md5sum)
+template = template.replace('@PROGRAM_SHA1SUM@', sha1sum)
+template = template.replace('@PROGRAM_SHA256SUM@', sha256sum)
+template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0])
+template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1])
+template = template.replace('@TAR_URL@', versions_info['versions'][version]['tar'][0])
+template = template.replace('@TAR_SHA256SUM@', versions_info['versions'][version]['tar'][1])
+with open('download.html', 'w', encoding='utf-8') as dlf:
+    dlf.write(template)

devscripts/gh-pages/sign-versions.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import rsa
+import json
+from binascii import hexlify
+
+try:
+    input = raw_input
+except NameError:
+    pass
+
+versions_info = json.load(open('update/versions.json'))
+if 'signature' in versions_info:
+	del versions_info['signature']
+
+print('Enter the PKCS1 private key, followed by a blank line:')
+privkey = b''
+while True:
+	try:
+		line = input()
+	except EOFError:
+		break
+	if line == '':
+		break
+	privkey += line.encode('ascii') + b'\n'
+privkey = rsa.PrivateKey.load_pkcs1(privkey)
+
+signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
+print('signature: ' + signature)
+
+versions_info['signature'] = signature
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)

devscripts/gh-pages/update-copyright.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import with_statement
+
+import datetime
+import glob
+import io # For Python 2 compatibilty
+import os
+import re
+
+year = str(datetime.datetime.now().year)
+for fn in glob.glob('*.html*'):
+    with io.open(fn, encoding='utf-8') as f:
+        content = f.read()
+    newc = re.sub(u'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', u'Copyright © 2006-' + year, content)
+    if content != newc:
+        tmpFn = fn + '.part'
+        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
+            outf.write(newc)
+        os.rename(tmpFn, fn)

devscripts/gh-pages/update-feed.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import datetime
+
+import textwrap
+
+import json
+
+atom_template=textwrap.dedent("""\
+								<?xml version='1.0' encoding='utf-8'?>
+								<atom:feed xmlns:atom="http://www.w3.org/2005/Atom">
+									<atom:title>youtube-dl releases</atom:title>
+									<atom:id>youtube-dl-updates-feed</atom:id>
+									<atom:updated>@TIMESTAMP@</atom:updated>
+									@ENTRIES@
+								</atom:feed>""")
+
+entry_template=textwrap.dedent("""
+								<atom:entry>
+									<atom:id>youtube-dl-@VERSION@</atom:id>
+									<atom:title>New version @VERSION@</atom:title>
+									<atom:link href="http://rg3.github.io/youtube-dl" />
+									<atom:content type="xhtml">
+										<div xmlns="http://www.w3.org/1999/xhtml">
+											Downloads available at <a href="http://youtube-dl.org/downloads/@VERSION@/">http://youtube-dl.org/downloads/@VERSION@/</a>
+										</div>
+									</atom:content>
+									<atom:author>
+										<atom:name>The youtube-dl maintainers</atom:name>
+									</atom:author>
+									<atom:updated>@TIMESTAMP@</atom:updated>
+								</atom:entry>
+								""")
+
+now = datetime.datetime.now()
+now_iso = now.isoformat()
+
+atom_template = atom_template.replace('@TIMESTAMP@',now_iso)
+
+entries=[]
+
+versions_info = json.load(open('update/versions.json'))
+versions = list(versions_info['versions'].keys())
+versions.sort()
+
+for v in versions:
+	entry = entry_template.replace('@TIMESTAMP@',v.replace('.','-'))
+	entry = entry.replace('@VERSION@',v)
+	entries.append(entry)
+
+entries_str = textwrap.indent(''.join(entries), '\t')
+atom_template = atom_template.replace('@ENTRIES@', entries_str)
+
+with open('update/releases.atom','w',encoding='utf-8') as atom_file:
+	atom_file.write(atom_template)
+
+

devscripts/make_readme.py

@@ -0,0 +1,20 @@
+import sys
+import re
+
+README_FILE = 'README.md'
+helptext = sys.stdin.read()
+
+with open(README_FILE) as f:
+    oldreadme = f.read()
+
+header = oldreadme[:oldreadme.index('# OPTIONS')]
+footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
+
+options = helptext[helptext.index('  General Options:')+19:]
+options = re.sub(r'^  (\w.+)$', r'## \1', options, flags=re.M)
+options = '# OPTIONS\n' + options + '\n'
+
+with open(README_FILE, 'w') as f:
+    f.write(header)
+    f.write(options)
+    f.write(footer)

devscripts/release.sh

@@ -1,11 +1,92 @@
-#!/bin/sh
+#!/bin/bash
+
+# IMPORTANT: the following assumptions are made
+# * the GH repo is on the origin remote
+# * the gh-pages branch is named so locally
+# * the git config user.signingkey is properly set
+
+# You will need
+# pip install coverage nose rsa
+
+# TODO
+# release notes
+# make hash on local files
+
+set -e
 
 if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
 version="$1"
 if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
-if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
-sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
+if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
-make all
+
-git add -A
+/bin/echo -e "\n### First of all, testing..."
+make cleanall
+nosetests --verbose --with-coverage --cover-package=youtube_dl --cover-html test --stop || exit 1
+
+/bin/echo -e "\n### Changing version in version.py..."
+sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
+
+/bin/echo -e "\n### Committing CHANGELOG README.md and youtube_dl/version.py..."
+make README.md
+git add CHANGELOG README.md youtube_dl/version.py
 git commit -m "release $version"
-git tag -m "Release $version" "$version"
+
+/bin/echo -e "\n### Now tagging, signing and pushing..."
+git tag -s -m "Release $version" "$version"
+git show "$version"
+read -p "Is it good, can I push? (y/n) " -n 1
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+echo
+MASTER=$(git rev-parse --abbrev-ref HEAD)
+git push origin $MASTER:master
+git push origin "$version"
+
+/bin/echo -e "\n### OK, now it is time to build the binaries..."
+REV=$(git rev-parse HEAD)
+make youtube-dl youtube-dl.tar.gz
+wget "http://jeromelaheurte.net:8142/download/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe || \
+	wget "http://jeromelaheurte.net:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
+mkdir -p "build/$version"
+mv youtube-dl youtube-dl.exe "build/$version"
+mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"
+RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz"
+(cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS)
+(cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS)
+(cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS)
+(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS)
+git checkout HEAD -- youtube-dl youtube-dl.exe
+
+/bin/echo -e "\n### Signing and uploading the new binaries to youtube-dl.org..."
+for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done
+scp -r "build/$version" ytdl@youtube-dl.org:html/downloads/
+
+/bin/echo -e "\n### Now switching to gh-pages..."
+git clone --branch gh-pages --single-branch . build/gh-pages
+ROOT=$(pwd)
+(
+    set -e
+    ORIGIN_URL=$(git config --get remote.origin.url)
+    cd build/gh-pages
+    "$ROOT/devscripts/gh-pages/add-version.py" $version
+    "$ROOT/devscripts/gh-pages/update-feed.py"
+    "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem"
+    "$ROOT/devscripts/gh-pages/generate-download.py"
+    "$ROOT/devscripts/gh-pages/update-copyright.py"
+    git add *.html *.html.in update
+    git commit -m "release $version"
+    git show HEAD
+    read -p "Is it good, can I push? (y/n) " -n 1
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
+    echo
+    git push "$ROOT" gh-pages
+    git push "$ORIGIN_URL" gh-pages
+)
+rm -rf build
+
+make pypi-files
+echo "Uploading to PyPi ..."
+python setup.py sdist upload
+make clean
+
+/bin/echo -e "\n### DONE!"

devscripts/transition_helper.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+
+import sys, os
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
+
+try:
+	raw_input()
+except NameError: # Python 3
+	input()
+
+filename = sys.argv[0]
+
+API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
+BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+try:
+    urlh = compat_urllib_request.urlopen(BIN_URL)
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+try:
+    with open(filename, 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

devscripts/transition_helper_exe/setup.py

@@ -0,0 +1,12 @@
+from distutils.core import setup
+import py2exe
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+
+setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None)

devscripts/transition_helper_exe/youtube-dl.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+
+import sys, os
+import urllib2
+import json, hashlib
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
+sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
+sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
+
+raw_input()
+
+filename = sys.argv[0]
+
+UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
+VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+JSON_URL = UPDATE_URL + 'versions.json'
+UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+if not os.access(filename, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % filename)
+
+exe = os.path.abspath(filename)
+directory = os.path.dirname(exe)
+if not os.access(directory, os.W_OK):
+    sys.exit('ERROR: no write permissions on %s' % directory)
+
+try:
+    versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8')
+    versions_info = json.loads(versions_info)
+except:
+    sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
+if not 'signature' in versions_info:
+    sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+signature = versions_info['signature']
+del versions_info['signature']
+if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
+    sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
+
+version = versions_info['versions'][versions_info['latest']]
+
+try:
+    urlh = urllib2.urlopen(version['exe'][0])
+    newcontent = urlh.read()
+    urlh.close()
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to download latest version')
+
+newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+if newcontent_hash != version['exe'][1]:
+    sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
+
+try:
+    with open(exe + '.new', 'wb') as outf:
+        outf.write(newcontent)
+except (IOError, OSError) as err:
+    sys.exit(u'ERROR: unable to write the new version')
+
+try:
+    bat = os.path.join(directory, 'youtube-dl-updater.bat')
+    b = open(bat, 'w')
+    b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+    \n""" %(exe, exe, bat))
+    b.close()
+
+    os.startfile(bat)
+except (IOError, OSError) as err:
+    sys.exit('ERROR: unable to overwrite current version')
+
+sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

setup.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from __future__ import print_function
+import pkg_resources
+import sys
+
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+try:
+    import py2exe
+    """This will create an exe that needs Microsoft Visual C++ 2008 Redistributable Package"""
+except ImportError:
+    if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+        print("Cannot import py2exe", file=sys.stderr)
+        exit(1)
+
+py2exe_options = {
+    "bundle_files": 1,
+    "compressed": 1,
+    "optimize": 2,
+    "dist_dir": '.',
+    "dll_excludes": ['w9xpopen.exe']
+}
+py2exe_console = [{
+    "script": "./youtube_dl/__main__.py",
+    "dest_base": "youtube-dl",
+}]
+py2exe_params = {
+    'console': py2exe_console,
+    'options': { "py2exe": py2exe_options },
+    'zipfile': None
+}
+
+if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
+    params = py2exe_params
+else:
+    params = {
+        'scripts': ['bin/youtube-dl'],
+        'data_files': [('etc/bash_completion.d', ['youtube-dl.bash-completion']), # Installing system-wide would require sudo...
+                       ('share/doc/youtube_dl', ['README.txt']),
+                       ('share/man/man1/', ['youtube-dl.1'])]
+    }
+
+# Get the version from youtube_dl/version.py without importing the package
+exec(compile(open('youtube_dl/version.py').read(), 'youtube_dl/version.py', 'exec'))
+
+setup(
+    name = 'youtube_dl',
+    version = __version__,
+    description = 'YouTube video downloader',
+    long_description = 'Small command-line program to download videos from YouTube.com and other video sites.',
+    url = 'https://github.com/rg3/youtube-dl',
+    author = 'Ricardo Garcia',
+    maintainer = 'Philipp Hagemeister',
+    maintainer_email = 'phihag@phihag.de',
+    packages = ['youtube_dl'],
+
+    # Provokes warning on most systems (why?!)
+    #test_suite = 'nose.collector',
+    #test_requires = ['nosetest'],
+
+    classifiers = [
+        "Topic :: Multimedia :: Video",
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "License :: Public Domain",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.3"
+    ],
+
+    **params
+)

test/parameters.json

@@ -1 +1,44 @@
-{"username": null, "listformats": null, "skip_download": false, "usenetrc": false, "max_downloads": null, "noprogress": false, "forcethumbnail": false, "forceformat": false, "format_limit": null, "ratelimit": null, "nooverwrites": false, "forceurl": false, "writeinfojson": false, "simulate": false, "playliststart": 1, "continuedl": true, "password": null, "prefer_free_formats": false, "nopart": false, "retries": 10, "updatetime": true, "consoletitle": false, "verbose": true, "forcefilename": false, "ignoreerrors": false, "logtostderr": false, "format": null, "subtitleslang": null, "quiet": false, "outtmpl": "%(id)s.%(ext)s", "rejecttitle": null, "playlistend": -1, "writedescription": false, "forcetitle": false, "forcedescription": false, "writesubtitles": false, "matchtitle": null}
+{
+    "consoletitle": false, 
+    "continuedl": true, 
+    "forcedescription": false, 
+    "forcefilename": false, 
+    "forceformat": false, 
+    "forcethumbnail": false, 
+    "forcetitle": false, 
+    "forceurl": false, 
+    "format": null, 
+    "format_limit": null, 
+    "ignoreerrors": false, 
+    "listformats": null, 
+    "logtostderr": false, 
+    "matchtitle": null, 
+    "max_downloads": null, 
+    "nooverwrites": false, 
+    "nopart": false, 
+    "noprogress": false, 
+    "outtmpl": "%(id)s.%(ext)s", 
+    "password": null, 
+    "playlistend": -1, 
+    "playliststart": 1, 
+    "prefer_free_formats": false, 
+    "quiet": false, 
+    "ratelimit": null, 
+    "rejecttitle": null, 
+    "retries": 10, 
+    "simulate": false, 
+    "skip_download": false, 
+    "subtitleslang": null, 
+    "subtitlesformat": "srt",
+    "test": true, 
+    "updatetime": true, 
+    "usenetrc": false, 
+    "username": null, 
+    "verbose": true, 
+    "writedescription": false, 
+    "writeinfojson": true, 
+    "writesubtitles": false,
+    "onlysubtitles": false,
+    "allsubtitles": false,
+    "listssubtitles": false
+}

test/test_all_urls.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE
+
+class TestAllURLsMatching(unittest.TestCase):
+    def test_youtube_playlist_matching(self):
+        self.assertTrue(YoutubePlaylistIE.suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'UUBABnxM4Ar9ten8Mdjj1j0Q')) #585
+        self.assertTrue(YoutubePlaylistIE.suitable(u'PL63F0C78739B09958'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC'))
+        self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
+        self.assertFalse(YoutubePlaylistIE.suitable(u'PLtS2H6bU1M'))
+
+    def test_youtube_matching(self):
+        self.assertTrue(YoutubeIE.suitable(u'PLtS2H6bU1M'))
+        self.assertFalse(YoutubeIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
+
+    def test_youtube_channel_matching(self):
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM'))
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec'))
+        self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM/videos'))
+
+    def test_justin_tv_channelid_matching(self):
+        self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"www.justin.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"www.twitch.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv/"))
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/"))
+
+    def test_justintv_videoid_matching(self):
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/b/328087483"))
+
+    def test_justin_tv_chapterid_matching(self):
+        self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/tsm_theoddone/c/2349361"))
+
+    def test_youtube_extract(self):
+        self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
+        self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc'), 'BaW_jenozKc')
+
+if __name__ == '__main__':
+    unittest.main()

test/test_download.py

@@ -1,198 +1,161 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
-import unittest
+
+import errno
 import hashlib
+import io
 import os
 import json
+import unittest
+import sys
+import socket
+import binascii
 
-from youtube_dl.FileDownloader import FileDownloader
+# Allow direct execution
-from youtube_dl.InfoExtractors  import YoutubeIE, DailymotionIE
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from youtube_dl.InfoExtractors import  MetacafeIE, BlipTVIE
+
-from youtube_dl.InfoExtractors import  XVideosIE, VimeoIE
+import youtube_dl.FileDownloader
-from youtube_dl.InfoExtractors import  SoundcloudIE, StanfordOpenClassroomIE
+import youtube_dl.InfoExtractors
-from youtube_dl.InfoExtractors import  CollegeHumorIE, XNXXIE
+from youtube_dl.utils import *
+
+DEF_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests.json')
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+RETRIES = 3
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+socket.setdefaulttimeout(10)
+
+def _try_rm(filename):
+    """ Remove a file if it exists """
+    try:
+        os.remove(filename)
+    except OSError as ose:
+        if ose.errno != errno.ENOENT:
+            raise
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        self.to_stderr = self.to_screen
+        self.processed_info_dicts = []
+        return youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+    def report_warning(self, message):
+        # Don't accept warnings during tests
+        raise ExtractorError(message)
+    def process_info(self, info_dict):
+        self.processed_info_dicts.append(info_dict)
+        return youtube_dl.FileDownloader.process_info(self, info_dict)
+
+def _file_md5(fn):
+    with open(fn, 'rb') as f:
+        return hashlib.md5(f.read()).hexdigest()
+
+with io.open(DEF_FILE, encoding='utf-8') as deff:
+    defs = json.load(deff)
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
 
 
-class DownloadTest(unittest.TestCase):
+class TestDownload(unittest.TestCase):
-	PARAMETERS_FILE = "test/parameters.json"
+    maxDiff = None
-	#calculated with md5sum:
+    def setUp(self):
-	#md5sum (GNU coreutils) 8.19
+        self.parameters = parameters
+        self.defs = defs
 
-	YOUTUBE_SIZE = 1993883
+### Dynamically generate tests
-	YOUTUBE_URL = "http://www.youtube.com/watch?v=BaW_jenozKc"
+def generator(test_case):
-	YOUTUBE_FILE = "BaW_jenozKc.mp4"
 
-	DAILYMOTION_MD5 = "d363a50e9eb4f22ce90d08d15695bb47"
+    def test_template(self):
-	DAILYMOTION_URL = "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech"
+        ie = youtube_dl.InfoExtractors.get_info_extractor(test_case['name'])
-	DAILYMOTION_FILE = "x33vw9.mp4"
+        if not ie._WORKING:
+            print('Skipping: IE marked as not _WORKING')
+            return
+        if 'playlist' not in test_case and not test_case['file']:
+            print('Skipping: No output file specified')
+            return
+        if 'skip' in test_case:
+            print('Skipping: {0}'.format(test_case['skip']))
+            return
 
-	METACAFE_SIZE = 5754305
+        params = self.parameters.copy()
-	METACAFE_URL = "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/"
+        params.update(test_case.get('params', {}))
-	METACAFE_FILE = "_aUehQsCQtM.flv"
 
-	BLIP_MD5 = "93c24d2f4e0782af13b8a7606ea97ba7"
+        fd = FileDownloader(params)
-	BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
+        for ie in youtube_dl.InfoExtractors.gen_extractors():
-	BLIP_FILE = "5779306.m4v"
+            fd.add_info_extractor(ie)
+        finished_hook_called = set()
+        def _hook(status):
+            if status['status'] == 'finished':
+                finished_hook_called.add(status['filename'])
+        fd.add_progress_hook(_hook)
 
-	XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
+        test_cases = test_case.get('playlist', [test_case])
-	XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1"
+        for tc in test_cases:
-	XVIDEO_FILE = "939581.flv"
+            _try_rm(tc['file'])
+            _try_rm(tc['file'] + '.part')
+            _try_rm(tc['file'] + '.info.json')
+        try:
+            for retry in range(1, RETRIES + 1):
+                try:
+                    fd.download([test_case['url']])
+                except (DownloadError, ExtractorError) as err:
+                    if retry == RETRIES: raise
 
-	VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
+                    # Check if the exception is not a network related one
-	VIMEO_URL = "http://vimeo.com/14160053"
+                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
-	VIMEO_FILE = ""
+                        raise
 
-	VIMEO2_MD5 = ""
+                    print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
-	VIMEO2_URL = "http://player.vimeo.com/video/47019590"
+                else:
-	VIMEO2_FILE = ""
+                    break
 
-	SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed"
+            for tc in test_cases:
-	SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy"
+                if not test_case.get('params', {}).get('skip_download', False):
-	SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3"
+                    self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
+                    self.assertTrue(tc['file'] in finished_hook_called)
+                self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
+                if 'md5' in tc:
+                    md5_for_file = _file_md5(tc['file'])
+                    self.assertEqual(md5_for_file, tc['md5'])
+                with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
+                    info_dict = json.load(infof)
+                for (info_field, value) in tc.get('info_dict', {}).items():
+                    if isinstance(value, compat_str) and value.startswith('md5:'):
+                        self.assertEqual(value, 'md5:' + md5(info_dict.get(info_field)))
+                    else:
+                        self.assertEqual(value, info_dict.get(info_field))
 
-	STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4"
+                # If checkable fields are missing from the test case, print the info_dict
-	STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100"
+                test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
-	STANDFORD_FILE = "PracticalUnix_intro-environment.mp4"
+                    for key, value in info_dict.items()
+                    if value and key in ('title', 'description', 'uploader', 'upload_date', 'uploader_id', 'location'))
+                if not all(key in tc.get('info_dict', {}).keys() for key in test_info_dict.keys()):
+                    sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=2) + u'\n')
 
-	COLLEGEHUMOR_MD5 = ""
+                # Check for the presence of mandatory fields
-	COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody"
+                for key in ('id', 'url', 'title', 'ext'):
-	COLLEGEHUMOR_FILE = ""
+                    self.assertTrue(key in info_dict.keys() and info_dict[key])
+        finally:
+            for tc in test_cases:
+                _try_rm(tc['file'])
+                _try_rm(tc['file'] + '.part')
+                _try_rm(tc['file'] + '.info.json')
 
-	XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d"
+    return test_template
-	XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_"
-	XNXX_FILE = "1135332.flv"
 
-	def test_youtube(self):
+### And add them to TestDownload
-		#let's download a file from youtube
+for test_case in defs:
-		with open(DownloadTest.PARAMETERS_FILE) as f:
+    test_method = generator(test_case)
-			fd = FileDownloader(json.load(f))
+    test_method.__name__ = "test_{0}".format(test_case["name"])
-		fd.add_info_extractor(YoutubeIE())
+    setattr(TestDownload, test_method.__name__, test_method)
-		fd.download([DownloadTest.YOUTUBE_URL])
+    del test_method
-		self.assertTrue(os.path.exists(DownloadTest.YOUTUBE_FILE))
-		self.assertEqual(os.path.getsize(DownloadTest.YOUTUBE_FILE), DownloadTest.YOUTUBE_SIZE)
 
-	def test_dailymotion(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(DailymotionIE())
-		fd.download([DownloadTest.DAILYMOTION_URL])
-		self.assertTrue(os.path.exists(DownloadTest.DAILYMOTION_FILE))
-		md5_down_file = md5_for_file(DownloadTest.DAILYMOTION_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.DAILYMOTION_MD5)
 
-	def test_metacafe(self):
+if __name__ == '__main__':
-		#this emulate a skip,to be 2.6 compatible
+    unittest.main()
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(MetacafeIE())
-		fd.add_info_extractor(YoutubeIE())
-		fd.download([DownloadTest.METACAFE_URL])
-		self.assertTrue(os.path.exists(DownloadTest.METACAFE_FILE))
-		self.assertEqual(os.path.getsize(DownloadTest.METACAFE_FILE), DownloadTest.METACAFE_SIZE)
-
-	def test_blip(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(BlipTVIE())
-		fd.download([DownloadTest.BLIP_URL])
-		self.assertTrue(os.path.exists(DownloadTest.BLIP_FILE))
-		md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
-
-	def test_xvideo(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(XVideosIE())
-		fd.download([DownloadTest.XVIDEO_URL])
-		self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE))
-		md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5)
-
-	def test_vimeo(self):
-		#skipped for the moment produce an error
-		return
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(VimeoIE())
-		fd.download([DownloadTest.VIMEO_URL])
-		self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE))
-		md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5)
-
-	def test_vimeo2(self):
-		#skipped for the moment produce an error
-		return
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(VimeoIE())
-		fd.download([DownloadTest.VIMEO2_URL])
-		self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE))
-		md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5)
-
-	def test_soundcloud(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(SoundcloudIE())
-		fd.download([DownloadTest.SOUNDCLOUD_URL])
-		self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE))
-		md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5)
-
-	def test_standford(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(StanfordOpenClassroomIE())
-		fd.download([DownloadTest.STANDFORD_URL])
-		self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE))
-		md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5)
-
-	def test_collegehumor(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(CollegeHumorIE())
-		fd.download([DownloadTest.COLLEGEHUMOR_URL])
-		self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE))
-		md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5)
-
-	def test_xnxx(self):
-		with open(DownloadTest.PARAMETERS_FILE) as f:
-			fd = FileDownloader(json.load(f))
-		fd.add_info_extractor(XNXXIE())
-		fd.download([DownloadTest.XNXX_URL])
-		self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE))
-		md5_down_file = md5_for_file(DownloadTest.XNXX_FILE)
-		self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5)
-
-	def tearDown(self):
-		if os.path.exists(DownloadTest.YOUTUBE_FILE):
-			os.remove(DownloadTest.YOUTUBE_FILE)
-		if os.path.exists(DownloadTest.DAILYMOTION_FILE):
-			os.remove(DownloadTest.DAILYMOTION_FILE)
-		if os.path.exists(DownloadTest.METACAFE_FILE):
-			os.remove(DownloadTest.METACAFE_FILE)
-		if os.path.exists(DownloadTest.BLIP_FILE):
-			os.remove(DownloadTest.BLIP_FILE)
-		if os.path.exists(DownloadTest.XVIDEO_FILE):
-			os.remove(DownloadTest.XVIDEO_FILE)
-		if os.path.exists(DownloadTest.VIMEO_FILE):
-			os.remove(DownloadTest.VIMEO_FILE)
-		if os.path.exists(DownloadTest.SOUNDCLOUD_FILE):
-			os.remove(DownloadTest.SOUNDCLOUD_FILE)
-		if os.path.exists(DownloadTest.STANDFORD_FILE):
-			os.remove(DownloadTest.STANDFORD_FILE)
-		if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE):
-			os.remove(DownloadTest.COLLEGEHUMOR_FILE)
-		if os.path.exists(DownloadTest.XNXX_FILE):
-			os.remove(DownloadTest.XNXX_FILE)
-
-def md5_for_file(filename, block_size=2**20):
-	with open(filename) as f:
-		md5 = hashlib.md5()
-		while True:
-			data = f.read(block_size)
-			if not data:
-				break
-			md5.update(data)
-			return md5.hexdigest()

test/test_execution.py

@@ -0,0 +1,26 @@
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+    _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+    _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+    def test_import(self):
+        subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+    def test_module_exec(self):
+        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+    def test_main_exec(self):
+        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_utils.py

@@ -1,79 +1,116 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python
 
 # Various small unit tests
 
+import sys
 import unittest
 
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 #from youtube_dl.utils import htmlentity_transform
 from youtube_dl.utils import timeconvert
 from youtube_dl.utils import sanitize_filename
 from youtube_dl.utils import unescapeHTML
 from youtube_dl.utils import orderedSet
+from youtube_dl.utils import DateRange
+from youtube_dl.utils import unified_strdate
+
+if sys.version_info < (3, 0):
+    _compat_str = lambda b: b.decode('unicode-escape')
+else:
+    _compat_str = lambda s: s
 
 
 class TestUtil(unittest.TestCase):
-	def test_timeconvert(self):
+    def test_timeconvert(self):
-		self.assertTrue(timeconvert('') is None)
+        self.assertTrue(timeconvert('') is None)
-		self.assertTrue(timeconvert('bougrg') is None)
+        self.assertTrue(timeconvert('bougrg') is None)
 
-	def test_sanitize_filename(self):
+    def test_sanitize_filename(self):
-		self.assertEqual(sanitize_filename(u'abc'), u'abc')
+        self.assertEqual(sanitize_filename('abc'), 'abc')
-		self.assertEqual(sanitize_filename(u'abc_d-e'), u'abc_d-e')
+        self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e')
 
-		self.assertEqual(sanitize_filename(u'123'), u'123')
+        self.assertEqual(sanitize_filename('123'), '123')
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/de'))
+        self.assertEqual('abc_de', sanitize_filename('abc/de'))
-		self.assertFalse(u'/' in sanitize_filename(u'abc/de///'))
+        self.assertFalse('/' in sanitize_filename('abc/de///'))
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de'))
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de'))
-		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|'))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|'))
-		self.assertEqual(u'yes no', sanitize_filename(u'yes? no'))
+        self.assertEqual('yes no', sanitize_filename('yes? no'))
-		self.assertEqual(u'this - that', sanitize_filename(u'this: that'))
+        self.assertEqual('this - that', sanitize_filename('this: that'))
 
-		self.assertEqual(sanitize_filename(u'AT&T'), u'AT&T')
+        self.assertEqual(sanitize_filename('AT&T'), 'AT&T')
-		self.assertEqual(sanitize_filename(u'ä'), u'ä')
+        aumlaut = _compat_str('\xe4')
-		self.assertEqual(sanitize_filename(u'кириллица'), u'кириллица')
+        self.assertEqual(sanitize_filename(aumlaut), aumlaut)
+        tests = _compat_str('\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430')
+        self.assertEqual(sanitize_filename(tests), tests)
 
-		forbidden = u'"\0\\/'
+        forbidden = '"\0\\/'
-		for fc in forbidden:
+        for fc in forbidden:
-			for fbc in forbidden:
+            for fbc in forbidden:
-				self.assertTrue(fbc not in sanitize_filename(fc))
+                self.assertTrue(fbc not in sanitize_filename(fc))
 
-	def test_sanitize_filename_restricted(self):
+    def test_sanitize_filename_restricted(self):
-		self.assertEqual(sanitize_filename(u'abc', restricted=True), u'abc')
+        self.assertEqual(sanitize_filename('abc', restricted=True), 'abc')
-		self.assertEqual(sanitize_filename(u'abc_d-e', restricted=True), u'abc_d-e')
+        self.assertEqual(sanitize_filename('abc_d-e', restricted=True), 'abc_d-e')
 
-		self.assertEqual(sanitize_filename(u'123', restricted=True), u'123')
+        self.assertEqual(sanitize_filename('123', restricted=True), '123')
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/de', restricted=True))
+        self.assertEqual('abc_de', sanitize_filename('abc/de', restricted=True))
-		self.assertFalse(u'/' in sanitize_filename(u'abc/de///', restricted=True))
+        self.assertFalse('/' in sanitize_filename('abc/de///', restricted=True))
 
-		self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de', restricted=True))
+        self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', restricted=True))
-		self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|', restricted=True))
+        self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', restricted=True))
-		self.assertEqual(u'yes_no', sanitize_filename(u'yes? no', restricted=True))
+        self.assertEqual('yes_no', sanitize_filename('yes? no', restricted=True))
-		self.assertEqual(u'this_-_that', sanitize_filename(u'this: that', restricted=True))
+        self.assertEqual('this_-_that', sanitize_filename('this: that', restricted=True))
 
-		self.assertEqual(sanitize_filename(u'aäb中国的c', restricted=True), u'a_b_c')
+        tests = _compat_str('a\xe4b\u4e2d\u56fd\u7684c')
-		self.assertTrue(sanitize_filename(u'ö', restricted=True) != u'') # No empty filename
+        self.assertEqual(sanitize_filename(tests, restricted=True), 'a_b_c')
+        self.assertTrue(sanitize_filename(_compat_str('\xf6'), restricted=True) != '')  # No empty filename
 
-		forbidden = u'"\0\\/&!: \'\t\n'
+        forbidden = '"\0\\/&!: \'\t\n()[]{}$;`^,#'
-		for fc in forbidden:
+        for fc in forbidden:
-			for fbc in forbidden:
+            for fbc in forbidden:
-				self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
+                self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
 
-		# Handle a common case more neatly
+        # Handle a common case more neatly
-		self.assertEqual(sanitize_filename(u'大声带 - Song', restricted=True), u'Song')
+        self.assertEqual(sanitize_filename(_compat_str('\u5927\u58f0\u5e26 - Song'), restricted=True), 'Song')
-		self.assertEqual(sanitize_filename(u'总统: Speech', restricted=True), u'Speech')
+        self.assertEqual(sanitize_filename(_compat_str('\u603b\u7edf: Speech'), restricted=True), 'Speech')
-		# .. but make sure the file name is never empty
+        # .. but make sure the file name is never empty
-		self.assertTrue(sanitize_filename(u'-', restricted=True) != u'')
+        self.assertTrue(sanitize_filename('-', restricted=True) != '')
-		self.assertTrue(sanitize_filename(u':', restricted=True) != u'')
+        self.assertTrue(sanitize_filename(':', restricted=True) != '')
 
-	def test_ordered_set(self):
+    def test_sanitize_ids(self):
-		self.assertEqual(orderedSet([1,1,2,3,4,4,5,6,7,3,5]), [1,2,3,4,5,6,7])
+        self.assertEqual(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
-		self.assertEqual(orderedSet([]), [])
+        self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
-		self.assertEqual(orderedSet([1]), [1])
+        self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
-		#keep the list ordered
-		self.assertEqual(orderedSet([135,1,1,1]), [135,1])
 
-	def test_unescape_html(self):
+    def test_ordered_set(self):
-		self.assertEqual(unescapeHTML(u"%20;"), u"%20;")
+        self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
+        self.assertEqual(orderedSet([]), [])
+        self.assertEqual(orderedSet([1]), [1])
+        #keep the list ordered
+        self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])
+
+    def test_unescape_html(self):
+        self.assertEqual(unescapeHTML(_compat_str('%20;')), _compat_str('%20;'))
+        
+    def test_daterange(self):
+        _20century = DateRange("19000101","20000101")
+        self.assertFalse("17890714" in _20century)
+        _ac = DateRange("00010101")
+        self.assertTrue("19690721" in _ac)
+        _firstmilenium = DateRange(end="10000101")
+        self.assertTrue("07110427" in _firstmilenium)
+
+    def test_unified_dates(self):
+        self.assertEqual(unified_strdate('December 21, 2010'), '20101221')
+        self.assertEqual(unified_strdate('8/7/2009'), '20090708')
+        self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
+        self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
+
+if __name__ == '__main__':
+    unittest.main()

test/test_write_info_json.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import json
+import os
+import sys
+import unittest
+
+# Allow direct execution
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl.FileDownloader
+import youtube_dl.InfoExtractors
+from youtube_dl.utils import *
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FileDownloader(youtube_dl.FileDownloader):
+    def __init__(self, *args, **kwargs):
+        youtube_dl.FileDownloader.__init__(self, *args, **kwargs)
+        self.to_stderr = self.to_screen
+
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    params = json.load(pf)
+params['writeinfojson'] = True
+params['skip_download'] = True
+params['writedescription'] = True
+
+TEST_ID = 'BaW_jenozKc'
+INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
+DESCRIPTION_FILE = TEST_ID + '.mp4.description'
+EXPECTED_DESCRIPTION = u'''test chars:  "'/\ä↭𝕐
+
+This is a test video for youtube-dl.
+
+For more information, contact phihag@phihag.de .'''
+
+class TestInfoJSON(unittest.TestCase):
+    def setUp(self):
+        # Clear old files
+        self.tearDown()
+
+    def test_info_json(self):
+        ie = youtube_dl.InfoExtractors.YoutubeIE()
+        fd = FileDownloader(params)
+        fd.add_info_extractor(ie)
+        fd.download([TEST_ID])
+        self.assertTrue(os.path.exists(INFO_JSON_FILE))
+        with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
+            jd = json.load(jsonf)
+        self.assertEqual(jd['upload_date'], u'20121002')
+        self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
+        self.assertEqual(jd['id'], TEST_ID)
+        self.assertEqual(jd['extractor'], 'youtube')
+        self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
+        self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
+
+        self.assertTrue(os.path.exists(DESCRIPTION_FILE))
+        with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
+            descr = descf.read()
+        self.assertEqual(descr, EXPECTED_DESCRIPTION)
+
+    def tearDown(self):
+        if os.path.exists(INFO_JSON_FILE):
+            os.remove(INFO_JSON_FILE)
+        if os.path.exists(DESCRIPTION_FILE):
+            os.remove(DESCRIPTION_FILE)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_lists.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE
+from youtube_dl.utils import *
+from youtube_dl.FileDownloader import FileDownloader
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(FileDownloader):
+    def __init__(self):
+        self.result = []
+        self.params = parameters
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s, tb=None):
+        raise Exception(s)
+    def extract_info(self, url):
+        self.result.append(url)
+        return url
+
+class TestYoutubeLists(unittest.TestCase):
+    def assertIsPlaylist(self,info):
+        """Make sure the info has '_type' set to 'playlist'"""
+        self.assertEqual(info['_type'], 'playlist')
+
+    def test_youtube_playlist(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')[0]
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['title'], 'ytdl test PL')
+        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
+        self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
+
+    def test_issue_673(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('PLBB231211A4F62143')[0]
+        self.assertTrue(len(result['entries']) > 25)
+
+    def test_youtube_playlist_long(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')[0]
+        self.assertIsPlaylist(result)
+        self.assertTrue(len(result['entries']) >= 799)
+
+    def test_youtube_playlist_with_deleted(self):
+        #651
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')[0]
+        ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
+        self.assertFalse('pElCt5oNDuI' in ytie_results)
+        self.assertFalse('KdPEApIVdWM' in ytie_results)
+        
+    def test_youtube_playlist_empty(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx')[0]
+        self.assertIsPlaylist(result)
+        self.assertEqual(len(result['entries']), 0)
+
+    def test_youtube_course(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        # TODO find a > 100 (paginating?) videos course
+        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')[0]
+        entries = result['entries']
+        self.assertEqual(YoutubeIE()._extract_id(entries[0]['url']), 'j9WZyLZCBzs')
+        self.assertEqual(len(entries), 25)
+        self.assertEqual(YoutubeIE()._extract_id(entries[-1]['url']), 'rYefUsYuEp0')
+
+    def test_youtube_channel(self):
+        dl = FakeDownloader()
+        ie = YoutubeChannelIE(dl)
+        #test paginated channel
+        result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w')[0]
+        self.assertTrue(len(result['entries']) > 90)
+        #test autogenerated channel
+        result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')[0]
+        self.assertTrue(len(result['entries']) >= 18)
+
+    def test_youtube_user(self):
+        dl = FakeDownloader()
+        ie = YoutubeUserIE(dl)
+        result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0]
+        self.assertTrue(len(result['entries']) >= 320)
+
+    def test_youtube_safe_search(self):
+        dl = FakeDownloader()
+        ie = YoutubePlaylistIE(dl)
+        result = ie.extract('PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl')[0]
+        self.assertEqual(len(result['entries']), 2)
+
+if __name__ == '__main__':
+    unittest.main()

test/test_youtube_subtitles.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import json
+import io
+import hashlib
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE
+from youtube_dl.utils import *
+from youtube_dl import FileDownloader
+
+PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    parameters = json.load(pf)
+
+# General configuration (from __init__, not very elegant...)
+jar = compat_cookiejar.CookieJar()
+cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
+proxy_handler = compat_urllib_request.ProxyHandler()
+opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
+compat_urllib_request.install_opener(opener)
+
+class FakeDownloader(FileDownloader):
+    def __init__(self):
+        self.result = []
+        # Different instances of the downloader can't share the same dictionary
+        # some test set the "sublang" parameter, which would break the md5 checks.
+        self.params = dict(parameters)
+    def to_screen(self, s):
+        print(s)
+    def trouble(self, s, tb=None):
+        raise Exception(s)
+    def download(self, x):
+        self.result.append(x)
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class TestYoutubeSubtitles(unittest.TestCase):
+    def setUp(self):
+        DL = FakeDownloader()
+        DL.params['allsubtitles'] = False
+        DL.params['writesubtitles'] = False
+        DL.params['subtitlesformat'] = 'srt'
+        DL.params['listsubtitles'] = False
+    def test_youtube_no_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = False
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        subtitles = info_dict[0]['subtitles']
+        self.assertEqual(subtitles, None)
+    def test_youtube_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260')
+    def test_youtube_subtitles_it(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitleslang'] = 'it'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '164a51f16f260476a05b50fe4c2f161d')
+    def test_youtube_onlysubtitles(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['onlysubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '4cd9278a35ba2305f47354ee13472260')
+    def test_youtube_allsubtitles(self):
+        DL = FakeDownloader()
+        DL.params['allsubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        subtitles = info_dict[0]['subtitles']
+        self.assertEqual(len(subtitles), 13)
+    def test_youtube_subtitles_format(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitlesformat'] = 'sbv'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertEqual(md5(sub[2]), '13aeaa0c245a8bed9a451cb643e3ad8b')
+    def test_youtube_list_subtitles(self):
+        DL = FakeDownloader()
+        DL.params['listsubtitles'] = True
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('QRS8MkLhQmM')
+        self.assertEqual(info_dict, None)
+    def test_youtube_automatic_captions(self):
+        DL = FakeDownloader()
+        DL.params['writesubtitles'] = True
+        DL.params['subtitleslang'] = 'it'
+        IE = YoutubeIE(DL)
+        info_dict = IE.extract('8YoUxe5ncPo')
+        sub = info_dict[0]['subtitles'][0]
+        self.assertTrue(sub[2] is not None)
+
+if __name__ == '__main__':
+    unittest.main()

test/tests.json

@@ -0,0 +1,634 @@
+[
+  {
+    "name": "Youtube",
+    "url":  "http://www.youtube.com/watch?v=BaW_jenozKc",
+    "file":  "BaW_jenozKc.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video \"'/\\ä↭𝕐",
+      "uploader": "Philipp Hagemeister",
+      "uploader_id": "phihag",
+      "upload_date": "20121002",
+      "description": "test chars:  \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
+    }
+  },
+  {
+    "name": "Dailymotion",
+    "md5":  "392c4b85a60a90dc4792da41ce3144eb",
+    "url":  "http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech",
+    "file":  "x33vw9.mp4",
+    "info_dict": {
+      "uploader": "Alex and Van .",
+      "title": "Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
+    }
+  },
+  {
+    "name": "Metacafe",
+    "add_ie": ["Youtube"],
+    "url":  "http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
+    "file":  "_aUehQsCQtM.flv",
+    "info_dict": {
+      "upload_date": "20090102",
+      "title": "The Electric Company | \"Short I\" | PBS KIDS GO!",
+      "description": "md5:2439a8ef6d5a70e380c22f5ad323e5a8",
+      "uploader": "PBS",
+      "uploader_id": "PBS"
+    }
+  },
+  {
+    "name": "BlipTV",
+    "md5":  "b2d849efcf7ee18917e4b4d9ff37cafe",
+    "url":  "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352",
+    "file":  "5779306.m4v",
+    "info_dict": {
+      "upload_date": "20111205",
+      "description": "md5:9bc31f227219cde65e47eeec8d2dc596",
+      "uploader": "Comic Book Resources - CBR TV",
+      "title": "CBR EXCLUSIVE: \"Gotham City Imposters\" Bats VS Jokerz Short 3"
+    }
+  },
+  {
+    "name": "XVideos",
+    "md5":  "1d0c835822f0a71a7bf011855db929d0",
+    "url":  "http://www.xvideos.com/video939581/funny_porns_by_s_-1",
+    "file":  "939581.flv",
+    "info_dict": {
+      "title": "Funny Porns By >>>>S<<<<<< -1"
+    }
+  },
+  {
+    "name": "YouPorn",
+    "md5": "c37ddbaaa39058c76a7e86c6813423c1",
+    "url": "http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/",
+    "file": "505835.mp4",
+    "info_dict": {
+      "upload_date": "20101221",
+      "description": "Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?",
+      "uploader": "Ask Dan And Jennifer",
+      "title": "Sex Ed: Is It Safe To Masturbate Daily?"
+    }
+  },
+  {
+    "name": "Pornotube",
+    "md5": "374dd6dcedd24234453b295209aa69b6",
+    "url": "http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing",
+    "file": "1689755.flv",
+    "info_dict": {
+      "upload_date": "20090708",
+      "title": "Marilyn-Monroe-Bathing"
+    }
+  },
+  {
+    "name": "YouJizz",
+    "md5": "07e15fa469ba384c7693fd246905547c",
+    "url": "http://www.youjizz.com/videos/zeichentrick-1-2189178.html",
+    "file": "2189178.flv",
+    "info_dict": {
+      "title": "Zeichentrick 1"
+    }
+  },
+  {
+    "name": "Vimeo",
+    "md5":  "8879b6cc097e987f02484baf890129e5",
+    "url":  "http://vimeo.com/56015672",
+    "file": "56015672.mp4",
+    "info_dict": {
+      "title": "youtube-dl test video - ★ \" ' 幸 / \\ ä ↭ 𝕐",
+      "uploader": "Filippo Valsorda",
+      "uploader_id": "user7108434",
+      "upload_date": "20121220",
+      "description": "This is a test case for youtube-dl.\nFor more information, see github.com/rg3/youtube-dl\nTest chars: ★ \" ' 幸 / \\ ä ↭ 𝕐"
+    }
+  },
+  {
+    "name": "Soundcloud",
+    "md5":  "ebef0a451b909710ed1d7787dddbf0d7",
+    "url":  "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
+    "file":  "62986583.mp3",
+    "info_dict": {
+      "upload_date": "20121011",
+      "description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
+      "uploader": "E.T. ExTerrestrial Music",
+      "title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
+    }
+  },
+  {
+    "name": "StanfordOpenClassroom",
+    "md5":  "544a9468546059d4e80d76265b0443b8",
+    "url":  "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100",
+    "file":  "PracticalUnix_intro-environment.mp4",
+    "info_dict": {
+      "title": "Intro Environment"
+    }
+  },
+  {
+    "name": "XNXX",
+    "md5":  "0831677e2b4761795f68d417e0b7b445",
+    "url":  "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
+    "file":  "1135332.flv",
+    "info_dict": {
+      "title": "lida » Naked Funny Actress  (5)"
+    }
+  },
+  {
+    "name": "Youku",
+    "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+    "file": "XNDgyMDQ2NTQw_part00.flv",
+    "md5": "ffe3f2e435663dc2d1eea34faeff5b5b",
+    "params": { "test": false },
+    "info_dict": {
+      "title": "youtube-dl test video \"'/\\ä↭𝕐"
+    }
+  },
+  {
+    "name": "NBA",
+    "url": "http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html",
+    "file": "0021200253-okc-bkn-recap.nba.mp4",
+    "md5": "c0edcfc37607344e2ff8f13c378c88a4",
+    "info_dict": {
+      "description": "Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.",
+      "title": "Thunder vs. Nets"
+    }
+  },
+  {
+    "name": "JustinTV",
+    "url": "http://www.twitch.tv/thegamedevhub/b/296128360",
+    "file": "296128360.flv",
+    "md5": "ecaa8a790c22a40770901460af191c9a",
+    "info_dict": {
+      "upload_date": "20110927",
+      "uploader_id": 25114803,
+      "uploader": "thegamedevhub",
+      "title": "Beginner Series - Scripting With Python Pt.1"
+    }
+  },
+  {
+    "name": "MyVideo",
+    "url": "http://www.myvideo.de/watch/8229274/bowling_fail_or_win",
+    "file": "8229274.flv",
+    "md5": "2d2753e8130479ba2cb7e0a37002053e",
+    "info_dict": {
+      "title": "bowling-fail-or-win"
+    }
+  },
+  {
+    "name": "Escapist",
+    "url": "http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate",
+    "file": "6618-Breaking-Down-Baldurs-Gate.mp4",
+    "md5": "c6793dbda81388f4264c1ba18684a74d",
+    "info_dict": {
+      "description": "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
+      "uploader": "the-escapist-presents",
+      "title": "Breaking Down Baldur's Gate"
+    }
+  },
+  {
+    "name": "GooglePlus",
+    "url": "https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH",
+    "file": "ZButuJc6CtH.flv",
+    "info_dict": {
+      "upload_date": "20120613",
+      "uploader": "井上ヨシマサ",
+      "title": "嘆きの天使 降臨"
+    }
+  },
+  {
+    "name": "FunnyOrDie",
+    "url": "http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version",
+    "file": "0732f586d7.mp4",
+    "md5": "f647e9e90064b53b6e046e75d0241fbd",
+    "info_dict": {
+      "description": "Lyrics changed to match the video. Spoken cameo by Obscurus Lupa (from ThatGuyWithTheGlasses.com). Based on a concept by Dustin McLean (DustFilms.com). Performed, edited, and written by David A. Scott.",
+      "title": "Heart-Shaped Box: Literal Video Version"
+    }
+  },
+  {
+    "name": "Steam",
+    "url": "http://store.steampowered.com/video/105600/",
+    "playlist": [
+      {
+        "file": "81300.flv",
+        "md5": "f870007cee7065d7c76b88f0a45ecc07",
+        "info_dict": {
+            "title": "Terraria 1.1 Trailer"
+        }
+      },
+      {
+        "file": "80859.flv",
+        "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+        "info_dict": {
+          "title": "Terraria Trailer"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Ustream",
+    "url": "http://www.ustream.tv/recorded/20274954",
+    "file": "20274954.flv",
+    "md5": "088f151799e8f572f84eb62f17d73e5c",
+    "info_dict": {
+        "title": "Young Americans for Liberty February 7, 2012 2:28 AM",
+        "uploader": "Young Americans for Liberty"
+    }
+  },
+  {
+    "name": "InfoQ",
+    "url": "http://www.infoq.com/presentations/A-Few-of-My-Favorite-Python-Things",
+    "file": "12-jan-pythonthings.mp4",
+    "info_dict": {
+      "description": "Mike Pirnat presents some tips and tricks, standard libraries and third party packages that make programming in Python a richer experience.",
+      "title": "A Few of My Favorite [Python] Things"
+    },
+    "params": {
+      "skip_download": true
+    }
+  },
+  {
+    "name": "ComedyCentral",
+    "url": "http://www.thedailyshow.com/watch/thu-december-13-2012/kristen-stewart",
+    "file": "422212.mp4",
+    "md5": "4e2f5cb088a83cd8cdb7756132f9739d",
+    "info_dict": {
+      "upload_date": "20121214",
+      "description": "Kristen Stewart",
+      "uploader": "thedailyshow",
+      "title": "thedailyshow-kristen-stewart part 1"
+    }
+  },
+  {
+    "name": "RBMARadio",
+    "url": "http://www.rbmaradio.com/shows/ford-lopatin-live-at-primavera-sound-2011",
+    "file": "ford-lopatin-live-at-primavera-sound-2011.mp3",
+    "md5": "6bc6f9bcb18994b4c983bc3bf4384d95",
+    "info_dict": {
+        "title": "Live at Primavera Sound 2011",
+        "description": "Joel Ford and Daniel \u2019Oneohtrix Point Never\u2019 Lopatin fly their midified pop extravaganza to Spain. Live at Primavera Sound 2011.",
+        "uploader": "Ford & Lopatin",
+        "uploader_id": "ford-lopatin",
+        "location": "Spain"
+    }
+  },
+  {
+    "name": "Facebook",
+    "url": "https://www.facebook.com/photo.php?v=120708114770723",
+    "file": "120708114770723.mp4",
+    "md5": "48975a41ccc4b7a581abd68651c1a5a8",
+    "info_dict": {
+      "title": "PEOPLE ARE AWESOME 2013",
+      "duration": 279
+    }
+  },
+  {
+    "name": "EightTracks",
+    "url": "http://8tracks.com/ytdl/youtube-dl-test-tracks-a",
+    "playlist": [
+      {
+        "file": "11885610.m4a",
+        "md5": "96ce57f24389fc8734ce47f4c1abcc55",
+        "info_dict": {
+          "title": "youtue-dl project<>\"' - youtube-dl test track 1 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885608.m4a",
+        "md5": "4ab26f05c1f7291ea460a3920be8021f",
+        "info_dict": {
+          "title": "youtube-dl project - youtube-dl test track 2 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+
+        }
+      },
+      {
+        "file": "11885679.m4a",
+        "md5": "d30b5b5f74217410f4689605c35d1fd7",
+        "info_dict": {
+          "title": "youtube-dl project as well - youtube-dl test track 3 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885680.m4a",
+        "md5": "4eb0a669317cd725f6bbd336a29f923a",
+        "info_dict": {
+          "title": "youtube-dl project as well - youtube-dl test track 4 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885682.m4a",
+        "md5": "1893e872e263a2705558d1d319ad19e8",
+        "info_dict": {
+          "title": "PH - youtube-dl test track 5 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885683.m4a",
+        "md5": "b673c46f47a216ab1741ae8836af5899",
+        "info_dict": {
+          "title": "PH - youtube-dl test track 6 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885684.m4a",
+        "md5": "1d74534e95df54986da7f5abf7d842b7",
+        "info_dict": {
+          "title": "phihag - youtube-dl test track 7 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      },
+      {
+        "file": "11885685.m4a",
+        "md5": "f081f47af8f6ae782ed131d38b9cd1c0",
+        "info_dict": {
+          "title": "phihag - youtube-dl test track 8 \"'/\\\u00e4\u21ad",
+          "uploader_id": "ytdl"
+        }
+      }
+    ]
+  },
+  {
+    "name": "Keek",
+    "url": "http://www.keek.com/ytdl/keeks/NODfbab",
+    "file": "NODfbab.mp4",
+    "md5": "9b0636f8c0f7614afa4ea5e4c6e57e83",
+    "info_dict": {
+      "uploader": "ytdl",
+      "title": "test chars: \"'/\\ä<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de ."
+    }
+  },
+  {
+    "name": "TED",
+    "url": "http://www.ted.com/talks/dan_dennett_on_our_consciousness.html",
+    "file": "102.mp4",
+    "md5": "8cd9dfa41ee000ce658fd48fb5d89a61",
+    "info_dict": {
+        "title": "Dan Dennett: The illusion of consciousness",
+        "description": "md5:c6fa72e6eedbd938c9caf6b2702f5922"
+    }
+  },
+  {
+    "name": "MySpass",
+    "url": "http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/",
+    "file": "11741.mp4",
+    "md5": "0b49f4844a068f8b33f4b7c88405862b",
+    "info_dict": {
+      "description": "Wer kann in die Fußstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
+      "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2"
+    }
+  },
+  {
+    "name": "Generic",
+    "url": "http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html",
+    "file": "13601338388002.mp4",
+    "md5": "85b90ccc9d73b4acd9138d3af4c27f89",
+    "info_dict": {
+      "uploader": "www.hodiho.fr",
+      "title": "Régis plante sa Jeep"
+    }
+  },
+  {
+    "name": "Spiegel",
+    "url": "http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html",
+    "file": "1259285.mp4",
+    "md5": "2c2754212136f35fb4b19767d242f66e",
+    "info_dict": {
+        "title": "Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
+    }
+  },
+  {
+    "name": "LiveLeak",
+    "md5":  "0813c2430bea7a46bf13acf3406992f4",
+    "url":  "http://www.liveleak.com/view?i=757_1364311680",
+    "file":  "757_1364311680.mp4",
+    "info_dict": {
+        "title": "Most unlucky car accident",
+        "description": "extremely bad day for this guy..!",
+        "uploader": "ljfriel2"
+    }
+  },
+  {
+    "name": "WorldStarHipHop",
+    "url": "http://www.worldstarhiphop.com/videos/video.php?v=wshh6a7q1ny0G34ZwuIO",
+    "file": "wshh6a7q1ny0G34ZwuIO.mp4",
+    "md5": "9d04de741161603bf7071bbf4e883186",
+    "info_dict": {
+        "title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+    }
+  },
+  {
+    "name": "ARD",
+    "url": "http://www.ardmediathek.de/das-erste/tagesschau-in-100-sek?documentId=14077640",
+    "file": "14077640.mp4",
+    "md5": "6ca8824255460c787376353f9e20bbd8",
+    "info_dict": {
+        "title": "11.04.2013 09:23 Uhr - Tagesschau in 100 Sekunden"
+    },
+    "skip": "Requires rtmpdump"
+  },
+  {
+    "name": "Tumblr",
+    "url": "http://resigno.tumblr.com/post/53364321212/e-de-extrema-importancia-que-esse-video-seja",
+    "file": "53364321212.mp4",
+    "md5": "0716d3dd51baf68a28b40fdf1251494e",
+    "info_dict": {
+        "title": "Rafael Lemos | Tumblr"
+    }
+  },
+  {
+    "name": "SoundcloudSet",
+    "url":"https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep",
+    "playlist":[
+      {
+        "file":"30510138.mp3",
+        "md5":"f9136bf103901728f29e419d2c70f55d",
+        "info_dict": {
+          "upload_date": "20111213",
+          "description": "The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "D-D-Dance"
+        }
+      },
+      {
+        "file":"47127625.mp3",
+        "md5":"09b6758a018470570f8fd423c9453dd8",
+        "info_dict": {
+          "upload_date": "20120521",
+          "description": "The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "The Royal Concept - Gimme Twice"
+        }
+      },
+      {
+        "file":"47127627.mp3",
+        "md5":"154abd4e418cea19c3b901f1e1306d9c",
+        "info_dict": {
+          "upload_date": "20120521",
+          "uploader": "The Royal Concept",
+          "title": "Goldrushed"
+        }
+      },
+      {
+        "file":"47127629.mp3",
+        "md5":"2f5471edc79ad3f33a683153e96a79c1",
+        "info_dict": {
+          "upload_date": "20120521",
+          "description": "The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "In the End"
+        }
+      },
+      {
+        "file":"47127631.mp3",
+        "md5":"f9ba87aa940af7213f98949254f1c6e2",
+        "info_dict": {
+          "upload_date": "20120521",
+          "description": "The Royal Concept from Stockholm\r\nFilip / David / Povel / Magnus\r\nwww.theroyalconceptband.com",
+          "uploader": "The Royal Concept",
+          "title": "Knocked Up"
+        }
+      },
+      {
+        "file":"75206121.mp3",
+        "md5":"f9d1fe9406717e302980c30de4af9353",
+        "info_dict": {
+          "upload_date": "20130116",
+          "description": "The unreleased track World on Fire premiered on the CW's hit show Arrow (8pm/7pm central).  \r\nAs a gift to our fans we would like to offer you a free download of the track!  ",
+          "uploader": "The Royal Concept",
+          "title": "World On Fire"
+        }
+      }
+    ]
+  },
+  {
+    "name":"Bandcamp",
+    "url":"http://youtube-dl.bandcamp.com/track/youtube-dl-test-song",
+    "file":"1812978515.mp3",
+    "md5":"cdeb30cdae1921719a3cbcab696ef53c",
+    "info_dict": {
+      "title":"youtube-dl test song \"'/\\ä↭"
+    },
+    "skip": "There is a limit of 200 free downloads / month for the test song"
+  },
+  {
+    "name": "RedTube",
+    "url": "http://www.redtube.com/66418",
+    "file": "66418.mp4",
+    "md5": "7b8c22b5e7098a3e1c09709df1126d2d",
+    "info_dict":{
+      "title":"Sucked on a toilet"
+    }
+  },
+  {
+    "name": "Photobucket",
+    "url": "http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0",
+    "file": "zpsc0c3b9fa.mp4",
+    "md5": "7dabfb92b0a31f6c16cebc0f8e60ff99",
+    "info_dict": {
+      "upload_date": "20130504",
+      "uploader": "rachaneronas",
+      "title": "Tired of Link Building? Try BacklinkMyDomain.com!"
+    }
+  },
+  {
+    "name": "Ina",
+    "url": "www.ina.fr/video/I12055569/francois-hollande-je-crois-que-c-est-clair-video.html",
+    "file": "I12055569.mp4",
+    "md5": "a667021bf2b41f8dc6049479d9bb38a3",
+    "info_dict":{
+      "title":"François Hollande \"Je crois que c'est clair\""
+    }
+  },
+  {
+    "name": "Yahoo",
+    "url": "http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html",
+    "file": "214727115.flv",
+    "md5": "2e717f169c1be93d84d3794a00d4a325",
+    "info_dict": {
+        "title": "Julian Smith & Travis Legg Watch Julian Smith"
+    },
+    "skip": "Requires rtmpdump"
+  },
+  {
+    "name": "Howcast",
+    "url": "http://www.howcast.com/videos/390161-How-to-Tie-a-Square-Knot-Properly",
+    "file": "390161.mp4",
+    "md5": "1d7ba54e2c9d7dc6935ef39e00529138",
+    "info_dict":{
+      "title":"How to Tie a Square Knot Properly",
+      "description":"The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here's the proper way to tie a square knot."
+    }
+  },
+  {
+    "name": "Vine",
+    "url": "https://vine.co/v/b9KOOWX7HUx",
+    "file": "b9KOOWX7HUx.mp4",
+    "md5": "2f36fed6235b16da96ce9b4dc890940d",
+    "info_dict":{
+      "title": "Chicken.",
+      "uploader": "Jack Dorsey"
+    }
+  },
+  {
+    "name": "Flickr",
+    "url": "http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/",
+    "file": "5645318632.mp4",
+    "md5": "6fdc01adbc89d72fc9c4f15b4a4ba87b",
+    "info_dict":{
+      "title": "Dark Hollow Waterfalls",
+      "uploader_id": "forestwander-nature-pictures",
+      "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up."
+    }
+  },
+  {
+    "name": "Teamcoco",
+    "url": "http://teamcoco.com/video/louis-ck-interview-george-w-bush",
+    "file": "19705.mp4",
+    "md5": "27b6f7527da5acf534b15f21b032656e",
+    "info_dict":{
+      "title": "Louis C.K. Interview Pt. 1 11/3/11",
+      "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one."
+    }
+  },
+  {
+    "name": "XHamster",
+    "url": "http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html",
+    "file": "1509445.flv",
+    "md5": "9f48e0e8d58e3076bb236ff412ab62fa",
+    "info_dict": {
+      "upload_date": "20121014",
+      "uploader_id": "Ruseful2011",
+      "title": "FemaleAgent Shy beauty takes the bait"
+    }
+  },
+  {
+    "name": "Hypem",
+    "url": "http://hypem.com/track/1v6ga/BODYWORK+-+TAME",
+    "file": "1v6ga.mp3",
+    "md5": "b9cc91b5af8995e9f0c1cee04c575828",
+    "info_dict":{
+      "title":"Tame"
+    }
+  },
+  {
+    "name": "Vbox7",
+    "url": "http://vbox7.com/play:249bb972c2",
+    "file": "249bb972c2.flv",
+    "md5": "9c70d6d956f888bdc08c124acc120cfe",
+    "info_dict":{
+      "title":"Смях! Чудо - чист за секунди - Скрита камера"
+    }
+  },
+  {
+    "name": "Gametrailers",
+    "url": "http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer",
+    "file": "zbvr8i.flv",
+    "md5": "c3edbc995ab4081976e16779bd96a878",
+    "info_dict": {
+        "title": "E3 2013: Debut Trailer"
+    },
+    "skip": "Requires rtmpdump"
+  }
+]

youtube-dl.1

@@ -1,306 +0,0 @@
-.TH YOUTUBE-DL 1 "" 
-.SH NAME
-.PP
-youtube-dl
-.SH SYNOPSIS
-.PP
-\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
-.SH DESCRIPTION
-.PP
-\f[B]youtube-dl\f[] is a small command-line program to download videos
-from YouTube.com and a few more sites.
-It requires the Python interpreter, version 2.x (x being at least 6),
-and it is not platform specific.
-It should work in your Unix box, in Windows or in Mac OS X.
-It is released to the public domain, which means you can modify it,
-redistribute it or use it however you like.
-.SH OPTIONS
-.IP
-.nf
-\f[C]
--h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
---version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
--U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
--i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
--r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
--R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
---buffer-size\ SIZE\ \ \ \ \ \ \ size\ of\ download\ buffer\ (e.g.\ 1024\ or\ 16k)\ (default
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ is\ 1024)
---no-resize-buffer\ \ \ \ \ \ \ do\ not\ automatically\ adjust\ the\ buffer\ size.\ By
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default,\ the\ buffer\ size\ is\ automatically\ resized
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ an\ initial\ value\ of\ SIZE.
---dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
---user-agent\ UA\ \ \ \ \ \ \ \ \ \ specify\ a\ custom\ user\ agent
---list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
-\f[]
-.fi
-.SS Video Selection:
-.IP
-.nf
-\f[C]
---playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
---playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
---match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
---reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
---max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
-\f[]
-.fi
-.SS Filesystem Options:
-.IP
-.nf
-\f[C]
--t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
---id\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ video\ ID\ in\ file\ name
--l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ [deprecated]\ alias\ of\ --title
--A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
--o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(title)s\ to\ get\ the
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
---restrict-filenames\ \ \ \ \ Restrict\ filenames\ to\ only\ ASCII\ characters,\ and
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avoid\ "&"\ and\ spaces\ in\ filenames
--a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
--w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
--c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
---no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
---cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
---no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
---no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
---write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
---write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
-\f[]
-.fi
-.SS Verbosity / Simulation Options:
-.IP
-.nf
-\f[C]
--q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
--s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
---skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
--g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
--e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
---get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
---get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
---get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
---get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
---no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
---console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
--v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
-\f[]
-.fi
-.SS Video Format Options:
-.IP
-.nf
-\f[C]
--f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
---all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
---prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
---max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
--F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
---write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
---srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
-\f[]
-.fi
-.SS Authentication Options:
-.IP
-.nf
-\f[C]
--u,\ --username\ USERNAME\ \ account\ username
--p,\ --password\ PASSWORD\ \ account\ password
--n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
-\f[]
-.fi
-.SS Post-processing Options:
-.IP
-.nf
-\f[C]
--x,\ --extract-audio\ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
---audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
---audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ quality\ specification,\ insert\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ value\ between\ 0\ (better)\ and\ 9\ (worse)\ for\ VBR\ or\ a
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ specific\ bitrate\ like\ 128K\ (default\ 5)
--k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
-\f[]
-.fi
-.SH CONFIGURATION
-.PP
-You can configure youtube-dl by placing default arguments (such as
-\f[C]--extract-audio\ --no-mtime\f[] to always extract the audio and not
-copy the mtime) into \f[C]/etc/youtube-dl.conf\f[] and/or
-\f[C]~/.local/config/youtube-dl.conf\f[].
-.SH OUTPUT TEMPLATE
-.PP
-The \f[C]-o\f[] option allows users to indicate a template for the
-output file names.
-The basic usage is not to set any template arguments when downloading a
-single file, like in
-\f[C]youtube-dl\ -o\ funny_video.flv\ "http://some/video"\f[].
-However, it may contain special sequences that will be replaced when
-downloading each video.
-The special sequences have the format \f[C]%(NAME)s\f[].
-To clarify, that is a percent symbol followed by a name in parenthesis,
-followed by a lowercase S.
-Allowed names are:
-.IP \[bu] 2
-\f[C]id\f[]: The sequence will be replaced by the video identifier.
-.IP \[bu] 2
-\f[C]url\f[]: The sequence will be replaced by the video URL.
-.IP \[bu] 2
-\f[C]uploader\f[]: The sequence will be replaced by the nickname of the
-person who uploaded the video.
-.IP \[bu] 2
-\f[C]upload_date\f[]: The sequence will be replaced by the upload date
-in YYYYMMDD format.
-.IP \[bu] 2
-\f[C]title\f[]: The sequence will be replaced by the video title.
-.IP \[bu] 2
-\f[C]ext\f[]: The sequence will be replaced by the appropriate extension
-(like flv or mp4).
-.IP \[bu] 2
-\f[C]epoch\f[]: The sequence will be replaced by the Unix epoch when
-creating the file.
-.IP \[bu] 2
-\f[C]autonumber\f[]: The sequence will be replaced by a five-digit
-number that will be increased with each download, starting at zero.
-.PP
-The current default template is \f[C]%(id)s.%(ext)s\f[], but that will
-be switchted to \f[C]%(title)s-%(id)s.%(ext)s\f[] (which can be
-requested with \f[C]-t\f[] at the moment).
-.PP
-In some cases, you don\[aq]t want special characters such as 中, spaces,
-or &, such as when transferring the downloaded filename to a Windows
-system or the filename through an 8bit-unsafe channel.
-In these cases, add the \f[C]--restrict-filenames\f[] flag to get a
-shorter title:
-.IP
-.nf
-\f[C]
-$\ youtube-dl\ --get-filename\ -o\ "%(title)s.%(ext)s"\ BaW_jenozKc
-youtube-dl\ test\ video\ \[aq]\[aq]_ä↭𝕐.mp4\ \ \ \ #\ All\ kinds\ of\ weird\ characters
-$\ youtube-dl\ --get-filename\ -o\ "%(title)s.%(ext)s"\ BaW_jenozKc\ --restrict-filenames
-youtube-dl_test_video_.mp4\ \ \ \ \ \ \ \ \ \ #\ A\ simple\ file\ name
-\f[]
-.fi
-.SH FAQ
-.SS Can you please put the -b option back?
-.PP
-Most people asking this question are not aware that youtube-dl now
-defaults to downloading the highest available quality as reported by
-YouTube, which will be 1080p or 720p in some cases, so you no longer
-need the -b option.
-For some specific videos, maybe YouTube does not report them to be
-available in a specific high quality format you\[aq]\[aq]re interested
-in.
-In that case, simply request it with the -f option and youtube-dl will
-try to download it.
-.SS I get HTTP error 402 when trying to download a video. What\[aq]s
-this?
-.PP
-Apparently YouTube requires you to pass a CAPTCHA test if you download
-too much.
-We\[aq]\[aq]re considering to provide a way to let you solve the
-CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
-moment, your best course of action is pointing a webbrowser to the
-youtube URL, solving the CAPTCHA, and restart youtube-dl.
-.SS I have downloaded a video but how can I play it?
-.PP
-Once the video is fully downloaded, use any video player, such as
-vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
-.SS The links provided by youtube-dl -g are not working anymore
-.PP
-The URLs youtube-dl outputs require the downloader to have the correct
-cookies.
-Use the \f[C]--cookies\f[] option to write the required cookies into a
-file, and advise your downloader to read cookies from that file.
-Some sites also require a common user agent to be used, use
-\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
-.SS ERROR: no fmt_url_map or conn information found in video info
-.PP
-youtube has switched to a new video info format in July 2011 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS ERROR: unable to download video
-.PP
-youtube requires an additional signature since September 2012 which is
-not supported by old versions of youtube-dl.
-You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
-.SS SyntaxError: Non-ASCII character
-.PP
-The error
-.IP
-.nf
-\f[C]
-File\ "youtube-dl",\ line\ 2
-SyntaxError:\ Non-ASCII\ character\ \[aq]\\x93\[aq]\ ...
-\f[]
-.fi
-.PP
-means you\[aq]re using an outdated version of Python.
-Please update to Python 2.6 or 2.7.
-.PP
-To run youtube-dl under Python 2.5, you\[aq]ll have to manually check it
-out like this:
-.IP
-.nf
-\f[C]
-git\ clone\ git://github.com/rg3/youtube-dl.git
-cd\ youtube-dl
-python\ -m\ youtube_dl\ --help
-\f[]
-.fi
-.PP
-Please note that Python 2.5 is not supported anymore.
-.SS What is this binary file? Where has the code gone?
-.PP
-Since June 2012 (#342) youtube-dl is packed as an executable zipfile,
-simply unzip it (might need renaming to \f[C]youtube-dl.zip\f[] first on
-some systems) or clone the git repository, as laid out above.
-If you modify the code, you can run it by executing the
-\f[C]__main__.py\f[] file.
-To recompile the executable, run \f[C]make\ youtube-dl\f[].
-.SS The exe throws a \f[I]Runtime error from Visual C++\f[]
-.PP
-To run the exe you need to install first the Microsoft Visual C++ 2008
-Redistributable
-Package (http://www.microsoft.com/en-us/download/details.aspx?id=29).
-.SH COPYRIGHT
-.PP
-youtube-dl is released into the public domain by the copyright holders.
-.PP
-This README file was originally written by Daniel Bolton
-(<https://github.com/dbbolton>) and is likewise released into the public
-domain.
-.SH BUGS
-.PP
-Bugs and suggestions should be reported at:
-<https://github.com/rg3/youtube-dl/issues>
-.PP
-Please include:
-.IP \[bu] 2
-Your exact command line, like
-\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
-A common mistake is not to escape the \f[C]&\f[].
-Putting URLs in quotes should solve this problem.
-.IP \[bu] 2
-The output of \f[C]youtube-dl\ --version\f[]
-.IP \[bu] 2
-The output of \f[C]python\ --version\f[]
-.IP \[bu] 2
-The name and version of your Operating System ("Ubuntu 11.04 x64" or
-"Windows 7 x64" is usually enough).

youtube-dl.bash-completion

@@ -1,14 +0,0 @@
-__youtube-dl()
-{
-    local cur prev opts
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --buffer-size --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --id --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --no-resize-buffer --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --restrict-filenames --retries --simulate --skip-download --srt-lang --title --update --user-agent --username --verbose --version --write-description --write-info-json --write-srt"
-
-    if [[ ${cur} == * ]] ; then
-        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
-        return 0
-    fi
-}
-
-complete -F __youtube-dl youtube-dl

youtube_dl/PostProcessor.py

@@ -1,198 +1,238 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
+
 import os
 import subprocess
 import sys
 import time
 
-from utils import *
+from .utils import *
 
 
 class PostProcessor(object):
-	"""Post Processor class.
+    """Post Processor class.
 
-	PostProcessor objects can be added to downloaders with their
+    PostProcessor objects can be added to downloaders with their
-	add_post_processor() method. When the downloader has finished a
+    add_post_processor() method. When the downloader has finished a
-	successful download, it will take its internal chain of PostProcessors
+    successful download, it will take its internal chain of PostProcessors
-	and start calling the run() method on each one of them, first with
+    and start calling the run() method on each one of them, first with
-	an initial argument and then with the returned value of the previous
+    an initial argument and then with the returned value of the previous
-	PostProcessor.
+    PostProcessor.
 
-	The chain will be stopped if one of them ever returns None or the end
+    The chain will be stopped if one of them ever returns None or the end
-	of the chain is reached.
+    of the chain is reached.
 
-	PostProcessor objects follow a "mutual registration" process similar
+    PostProcessor objects follow a "mutual registration" process similar
-	to InfoExtractor objects.
+    to InfoExtractor objects.
-	"""
+    """
 
-	_downloader = None
+    _downloader = None
 
-	def __init__(self, downloader=None):
+    def __init__(self, downloader=None):
-		self._downloader = downloader
+        self._downloader = downloader
 
-	def set_downloader(self, downloader):
+    def set_downloader(self, downloader):
-		"""Sets the downloader for this PP."""
+        """Sets the downloader for this PP."""
-		self._downloader = downloader
+        self._downloader = downloader
 
-	def run(self, information):
+    def run(self, information):
-		"""Run the PostProcessor.
+        """Run the PostProcessor.
 
-		The "information" argument is a dictionary like the ones
+        The "information" argument is a dictionary like the ones
-		composed by InfoExtractors. The only difference is that this
+        composed by InfoExtractors. The only difference is that this
-		one has an extra field called "filepath" that points to the
+        one has an extra field called "filepath" that points to the
-		downloaded file.
+        downloaded file.
 
-		When this method returns None, the postprocessing chain is
+        This method returns a tuple, the first element of which describes
-		stopped. However, this method may return an information
+        whether the original file should be kept (i.e. not deleted - None for
-		dictionary that will be passed to the next postprocessing
+        no preference), and the second of which is the updated information.
-		object in the chain. It can be the one it received after
-		changing some fields.
 
-		In addition, this method may raise a PostProcessingError
+        In addition, this method may raise a PostProcessingError
-		exception that will be taken into account by the downloader
+        exception if post processing fails.
-		it was called from.
+        """
-		"""
+        return None, information # by default, keep file and do nothing
-		return information # by default, do nothing
 
-class AudioConversionError(BaseException):
+class FFmpegPostProcessorError(PostProcessingError):
-	def __init__(self, message):
+    pass
-		self.message = message
 
-class FFmpegExtractAudioPP(PostProcessor):
+class AudioConversionError(PostProcessingError):
-	def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
+    pass
-		PostProcessor.__init__(self, downloader)
-		if preferredcodec is None:
-			preferredcodec = 'best'
-		self._preferredcodec = preferredcodec
-		self._preferredquality = preferredquality
-		self._keepvideo = keepvideo
-		self._exes = self.detect_executables()
 
-	@staticmethod
+class FFmpegPostProcessor(PostProcessor):
-	def detect_executables():
+    def __init__(self,downloader=None):
-		def executable(exe):
+        PostProcessor.__init__(self, downloader)
-			try:
+        self._exes = self.detect_executables()
-				subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-			except OSError:
-				return False
-			return exe
-		programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-		return dict((program, executable(program)) for program in programs)
 
-	def get_audio_codec(self, path):
+    @staticmethod
-		if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
+    def detect_executables():
-		try:
+        def executable(exe):
-			cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)]
+            try:
-			handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
+                subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-			output = handle.communicate()[0]
+            except OSError:
-			if handle.wait() != 0:
+                return False
-				return None
+            return exe
-		except (IOError, OSError):
+        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-			return None
+        return dict((program, executable(program)) for program in programs)
-		audio_codec = None
-		for line in output.split('\n'):
-			if line.startswith('codec_name='):
-				audio_codec = line.split('=')[1].strip()
-			elif line.strip() == 'codec_type=audio' and audio_codec is not None:
-				return audio_codec
-		return None
 
-	def run_ffmpeg(self, path, out_path, codec, more_opts):
+    def run_ffmpeg(self, path, out_path, opts):
-		if not self._exes['ffmpeg'] and not self._exes['avconv']:
+        if not self._exes['ffmpeg'] and not self._exes['avconv']:
-			raise AudioConversionError('ffmpeg or avconv not found. Please install one.')	
+            raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
-		if codec is None:
+        cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path)]
-			acodec_opts = []
+               + opts +
-		else:
+               [encodeFilename(self._ffmpeg_filename_argument(out_path))])
-			acodec_opts = ['-acodec', codec]
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-		cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn']
+        stdout,stderr = p.communicate()
-			   + acodec_opts + more_opts +
+        if p.returncode != 0:
-			   ['--', encodeFilename(out_path)])
+            stderr = stderr.decode('utf-8', 'replace')
-		p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            msg = stderr.strip().split('\n')[-1]
-		stdout,stderr = p.communicate()
+            raise FFmpegPostProcessorError(msg)
-		if p.returncode != 0:
-			msg = stderr.strip().split('\n')[-1]
-			raise AudioConversionError(msg)
 
-	def run(self, information):
+    def _ffmpeg_filename_argument(self, fn):
-		path = information['filepath']
+        # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
+        if fn.startswith(u'-'):
+            return u'./' + fn
+        return fn
 
-		filecodec = self.get_audio_codec(path)
+class FFmpegExtractAudioPP(FFmpegPostProcessor):
-		if filecodec is None:
+    def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
-			self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
+        FFmpegPostProcessor.__init__(self, downloader)
-			return None
+        if preferredcodec is None:
+            preferredcodec = 'best'
+        self._preferredcodec = preferredcodec
+        self._preferredquality = preferredquality
+        self._nopostoverwrites = nopostoverwrites
 
-		more_opts = []
+    def get_audio_codec(self, path):
-		if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+        if not self._exes['ffprobe'] and not self._exes['avprobe']: return None
-			if self._preferredcodec == 'm4a' and filecodec == 'aac':
+        try:
-				# Lossless, but in another container
+            cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', encodeFilename(self._ffmpeg_filename_argument(path))]
-				acodec = 'copy'
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
-				extension = self._preferredcodec
+            output = handle.communicate()[0]
-				more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            if handle.wait() != 0:
-			elif filecodec in ['aac', 'mp3', 'vorbis']:
+                return None
-				# Lossless if possible
+        except (IOError, OSError):
-				acodec = 'copy'
+            return None
-				extension = filecodec
+        audio_codec = None
-				if filecodec == 'aac':
+        for line in output.decode('ascii', 'ignore').split('\n'):
-					more_opts = ['-f', 'adts']
+            if line.startswith('codec_name='):
-				if filecodec == 'vorbis':
+                audio_codec = line.split('=')[1].strip()
-					extension = 'ogg'
+            elif line.strip() == 'codec_type=audio' and audio_codec is not None:
-			else:
+                return audio_codec
-				# MP3 otherwise.
+        return None
-				acodec = 'libmp3lame'
-				extension = 'mp3'
-				more_opts = []
-				if self._preferredquality is not None:
-					if int(self._preferredquality) < 10:
-						more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
-					else:
-						more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
-		else:
-			# We convert the audio (lossy)
-			acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
-			extension = self._preferredcodec
-			more_opts = []
-			if self._preferredquality is not None:
-				if int(self._preferredquality) < 10:
-					more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
-				else:
-					more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
-			if self._preferredcodec == 'aac':
-				more_opts += ['-f', 'adts']
-			if self._preferredcodec == 'm4a':
-				more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
-			if self._preferredcodec == 'vorbis':
-				extension = 'ogg'
-			if self._preferredcodec == 'wav':
-				extension = 'wav'
-				more_opts += ['-f', 'wav']
 
-		prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+    def run_ffmpeg(self, path, out_path, codec, more_opts):
-		new_path = prefix + sep + extension
+        if not self._exes['ffmpeg'] and not self._exes['avconv']:
-		self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+            raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
-		try:
+        if codec is None:
-			self.run_ffmpeg(path, new_path, acodec, more_opts)
+            acodec_opts = []
-		except:
+        else:
-			etype,e,tb = sys.exc_info()
+            acodec_opts = ['-acodec', codec]
-			if isinstance(e, AudioConversionError):
+        opts = ['-vn'] + acodec_opts + more_opts
-				self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+        try:
-			else:
+            FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts)
-				self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg'))
+        except FFmpegPostProcessorError as err:
-			return None
+            raise AudioConversionError(err.message)
 
- 		# Try to update the date time for extracted audio file.
+    def run(self, information):
-		if information.get('filetime') is not None:
+        path = information['filepath']
-			try:
-				os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
-			except:
-				self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
 
-		if not self._keepvideo:
+        filecodec = self.get_audio_codec(path)
-			try:
+        if filecodec is None:
-				os.remove(encodeFilename(path))
+            raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe')
-			except (IOError, OSError):
-				self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
-				return None
 
-		information['filepath'] = new_path
+        more_opts = []
-		return information
+        if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+            if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
+                # Lossless, but in another container
+                acodec = 'copy'
+                extension = 'm4a'
+                more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
+                # Lossless if possible
+                acodec = 'copy'
+                extension = filecodec
+                if filecodec == 'aac':
+                    more_opts = ['-f', 'adts']
+                if filecodec == 'vorbis':
+                    extension = 'ogg'
+            else:
+                # MP3 otherwise.
+                acodec = 'libmp3lame'
+                extension = 'mp3'
+                more_opts = []
+                if self._preferredquality is not None:
+                    if int(self._preferredquality) < 10:
+                        more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                    else:
+                        more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+        else:
+            # We convert the audio (lossy)
+            acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
+            extension = self._preferredcodec
+            more_opts = []
+            if self._preferredquality is not None:
+                if int(self._preferredquality) < 10:
+                    more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
+                else:
+                    more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
+            if self._preferredcodec == 'aac':
+                more_opts += ['-f', 'adts']
+            if self._preferredcodec == 'm4a':
+                more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
+            if self._preferredcodec == 'vorbis':
+                extension = 'ogg'
+            if self._preferredcodec == 'wav':
+                extension = 'wav'
+                more_opts += ['-f', 'wav']
+
+        prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+        new_path = prefix + sep + extension
+
+        # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
+        if new_path == path:
+            self._nopostoverwrites = True
+
+        try:
+            if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
+                self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
+            else:
+                self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
+                self.run_ffmpeg(path, new_path, acodec, more_opts)
+        except:
+            etype,e,tb = sys.exc_info()
+            if isinstance(e, AudioConversionError):
+                msg = u'audio conversion failed: ' + e.message
+            else:
+                msg = u'error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')
+            raise PostProcessingError(msg)
+
+        # Try to update the date time for extracted audio file.
+        if information.get('filetime') is not None:
+            try:
+                os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
+            except:
+                self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
+
+        information['filepath'] = new_path
+        return self._nopostoverwrites,information
+
+class FFmpegVideoConvertor(FFmpegPostProcessor):
+    def __init__(self, downloader=None,preferedformat=None):
+        super(FFmpegVideoConvertor, self).__init__(downloader)
+        self._preferedformat=preferedformat
+
+    def run(self, information):
+        path = information['filepath']
+        prefix, sep, ext = path.rpartition(u'.')
+        outpath = prefix + sep + self._preferedformat
+        if information['ext'] == self._preferedformat:
+            self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
+            return True,information
+        self._downloader.to_screen(u'['+'ffmpeg'+'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) +outpath)
+        self.run_ffmpeg(path, outpath, [])
+        information['filepath'] = outpath
+        information['format'] = self._preferedformat
+        information['ext'] = self._preferedformat
+        return False,information

youtube_dl/__main__.py

@@ -1,7 +1,18 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
-import __init__
+# Execute with
+# $ python youtube_dl/__main__.py (2.6+)
+# $ python -m youtube_dl          (2.7+)
+
+import sys
+
+if __package__ is None and not hasattr(sys, "frozen"):
+    # direct call of __main__.py
+    import os.path
+    path = os.path.realpath(os.path.abspath(__file__))
+    sys.path.append(os.path.dirname(os.path.dirname(path)))
+
+import youtube_dl
 
 if __name__ == '__main__':
-	__init__.main()
+    youtube_dl.main()

youtube_dl/update.py

@@ -0,0 +1,172 @@
+import json
+import traceback
+import hashlib
+from zipimport import zipimporter
+
+from .utils import *
+from .version import __version__
+
+def rsa_verify(message, signature, key):
+    from struct import pack
+    from hashlib import sha256
+    from sys import version_info
+    def b(x):
+        if version_info[0] == 2: return x
+        else: return x.encode('latin1')
+    assert(type(message) == type(b('')))
+    block_size = 0
+    n = key[0]
+    while n:
+        block_size += 1
+        n >>= 8
+    signature = pow(int(signature, 16), key[1], key[0])
+    raw_bytes = []
+    while signature:
+        raw_bytes.insert(0, pack("B", signature & 0xFF))
+        signature >>= 8
+    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
+    if signature[0:2] != b('\x00\x01'): return False
+    signature = signature[2:]
+    if not b('\x00') in signature: return False
+    signature = signature[signature.index(b('\x00'))+1:]
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    signature = signature[19:]
+    if signature != sha256(message).digest(): return False
+    return True
+
+def update_self(to_screen, verbose, filename):
+    """Update the program file with the latest version from the repository"""
+
+    UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
+    VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
+    JSON_URL = UPDATE_URL + 'versions.json'
+    UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
+
+
+    if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
+        to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.')
+        return
+
+    # Check if there is a new version
+    try:
+        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+        return
+    if newversion == __version__:
+        to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+        return
+
+    # Download and check versions info
+    try:
+        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = json.loads(versions_info)
+    except:
+        if verbose: to_screen(compat_str(traceback.format_exc()))
+        to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
+        return
+    if not 'signature' in versions_info:
+        to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+        return
+    signature = versions_info['signature']
+    del versions_info['signature']
+    if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
+        to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
+        return
+
+    to_screen(u'Updating to version ' + versions_info['latest'] + '...')
+    version = versions_info['versions'][versions_info['latest']]
+
+    print_notes(to_screen, versions_info['versions'])
+
+    if not os.access(filename, os.W_OK):
+        to_screen(u'ERROR: no write permissions on %s' % filename)
+        return
+
+    # Py2EXE
+    if hasattr(sys, "frozen"):
+        exe = os.path.abspath(filename)
+        directory = os.path.dirname(exe)
+        if not os.access(directory, os.W_OK):
+            to_screen(u'ERROR: no write permissions on %s' % directory)
+            return
+
+        try:
+            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['exe'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(exe + '.new', 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to write the new version')
+            return
+
+        try:
+            bat = os.path.join(directory, 'youtube-dl-updater.bat')
+            b = open(bat, 'w')
+            b.write("""
+echo Updating youtube-dl...
+ping 127.0.0.1 -n 5 -w 1000 > NUL
+move /Y "%s.new" "%s"
+del "%s"
+            \n""" %(exe, exe, bat))
+            b.close()
+
+            os.startfile(bat)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    # Zip unix package
+    elif isinstance(globals().get('__loader__'), zipimporter):
+        try:
+            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            newcontent = urlh.read()
+            urlh.close()
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to download latest version')
+            return
+
+        newcontent_hash = hashlib.sha256(newcontent).hexdigest()
+        if newcontent_hash != version['bin'][1]:
+            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            return
+
+        try:
+            with open(filename, 'wb') as outf:
+                outf.write(newcontent)
+        except (IOError, OSError) as err:
+            if verbose: to_screen(compat_str(traceback.format_exc()))
+            to_screen(u'ERROR: unable to overwrite current version')
+            return
+
+    to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+
+def get_notes(versions, fromVersion):
+    notes = []
+    for v,vdata in sorted(versions.items()):
+        if v > fromVersion:
+            notes.extend(vdata.get('notes', []))
+    return notes
+
+def print_notes(to_screen, versions, fromVersion=__version__):
+    notes = get_notes(versions, fromVersion)
+    if notes:
+        to_screen(u'PLEASE NOTE:')
+        for note in notes:
+            to_screen(note)

youtube_dl/utils.py

@@ -1,380 +1,680 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import errno
 import gzip
-import htmlentitydefs
+import io
-import HTMLParser
+import json
 import locale
 import os
 import re
 import sys
+import traceback
 import zlib
-import urllib2
 import email.utils
-import json
+import socket
+import datetime
 
 try:
-	import cStringIO as StringIO
+    import urllib.request as compat_urllib_request
-except ImportError:
+except ImportError: # Python 2
-	import StringIO
+    import urllib2 as compat_urllib_request
 
-std_headers = {
+try:
-	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
+    import urllib.error as compat_urllib_error
-	'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
+except ImportError: # Python 2
-	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    import urllib2 as compat_urllib_error
-	'Accept-Encoding': 'gzip, deflate',
+
-	'Accept-Language': 'en-us,en;q=0.5',
+try:
-}
+    import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+    import urllib as compat_urllib_parse
+
+try:
+    from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+    from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+    import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+    import cookielib as compat_cookiejar
+
+try:
+    import html.entities as compat_html_entities
+except ImportError: # Python 2
+    import htmlentitydefs as compat_html_entities
+
+try:
+    import html.parser as compat_html_parser
+except ImportError: # Python 2
+    import HTMLParser as compat_html_parser
+
+try:
+    import http.client as compat_http_client
+except ImportError: # Python 2
+    import httplib as compat_http_client
+
+try:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
+    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+    def _unquote(string, encoding='utf-8', errors='replace'):
+        if string == '':
+            return string
+        res = string.split('%')
+        if len(res) == 1:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+        pct_sequence = b''
+        string = res[0]
+        for item in res[1:]:
+            try:
+                if not item:
+                    raise ValueError
+                pct_sequence += item[:2].decode('hex')
+                rest = item[2:]
+                if not rest:
+                    # This segment was just a single percent-encoded character.
+                    # May be part of a sequence of code units, so delay decoding.
+                    # (Stored in pct_sequence).
+                    continue
+            except ValueError:
+                rest = '%' + item
+            # Encountered non-percent-encoded characters. Flush the current
+            # pct_sequence.
+            string += pct_sequence.decode(encoding, errors) + rest
+            pct_sequence = b''
+        if pct_sequence:
+            # Flush the final pct_sequence
+            string += pct_sequence.decode(encoding, errors)
+        return string
+
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, unicode
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = _unquote(name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = _unquote(value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                        encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
 
 try:
     compat_str = unicode # Python 2
 except NameError:
     compat_str = str
 
+try:
+    compat_chr = unichr # Python 2
+except NameError:
+    compat_chr = chr
+
+def compat_ord(c):
+    if type(c) is int: return c
+    else: return ord(c)
+
+# This is not clearly defined otherwise
+compiled_regex_type = type(re.compile(''))
+
+std_headers = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
+    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Encoding': 'gzip, deflate',
+    'Accept-Language': 'en-us,en;q=0.5',
+}
+
 def preferredencoding():
-	"""Get preferred encoding.
+    """Get preferred encoding.
 
-	Returns the best encoding scheme for the system, based on
+    Returns the best encoding scheme for the system, based on
-	locale.getpreferredencoding() and some further tweaks.
+    locale.getpreferredencoding() and some further tweaks.
-	"""
+    """
-	def yield_preferredencoding():
+    try:
-		try:
+        pref = locale.getpreferredencoding()
-			pref = locale.getpreferredencoding()
+        u'TEST'.encode(pref)
-			u'TEST'.encode(pref)
+    except:
-		except:
+        pref = 'UTF-8'
-			pref = 'UTF-8'
-		while True:
-			yield pref
-	return yield_preferredencoding().next()
 
+    return pref
+
+if sys.version_info < (3,0):
+    def compat_print(s):
+        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+    def compat_print(s):
+        assert type(s) == type(u'')
+        print(s)
+
+# In Python 2.x, json.dump expects a bytestream.
+# In Python 3.x, it writes to a character stream
+if sys.version_info < (3,0):
+    def write_json_file(obj, fn):
+        with open(fn, 'wb') as f:
+            json.dump(obj, f)
+else:
+    def write_json_file(obj, fn):
+        with open(fn, 'w', encoding='utf-8') as f:
+            json.dump(obj, f)
 
 def htmlentity_transform(matchobj):
-	"""Transforms an HTML entity to a Unicode character.
+    """Transforms an HTML entity to a character.
 
-	This function receives a match object and is intended to be used with
+    This function receives a match object and is intended to be used with
-	the re.sub() function.
+    the re.sub() function.
-	"""
+    """
-	entity = matchobj.group(1)
+    entity = matchobj.group(1)
 
-	# Known non-numeric HTML entity
+    # Known non-numeric HTML entity
-	if entity in htmlentitydefs.name2codepoint:
+    if entity in compat_html_entities.name2codepoint:
-		return unichr(htmlentitydefs.name2codepoint[entity])
+        return compat_chr(compat_html_entities.name2codepoint[entity])
 
-	# Unicode character
+    mobj = re.match(u'(?u)#(x?\\d+)', entity)
-	mobj = re.match(ur'(?u)#(x?\d+)', entity)
+    if mobj is not None:
-	if mobj is not None:
+        numstr = mobj.group(1)
-		numstr = mobj.group(1)
+        if numstr.startswith(u'x'):
-		if numstr.startswith(u'x'):
+            base = 16
-			base = 16
+            numstr = u'0%s' % numstr
-			numstr = u'0%s' % numstr
+        else:
-		else:
+            base = 10
-			base = 10
+        return compat_chr(int(numstr, base))
-		return unichr(long(numstr, base))
 
-	# Unknown entity in name, return its literal representation
+    # Unknown entity in name, return its literal representation
-	return (u'&%s;' % entity)
+    return (u'&%s;' % entity)
 
-HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-class IDParser(HTMLParser.HTMLParser):
+class AttrParser(compat_html_parser.HTMLParser):
-	"""Modified HTMLParser that isolates a tag with the specified id"""
+    """Modified HTMLParser that isolates a tag with the specified attribute"""
-	def __init__(self, id):
+    def __init__(self, attribute, value):
-		self.id = id
+        self.attribute = attribute
-		self.result = None
+        self.value = value
-		self.started = False
+        self.result = None
-		self.depth = {}
+        self.started = False
-		self.html = None
+        self.depth = {}
-		self.watch_startpos = False
+        self.html = None
-		self.error_count = 0
+        self.watch_startpos = False
-		HTMLParser.HTMLParser.__init__(self)
+        self.error_count = 0
+        compat_html_parser.HTMLParser.__init__(self)
 
-	def error(self, message):
+    def error(self, message):
-		if self.error_count > 10 or self.started:
+        if self.error_count > 10 or self.started:
-			raise HTMLParser.HTMLParseError(message, self.getpos())
+            raise compat_html_parser.HTMLParseError(message, self.getpos())
-		self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
+        self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
-		self.error_count += 1
+        self.error_count += 1
-		self.goahead(1)
+        self.goahead(1)
 
-	def loads(self, html):
+    def loads(self, html):
-		self.html = html
+        self.html = html
-		self.feed(html)
+        self.feed(html)
-		self.close()
+        self.close()
 
-	def handle_starttag(self, tag, attrs):
+    def handle_starttag(self, tag, attrs):
-		attrs = dict(attrs)
+        attrs = dict(attrs)
-		if self.started:
+        if self.started:
-			self.find_startpos(None)
+            self.find_startpos(None)
-		if 'id' in attrs and attrs['id'] == self.id:
+        if self.attribute in attrs and attrs[self.attribute] == self.value:
-			self.result = [tag]
+            self.result = [tag]
-			self.started = True
+            self.started = True
-			self.watch_startpos = True
+            self.watch_startpos = True
-		if self.started:
+        if self.started:
-			if not tag in self.depth: self.depth[tag] = 0
+            if not tag in self.depth: self.depth[tag] = 0
-			self.depth[tag] += 1
+            self.depth[tag] += 1
 
-	def handle_endtag(self, tag):
+    def handle_endtag(self, tag):
-		if self.started:
+        if self.started:
-			if tag in self.depth: self.depth[tag] -= 1
+            if tag in self.depth: self.depth[tag] -= 1
-			if self.depth[self.result[0]] == 0:
+            if self.depth[self.result[0]] == 0:
-				self.started = False
+                self.started = False
-				self.result.append(self.getpos())
+                self.result.append(self.getpos())
 
-	def find_startpos(self, x):
+    def find_startpos(self, x):
-		"""Needed to put the start position of the result (self.result[1])
+        """Needed to put the start position of the result (self.result[1])
-		after the opening tag with the requested id"""
+        after the opening tag with the requested id"""
-		if self.watch_startpos:
+        if self.watch_startpos:
-			self.watch_startpos = False
+            self.watch_startpos = False
-			self.result.append(self.getpos())
+            self.result.append(self.getpos())
-	handle_entityref = handle_charref = handle_data = handle_comment = \
+    handle_entityref = handle_charref = handle_data = handle_comment = \
-	handle_decl = handle_pi = unknown_decl = find_startpos
+    handle_decl = handle_pi = unknown_decl = find_startpos
 
-	def get_result(self):
+    def get_result(self):
-		if self.result == None: return None
+        if self.result is None:
-		if len(self.result) != 3: return None
+            return None
-		lines = self.html.split('\n')
+        if len(self.result) != 3:
-		lines = lines[self.result[1][0]-1:self.result[2][0]]
+            return None
-		lines[0] = lines[0][self.result[1][1]:]
+        lines = self.html.split('\n')
-		if len(lines) == 1:
+        lines = lines[self.result[1][0]-1:self.result[2][0]]
-			lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
+        lines[0] = lines[0][self.result[1][1]:]
-		lines[-1] = lines[-1][:self.result[2][1]]
+        if len(lines) == 1:
-		return '\n'.join(lines).strip()
+            lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
+        lines[-1] = lines[-1][:self.result[2][1]]
+        return '\n'.join(lines).strip()
+# Hack for https://github.com/rg3/youtube-dl/issues/662
+if sys.version_info < (2, 7, 3):
+    AttrParser.parse_endtag = (lambda self, i:
+        i + len("</scr'+'ipt>")
+        if self.rawdata[i:].startswith("</scr'+'ipt>")
+        else compat_html_parser.HTMLParser.parse_endtag(self, i))
 
 def get_element_by_id(id, html):
-	"""Return the content of the tag with the specified id in the passed HTML document"""
+    """Return the content of the tag with the specified ID in the passed HTML document"""
-	parser = IDParser(id)
+    return get_element_by_attribute("id", id, html)
-	try:
+
-		parser.loads(html)
+def get_element_by_attribute(attribute, value, html):
-	except HTMLParser.HTMLParseError:
+    """Return the content of the tag with the specified attribute in the passed HTML document"""
-		pass
+    parser = AttrParser(attribute, value)
-	return parser.get_result()
+    try:
+        parser.loads(html)
+    except compat_html_parser.HTMLParseError:
+        pass
+    return parser.get_result()
 
 
 def clean_html(html):
-	"""Clean an HTML snippet into a readable string"""
+    """Clean an HTML snippet into a readable string"""
-	# Newline vs <br />
+    # Newline vs <br />
-	html = html.replace('\n', ' ')
+    html = html.replace('\n', ' ')
-	html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
-	# Strip html tags
+    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
-	html = re.sub('<.*?>', '', html)
+    # Strip html tags
-	# Replace html entities
+    html = re.sub('<.*?>', '', html)
-	html = unescapeHTML(html)
+    # Replace html entities
-	return html
+    html = unescapeHTML(html)
+    return html.strip()
 
 
 def sanitize_open(filename, open_mode):
-	"""Try to open the given filename, and slightly tweak it if this fails.
+    """Try to open the given filename, and slightly tweak it if this fails.
 
-	Attempts to open the given filename. If this fails, it tries to change
+    Attempts to open the given filename. If this fails, it tries to change
-	the filename slightly, step by step, until it's either able to open it
+    the filename slightly, step by step, until it's either able to open it
-	or it fails and raises a final exception, like the standard open()
+    or it fails and raises a final exception, like the standard open()
-	function.
+    function.
 
-	It returns the tuple (stream, definitive_file_name).
+    It returns the tuple (stream, definitive_file_name).
-	"""
+    """
-	try:
+    try:
-		if filename == u'-':
+        if filename == u'-':
-			if sys.platform == 'win32':
+            if sys.platform == 'win32':
-				import msvcrt
+                import msvcrt
-				msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+                msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
-			return (sys.stdout, filename)
+            return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
-		stream = open(encodeFilename(filename), open_mode)
+        stream = open(encodeFilename(filename), open_mode)
-		return (stream, filename)
+        return (stream, filename)
-	except (IOError, OSError), err:
+    except (IOError, OSError) as err:
-		# In case of error, try to remove win32 forbidden chars
+        if err.errno in (errno.EACCES,):
-		filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
+            raise
 
-		# An exception here should be caught in the caller
+        # In case of error, try to remove win32 forbidden chars
-		stream = open(encodeFilename(filename), open_mode)
+        alt_filename = os.path.join(
-		return (stream, filename)
+                        re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
+                        for path_part in os.path.split(filename)
+                       )
+        if alt_filename == filename:
+            raise
+        else:
+            # An exception here should be caught in the caller
+            stream = open(encodeFilename(filename), open_mode)
+            return (stream, alt_filename)
 
 
 def timeconvert(timestr):
-	"""Convert RFC 2822 defined time string into system timestamp"""
+    """Convert RFC 2822 defined time string into system timestamp"""
-	timestamp = None
+    timestamp = None
-	timetuple = email.utils.parsedate_tz(timestr)
+    timetuple = email.utils.parsedate_tz(timestr)
-	if timetuple is not None:
+    if timetuple is not None:
-		timestamp = email.utils.mktime_tz(timetuple)
+        timestamp = email.utils.mktime_tz(timetuple)
-	return timestamp
+    return timestamp
 
-def sanitize_filename(s, restricted=False):
+def sanitize_filename(s, restricted=False, is_id=False):
-	"""Sanitizes a string so it could be used as part of a filename.
+    """Sanitizes a string so it could be used as part of a filename.
-	If restricted is set, use a stricter subset of allowed characters.
+    If restricted is set, use a stricter subset of allowed characters.
-	"""
+    Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
-	def replace_insane(char):
+    """
-		if char == '?' or ord(char) < 32 or ord(char) == 127:
+    def replace_insane(char):
-			return ''
+        if char == '?' or ord(char) < 32 or ord(char) == 127:
-		elif char == '"':
+            return ''
-			return '' if restricted else '\''
+        elif char == '"':
-		elif char == ':':
+            return '' if restricted else '\''
-			return '_-' if restricted else ' -'
+        elif char == ':':
-		elif char in '\\/|*<>':
+            return '_-' if restricted else ' -'
-			return '_'
+        elif char in '\\/|*<>':
-		if restricted and (char in '!&\'' or char.isspace()):
+            return '_'
-			return '_'
+        if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
-		if restricted and ord(char) > 127:
+            return '_'
-			return '_'
+        if restricted and ord(char) > 127:
-		return char
+            return '_'
+        return char
 
-	result = u''.join(map(replace_insane, s))
+    result = u''.join(map(replace_insane, s))
-	while '__' in result:
+    if not is_id:
-		result = result.replace('__', '_')
+        while '__' in result:
-	result = result.strip('_')
+            result = result.replace('__', '_')
-	# Common case of "Foreign band name - English song title"
+        result = result.strip('_')
-	if restricted and result.startswith('-_'):
+        # Common case of "Foreign band name - English song title"
-		result = result[2:]
+        if restricted and result.startswith('-_'):
-	if not result:
+            result = result[2:]
-		result = '_'
+        if not result:
-	return result
+            result = '_'
+    return result
 
 def orderedSet(iterable):
-	""" Remove all duplicates from the input iterable """
+    """ Remove all duplicates from the input iterable """
-	res = []
+    res = []
-	for el in iterable:
+    for el in iterable:
-		if el not in res:
+        if el not in res:
-			res.append(el)
+            res.append(el)
-	return res
+    return res
 
 def unescapeHTML(s):
-	"""
+    """
-	@param s a string (of type unicode)
+    @param s a string
-	"""
+    """
-	assert type(s) == type(u'')
+    assert type(s) == type(u'')
 
-	result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
+    result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
-	return result
+    return result
 
 def encodeFilename(s):
-	"""
+    """
-	@param s The name of the file (of type unicode)
+    @param s The name of the file
-	"""
+    """
 
-	assert type(s) == type(u'')
+    assert type(s) == type(u'')
+
+    # Python 3 has a Unicode API
+    if sys.version_info >= (3, 0):
+        return s
+
+    if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
+        # Pass u'' directly to use Unicode APIs on Windows 2000 and up
+        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+        # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+        return s
+    else:
+        encoding = sys.getfilesystemencoding()
+        if encoding is None:
+            encoding = 'utf-8'
+        return s.encode(encoding, 'ignore')
+
+def decodeOption(optval):
+    if optval is None:
+        return optval
+    if isinstance(optval, bytes):
+        optval = optval.decode(preferredencoding())
+
+    assert isinstance(optval, compat_str)
+    return optval
+
+def formatSeconds(secs):
+    if secs > 3600:
+        return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
+    elif secs > 60:
+        return '%d:%02d' % (secs // 60, secs % 60)
+    else:
+        return '%d' % secs
+
+def make_HTTPS_handler(opts):
+    if sys.version_info < (3,2):
+        # Python's 2.x handler is very simplistic
+        return compat_urllib_request.HTTPSHandler()
+    else:
+        import ssl
+        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context.set_default_verify_paths()
+        
+        context.verify_mode = (ssl.CERT_NONE
+                               if opts.no_check_certificate
+                               else ssl.CERT_REQUIRED)
+        return compat_urllib_request.HTTPSHandler(context=context)
+
+class ExtractorError(Exception):
+    """Error during info extraction."""
+    def __init__(self, msg, tb=None):
+        """ tb, if given, is the original traceback (so that it can be printed out). """
+
+        if not sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+            msg = msg + u'; please report this issue on GitHub.'
+        super(ExtractorError, self).__init__(msg)
+
+        self.traceback = tb
+        self.exc_info = sys.exc_info()  # preserve original exception
+
+    def format_traceback(self):
+        if self.traceback is None:
+            return None
+        return u''.join(traceback.format_tb(self.traceback))
 
-	if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-		# Pass u'' directly to use Unicode APIs on Windows 2000 and up
-		# (Detecting Windows NT 4 is tricky because 'major >= 4' would
-		# match Windows 9x series as well. Besides, NT 4 is obsolete.)
-		return s
-	else:
-		return s.encode(sys.getfilesystemencoding(), 'ignore')
 
 class DownloadError(Exception):
-	"""Download Error exception.
+    """Download Error exception.
 
-	This exception may be thrown by FileDownloader objects if they are not
+    This exception may be thrown by FileDownloader objects if they are not
-	configured to continue on errors. They will contain the appropriate
+    configured to continue on errors. They will contain the appropriate
-	error message.
+    error message.
-	"""
+    """
-	pass
+    def __init__(self, msg, exc_info=None):
+        """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
+        super(DownloadError, self).__init__(msg)
+        self.exc_info = exc_info
 
 
 class SameFileError(Exception):
-	"""Same File exception.
+    """Same File exception.
 
-	This exception will be thrown by FileDownloader objects if they detect
+    This exception will be thrown by FileDownloader objects if they detect
-	multiple files would have to be downloaded to the same file on disk.
+    multiple files would have to be downloaded to the same file on disk.
-	"""
+    """
-	pass
+    pass
 
 
 class PostProcessingError(Exception):
-	"""Post Processing exception.
+    """Post Processing exception.
 
-	This exception may be raised by PostProcessor's .run() method to
+    This exception may be raised by PostProcessor's .run() method to
-	indicate an error in the postprocessing task.
+    indicate an error in the postprocessing task.
-	"""
+    """
-	pass
+    def __init__(self, msg):
+        self.msg = msg
 
 class MaxDownloadsReached(Exception):
-	""" --max-downloads limit has been reached. """
+    """ --max-downloads limit has been reached. """
-	pass
+    pass
 
 
 class UnavailableVideoError(Exception):
-	"""Unavailable Format exception.
+    """Unavailable Format exception.
 
-	This exception will be thrown when a video is requested
+    This exception will be thrown when a video is requested
-	in a format that is not available for that video.
+    in a format that is not available for that video.
-	"""
+    """
-	pass
+    pass
 
 
 class ContentTooShortError(Exception):
-	"""Content Too Short exception.
+    """Content Too Short exception.
 
-	This exception may be raised by FileDownloader objects when a file they
+    This exception may be raised by FileDownloader objects when a file they
-	download is too small for what the server announced first, indicating
+    download is too small for what the server announced first, indicating
-	the connection was probably interrupted.
+    the connection was probably interrupted.
-	"""
+    """
-	# Both in bytes
+    # Both in bytes
-	downloaded = None
+    downloaded = None
-	expected = None
+    expected = None
 
-	def __init__(self, downloaded, expected):
+    def __init__(self, downloaded, expected):
-		self.downloaded = downloaded
+        self.downloaded = downloaded
-		self.expected = expected
+        self.expected = expected
 
+class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
+    """Handler for HTTP requests and responses.
 
-class Trouble(Exception):
+    This class, when installed with an OpenerDirector, automatically adds
-	"""Trouble helper exception
+    the standard headers to every HTTP request and handles gzipped and
+    deflated responses from web servers. If compression is to be avoided in
+    a particular request, the original request in the program code only has
+    to include the HTTP header "Youtubedl-No-Compression", which will be
+    removed before making the real request.
 
-	This is an exception to be handled with
+    Part of this code was copied from:
-	FileDownloader.trouble
-	"""
 
-class YoutubeDLHandler(urllib2.HTTPHandler):
+    http://techknack.net/python-urllib2-handlers/
-	"""Handler for HTTP requests and responses.
 
-	This class, when installed with an OpenerDirector, automatically adds
+    Andrew Rowls, the author of that code, agreed to release it to the
-	the standard headers to every HTTP request and handles gzipped and
+    public domain.
-	deflated responses from web servers. If compression is to be avoided in
+    """
-	a particular request, the original request in the program code only has
-	to include the HTTP header "Youtubedl-No-Compression", which will be
-	removed before making the real request.
 
-	Part of this code was copied from:
+    @staticmethod
+    def deflate(data):
+        try:
+            return zlib.decompress(data, -zlib.MAX_WBITS)
+        except zlib.error:
+            return zlib.decompress(data)
 
-	http://techknack.net/python-urllib2-handlers/
+    @staticmethod
+    def addinfourl_wrapper(stream, headers, url, code):
+        if hasattr(compat_urllib_request.addinfourl, 'getcode'):
+            return compat_urllib_request.addinfourl(stream, headers, url, code)
+        ret = compat_urllib_request.addinfourl(stream, headers, url)
+        ret.code = code
+        return ret
 
-	Andrew Rowls, the author of that code, agreed to release it to the
+    def http_request(self, req):
-	public domain.
+        for h,v in std_headers.items():
-	"""
+            if h in req.headers:
+                del req.headers[h]
+            req.add_header(h, v)
+        if 'Youtubedl-no-compression' in req.headers:
+            if 'Accept-encoding' in req.headers:
+                del req.headers['Accept-encoding']
+            del req.headers['Youtubedl-no-compression']
+        if 'Youtubedl-user-agent' in req.headers:
+            if 'User-agent' in req.headers:
+                del req.headers['User-agent']
+            req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
+            del req.headers['Youtubedl-user-agent']
+        return req
 
-	@staticmethod
+    def http_response(self, req, resp):
-	def deflate(data):
+        old_resp = resp
-		try:
+        # gzip
-			return zlib.decompress(data, -zlib.MAX_WBITS)
+        if resp.headers.get('Content-encoding', '') == 'gzip':
-		except zlib.error:
+            gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
-			return zlib.decompress(data)
+            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        # deflate
+        if resp.headers.get('Content-encoding', '') == 'deflate':
+            gz = io.BytesIO(self.deflate(resp.read()))
+            resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
 
-	@staticmethod
+    https_request = http_request
-	def addinfourl_wrapper(stream, headers, url, code):
+    https_response = http_response
-		if hasattr(urllib2.addinfourl, 'getcode'):
-			return urllib2.addinfourl(stream, headers, url, code)
-		ret = urllib2.addinfourl(stream, headers, url)
-		ret.code = code
-		return ret
 
-	def http_request(self, req):
+def unified_strdate(date_str):
-		for h in std_headers:
+    """Return a string with the date in the format YYYYMMDD"""
-			if h in req.headers:
+    upload_date = None
-				del req.headers[h]
+    #Replace commas
-			req.add_header(h, std_headers[h])
+    date_str = date_str.replace(',',' ')
-		if 'Youtubedl-no-compression' in req.headers:
+    # %z (UTC offset) is only supported in python>=3.2
-			if 'Accept-encoding' in req.headers:
+    date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
-				del req.headers['Accept-encoding']
+    format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
-			del req.headers['Youtubedl-no-compression']
+    for expression in format_expressions:
-		return req
+        try:
+            upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
+        except:
+            pass
+    return upload_date
 
-	def http_response(self, req, resp):
+def date_from_str(date_str):
-		old_resp = resp
+    """
-		# gzip
+    Return a datetime object from a string in the format YYYYMMDD or
-		if resp.headers.get('Content-encoding', '') == 'gzip':
+    (now|today)[+-][0-9](day|week|month|year)(s)?"""
-			gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
+    today = datetime.date.today()
-			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+    if date_str == 'now'or date_str == 'today':
-			resp.msg = old_resp.msg
+        return today
-		# deflate
+    match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
-		if resp.headers.get('Content-encoding', '') == 'deflate':
+    if match is not None:
-			gz = StringIO.StringIO(self.deflate(resp.read()))
+        sign = match.group('sign')
-			resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
+        time = int(match.group('time'))
-			resp.msg = old_resp.msg
+        if sign == '-':
-		return resp
+            time = -time
+        unit = match.group('unit')
+        #A bad aproximation?
+        if unit == 'month':
+            unit = 'day'
+            time *= 30
+        elif unit == 'year':
+            unit = 'day'
+            time *= 365
+        unit += 's'
+        delta = datetime.timedelta(**{unit: time})
+        return today + delta
+    return datetime.datetime.strptime(date_str, "%Y%m%d").date()
+    
+class DateRange(object):
+    """Represents a time interval between two dates"""
+    def __init__(self, start=None, end=None):
+        """start and end must be strings in the format accepted by date"""
+        if start is not None:
+            self.start = date_from_str(start)
+        else:
+            self.start = datetime.datetime.min.date()
+        if end is not None:
+            self.end = date_from_str(end)
+        else:
+            self.end = datetime.datetime.max.date()
+        if self.start > self.end:
+            raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
+    @classmethod
+    def day(cls, day):
+        """Returns a range that only contains the given day"""
+        return cls(day,day)
+    def __contains__(self, date):
+        """Check if the date is in the range"""
+        if not isinstance(date, datetime.date):
+            date = date_from_str(date)
+        return self.start <= date <= self.end
+    def __str__(self):
+        return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())

youtube_dl/version.py

@@ -0,0 +1,2 @@
+
+__version__ = '2013.06.21'