Merge branch 'master' into prefer-webm
This commit is contained in:
Rogério Brito
commit
9e1ee3364a
@ -1 +1 @@
|
||||
2010.11.19
|
||||
2010.12.09
|
||||
|
||||
436
youtube-dl
436
youtube-dl
@ -3,8 +3,14 @@
|
||||
# Author: Ricardo Garcia Gonzalez
|
||||
# Author: Danny Colligan
|
||||
# Author: Benjamin Johnson
|
||||
# Author: Vasyl' Vavrychuk
|
||||
# Author: Witold Baryluk
|
||||
# License: Public domain code
|
||||
import cookielib
|
||||
import ctypes
|
||||
import datetime
|
||||
import email.utils
|
||||
import gzip
|
||||
import htmlentitydefs
|
||||
import httplib
|
||||
import locale
|
||||
@ -15,11 +21,13 @@ import os.path
|
||||
import re
|
||||
import socket
|
||||
import string
|
||||
import StringIO
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib
|
||||
import urllib2
|
||||
import zlib
|
||||
|
||||
# parse_qs was moved from the cgi module to the urlparse module recently.
|
||||
try:
|
||||
@ -31,26 +39,12 @@ std_headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12',
|
||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'Accept-Language': 'en-us,en;q=0.5',
|
||||
}
|
||||
|
||||
simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
|
||||
|
||||
month_name_to_number = {
|
||||
'January': '01',
|
||||
'February': '02',
|
||||
'March': '03',
|
||||
'April': '04',
|
||||
'May': '05',
|
||||
'June': '06',
|
||||
'July': '07',
|
||||
'August': '08',
|
||||
'September': '09',
|
||||
'October': '10',
|
||||
'November': '11',
|
||||
'December': '12',
|
||||
}
|
||||
|
||||
def preferredencoding():
|
||||
"""Get preferred encoding.
|
||||
|
||||
@ -124,6 +118,13 @@ def sanitize_open(filename, open_mode):
|
||||
stream = open(filename, open_mode)
|
||||
return (stream, filename)
|
||||
|
||||
def timeconvert(timestr):
|
||||
"""Convert RFC 2822 defined time string into system timestamp"""
|
||||
timestamp = None
|
||||
timetuple = email.utils.parsedate_tz(timestr)
|
||||
if timetuple is not None:
|
||||
timestamp = email.utils.mktime_tz(timetuple)
|
||||
return timestamp
|
||||
|
||||
class DownloadError(Exception):
|
||||
"""Download Error exception.
|
||||
@ -173,6 +174,64 @@ class ContentTooShortError(Exception):
|
||||
self.downloaded = downloaded
|
||||
self.expected = expected
|
||||
|
||||
class YoutubeDLHandler(urllib2.HTTPHandler):
|
||||
"""Handler for HTTP requests and responses.
|
||||
|
||||
This class, when installed with an OpenerDirector, automatically adds
|
||||
the standard headers to every HTTP request and handles gzipped and
|
||||
deflated responses from web servers. If compression is to be avoided in
|
||||
a particular request, the original request in the program code only has
|
||||
to include the HTTP header "Youtubedl-No-Compression", which will be
|
||||
removed before making the real request.
|
||||
|
||||
Part of this code was copied from:
|
||||
|
||||
http://techknack.net/python-urllib2-handlers/
|
||||
|
||||
Andrew Rowls, the author of that code, agreed to release it to the
|
||||
public domain.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def deflate(data):
|
||||
try:
|
||||
return zlib.decompress(data, -zlib.MAX_WBITS)
|
||||
except zlib.error:
|
||||
return zlib.decompress(data)
|
||||
|
||||
@staticmethod
|
||||
def addinfourl_wrapper(stream, headers, url, code):
|
||||
if hasattr(urllib2.addinfourl, 'getcode'):
|
||||
return urllib2.addinfourl(stream, headers, url, code)
|
||||
ret = urllib2.addinfourl(stream, headers, url)
|
||||
ret.code = code
|
||||
return ret
|
||||
|
||||
def http_request(self, req):
|
||||
for h in std_headers:
|
||||
if h in req.headers:
|
||||
del req.headers[h]
|
||||
req.add_header(h, std_headers[h])
|
||||
if 'Youtubedl-no-compression' in req.headers:
|
||||
if 'Accept-encoding' in req.headers:
|
||||
del req.headers['Accept-encoding']
|
||||
del req.headers['Youtubedl-no-compression']
|
||||
return req
|
||||
|
||||
def http_response(self, req, resp):
|
||||
old_resp = resp
|
||||
# gzip
|
||||
if resp.headers.get('Content-encoding', '') == 'gzip':
|
||||
gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
# deflate
|
||||
if resp.headers.get('Content-encoding', '') == 'deflate':
|
||||
gz = StringIO.StringIO(self.deflate(resp.read()))
|
||||
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
|
||||
resp.msg = old_resp.msg
|
||||
return resp
|
||||
|
||||
class FileDownloader(object):
|
||||
"""File Downloader class.
|
||||
|
||||
@ -208,6 +267,7 @@ class FileDownloader(object):
|
||||
forcetitle: Force printing title.
|
||||
forcethumbnail: Force printing thumbnail URL.
|
||||
forcedescription: Force printing description.
|
||||
forcefilename: Force printing final filename.
|
||||
simulate: Do not download the video files.
|
||||
format: Video format code.
|
||||
format_limit: Highest quality format to try.
|
||||
@ -221,6 +281,9 @@ class FileDownloader(object):
|
||||
playliststart: Playlist item to start at.
|
||||
playlistend: Playlist item to end at.
|
||||
logtostderr: Log messages to stderr instead of stdout.
|
||||
consoletitle: Display progress in console window's titlebar.
|
||||
nopart: Do not use temporary .part files.
|
||||
updatetime: Use the Last-modified header to set output file timestamps.
|
||||
"""
|
||||
|
||||
params = None
|
||||
@ -338,6 +401,17 @@ class FileDownloader(object):
|
||||
"""Print message to stderr."""
|
||||
print >>sys.stderr, message.encode(preferredencoding())
|
||||
|
||||
def to_cons_title(self, message):
|
||||
"""Set console/terminal window title to message."""
|
||||
if not self.params.get('consoletitle', False):
|
||||
return
|
||||
if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
|
||||
# c_wchar_p() might not be necessary if `message` is
|
||||
# already of type unicode()
|
||||
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
|
||||
elif 'TERM' in os.environ:
|
||||
sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
|
||||
|
||||
def fixed_template(self):
|
||||
"""Checks if the output template is fixed."""
|
||||
return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
|
||||
@ -368,6 +442,43 @@ class FileDownloader(object):
|
||||
if speed > rate_limit:
|
||||
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
|
||||
|
||||
def temp_name(self, filename):
|
||||
"""Returns a temporary filename for the given filename."""
|
||||
if self.params.get('nopart', False) or filename == u'-' or \
|
||||
(os.path.exists(filename) and not os.path.isfile(filename)):
|
||||
return filename
|
||||
return filename + u'.part'
|
||||
|
||||
def undo_temp_name(self, filename):
|
||||
if filename.endswith(u'.part'):
|
||||
return filename[:-len(u'.part')]
|
||||
return filename
|
||||
|
||||
def try_rename(self, old_filename, new_filename):
|
||||
try:
|
||||
if old_filename == new_filename:
|
||||
return
|
||||
os.rename(old_filename, new_filename)
|
||||
except (IOError, OSError), err:
|
||||
self.trouble(u'ERROR: unable to rename file')
|
||||
|
||||
def try_utime(self, filename, last_modified_hdr):
|
||||
"""Try to set the last-modified time of the given file."""
|
||||
if last_modified_hdr is None:
|
||||
return
|
||||
if not os.path.isfile(filename):
|
||||
return
|
||||
timestr = last_modified_hdr
|
||||
if timestr is None:
|
||||
return
|
||||
filetime = timeconvert(timestr)
|
||||
if filetime is None:
|
||||
return
|
||||
try:
|
||||
os.utime(filename,(time.time(), filetime))
|
||||
except:
|
||||
pass
|
||||
|
||||
def report_destination(self, filename):
|
||||
"""Report destination filename."""
|
||||
self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
|
||||
@ -378,6 +489,8 @@ class FileDownloader(object):
|
||||
return
|
||||
self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
|
||||
(percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
|
||||
self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
|
||||
(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
|
||||
|
||||
def report_resuming_byte(self, resume_len):
|
||||
"""Report attempt to resume at given byte."""
|
||||
@ -409,8 +522,21 @@ class FileDownloader(object):
|
||||
"""Increment the ordinal that assigns a number to each file."""
|
||||
self._num_downloads += 1
|
||||
|
||||
def prepare_filename(self, info_dict):
|
||||
"""Generate the output filename."""
|
||||
try:
|
||||
template_dict = dict(info_dict)
|
||||
template_dict['epoch'] = unicode(long(time.time()))
|
||||
template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
|
||||
filename = self.params['outtmpl'] % template_dict
|
||||
return filename
|
||||
except (ValueError, KeyError), err:
|
||||
self.trouble(u'ERROR: invalid system charset or erroneous output template')
|
||||
return None
|
||||
|
||||
def process_info(self, info_dict):
|
||||
"""Process a single dictionary returned by an InfoExtractor."""
|
||||
filename = self.prepare_filename(info_dict)
|
||||
# Do nothing else if in simulate mode
|
||||
if self.params.get('simulate', False):
|
||||
# Forced printings
|
||||
@ -422,16 +548,12 @@ class FileDownloader(object):
|
||||
print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
|
||||
if self.params.get('forcedescription', False) and 'description' in info_dict:
|
||||
print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
|
||||
if self.params.get('forcefilename', False) and filename is not None:
|
||||
print filename.encode(preferredencoding(), 'xmlcharrefreplace')
|
||||
|
||||
return
|
||||
|
||||
try:
|
||||
template_dict = dict(info_dict)
|
||||
template_dict['epoch'] = unicode(long(time.time()))
|
||||
template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
|
||||
filename = self.params['outtmpl'] % template_dict
|
||||
except (ValueError, KeyError), err:
|
||||
self.trouble(u'ERROR: invalid system charset or erroneous output template')
|
||||
if filename is None:
|
||||
return
|
||||
if self.params.get('nooverwrites', False) and os.path.exists(filename):
|
||||
self.to_stderr(u'WARNING: file exists and will be skipped')
|
||||
@ -498,6 +620,7 @@ class FileDownloader(object):
|
||||
|
||||
def _download_with_rtmpdump(self, filename, url, player_url):
|
||||
self.report_destination(filename)
|
||||
tmpfilename = self.temp_name(filename)
|
||||
|
||||
# Check for rtmpdump first
|
||||
try:
|
||||
@ -509,36 +632,46 @@ class FileDownloader(object):
|
||||
# Download using rtmpdump. rtmpdump returns exit code 2 when
|
||||
# the connection was interrumpted and resuming appears to be
|
||||
# possible. This is part of rtmpdump's normal usage, AFAIK.
|
||||
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
|
||||
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
|
||||
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
|
||||
while retval == 2 or retval == 1:
|
||||
prevsize = os.path.getsize(filename)
|
||||
prevsize = os.path.getsize(tmpfilename)
|
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
|
||||
time.sleep(5.0) # This seems to be needed
|
||||
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
|
||||
cursize = os.path.getsize(filename)
|
||||
cursize = os.path.getsize(tmpfilename)
|
||||
if prevsize == cursize and retval == 1:
|
||||
break
|
||||
if retval == 0:
|
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
|
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
|
||||
self.try_rename(tmpfilename, filename)
|
||||
return True
|
||||
else:
|
||||
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
|
||||
return False
|
||||
|
||||
def _do_download(self, filename, url, player_url):
|
||||
# Check file already present
|
||||
if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
|
||||
self.report_file_already_downloaded(filename)
|
||||
return True
|
||||
|
||||
# Attempt to download using rtmpdump
|
||||
if url.startswith('rtmp'):
|
||||
return self._download_with_rtmpdump(filename, url, player_url)
|
||||
|
||||
tmpfilename = self.temp_name(filename)
|
||||
stream = None
|
||||
open_mode = 'wb'
|
||||
basic_request = urllib2.Request(url, None, std_headers)
|
||||
request = urllib2.Request(url, None, std_headers)
|
||||
|
||||
# Do not include the Accept-Encoding header
|
||||
headers = {'Youtubedl-no-compression': 'True'}
|
||||
basic_request = urllib2.Request(url, None, headers)
|
||||
request = urllib2.Request(url, None, headers)
|
||||
|
||||
# Establish possible resume length
|
||||
if os.path.isfile(filename):
|
||||
resume_len = os.path.getsize(filename)
|
||||
if os.path.isfile(tmpfilename):
|
||||
resume_len = os.path.getsize(tmpfilename)
|
||||
else:
|
||||
resume_len = 0
|
||||
|
||||
@ -580,6 +713,7 @@ class FileDownloader(object):
|
||||
# completely downloaded if the file size differs less than 100 bytes from
|
||||
# the one in the hard drive.
|
||||
self.report_file_already_downloaded(filename)
|
||||
self.try_rename(tmpfilename, filename)
|
||||
return True
|
||||
else:
|
||||
# The length does not match, we start the download over
|
||||
@ -596,8 +730,10 @@ class FileDownloader(object):
|
||||
return False
|
||||
|
||||
data_len = data.info().get('Content-length', None)
|
||||
if data_len is not None:
|
||||
data_len = long(data_len) + resume_len
|
||||
data_len_str = self.format_bytes(data_len)
|
||||
byte_counter = 0
|
||||
byte_counter = 0 + resume_len
|
||||
block_size = 1024
|
||||
start = time.time()
|
||||
while True:
|
||||
@ -605,15 +741,15 @@ class FileDownloader(object):
|
||||
before = time.time()
|
||||
data_block = data.read(block_size)
|
||||
after = time.time()
|
||||
data_block_len = len(data_block)
|
||||
if data_block_len == 0:
|
||||
if len(data_block) == 0:
|
||||
break
|
||||
byte_counter += data_block_len
|
||||
byte_counter += len(data_block)
|
||||
|
||||
# Open file just in time
|
||||
if stream is None:
|
||||
try:
|
||||
(stream, filename) = sanitize_open(filename, open_mode)
|
||||
(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
|
||||
filename = self.undo_temp_name(tmpfilename)
|
||||
self.report_destination(filename)
|
||||
except (OSError, IOError), err:
|
||||
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
|
||||
@ -623,20 +759,27 @@ class FileDownloader(object):
|
||||
except (IOError, OSError), err:
|
||||
self.trouble(u'\nERROR: unable to write data: %s' % str(err))
|
||||
return False
|
||||
block_size = self.best_block_size(after - before, data_block_len)
|
||||
block_size = self.best_block_size(after - before, len(data_block))
|
||||
|
||||
# Progress message
|
||||
percent_str = self.calc_percent(byte_counter, data_len)
|
||||
eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
|
||||
speed_str = self.calc_speed(start, time.time(), byte_counter)
|
||||
eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
|
||||
speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
|
||||
self.report_progress(percent_str, data_len_str, speed_str, eta_str)
|
||||
|
||||
# Apply rate limit
|
||||
self.slow_down(start, byte_counter)
|
||||
self.slow_down(start, byte_counter - resume_len)
|
||||
|
||||
stream.close()
|
||||
self.report_finish()
|
||||
if data_len is not None and str(byte_counter) != data_len:
|
||||
if data_len is not None and byte_counter != data_len:
|
||||
raise ContentTooShortError(byte_counter, long(data_len))
|
||||
self.try_rename(tmpfilename, filename)
|
||||
|
||||
# Update file modification time
|
||||
if self.params.get('updatetime', True):
|
||||
self.try_utime(filename, data.info().get('last-modified', None))
|
||||
|
||||
return True
|
||||
|
||||
class InfoExtractor(object):
|
||||
@ -713,7 +856,7 @@ class InfoExtractor(object):
|
||||
class YoutubeIE(InfoExtractor):
|
||||
"""Information extractor for youtube.com."""
|
||||
|
||||
_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
|
||||
_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:(?:v|embed)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
|
||||
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
|
||||
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
|
||||
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
|
||||
@ -792,7 +935,7 @@ class YoutubeIE(InfoExtractor):
|
||||
return
|
||||
|
||||
# Set language
|
||||
request = urllib2.Request(self._LANG_URL, None, std_headers)
|
||||
request = urllib2.Request(self._LANG_URL)
|
||||
try:
|
||||
self.report_lang()
|
||||
urllib2.urlopen(request).read()
|
||||
@ -812,7 +955,7 @@ class YoutubeIE(InfoExtractor):
|
||||
'username': username,
|
||||
'password': password,
|
||||
}
|
||||
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
|
||||
request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
|
||||
try:
|
||||
self.report_login()
|
||||
login_results = urllib2.urlopen(request).read()
|
||||
@ -828,7 +971,7 @@ class YoutubeIE(InfoExtractor):
|
||||
'next_url': '/',
|
||||
'action_confirm': 'Confirm',
|
||||
}
|
||||
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
|
||||
request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
|
||||
try:
|
||||
self.report_age_confirmation()
|
||||
age_results = urllib2.urlopen(request).read()
|
||||
@ -846,7 +989,7 @@ class YoutubeIE(InfoExtractor):
|
||||
|
||||
# Get video webpage
|
||||
self.report_video_webpage_download(video_id)
|
||||
request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers)
|
||||
request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
|
||||
try:
|
||||
video_webpage = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -865,7 +1008,7 @@ class YoutubeIE(InfoExtractor):
|
||||
for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
|
||||
video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
|
||||
% (video_id, el_type))
|
||||
request = urllib2.Request(video_info_url, None, std_headers)
|
||||
request = urllib2.Request(video_info_url)
|
||||
try:
|
||||
video_info_webpage = urllib2.urlopen(request).read()
|
||||
video_info = parse_qs(video_info_webpage)
|
||||
@ -913,18 +1056,13 @@ class YoutubeIE(InfoExtractor):
|
||||
upload_date = u'NA'
|
||||
mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
|
||||
if mobj is not None:
|
||||
upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
|
||||
format_expressions = ['%d %B %Y', '%B %d %Y']
|
||||
for expression in format_expressions:
|
||||
try:
|
||||
if ',' in mobj.group(1):
|
||||
# Month Day, Year
|
||||
m, d, y = mobj.group(1).replace(',', '').split()
|
||||
else:
|
||||
# Day Month Year, we'll suppose
|
||||
d, m, y = mobj.group(1).split()
|
||||
m = month_name_to_number[m]
|
||||
d = '%02d' % (long(d))
|
||||
upload_date = '%s%s%s' % (y, m, d)
|
||||
upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
|
||||
except:
|
||||
upload_date = u'NA'
|
||||
pass
|
||||
|
||||
# description
|
||||
video_description = 'No description available.'
|
||||
@ -937,8 +1075,7 @@ class YoutubeIE(InfoExtractor):
|
||||
video_token = urllib.unquote_plus(video_info['token'][0])
|
||||
|
||||
# Decide which formats to download
|
||||
requested_format = self._downloader.params.get('format', None)
|
||||
get_video_template = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=&ps=&asv=&fmt=%%s' % (video_id, video_token)
|
||||
req_format = self._downloader.params.get('format', None)
|
||||
|
||||
if 'fmt_url_map' in video_info:
|
||||
url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(','))
|
||||
@ -951,12 +1088,16 @@ class YoutubeIE(InfoExtractor):
|
||||
if len(existing_formats) == 0:
|
||||
self._downloader.trouble(u'ERROR: no known formats available for video')
|
||||
return
|
||||
if requested_format is None:
|
||||
video_url_list = [(existing_formats[0], get_video_template % existing_formats[0])] # Best quality
|
||||
elif requested_format == '-1':
|
||||
video_url_list = [(f, get_video_template % f) for f in existing_formats] # All formats
|
||||
if req_format is None:
|
||||
video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
|
||||
elif req_format == '-1':
|
||||
video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
|
||||
else:
|
||||
video_url_list = [(requested_format, get_video_template % requested_format)] # Specific format
|
||||
# Specific format
|
||||
if req_format not in url_map:
|
||||
self._downloader.trouble(u'ERROR: requested format not available')
|
||||
return
|
||||
video_url_list = [(req_format, url_map[req_format])] # Specific format
|
||||
|
||||
elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
|
||||
self.report_rtmp_download()
|
||||
@ -990,7 +1131,7 @@ class YoutubeIE(InfoExtractor):
|
||||
'player_url': player_url,
|
||||
})
|
||||
except UnavailableVideoError, err:
|
||||
self._downloader.trouble(u'ERROR: unable to download video (format may not be available)')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class MetacafeIE(InfoExtractor):
|
||||
@ -1027,7 +1168,7 @@ class MetacafeIE(InfoExtractor):
|
||||
|
||||
def _real_initialize(self):
|
||||
# Retrieve disclaimer
|
||||
request = urllib2.Request(self._DISCLAIMER, None, std_headers)
|
||||
request = urllib2.Request(self._DISCLAIMER)
|
||||
try:
|
||||
self.report_disclaimer()
|
||||
disclaimer = urllib2.urlopen(request).read()
|
||||
@ -1040,7 +1181,7 @@ class MetacafeIE(InfoExtractor):
|
||||
'filters': '0',
|
||||
'submit': "Continue - I'm over 18",
|
||||
}
|
||||
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
|
||||
request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
|
||||
try:
|
||||
self.report_age_confirmation()
|
||||
disclaimer = urllib2.urlopen(request).read()
|
||||
@ -1135,7 +1276,7 @@ class MetacafeIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class DailymotionIE(InfoExtractor):
|
||||
@ -1204,7 +1345,7 @@ class DailymotionIE(InfoExtractor):
|
||||
video_title = mobj.group(1).decode('utf-8')
|
||||
video_title = sanitize_title(video_title)
|
||||
|
||||
mobj = re.search(r'(?im)<div class="dmco_html owner">.*?<a class="name" href="/.+?">(.+?)</a>', webpage)
|
||||
mobj = re.search(r'(?im)<Attribute name="owner">(.+?)</Attribute>', webpage)
|
||||
if mobj is None:
|
||||
self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
|
||||
return
|
||||
@ -1224,7 +1365,7 @@ class DailymotionIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
class GoogleIE(InfoExtractor):
|
||||
"""Information extractor for video.google.com."""
|
||||
@ -1334,7 +1475,7 @@ class GoogleIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class PhotobucketIE(InfoExtractor):
|
||||
@ -1416,7 +1557,7 @@ class PhotobucketIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class YahooIE(InfoExtractor):
|
||||
@ -1574,7 +1715,7 @@ class YahooIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class GenericIE(InfoExtractor):
|
||||
@ -1617,6 +1758,7 @@ class GenericIE(InfoExtractor):
|
||||
self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
|
||||
return
|
||||
|
||||
self.report_extraction(video_id)
|
||||
# Start with something easy: JW Player in SWFObject
|
||||
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
|
||||
if mobj is None:
|
||||
@ -1674,7 +1816,7 @@ class GenericIE(InfoExtractor):
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError, err:
|
||||
self._downloader.trouble(u'ERROR: unable to download video')
|
||||
self._downloader.trouble(u'\nERROR: unable to download video')
|
||||
|
||||
|
||||
class YoutubeSearchIE(InfoExtractor):
|
||||
@ -1742,7 +1884,7 @@ class YoutubeSearchIE(InfoExtractor):
|
||||
while True:
|
||||
self.report_download_page(query, pagenum)
|
||||
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
|
||||
request = urllib2.Request(result_url, None, std_headers)
|
||||
request = urllib2.Request(result_url)
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -1833,7 +1975,7 @@ class GoogleSearchIE(InfoExtractor):
|
||||
while True:
|
||||
self.report_download_page(query, pagenum)
|
||||
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
|
||||
request = urllib2.Request(result_url, None, std_headers)
|
||||
request = urllib2.Request(result_url)
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -1924,7 +2066,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
while True:
|
||||
self.report_download_page(query, pagenum)
|
||||
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
|
||||
request = urllib2.Request(result_url, None, std_headers)
|
||||
request = urllib2.Request(result_url)
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -1953,7 +2095,7 @@ class YahooSearchIE(InfoExtractor):
|
||||
class YoutubePlaylistIE(InfoExtractor):
|
||||
"""Information Extractor for YouTube playlists."""
|
||||
|
||||
_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
|
||||
_VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/|p/)([^&]+).*'
|
||||
_TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
|
||||
_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
|
||||
_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
|
||||
@ -1988,7 +2130,7 @@ class YoutubePlaylistIE(InfoExtractor):
|
||||
|
||||
while True:
|
||||
self.report_download_page(playlist_id, pagenum)
|
||||
request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
|
||||
request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum))
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -2050,7 +2192,7 @@ class YoutubeUserIE(InfoExtractor):
|
||||
pagenum = 1
|
||||
|
||||
self.report_download_page(username)
|
||||
request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
|
||||
request = urllib2.Request(self._TEMPLATE_URL % (username))
|
||||
try:
|
||||
page = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
@ -2073,6 +2215,85 @@ class YoutubeUserIE(InfoExtractor):
|
||||
self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
|
||||
return
|
||||
|
||||
class DepositFilesIE(InfoExtractor):
|
||||
"""Information extractor for depositfiles.com"""
|
||||
|
||||
_VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)'
|
||||
|
||||
def __init__(self, downloader=None):
|
||||
InfoExtractor.__init__(self, downloader)
|
||||
|
||||
@staticmethod
|
||||
def suitable(url):
|
||||
return (re.match(DepositFilesIE._VALID_URL, url) is not None)
|
||||
|
||||
def report_download_webpage(self, file_id):
|
||||
"""Report webpage download."""
|
||||
self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
|
||||
|
||||
def report_extraction(self, file_id):
|
||||
"""Report information extraction."""
|
||||
self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
|
||||
|
||||
def _real_initialize(self):
|
||||
return
|
||||
|
||||
def _real_extract(self, url):
|
||||
# At this point we have a new file
|
||||
self._downloader.increment_downloads()
|
||||
|
||||
file_id = url.split('/')[-1]
|
||||
# Rebuild url in english locale
|
||||
url = 'http://depositfiles.com/en/files/' + file_id
|
||||
|
||||
# Retrieve file webpage with 'Free download' button pressed
|
||||
free_download_indication = { 'gateway_result' : '1' }
|
||||
request = urllib2.Request(url, urllib.urlencode(free_download_indication))
|
||||
try:
|
||||
self.report_download_webpage(file_id)
|
||||
webpage = urllib2.urlopen(request).read()
|
||||
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
|
||||
self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
|
||||
return
|
||||
|
||||
# Search for the real file URL
|
||||
mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
|
||||
if (mobj is None) or (mobj.group(1) is None):
|
||||
# Try to figure out reason of the error.
|
||||
mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
|
||||
if (mobj is not None) and (mobj.group(1) is not None):
|
||||
restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
|
||||
self._downloader.trouble(u'ERROR: %s' % restriction_message)
|
||||
else:
|
||||
self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
|
||||
return
|
||||
|
||||
file_url = mobj.group(1)
|
||||
file_extension = os.path.splitext(file_url)[1][1:]
|
||||
|
||||
# Search for file title
|
||||
mobj = re.search(r'<b title="(.*?)">', webpage)
|
||||
if mobj is None:
|
||||
self._downloader.trouble(u'ERROR: unable to extract title')
|
||||
return
|
||||
file_title = mobj.group(1).decode('utf-8')
|
||||
|
||||
try:
|
||||
# Process file information
|
||||
self._downloader.process_info({
|
||||
'id': file_id.decode('utf-8'),
|
||||
'url': file_url.decode('utf-8'),
|
||||
'uploader': u'NA',
|
||||
'upload_date': u'NA',
|
||||
'title': file_title,
|
||||
'stitle': file_title,
|
||||
'ext': file_extension.decode('utf-8'),
|
||||
'format': u'NA',
|
||||
'player_url': None,
|
||||
})
|
||||
except UnavailableVideoError, err:
|
||||
self._downloader.trouble(u'ERROR: unable to download file')
|
||||
|
||||
class PostProcessor(object):
|
||||
"""Post Processor class.
|
||||
|
||||
@ -2126,26 +2347,32 @@ if __name__ == '__main__':
|
||||
import getpass
|
||||
import optparse
|
||||
|
||||
# Function to update the program file with the latest version from bitbucket.org
|
||||
# Function to update the program file with the latest version from the repository.
|
||||
def update_self(downloader, filename):
|
||||
# Note: downloader only used for options
|
||||
if not os.access(filename, os.W_OK):
|
||||
sys.exit('ERROR: no write permissions on %s' % filename)
|
||||
|
||||
downloader.to_screen('Updating to latest stable version...')
|
||||
try:
|
||||
latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
|
||||
latest_version = urllib.urlopen(latest_url).read().strip()
|
||||
prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
|
||||
newcontent = urllib.urlopen(prog_url).read()
|
||||
except (IOError, OSError), err:
|
||||
sys.exit('ERROR: unable to download latest version')
|
||||
try:
|
||||
stream = open(filename, 'w')
|
||||
stream.write(newcontent)
|
||||
stream.close()
|
||||
except (IOError, OSError), err:
|
||||
sys.exit('ERROR: unable to overwrite current version')
|
||||
downloader.to_screen('Updated to version %s' % latest_version)
|
||||
|
||||
# Parse command line
|
||||
parser = optparse.OptionParser(
|
||||
usage='Usage: %prog [options] url...',
|
||||
version='2010.11.19',
|
||||
version='2010.12.09',
|
||||
conflict_handler='resolve',
|
||||
)
|
||||
|
||||
@ -2165,6 +2392,9 @@ if __name__ == '__main__':
|
||||
dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
|
||||
parser.add_option('--playlist-end',
|
||||
dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
|
||||
parser.add_option('--dump-user-agent',
|
||||
action='store_true', dest='dump_user_agent',
|
||||
help='display the current browser identification', default=False)
|
||||
|
||||
authentication = optparse.OptionGroup(parser, 'Authentication Options')
|
||||
authentication.add_option('-u', '--username',
|
||||
@ -2178,14 +2408,10 @@ if __name__ == '__main__':
|
||||
video_format = optparse.OptionGroup(parser, 'Video Format Options')
|
||||
video_format.add_option('-f', '--format',
|
||||
action='store', dest='format', metavar='FORMAT', help='video format code')
|
||||
video_format.add_option('-m', '--mobile-version',
|
||||
action='store_const', dest='format', help='alias for -f 17', const='17')
|
||||
video_format.add_option('--all-formats',
|
||||
action='store_const', dest='format', help='download all available video formats', const='-1')
|
||||
video_format.add_option('--max-quality',
|
||||
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
|
||||
video_format.add_option('-b', '--best-quality',
|
||||
action='store_true', dest='bestquality', help='download the best video quality (DEPRECATED)')
|
||||
parser.add_option_group(video_format)
|
||||
|
||||
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
|
||||
@ -2198,11 +2424,19 @@ if __name__ == '__main__':
|
||||
verbosity.add_option('-e', '--get-title',
|
||||
action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
|
||||
verbosity.add_option('--get-thumbnail',
|
||||
action='store_true', dest='getthumbnail', help='simulate, quiet but print thumbnail URL', default=False)
|
||||
action='store_true', dest='getthumbnail',
|
||||
help='simulate, quiet but print thumbnail URL', default=False)
|
||||
verbosity.add_option('--get-description',
|
||||
action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False)
|
||||
action='store_true', dest='getdescription',
|
||||
help='simulate, quiet but print video description', default=False)
|
||||
verbosity.add_option('--get-filename',
|
||||
action='store_true', dest='getfilename',
|
||||
help='simulate, quiet but print output filename', default=False)
|
||||
verbosity.add_option('--no-progress',
|
||||
action='store_true', dest='noprogress', help='do not print progress bar', default=False)
|
||||
verbosity.add_option('--console-title',
|
||||
action='store_true', dest='consoletitle',
|
||||
help='display progress in console titlebar', default=False)
|
||||
parser.add_option_group(verbosity)
|
||||
|
||||
filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
|
||||
@ -2211,7 +2445,8 @@ if __name__ == '__main__':
|
||||
filesystem.add_option('-l', '--literal',
|
||||
action='store_true', dest='useliteral', help='use literal title in file name', default=False)
|
||||
filesystem.add_option('-A', '--auto-number',
|
||||
action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False)
|
||||
action='store_true', dest='autonumber',
|
||||
help='number downloaded files starting from 00000', default=False)
|
||||
filesystem.add_option('-o', '--output',
|
||||
dest='outtmpl', metavar='TEMPLATE', help='output filename template')
|
||||
filesystem.add_option('-a', '--batch-file',
|
||||
@ -2222,6 +2457,11 @@ if __name__ == '__main__':
|
||||
action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
|
||||
filesystem.add_option('--cookies',
|
||||
dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
|
||||
filesystem.add_option('--no-part',
|
||||
action='store_true', dest='nopart', help='do not use .part files', default=False)
|
||||
filesystem.add_option('--no-mtime',
|
||||
action='store_false', dest='updatetime',
|
||||
help='do not use the Last-modified header to set the file modification time', default=True)
|
||||
parser.add_option_group(filesystem)
|
||||
|
||||
(opts, args) = parser.parse_args()
|
||||
@ -2237,10 +2477,14 @@ if __name__ == '__main__':
|
||||
except (IOError, OSError), err:
|
||||
sys.exit(u'ERROR: unable to open cookie file')
|
||||
|
||||
# Dump user agent
|
||||
if opts.dump_user_agent:
|
||||
print std_headers['User-Agent']
|
||||
sys.exit(0)
|
||||
|
||||
# General configuration
|
||||
cookie_processor = urllib2.HTTPCookieProcessor(jar)
|
||||
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
|
||||
urllib2.install_opener(urllib2.build_opener(cookie_processor))
|
||||
urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
|
||||
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
|
||||
|
||||
# Batch file verification
|
||||
@ -2259,8 +2503,6 @@ if __name__ == '__main__':
|
||||
all_urls = batchurls + args
|
||||
|
||||
# Conflicting, missing and erroneous options
|
||||
if opts.bestquality:
|
||||
print >>sys.stderr, u'\nWARNING: -b/--best-quality IS DEPRECATED AS IT IS THE DEFAULT BEHAVIOR NOW\n'
|
||||
if opts.usenetrc and (opts.username is not None or opts.password is not None):
|
||||
parser.error(u'using .netrc conflicts with giving username/password')
|
||||
if opts.password is not None and opts.username is None:
|
||||
@ -2306,6 +2548,7 @@ if __name__ == '__main__':
|
||||
photobucket_ie = PhotobucketIE()
|
||||
yahoo_ie = YahooIE()
|
||||
yahoo_search_ie = YahooSearchIE(yahoo_ie)
|
||||
deposit_files_ie = DepositFilesIE()
|
||||
generic_ie = GenericIE()
|
||||
|
||||
# File downloader
|
||||
@ -2313,12 +2556,13 @@ if __name__ == '__main__':
|
||||
'usenetrc': opts.usenetrc,
|
||||
'username': opts.username,
|
||||
'password': opts.password,
|
||||
'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
|
||||
'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
|
||||
'forceurl': opts.geturl,
|
||||
'forcetitle': opts.gettitle,
|
||||
'forcethumbnail': opts.getthumbnail,
|
||||
'forcedescription': opts.getdescription,
|
||||
'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription),
|
||||
'forcefilename': opts.getfilename,
|
||||
'simulate': (opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename),
|
||||
'format': opts.format,
|
||||
'format_limit': opts.format_limit,
|
||||
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
|
||||
@ -2340,6 +2584,9 @@ if __name__ == '__main__':
|
||||
'playliststart': opts.playliststart,
|
||||
'playlistend': opts.playlistend,
|
||||
'logtostderr': opts.outtmpl == '-',
|
||||
'consoletitle': opts.consoletitle,
|
||||
'nopart': opts.nopart,
|
||||
'updatetime': opts.updatetime,
|
||||
})
|
||||
fd.add_info_extractor(youtube_search_ie)
|
||||
fd.add_info_extractor(youtube_pl_ie)
|
||||
@ -2352,6 +2599,7 @@ if __name__ == '__main__':
|
||||
fd.add_info_extractor(photobucket_ie)
|
||||
fd.add_info_extractor(yahoo_ie)
|
||||
fd.add_info_extractor(yahoo_search_ie)
|
||||
fd.add_info_extractor(deposit_files_ie)
|
||||
|
||||
# This must come last since it's the
|
||||
# fallback if none of the others work
|
||||
|
||||
Loading…
Reference in New Issue
Block a user