Compare commits
228 Commits
2013.09.06
...
2013.10.06
Author | SHA1 | Date | |
---|---|---|---|
7e5e8306fd | |||
41e8bca4d0 | |||
f4aac741d5 | |||
226113c880 | |||
8932a66e49 | |||
79cfb46d42 | |||
00fcc17aee | |||
e94b783c74 | |||
97dae9ae07 | |||
c3fef636b5 | |||
46e28a84ca | |||
17ad2b3fb1 | |||
5e2a60db4a | |||
cd214418f6 | |||
ba2d9f213e | |||
7f8ae73a5d | |||
466880f531 | |||
9f1f6d2437 | |||
9e0f897f6b | |||
c0f6aa876f | |||
d93bdee9a6 | |||
f13d09332d | |||
2f5865cc6d | |||
deefc05b88 | |||
0d8cb1cc14 | |||
a90b9fd209 | |||
829493439a | |||
3cd022f6e6 | |||
abefd1f7c4 | |||
c21315f273 | |||
9ab1018b1a | |||
da0a5d2d6e | |||
ee6adb166c | |||
be8fe32c92 | |||
c38b1e776d | |||
4f8bf17f23 | |||
ca40186c75 | |||
a8c6b24155 | |||
bd8e5c7ca2 | |||
7c61bd36bb | |||
c54283824c | |||
52f15da2ca | |||
44d466559e | |||
05751eb047 | |||
f10503db67 | |||
adfeafe9e1 | |||
4c62a16f4f | |||
c0de39e6d4 | |||
fa55675593 | |||
d4d9920a26 | |||
47192f92d8 | |||
722076a123 | |||
bb4aa62cf7 | |||
843530568f | |||
138a5454b5 | |||
d279037036 | |||
46353f6783 | |||
70922df8b5 | |||
9c15e9de84 | |||
123c10608d | |||
0b7c2485b6 | |||
9abb32045a | |||
f490e77e77 | |||
2dc592991a | |||
0a60edcfa9 | |||
c53f9d30c8 | |||
509f398292 | |||
74bab3f0a4 | |||
8574862991 | |||
2de957c7e1 | |||
920de7a27d | |||
63efc427cd | |||
ce65fb6c76 | |||
4de1994b6e | |||
592882aa9f | |||
b98d6a1e19 | |||
29c7a63df8 | |||
8b25323ae2 | |||
f426de8460 | |||
695dc094ab | |||
e80d861064 | |||
2cdeb20135 | |||
7f74773254 | |||
f2c327fd39 | |||
e35e4ddc9a | |||
c3c88a2664 | |||
bb0eee71e7 | |||
6f56389b88 | |||
5b333c1ce6 | |||
a825f33030 | |||
92f618f2e2 | |||
81ec7c7901 | |||
dd5d2eb03c | |||
4ae720042c | |||
c705320f48 | |||
d2d8f89531 | |||
bdde940e90 | |||
45f4a76dbc | |||
13dc64ce74 | |||
c35f9e72ce | |||
f8061589e6 | |||
0ca96d48c7 | |||
4ba146f35d | |||
edf3e38ebd | |||
c4417ddb61 | |||
4a2080e407 | |||
2f2ffea9ca | |||
ba552f542f | |||
8379969834 | |||
95dbd2f990 | |||
a7177865b1 | |||
e0df6211cc | |||
b00ca882a4 | |||
39baacc49f | |||
3a1d48d6de | |||
34308b30d6 | |||
bc1506f8c0 | |||
b61067fa4f | |||
69b227a9bc | |||
0fd49457f5 | |||
58f289d013 | |||
3d60bb96e1 | |||
38d025b3f0 | |||
c40c6aaaaa | |||
1a810f0d4e | |||
63037593c0 | |||
7a878d47fa | |||
bc4b900898 | |||
c5e743f66f | |||
6c36d8d6fb | |||
71c82637e7 | |||
2dad310e2c | |||
d0ae9e3a8d | |||
a19413c311 | |||
1ef80b55dd | |||
eb03f4dad3 | |||
830dd1944a | |||
1237c9a3a5 | |||
5d13df79a5 | |||
6523223a4c | |||
4a67aafb7e | |||
f3f34c5b0f | |||
6ae8ee3f54 | |||
e8f8e80097 | |||
4dc0ff3ecf | |||
4b6462fc1e | |||
c4ece78564 | |||
0761d02b0b | |||
71c107fc57 | |||
7459e3a290 | |||
f9e66fb993 | |||
6c603ccce3 | |||
ef66b0c6ef | |||
22b50ecb2f | |||
5a6fecc3de | |||
cdbccafed9 | |||
e69ae5b9e7 | |||
92790f4e54 | |||
471a5ee908 | |||
19e1d35989 | |||
0b7f31184d | |||
fad84d50fe | |||
9a1c32dc54 | |||
a921f40799 | |||
74ac9bdd82 | |||
94518f2087 | |||
535f59bbcf | |||
71cedb3c0c | |||
dd01d6558a | |||
ce85f022d2 | |||
ad94a6fe44 | |||
353ba14060 | |||
83de794223 | |||
bfd5c93af9 | |||
c247d87ef3 | |||
07ac9e2cc2 | |||
6bc520c207 | |||
f1d20fa39f | |||
e3dc22ca3a | |||
d665f8d3cb | |||
055e6f3657 | |||
ac4f319ba1 | |||
542cca0e8c | |||
6a2449df3b | |||
7fad1c6328 | |||
d82134c339 | |||
54d39d8b2f | |||
de7f3446e0 | |||
f8e52269c1 | |||
cf1dd0c59e | |||
22c8b52545 | |||
1f7dc42cd0 | |||
aa8f2641da | |||
648d25d43d | |||
df3e61003a | |||
6b361ad5ee | |||
5d8afe69f7 | |||
a1ab553858 | |||
07463ea162 | |||
6d2d21f713 | |||
061b2889a9 | |||
8963d9c266 | |||
890f62e868 | |||
8f362589a5 | |||
a27a2470cd | |||
72836fcee4 | |||
a7130543fa | |||
a490fda746 | |||
7e77275293 | |||
d6e203b3dc | |||
e3ea479087 | |||
faab1d3836 | |||
8851a574a3 | |||
b3f0e53048 | |||
06a401c845 | |||
bd2dee6c67 | |||
18b4e04f1c | |||
d80a064eff | |||
d55de6eec2 | |||
69df680b97 | |||
447591e1ae | |||
33eb0ce4c4 | |||
505c28aac9 | |||
8377574c9c | |||
372297e713 | |||
953e32b2c1 | |||
5898e28272 | |||
67dfbc0cb9 |
10
.gitignore
vendored
10
.gitignore
vendored
@ -17,4 +17,12 @@ youtube-dl.tar.gz
|
|||||||
.coverage
|
.coverage
|
||||||
cover/
|
cover/
|
||||||
updates_key.pem
|
updates_key.pem
|
||||||
*.egg-info
|
*.egg-info
|
||||||
|
*.srt
|
||||||
|
*.sbv
|
||||||
|
*.vtt
|
||||||
|
*.flv
|
||||||
|
*.mp4
|
||||||
|
*.part
|
||||||
|
test/testdata
|
||||||
|
.tox
|
||||||
|
15
README.md
15
README.md
@ -19,7 +19,8 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
-U, --update update this program to latest version. Make sure
|
-U, --update update this program to latest version. Make sure
|
||||||
that you have sufficient permissions (run with
|
that you have sufficient permissions (run with
|
||||||
sudo if needed)
|
sudo if needed)
|
||||||
-i, --ignore-errors continue on download errors
|
-i, --ignore-errors continue on download errors, for example to to
|
||||||
|
skip unavailable videos in a playlist
|
||||||
--dump-user-agent display the current browser identification
|
--dump-user-agent display the current browser identification
|
||||||
--user-agent UA specify a custom user agent
|
--user-agent UA specify a custom user agent
|
||||||
--referer REF specify a custom referer, use if the video access
|
--referer REF specify a custom referer, use if the video access
|
||||||
@ -29,6 +30,11 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
--extractor-descriptions Output descriptions of all supported extractors
|
--extractor-descriptions Output descriptions of all supported extractors
|
||||||
--proxy URL Use the specified HTTP/HTTPS proxy
|
--proxy URL Use the specified HTTP/HTTPS proxy
|
||||||
--no-check-certificate Suppress HTTPS certificate validation.
|
--no-check-certificate Suppress HTTPS certificate validation.
|
||||||
|
--cache-dir None Location in the filesystem where youtube-dl can
|
||||||
|
store downloaded information permanently. By
|
||||||
|
default $XDG_CACHE_HOME/youtube-dl or ~/.cache
|
||||||
|
/youtube-dl .
|
||||||
|
--no-cache-dir Disable filesystem caching
|
||||||
|
|
||||||
## Video Selection:
|
## Video Selection:
|
||||||
--playlist-start NUMBER playlist video to start at (default is 1)
|
--playlist-start NUMBER playlist video to start at (default is 1)
|
||||||
@ -45,6 +51,7 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
--date DATE download only videos uploaded in this date
|
--date DATE download only videos uploaded in this date
|
||||||
--datebefore DATE download only videos uploaded before this date
|
--datebefore DATE download only videos uploaded before this date
|
||||||
--dateafter DATE download only videos uploaded after this date
|
--dateafter DATE download only videos uploaded after this date
|
||||||
|
--no-playlist download only the currently playing video
|
||||||
|
|
||||||
## Download Options:
|
## Download Options:
|
||||||
-r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m)
|
-r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m)
|
||||||
@ -123,10 +130,8 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
only)
|
only)
|
||||||
|
|
||||||
## Subtitle Options:
|
## Subtitle Options:
|
||||||
--write-sub write subtitle file (currently youtube only)
|
--write-sub write subtitle file
|
||||||
--write-auto-sub write automatic subtitle file (currently youtube
|
--write-auto-sub write automatic subtitle file (youtube only)
|
||||||
only)
|
|
||||||
--only-sub [deprecated] alias of --skip-download
|
|
||||||
--all-subs downloads all the available subtitles of the
|
--all-subs downloads all the available subtitles of the
|
||||||
video
|
video
|
||||||
--list-subs lists all available subtitles for the video
|
--list-subs lists all available subtitles for the video
|
||||||
|
405
devscripts/buildserver.py
Normal file
405
devscripts/buildserver.py
Normal file
@ -0,0 +1,405 @@
|
|||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||||
|
from socketserver import ThreadingMixIn
|
||||||
|
import argparse
|
||||||
|
import ctypes
|
||||||
|
import functools
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import traceback
|
||||||
|
import os.path
|
||||||
|
|
||||||
|
|
||||||
|
class BuildHTTPServer(ThreadingMixIn, HTTPServer):
|
||||||
|
allow_reuse_address = True
|
||||||
|
|
||||||
|
|
||||||
|
advapi32 = ctypes.windll.advapi32
|
||||||
|
|
||||||
|
SC_MANAGER_ALL_ACCESS = 0xf003f
|
||||||
|
SC_MANAGER_CREATE_SERVICE = 0x02
|
||||||
|
SERVICE_WIN32_OWN_PROCESS = 0x10
|
||||||
|
SERVICE_AUTO_START = 0x2
|
||||||
|
SERVICE_ERROR_NORMAL = 0x1
|
||||||
|
DELETE = 0x00010000
|
||||||
|
SERVICE_STATUS_START_PENDING = 0x00000002
|
||||||
|
SERVICE_STATUS_RUNNING = 0x00000004
|
||||||
|
SERVICE_ACCEPT_STOP = 0x1
|
||||||
|
|
||||||
|
SVCNAME = 'youtubedl_builder'
|
||||||
|
|
||||||
|
LPTSTR = ctypes.c_wchar_p
|
||||||
|
START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
|
||||||
|
|
||||||
|
|
||||||
|
class SERVICE_TABLE_ENTRY(ctypes.Structure):
|
||||||
|
_fields_ = [
|
||||||
|
('lpServiceName', LPTSTR),
|
||||||
|
('lpServiceProc', START_CALLBACK)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
HandlerEx = ctypes.WINFUNCTYPE(
|
||||||
|
ctypes.c_int, # return
|
||||||
|
ctypes.c_int, # dwControl
|
||||||
|
ctypes.c_int, # dwEventType
|
||||||
|
ctypes.c_void_p, # lpEventData,
|
||||||
|
ctypes.c_void_p, # lpContext,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ctypes_array(c_type, py_array):
|
||||||
|
ar = (c_type * len(py_array))()
|
||||||
|
ar[:] = py_array
|
||||||
|
return ar
|
||||||
|
|
||||||
|
|
||||||
|
def win_OpenSCManager():
|
||||||
|
res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
|
||||||
|
if not res:
|
||||||
|
raise Exception('Opening service manager failed - '
|
||||||
|
'are you running this as administrator?')
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def win_install_service(service_name, cmdline):
|
||||||
|
manager = win_OpenSCManager()
|
||||||
|
try:
|
||||||
|
h = advapi32.CreateServiceW(
|
||||||
|
manager, service_name, None,
|
||||||
|
SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
|
||||||
|
SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
|
||||||
|
cmdline, None, None, None, None, None)
|
||||||
|
if not h:
|
||||||
|
raise OSError('Service creation failed: %s' % ctypes.FormatError())
|
||||||
|
|
||||||
|
advapi32.CloseServiceHandle(h)
|
||||||
|
finally:
|
||||||
|
advapi32.CloseServiceHandle(manager)
|
||||||
|
|
||||||
|
|
||||||
|
def win_uninstall_service(service_name):
|
||||||
|
manager = win_OpenSCManager()
|
||||||
|
try:
|
||||||
|
h = advapi32.OpenServiceW(manager, service_name, DELETE)
|
||||||
|
if not h:
|
||||||
|
raise OSError('Could not find service %s: %s' % (
|
||||||
|
service_name, ctypes.FormatError()))
|
||||||
|
|
||||||
|
try:
|
||||||
|
if not advapi32.DeleteService(h):
|
||||||
|
raise OSError('Deletion failed: %s' % ctypes.FormatError())
|
||||||
|
finally:
|
||||||
|
advapi32.CloseServiceHandle(h)
|
||||||
|
finally:
|
||||||
|
advapi32.CloseServiceHandle(manager)
|
||||||
|
|
||||||
|
|
||||||
|
def win_service_report_event(service_name, msg, is_error=True):
|
||||||
|
with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
|
||||||
|
f.write(msg + '\n')
|
||||||
|
|
||||||
|
event_log = advapi32.RegisterEventSourceW(None, service_name)
|
||||||
|
if not event_log:
|
||||||
|
raise OSError('Could not report event: %s' % ctypes.FormatError())
|
||||||
|
|
||||||
|
try:
|
||||||
|
type_id = 0x0001 if is_error else 0x0004
|
||||||
|
event_id = 0xc0000000 if is_error else 0x40000000
|
||||||
|
lines = _ctypes_array(LPTSTR, [msg])
|
||||||
|
|
||||||
|
if not advapi32.ReportEventW(
|
||||||
|
event_log, type_id, 0, event_id, None, len(lines), 0,
|
||||||
|
lines, None):
|
||||||
|
raise OSError('Event reporting failed: %s' % ctypes.FormatError())
|
||||||
|
finally:
|
||||||
|
advapi32.DeregisterEventSource(event_log)
|
||||||
|
|
||||||
|
|
||||||
|
def win_service_handler(stop_event, *args):
|
||||||
|
try:
|
||||||
|
raise ValueError('Handler called with args ' + repr(args))
|
||||||
|
TODO
|
||||||
|
except Exception as e:
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
msg = str(e) + '\n' + tb
|
||||||
|
win_service_report_event(service_name, msg, is_error=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def win_service_set_status(handle, status_code):
|
||||||
|
svcStatus = SERVICE_STATUS()
|
||||||
|
svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
|
||||||
|
svcStatus.dwCurrentState = status_code
|
||||||
|
svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
|
||||||
|
|
||||||
|
svcStatus.dwServiceSpecificExitCode = 0
|
||||||
|
|
||||||
|
if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
|
||||||
|
raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
|
||||||
|
|
||||||
|
|
||||||
|
def win_service_main(service_name, real_main, argc, argv_raw):
|
||||||
|
try:
|
||||||
|
#args = [argv_raw[i].value for i in range(argc)]
|
||||||
|
stop_event = threading.Event()
|
||||||
|
handler = HandlerEx(functools.partial(stop_event, win_service_handler))
|
||||||
|
h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
|
||||||
|
if not h:
|
||||||
|
raise OSError('Handler registration failed: %s' %
|
||||||
|
ctypes.FormatError())
|
||||||
|
|
||||||
|
TODO
|
||||||
|
except Exception as e:
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
msg = str(e) + '\n' + tb
|
||||||
|
win_service_report_event(service_name, msg, is_error=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def win_service_start(service_name, real_main):
|
||||||
|
try:
|
||||||
|
cb = START_CALLBACK(
|
||||||
|
functools.partial(win_service_main, service_name, real_main))
|
||||||
|
dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
|
||||||
|
SERVICE_TABLE_ENTRY(
|
||||||
|
service_name,
|
||||||
|
cb
|
||||||
|
),
|
||||||
|
SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
|
||||||
|
])
|
||||||
|
|
||||||
|
if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
|
||||||
|
raise OSError('ctypes start failed: %s' % ctypes.FormatError())
|
||||||
|
except Exception as e:
|
||||||
|
tb = traceback.format_exc()
|
||||||
|
msg = str(e) + '\n' + tb
|
||||||
|
win_service_report_event(service_name, msg, is_error=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(args=None):
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('-i', '--install',
|
||||||
|
action='store_const', dest='action', const='install',
|
||||||
|
help='Launch at Windows startup')
|
||||||
|
parser.add_argument('-u', '--uninstall',
|
||||||
|
action='store_const', dest='action', const='uninstall',
|
||||||
|
help='Remove Windows service')
|
||||||
|
parser.add_argument('-s', '--service',
|
||||||
|
action='store_const', dest='action', const='service',
|
||||||
|
help='Run as a Windows service')
|
||||||
|
parser.add_argument('-b', '--bind', metavar='<host:port>',
|
||||||
|
action='store', default='localhost:8142',
|
||||||
|
help='Bind to host:port (default %default)')
|
||||||
|
options = parser.parse_args(args=args)
|
||||||
|
|
||||||
|
if options.action == 'install':
|
||||||
|
fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
|
||||||
|
cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
|
||||||
|
win_install_service(SVCNAME, cmdline)
|
||||||
|
return
|
||||||
|
|
||||||
|
if options.action == 'uninstall':
|
||||||
|
win_uninstall_service(SVCNAME)
|
||||||
|
return
|
||||||
|
|
||||||
|
if options.action == 'service':
|
||||||
|
win_service_start(SVCNAME, main)
|
||||||
|
return
|
||||||
|
|
||||||
|
host, port_str = options.bind.split(':')
|
||||||
|
port = int(port_str)
|
||||||
|
|
||||||
|
print('Listening on %s:%d' % (host, port))
|
||||||
|
srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
|
||||||
|
thr = threading.Thread(target=srv.serve_forever)
|
||||||
|
thr.start()
|
||||||
|
input('Press ENTER to shut down')
|
||||||
|
srv.shutdown()
|
||||||
|
thr.join()
|
||||||
|
|
||||||
|
|
||||||
|
def rmtree(path):
|
||||||
|
for name in os.listdir(path):
|
||||||
|
fname = os.path.join(path, name)
|
||||||
|
if os.path.isdir(fname):
|
||||||
|
rmtree(fname)
|
||||||
|
else:
|
||||||
|
os.chmod(fname, 0o666)
|
||||||
|
os.remove(fname)
|
||||||
|
os.rmdir(path)
|
||||||
|
|
||||||
|
#==============================================================================
|
||||||
|
|
||||||
|
class BuildError(Exception):
|
||||||
|
def __init__(self, output, code=500):
|
||||||
|
self.output = output
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.output
|
||||||
|
|
||||||
|
|
||||||
|
class HTTPError(BuildError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class PythonBuilder(object):
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
pythonVersion = kwargs.pop('python', '2.7')
|
||||||
|
try:
|
||||||
|
key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % pythonVersion)
|
||||||
|
try:
|
||||||
|
self.pythonPath, _ = _winreg.QueryValueEx(key, '')
|
||||||
|
finally:
|
||||||
|
_winreg.CloseKey(key)
|
||||||
|
except Exception:
|
||||||
|
raise BuildError('No such Python version: %s' % pythonVersion)
|
||||||
|
|
||||||
|
super(PythonBuilder, self).__init__(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GITInfoBuilder(object):
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
try:
|
||||||
|
self.user, self.repoName = kwargs['path'][:2]
|
||||||
|
self.rev = kwargs.pop('rev')
|
||||||
|
except ValueError:
|
||||||
|
raise BuildError('Invalid path')
|
||||||
|
except KeyError as e:
|
||||||
|
raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
|
||||||
|
|
||||||
|
path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
|
||||||
|
if not os.path.exists(path):
|
||||||
|
os.makedirs(path)
|
||||||
|
self.basePath = tempfile.mkdtemp(dir=path)
|
||||||
|
self.buildPath = os.path.join(self.basePath, 'build')
|
||||||
|
|
||||||
|
super(GITInfoBuilder, self).__init__(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class GITBuilder(GITInfoBuilder):
|
||||||
|
def build(self):
|
||||||
|
try:
|
||||||
|
subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
|
||||||
|
subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
raise BuildError(e.output)
|
||||||
|
|
||||||
|
super(GITBuilder, self).build()
|
||||||
|
|
||||||
|
|
||||||
|
class YoutubeDLBuilder(object):
|
||||||
|
authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile']
|
||||||
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
if self.repoName != 'youtube-dl':
|
||||||
|
raise BuildError('Invalid repository "%s"' % self.repoName)
|
||||||
|
if self.user not in self.authorizedUsers:
|
||||||
|
raise HTTPError('Unauthorized user "%s"' % self.user, 401)
|
||||||
|
|
||||||
|
super(YoutubeDLBuilder, self).__init__(**kwargs)
|
||||||
|
|
||||||
|
def build(self):
|
||||||
|
try:
|
||||||
|
subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
|
||||||
|
cwd=self.buildPath)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
raise BuildError(e.output)
|
||||||
|
|
||||||
|
super(YoutubeDLBuilder, self).build()
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadBuilder(object):
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
self.handler = kwargs.pop('handler')
|
||||||
|
self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
|
||||||
|
self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
|
||||||
|
if not self.srcPath.startswith(self.buildPath):
|
||||||
|
raise HTTPError(self.srcPath, 401)
|
||||||
|
|
||||||
|
super(DownloadBuilder, self).__init__(**kwargs)
|
||||||
|
|
||||||
|
def build(self):
|
||||||
|
if not os.path.exists(self.srcPath):
|
||||||
|
raise HTTPError('No such file', 404)
|
||||||
|
if os.path.isdir(self.srcPath):
|
||||||
|
raise HTTPError('Is a directory: %s' % self.srcPath, 401)
|
||||||
|
|
||||||
|
self.handler.send_response(200)
|
||||||
|
self.handler.send_header('Content-Type', 'application/octet-stream')
|
||||||
|
self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
|
||||||
|
self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
|
||||||
|
self.handler.end_headers()
|
||||||
|
|
||||||
|
with open(self.srcPath, 'rb') as src:
|
||||||
|
shutil.copyfileobj(src, self.handler.wfile)
|
||||||
|
|
||||||
|
super(DownloadBuilder, self).build()
|
||||||
|
|
||||||
|
|
||||||
|
class CleanupTempDir(object):
|
||||||
|
def build(self):
|
||||||
|
try:
|
||||||
|
rmtree(self.basePath)
|
||||||
|
except Exception as e:
|
||||||
|
print('WARNING deleting "%s": %s' % (self.basePath, e))
|
||||||
|
|
||||||
|
super(CleanupTempDir, self).build()
|
||||||
|
|
||||||
|
|
||||||
|
class Null(object):
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def build(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
|
||||||
|
actionDict = { 'build': Builder, 'download': Builder } # They're the same, no more caching.
|
||||||
|
|
||||||
|
def do_GET(self):
|
||||||
|
path = urlparse.urlparse(self.path)
|
||||||
|
paramDict = dict([(key, value[0]) for key, value in urlparse.parse_qs(path.query).items()])
|
||||||
|
action, _, path = path.path.strip('/').partition('/')
|
||||||
|
if path:
|
||||||
|
path = path.split('/')
|
||||||
|
if action in self.actionDict:
|
||||||
|
try:
|
||||||
|
builder = self.actionDict[action](path=path, handler=self, **paramDict)
|
||||||
|
builder.start()
|
||||||
|
try:
|
||||||
|
builder.build()
|
||||||
|
finally:
|
||||||
|
builder.close()
|
||||||
|
except BuildError as e:
|
||||||
|
self.send_response(e.code)
|
||||||
|
msg = unicode(e).encode('UTF-8')
|
||||||
|
self.send_header('Content-Type', 'text/plain; charset=UTF-8')
|
||||||
|
self.send_header('Content-Length', len(msg))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(msg)
|
||||||
|
except HTTPError as e:
|
||||||
|
self.send_response(e.code, str(e))
|
||||||
|
else:
|
||||||
|
self.send_response(500, 'Unknown build method "%s"' % action)
|
||||||
|
else:
|
||||||
|
self.send_response(500, 'Malformed URL')
|
||||||
|
|
||||||
|
#==============================================================================
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
@ -3,7 +3,8 @@
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import hashlib
|
import hashlib
|
||||||
import urllib.request
|
import os.path
|
||||||
|
|
||||||
|
|
||||||
if len(sys.argv) <= 1:
|
if len(sys.argv) <= 1:
|
||||||
print('Specify the version number as parameter')
|
print('Specify the version number as parameter')
|
||||||
@ -25,6 +26,7 @@ filenames = {
|
|||||||
'tar': 'youtube-dl-%s.tar.gz' % version}
|
'tar': 'youtube-dl-%s.tar.gz' % version}
|
||||||
build_dir = os.path.join('..', '..', 'build', version)
|
build_dir = os.path.join('..', '..', 'build', version)
|
||||||
for key, filename in filenames.items():
|
for key, filename in filenames.items():
|
||||||
|
url = 'https://yt-dl.org/downloads/%s/%s' % (version, filename)
|
||||||
fn = os.path.join(build_dir, filename)
|
fn = os.path.join(build_dir, filename)
|
||||||
with open(fn, 'rb') as f:
|
with open(fn, 'rb') as f:
|
||||||
data = f.read()
|
data = f.read()
|
||||||
|
@ -14,7 +14,7 @@ def main():
|
|||||||
template = tmplf.read()
|
template = tmplf.read()
|
||||||
|
|
||||||
ie_htmls = []
|
ie_htmls = []
|
||||||
for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME):
|
for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()):
|
||||||
ie_html = '<b>{}</b>'.format(ie.IE_NAME)
|
ie_html = '<b>{}</b>'.format(ie.IE_NAME)
|
||||||
try:
|
try:
|
||||||
ie_html += ': {}'.format(ie.IE_DESC)
|
ie_html += ': {}'.format(ie.IE_DESC)
|
||||||
|
@ -55,8 +55,8 @@ git push origin "$version"
|
|||||||
/bin/echo -e "\n### OK, now it is time to build the binaries..."
|
/bin/echo -e "\n### OK, now it is time to build the binaries..."
|
||||||
REV=$(git rev-parse HEAD)
|
REV=$(git rev-parse HEAD)
|
||||||
make youtube-dl youtube-dl.tar.gz
|
make youtube-dl youtube-dl.tar.gz
|
||||||
wget "http://jeromelaheurte.net:8142/download/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe || \
|
read -p "VM running? (y/n) " -n 1
|
||||||
wget "http://jeromelaheurte.net:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
|
wget "http://localhost:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe
|
||||||
mkdir -p "build/$version"
|
mkdir -p "build/$version"
|
||||||
mv youtube-dl youtube-dl.exe "build/$version"
|
mv youtube-dl youtube-dl.exe "build/$version"
|
||||||
mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"
|
mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz"
|
||||||
|
@ -1,109 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Generate youtube signature algorithm from test cases
|
|
||||||
|
|
||||||
import sys
|
|
||||||
|
|
||||||
tests = [
|
|
||||||
# 92 - vflQw-fB4 2013/07/17
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`~\"",
|
|
||||||
"mrtyuioplkjhgfdsazxcvbnq1234567890QWERTY}IOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]\"|:;"),
|
|
||||||
# 90
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`",
|
|
||||||
"mrtyuioplkjhgfdsazxcvbne1234567890QWER[YUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={`]}|"),
|
|
||||||
# 89
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'",
|
|
||||||
"/?;:|}<[{=+-_)(*&^%$#@!MqBVCXZASDFGHJKLPOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuyt"),
|
|
||||||
# 88 - vflapUV9V 2013/08/28
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<",
|
|
||||||
"ioplkjhgfdsazxcvbnm12<4567890QWERTYUIOZLKJHGFDSAeXCVBNM!@#$%^&*()_-+={[]}|:;?/>.3"),
|
|
||||||
# 87
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$^&*()_-+={[]}|:;?/>.<",
|
|
||||||
"uioplkjhgfdsazxcvbnm1t34567890QWE2TYUIOPLKJHGFDSAZXCVeNM!@#$^&*()_-+={[]}|:;?/>.<"),
|
|
||||||
# 86 - vfluy6kdb 2013/09/06
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
|
|
||||||
"yuioplkjhgfdsazxcvbnm12345678q0QWrRTYUIOELKJHGFD-AZXCVBNM!@#$%^&*()_<+={[|};?/>.S"),
|
|
||||||
# 85
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?/>.<",
|
|
||||||
".>/?;}[{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWQ0q876543r1mnbvcx9asdfghjklpoiuyt2"),
|
|
||||||
# 84 - vflg0g8PQ 2013/08/29 (sporadic)
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?>.<",
|
|
||||||
">?;}[{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWq0987654321mnbvcxzasdfghjklpoiuytr"),
|
|
||||||
# 83
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!#$%^&*()_+={[};?/>.<",
|
|
||||||
".>/?;}[{=+_)(*&^%<#!MNBVCXZASPFGHJKLwOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuytreq"),
|
|
||||||
# 82 - vflZK4ZYR 2013/08/23
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.<",
|
|
||||||
"wertyuioplkjhgfdsaqxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&z(-+={[};?/>.<"),
|
|
||||||
# 81 - vflLC8JvQ 2013/07/25
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.",
|
|
||||||
"C>/?;}[{=+-(*&^%$#@!MNBVYXZASDFGHKLPOIU.TREWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
|
|
||||||
# 80 - vflZK4ZYR 2013/08/23 (sporadic)
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>",
|
|
||||||
"wertyuioplkjhgfdsaqxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&z(-+={[};?/>"),
|
|
||||||
# 79 - vflLC8JvQ 2013/07/25 (sporadic)
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/",
|
|
||||||
"Z?;}[{=+-(*&^%$#@!MNBVCXRASDFGHKLPOIUYT/EWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
|
|
||||||
]
|
|
||||||
|
|
||||||
tests_age_gate = [
|
|
||||||
# 86 - vflqinMWD
|
|
||||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
|
|
||||||
"ertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!/#$%^&*()_-+={[|};?@"),
|
|
||||||
]
|
|
||||||
|
|
||||||
def find_matching(wrong, right):
|
|
||||||
idxs = [wrong.index(c) for c in right]
|
|
||||||
return compress(idxs)
|
|
||||||
return ('s[%d]' % i for i in idxs)
|
|
||||||
|
|
||||||
def compress(idxs):
|
|
||||||
def _genslice(start, end, step):
|
|
||||||
starts = '' if start == 0 else str(start)
|
|
||||||
ends = ':%d' % (end+step)
|
|
||||||
steps = '' if step == 1 else (':%d' % step)
|
|
||||||
return 's[%s%s%s]' % (starts, ends, steps)
|
|
||||||
|
|
||||||
step = None
|
|
||||||
for i, prev in zip(idxs[1:], idxs[:-1]):
|
|
||||||
if step is not None:
|
|
||||||
if i - prev == step:
|
|
||||||
continue
|
|
||||||
yield _genslice(start, prev, step)
|
|
||||||
step = None
|
|
||||||
continue
|
|
||||||
if i - prev in [-1, 1]:
|
|
||||||
step = i - prev
|
|
||||||
start = prev
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
yield 's[%d]' % prev
|
|
||||||
if step is None:
|
|
||||||
yield 's[%d]' % i
|
|
||||||
else:
|
|
||||||
yield _genslice(start, i, step)
|
|
||||||
|
|
||||||
def _assert_compress(inp, exp):
|
|
||||||
res = list(compress(inp))
|
|
||||||
if res != exp:
|
|
||||||
print('Got %r, expected %r' % (res, exp))
|
|
||||||
assert res == exp
|
|
||||||
_assert_compress([0,2,4,6], ['s[0]', 's[2]', 's[4]', 's[6]'])
|
|
||||||
_assert_compress([0,1,2,4,6,7], ['s[:3]', 's[4]', 's[6:8]'])
|
|
||||||
_assert_compress([8,0,1,2,4,7,6,9], ['s[8]', 's[:3]', 's[4]', 's[7:5:-1]', 's[9]'])
|
|
||||||
|
|
||||||
def gen(wrong, right, indent):
|
|
||||||
code = ' + '.join(find_matching(wrong, right))
|
|
||||||
return 'if len(s) == %d:\n%s return %s\n' % (len(wrong), indent, code)
|
|
||||||
|
|
||||||
def genall(tests):
|
|
||||||
indent = ' ' * 8
|
|
||||||
return indent + (indent + 'el').join(gen(wrong, right, indent) for wrong,right in tests)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
print(genall(tests))
|
|
||||||
print(u' Age gate:')
|
|
||||||
print(genall(tests_age_gate))
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
@ -1,6 +1,9 @@
|
|||||||
|
import errno
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import os.path
|
import os.path
|
||||||
|
import re
|
||||||
|
import types
|
||||||
|
|
||||||
import youtube_dl.extractor
|
import youtube_dl.extractor
|
||||||
from youtube_dl import YoutubeDL, YoutubeDLHandler
|
from youtube_dl import YoutubeDL, YoutubeDLHandler
|
||||||
@ -20,19 +23,41 @@ PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "para
|
|||||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||||
parameters = json.load(pf)
|
parameters = json.load(pf)
|
||||||
|
|
||||||
|
|
||||||
|
def try_rm(filename):
|
||||||
|
""" Remove a file if it exists """
|
||||||
|
try:
|
||||||
|
os.remove(filename)
|
||||||
|
except OSError as ose:
|
||||||
|
if ose.errno != errno.ENOENT:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
class FakeYDL(YoutubeDL):
|
class FakeYDL(YoutubeDL):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.result = []
|
|
||||||
# Different instances of the downloader can't share the same dictionary
|
# Different instances of the downloader can't share the same dictionary
|
||||||
# some test set the "sublang" parameter, which would break the md5 checks.
|
# some test set the "sublang" parameter, which would break the md5 checks.
|
||||||
self.params = dict(parameters)
|
params = dict(parameters)
|
||||||
def to_screen(self, s):
|
super(FakeYDL, self).__init__(params)
|
||||||
|
self.result = []
|
||||||
|
|
||||||
|
def to_screen(self, s, skip_eol=None):
|
||||||
print(s)
|
print(s)
|
||||||
|
|
||||||
def trouble(self, s, tb=None):
|
def trouble(self, s, tb=None):
|
||||||
raise Exception(s)
|
raise Exception(s)
|
||||||
|
|
||||||
def download(self, x):
|
def download(self, x):
|
||||||
self.result.append(x)
|
self.result.append(x)
|
||||||
|
|
||||||
|
def expect_warning(self, regex):
|
||||||
|
# Silence an expected warning matching a regex
|
||||||
|
old_report_warning = self.report_warning
|
||||||
|
def report_warning(self, message):
|
||||||
|
if re.match(regex, message): return
|
||||||
|
old_report_warning(message)
|
||||||
|
self.report_warning = types.MethodType(report_warning, self)
|
||||||
|
|
||||||
def get_testcases():
|
def get_testcases():
|
||||||
for ie in youtube_dl.extractor.gen_extractors():
|
for ie in youtube_dl.extractor.gen_extractors():
|
||||||
t = getattr(ie, '_TEST', None)
|
t = getattr(ie, '_TEST', None)
|
||||||
|
@ -38,7 +38,6 @@
|
|||||||
"writedescription": false,
|
"writedescription": false,
|
||||||
"writeinfojson": true,
|
"writeinfojson": true,
|
||||||
"writesubtitles": false,
|
"writesubtitles": false,
|
||||||
"onlysubtitles": false,
|
|
||||||
"allsubtitles": false,
|
"allsubtitles": false,
|
||||||
"listssubtitles": false
|
"listssubtitles": false
|
||||||
}
|
}
|
||||||
|
@ -21,29 +21,41 @@ class TestAllURLsMatching(unittest.TestCase):
|
|||||||
self.assertEqual(self.matching_ies(url), ie_list)
|
self.assertEqual(self.matching_ies(url), ie_list)
|
||||||
|
|
||||||
def test_youtube_playlist_matching(self):
|
def test_youtube_playlist_matching(self):
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
|
assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist'])
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'UUBABnxM4Ar9ten8Mdjj1j0Q')) #585
|
assertPlaylist(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'PL63F0C78739B09958'))
|
assertPlaylist(u'UUBABnxM4Ar9ten8Mdjj1j0Q') #585
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q'))
|
assertPlaylist(u'PL63F0C78739B09958')
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8'))
|
assertPlaylist(u'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC'))
|
assertPlaylist(u'https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
|
||||||
self.assertTrue(YoutubePlaylistIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
|
assertPlaylist(u'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
|
||||||
self.assertFalse(YoutubePlaylistIE.suitable(u'PLtS2H6bU1M'))
|
assertPlaylist(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012') #668
|
||||||
|
self.assertFalse('youtube:playlist' in self.matching_ies(u'PLtS2H6bU1M'))
|
||||||
|
|
||||||
def test_youtube_matching(self):
|
def test_youtube_matching(self):
|
||||||
self.assertTrue(YoutubeIE.suitable(u'PLtS2H6bU1M'))
|
self.assertTrue(YoutubeIE.suitable(u'PLtS2H6bU1M'))
|
||||||
self.assertFalse(YoutubeIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
|
self.assertFalse(YoutubeIE.suitable(u'https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
|
||||||
self.assertMatch('http://youtu.be/BaW_jenozKc', ['youtube'])
|
self.assertMatch('http://youtu.be/BaW_jenozKc', ['youtube'])
|
||||||
self.assertMatch('http://www.youtube.com/v/BaW_jenozKc', ['youtube'])
|
self.assertMatch('http://www.youtube.com/v/BaW_jenozKc', ['youtube'])
|
||||||
|
self.assertMatch('https://youtube.googleapis.com/v/BaW_jenozKc', ['youtube'])
|
||||||
|
|
||||||
def test_youtube_channel_matching(self):
|
def test_youtube_channel_matching(self):
|
||||||
self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM'))
|
assertChannel = lambda url: self.assertMatch(url, ['youtube:channel'])
|
||||||
self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec'))
|
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM')
|
||||||
self.assertTrue(YoutubeChannelIE.suitable('https://www.youtube.com/channel/HCtnHdj3df7iM/videos'))
|
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM?feature=gb_ch_rec')
|
||||||
|
assertChannel('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')
|
||||||
|
|
||||||
def test_youtube_user_matching(self):
|
def test_youtube_user_matching(self):
|
||||||
self.assertMatch('www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
|
self.assertMatch('www.youtube.com/NASAgovVideo/videos', ['youtube:user'])
|
||||||
|
|
||||||
|
def test_youtube_feeds(self):
|
||||||
|
self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:watch_later'])
|
||||||
|
self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:subscriptions'])
|
||||||
|
self.assertMatch('https://www.youtube.com/feed/recommended', ['youtube:recommended'])
|
||||||
|
self.assertMatch('https://www.youtube.com/my_favorites', ['youtube:favorites'])
|
||||||
|
|
||||||
|
def test_youtube_show_matching(self):
|
||||||
|
self.assertMatch('http://www.youtube.com/show/airdisasters', ['youtube:show'])
|
||||||
|
|
||||||
def test_justin_tv_channelid_matching(self):
|
def test_justin_tv_channelid_matching(self):
|
||||||
self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
|
self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
|
||||||
self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))
|
self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))
|
||||||
@ -61,10 +73,13 @@ class TestAllURLsMatching(unittest.TestCase):
|
|||||||
self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/tsm_theoddone/c/2349361"))
|
self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/tsm_theoddone/c/2349361"))
|
||||||
|
|
||||||
def test_youtube_extract(self):
|
def test_youtube_extract(self):
|
||||||
self.assertEqual(YoutubeIE()._extract_id('http://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
|
assertExtractId = lambda url, id: self.assertEqual(YoutubeIE()._extract_id(url), id)
|
||||||
self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?&v=BaW_jenozKc'), 'BaW_jenozKc')
|
assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc'), 'BaW_jenozKc')
|
assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
self.assertEqual(YoutubeIE()._extract_id('https://www.youtube.com/watch_popup?v=BaW_jenozKc'), 'BaW_jenozKc')
|
assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc')
|
||||||
|
assertExtractId('BaW_jenozKc', 'BaW_jenozKc')
|
||||||
|
|
||||||
def test_no_duplicates(self):
|
def test_no_duplicates(self):
|
||||||
ies = gen_extractors()
|
ies = gen_extractors()
|
||||||
|
72
test/test_dailymotion_subtitles.py
Normal file
72
test/test_dailymotion_subtitles.py
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from youtube_dl.extractor import DailymotionIE
|
||||||
|
from youtube_dl.utils import *
|
||||||
|
from helper import FakeYDL
|
||||||
|
|
||||||
|
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||||
|
|
||||||
|
class TestDailymotionSubtitles(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.DL = FakeYDL()
|
||||||
|
self.url = 'http://www.dailymotion.com/video/xczg00'
|
||||||
|
def getInfoDict(self):
|
||||||
|
IE = DailymotionIE(self.DL)
|
||||||
|
info_dict = IE.extract(self.url)
|
||||||
|
return info_dict
|
||||||
|
def getSubtitles(self):
|
||||||
|
info_dict = self.getInfoDict()
|
||||||
|
return info_dict[0]['subtitles']
|
||||||
|
def test_no_writesubtitles(self):
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(subtitles, None)
|
||||||
|
def test_subtitles(self):
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
|
||||||
|
def test_subtitles_lang(self):
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
self.DL.params['subtitleslangs'] = ['fr']
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
|
||||||
|
def test_allsubtitles(self):
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
self.DL.params['allsubtitles'] = True
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(len(subtitles.keys()), 5)
|
||||||
|
def test_list_subtitles(self):
|
||||||
|
self.DL.expect_warning(u'Automatic Captions not supported by this server')
|
||||||
|
self.DL.params['listsubtitles'] = True
|
||||||
|
info_dict = self.getInfoDict()
|
||||||
|
self.assertEqual(info_dict, None)
|
||||||
|
def test_automatic_captions(self):
|
||||||
|
self.DL.expect_warning(u'Automatic Captions not supported by this server')
|
||||||
|
self.DL.params['writeautomaticsub'] = True
|
||||||
|
self.DL.params['subtitleslang'] = ['en']
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertTrue(len(subtitles.keys()) == 0)
|
||||||
|
def test_nosubtitles(self):
|
||||||
|
self.DL.expect_warning(u'video doesn\'t have subtitles')
|
||||||
|
self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
self.DL.params['allsubtitles'] = True
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(len(subtitles), 0)
|
||||||
|
def test_multiple_langs(self):
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
langs = ['es', 'fr', 'de']
|
||||||
|
self.DL.params['subtitleslangs'] = langs
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
for lang in langs:
|
||||||
|
self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
@ -1,6 +1,5 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
import errno
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
@ -28,14 +27,6 @@ opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, You
|
|||||||
compat_urllib_request.install_opener(opener)
|
compat_urllib_request.install_opener(opener)
|
||||||
socket.setdefaulttimeout(10)
|
socket.setdefaulttimeout(10)
|
||||||
|
|
||||||
def _try_rm(filename):
|
|
||||||
""" Remove a file if it exists """
|
|
||||||
try:
|
|
||||||
os.remove(filename)
|
|
||||||
except OSError as ose:
|
|
||||||
if ose.errno != errno.ENOENT:
|
|
||||||
raise
|
|
||||||
|
|
||||||
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||||
|
|
||||||
class YoutubeDL(youtube_dl.YoutubeDL):
|
class YoutubeDL(youtube_dl.YoutubeDL):
|
||||||
@ -54,7 +45,7 @@ def _file_md5(fn):
|
|||||||
with open(fn, 'rb') as f:
|
with open(fn, 'rb') as f:
|
||||||
return hashlib.md5(f.read()).hexdigest()
|
return hashlib.md5(f.read()).hexdigest()
|
||||||
|
|
||||||
from helper import get_testcases
|
from helper import get_testcases, try_rm
|
||||||
defs = get_testcases()
|
defs = get_testcases()
|
||||||
|
|
||||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||||
@ -97,9 +88,9 @@ def generator(test_case):
|
|||||||
|
|
||||||
test_cases = test_case.get('playlist', [test_case])
|
test_cases = test_case.get('playlist', [test_case])
|
||||||
for tc in test_cases:
|
for tc in test_cases:
|
||||||
_try_rm(tc['file'])
|
try_rm(tc['file'])
|
||||||
_try_rm(tc['file'] + '.part')
|
try_rm(tc['file'] + '.part')
|
||||||
_try_rm(tc['file'] + '.info.json')
|
try_rm(tc['file'] + '.info.json')
|
||||||
try:
|
try:
|
||||||
for retry in range(1, RETRIES + 1):
|
for retry in range(1, RETRIES + 1):
|
||||||
try:
|
try:
|
||||||
@ -145,9 +136,9 @@ def generator(test_case):
|
|||||||
self.assertTrue(key in info_dict.keys() and info_dict[key])
|
self.assertTrue(key in info_dict.keys() and info_dict[key])
|
||||||
finally:
|
finally:
|
||||||
for tc in test_cases:
|
for tc in test_cases:
|
||||||
_try_rm(tc['file'])
|
try_rm(tc['file'])
|
||||||
_try_rm(tc['file'] + '.part')
|
try_rm(tc['file'] + '.part')
|
||||||
_try_rm(tc['file'] + '.info.json')
|
try_rm(tc['file'] + '.info.json')
|
||||||
|
|
||||||
return test_template
|
return test_template
|
||||||
|
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
# encoding: utf-8
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
@ -8,7 +9,14 @@ import json
|
|||||||
import os
|
import os
|
||||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
from youtube_dl.extractor import DailymotionPlaylistIE, VimeoChannelIE
|
from youtube_dl.extractor import (
|
||||||
|
DailymotionPlaylistIE,
|
||||||
|
DailymotionUserIE,
|
||||||
|
VimeoChannelIE,
|
||||||
|
UstreamChannelIE,
|
||||||
|
SoundcloudUserIE,
|
||||||
|
LivestreamIE,
|
||||||
|
)
|
||||||
from youtube_dl.utils import *
|
from youtube_dl.utils import *
|
||||||
|
|
||||||
from helper import FakeYDL
|
from helper import FakeYDL
|
||||||
@ -26,6 +34,14 @@ class TestPlaylists(unittest.TestCase):
|
|||||||
self.assertEqual(result['title'], u'SPORT')
|
self.assertEqual(result['title'], u'SPORT')
|
||||||
self.assertTrue(len(result['entries']) > 20)
|
self.assertTrue(len(result['entries']) > 20)
|
||||||
|
|
||||||
|
def test_dailymotion_user(self):
|
||||||
|
dl = FakeYDL()
|
||||||
|
ie = DailymotionUserIE(dl)
|
||||||
|
result = ie.extract('http://www.dailymotion.com/user/generation-quoi/')
|
||||||
|
self.assertIsPlaylist(result)
|
||||||
|
self.assertEqual(result['title'], u'Génération Quoi')
|
||||||
|
self.assertTrue(len(result['entries']) >= 26)
|
||||||
|
|
||||||
def test_vimeo_channel(self):
|
def test_vimeo_channel(self):
|
||||||
dl = FakeYDL()
|
dl = FakeYDL()
|
||||||
ie = VimeoChannelIE(dl)
|
ie = VimeoChannelIE(dl)
|
||||||
@ -34,5 +50,29 @@ class TestPlaylists(unittest.TestCase):
|
|||||||
self.assertEqual(result['title'], u'Vimeo Tributes')
|
self.assertEqual(result['title'], u'Vimeo Tributes')
|
||||||
self.assertTrue(len(result['entries']) > 24)
|
self.assertTrue(len(result['entries']) > 24)
|
||||||
|
|
||||||
|
def test_ustream_channel(self):
|
||||||
|
dl = FakeYDL()
|
||||||
|
ie = UstreamChannelIE(dl)
|
||||||
|
result = ie.extract('http://www.ustream.tv/channel/young-americans-for-liberty')
|
||||||
|
self.assertIsPlaylist(result)
|
||||||
|
self.assertEqual(result['id'], u'5124905')
|
||||||
|
self.assertTrue(len(result['entries']) >= 11)
|
||||||
|
|
||||||
|
def test_soundcloud_user(self):
|
||||||
|
dl = FakeYDL()
|
||||||
|
ie = SoundcloudUserIE(dl)
|
||||||
|
result = ie.extract('https://soundcloud.com/the-concept-band')
|
||||||
|
self.assertIsPlaylist(result)
|
||||||
|
self.assertEqual(result['id'], u'9615865')
|
||||||
|
self.assertTrue(len(result['entries']) >= 12)
|
||||||
|
|
||||||
|
def test_livestream_event(self):
|
||||||
|
dl = FakeYDL()
|
||||||
|
ie = LivestreamIE(dl)
|
||||||
|
result = ie.extract('http://new.livestream.com/tedx/cityenglish')
|
||||||
|
self.assertIsPlaylist(result)
|
||||||
|
self.assertEqual(result['title'], u'TEDCity2.0 (English)')
|
||||||
|
self.assertTrue(len(result['entries']) >= 4)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
@ -11,13 +11,16 @@ import os
|
|||||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
#from youtube_dl.utils import htmlentity_transform
|
#from youtube_dl.utils import htmlentity_transform
|
||||||
from youtube_dl.utils import timeconvert
|
from youtube_dl.utils import (
|
||||||
from youtube_dl.utils import sanitize_filename
|
timeconvert,
|
||||||
from youtube_dl.utils import unescapeHTML
|
sanitize_filename,
|
||||||
from youtube_dl.utils import orderedSet
|
unescapeHTML,
|
||||||
from youtube_dl.utils import DateRange
|
orderedSet,
|
||||||
from youtube_dl.utils import unified_strdate
|
DateRange,
|
||||||
from youtube_dl.utils import find_xpath_attr
|
unified_strdate,
|
||||||
|
find_xpath_attr,
|
||||||
|
get_meta_content,
|
||||||
|
)
|
||||||
|
|
||||||
if sys.version_info < (3, 0):
|
if sys.version_info < (3, 0):
|
||||||
_compat_str = lambda b: b.decode('unicode-escape')
|
_compat_str = lambda b: b.decode('unicode-escape')
|
||||||
@ -127,5 +130,16 @@ class TestUtil(unittest.TestCase):
|
|||||||
self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
|
self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
|
||||||
self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
|
self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
|
||||||
|
|
||||||
|
def test_meta_parser(self):
|
||||||
|
testhtml = u'''
|
||||||
|
<head>
|
||||||
|
<meta name="description" content="foo & bar">
|
||||||
|
<meta content='Plato' name='author'/>
|
||||||
|
</head>
|
||||||
|
'''
|
||||||
|
get_meta = lambda name: get_meta_content(name, testhtml)
|
||||||
|
self.assertEqual(get_meta('description'), u'foo & bar')
|
||||||
|
self.assertEqual(get_meta('author'), 'Plato')
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
@ -27,6 +27,14 @@ class TestYoutubeLists(unittest.TestCase):
|
|||||||
ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
|
ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
|
||||||
self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
|
self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
|
||||||
|
|
||||||
|
def test_youtube_playlist_noplaylist(self):
|
||||||
|
dl = FakeYDL()
|
||||||
|
dl.params['noplaylist'] = True
|
||||||
|
ie = YoutubePlaylistIE(dl)
|
||||||
|
result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
|
||||||
|
self.assertEqual(result['_type'], 'url')
|
||||||
|
self.assertEqual(YoutubeIE()._extract_id(result['url']), 'FXxLjLQi3Fg')
|
||||||
|
|
||||||
def test_issue_673(self):
|
def test_issue_673(self):
|
||||||
dl = FakeYDL()
|
dl = FakeYDL()
|
||||||
ie = YoutubePlaylistIE(dl)
|
ie = YoutubePlaylistIE(dl)
|
||||||
|
80
test/test_youtube_signature.py
Normal file
80
test/test_youtube_signature.py
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import io
|
||||||
|
import re
|
||||||
|
import string
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
# Allow direct execution
|
||||||
|
import os
|
||||||
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from youtube_dl.extractor import YoutubeIE
|
||||||
|
from youtube_dl.utils import compat_str, compat_urlretrieve
|
||||||
|
|
||||||
|
_TESTS = [
|
||||||
|
(
|
||||||
|
u'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
|
||||||
|
u'js',
|
||||||
|
86,
|
||||||
|
u'>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
u'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
|
||||||
|
u'js',
|
||||||
|
85,
|
||||||
|
u'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
|
||||||
|
),
|
||||||
|
(
|
||||||
|
u'https://s.ytimg.com/yts/swfbin/watch_as3-vflg5GhxU.swf',
|
||||||
|
u'swf',
|
||||||
|
82,
|
||||||
|
u':/.-,+*)=\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBAzyxw>utsrqponmlkjihgfedcba987654321'
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestSignature(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
|
||||||
|
if not os.path.exists(self.TESTDATA_DIR):
|
||||||
|
os.mkdir(self.TESTDATA_DIR)
|
||||||
|
|
||||||
|
|
||||||
|
def make_tfunc(url, stype, sig_length, expected_sig):
|
||||||
|
basename = url.rpartition('/')[2]
|
||||||
|
m = re.match(r'.*-([a-zA-Z0-9_-]+)\.[a-z]+$', basename)
|
||||||
|
assert m, '%r should follow URL format' % basename
|
||||||
|
test_id = m.group(1)
|
||||||
|
|
||||||
|
def test_func(self):
|
||||||
|
fn = os.path.join(self.TESTDATA_DIR, basename)
|
||||||
|
|
||||||
|
if not os.path.exists(fn):
|
||||||
|
compat_urlretrieve(url, fn)
|
||||||
|
|
||||||
|
ie = YoutubeIE()
|
||||||
|
if stype == 'js':
|
||||||
|
with io.open(fn, encoding='utf-8') as testf:
|
||||||
|
jscode = testf.read()
|
||||||
|
func = ie._parse_sig_js(jscode)
|
||||||
|
else:
|
||||||
|
assert stype == 'swf'
|
||||||
|
with open(fn, 'rb') as testf:
|
||||||
|
swfcode = testf.read()
|
||||||
|
func = ie._parse_sig_swf(swfcode)
|
||||||
|
src_sig = compat_str(string.printable[:sig_length])
|
||||||
|
got_sig = func(src_sig)
|
||||||
|
self.assertEqual(got_sig, expected_sig)
|
||||||
|
|
||||||
|
test_func.__name__ = str('test_signature_' + stype + '_' + test_id)
|
||||||
|
setattr(TestSignature, test_func.__name__, test_func)
|
||||||
|
|
||||||
|
for test_spec in _TESTS:
|
||||||
|
make_tfunc(*test_spec)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
import json
|
|
||||||
import io
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
||||||
# Allow direct execution
|
# Allow direct execution
|
||||||
@ -18,85 +16,67 @@ md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
|||||||
|
|
||||||
class TestYoutubeSubtitles(unittest.TestCase):
|
class TestYoutubeSubtitles(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
DL = FakeYDL()
|
self.DL = FakeYDL()
|
||||||
DL.params['allsubtitles'] = False
|
self.url = 'QRS8MkLhQmM'
|
||||||
DL.params['writesubtitles'] = False
|
def getInfoDict(self):
|
||||||
DL.params['subtitlesformat'] = 'srt'
|
IE = YoutubeIE(self.DL)
|
||||||
DL.params['listsubtitles'] = False
|
info_dict = IE.extract(self.url)
|
||||||
def test_youtube_no_subtitles(self):
|
return info_dict
|
||||||
DL = FakeYDL()
|
def getSubtitles(self):
|
||||||
DL.params['writesubtitles'] = False
|
info_dict = self.getInfoDict()
|
||||||
IE = YoutubeIE(DL)
|
return info_dict[0]['subtitles']
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
def test_youtube_no_writesubtitles(self):
|
||||||
subtitles = info_dict[0]['subtitles']
|
self.DL.params['writesubtitles'] = False
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
self.assertEqual(subtitles, None)
|
self.assertEqual(subtitles, None)
|
||||||
def test_youtube_subtitles(self):
|
def test_youtube_subtitles(self):
|
||||||
DL = FakeYDL()
|
self.DL.params['writesubtitles'] = True
|
||||||
DL.params['writesubtitles'] = True
|
subtitles = self.getSubtitles()
|
||||||
IE = YoutubeIE(DL)
|
self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
def test_youtube_subtitles_lang(self):
|
||||||
sub = info_dict[0]['subtitles']['en']
|
self.DL.params['writesubtitles'] = True
|
||||||
self.assertEqual(md5(sub), '4cd9278a35ba2305f47354ee13472260')
|
self.DL.params['subtitleslangs'] = ['it']
|
||||||
def test_youtube_subtitles_it(self):
|
subtitles = self.getSubtitles()
|
||||||
DL = FakeYDL()
|
self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
|
||||||
DL.params['writesubtitles'] = True
|
|
||||||
DL.params['subtitleslangs'] = ['it']
|
|
||||||
IE = YoutubeIE(DL)
|
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
|
||||||
sub = info_dict[0]['subtitles']['it']
|
|
||||||
self.assertEqual(md5(sub), '164a51f16f260476a05b50fe4c2f161d')
|
|
||||||
def test_youtube_onlysubtitles(self):
|
|
||||||
DL = FakeYDL()
|
|
||||||
DL.params['writesubtitles'] = True
|
|
||||||
DL.params['onlysubtitles'] = True
|
|
||||||
IE = YoutubeIE(DL)
|
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
|
||||||
sub = info_dict[0]['subtitles']['en']
|
|
||||||
self.assertEqual(md5(sub), '4cd9278a35ba2305f47354ee13472260')
|
|
||||||
def test_youtube_allsubtitles(self):
|
def test_youtube_allsubtitles(self):
|
||||||
DL = FakeYDL()
|
self.DL.params['writesubtitles'] = True
|
||||||
DL.params['allsubtitles'] = True
|
self.DL.params['allsubtitles'] = True
|
||||||
IE = YoutubeIE(DL)
|
subtitles = self.getSubtitles()
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
|
||||||
subtitles = info_dict[0]['subtitles']
|
|
||||||
self.assertEqual(len(subtitles.keys()), 13)
|
self.assertEqual(len(subtitles.keys()), 13)
|
||||||
def test_youtube_subtitles_sbv_format(self):
|
def test_youtube_subtitles_sbv_format(self):
|
||||||
DL = FakeYDL()
|
self.DL.params['writesubtitles'] = True
|
||||||
DL.params['writesubtitles'] = True
|
self.DL.params['subtitlesformat'] = 'sbv'
|
||||||
DL.params['subtitlesformat'] = 'sbv'
|
subtitles = self.getSubtitles()
|
||||||
IE = YoutubeIE(DL)
|
self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
|
||||||
sub = info_dict[0]['subtitles']['en']
|
|
||||||
self.assertEqual(md5(sub), '13aeaa0c245a8bed9a451cb643e3ad8b')
|
|
||||||
def test_youtube_subtitles_vtt_format(self):
|
def test_youtube_subtitles_vtt_format(self):
|
||||||
DL = FakeYDL()
|
self.DL.params['writesubtitles'] = True
|
||||||
DL.params['writesubtitles'] = True
|
self.DL.params['subtitlesformat'] = 'vtt'
|
||||||
DL.params['subtitlesformat'] = 'vtt'
|
subtitles = self.getSubtitles()
|
||||||
IE = YoutubeIE(DL)
|
self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
|
||||||
sub = info_dict[0]['subtitles']['en']
|
|
||||||
self.assertEqual(md5(sub), '356cdc577fde0c6783b9b822e7206ff7')
|
|
||||||
def test_youtube_list_subtitles(self):
|
def test_youtube_list_subtitles(self):
|
||||||
DL = FakeYDL()
|
self.DL.expect_warning(u'Video doesn\'t have automatic captions')
|
||||||
DL.params['listsubtitles'] = True
|
self.DL.params['listsubtitles'] = True
|
||||||
IE = YoutubeIE(DL)
|
info_dict = self.getInfoDict()
|
||||||
info_dict = IE.extract('QRS8MkLhQmM')
|
|
||||||
self.assertEqual(info_dict, None)
|
self.assertEqual(info_dict, None)
|
||||||
def test_youtube_automatic_captions(self):
|
def test_youtube_automatic_captions(self):
|
||||||
DL = FakeYDL()
|
self.url = '8YoUxe5ncPo'
|
||||||
DL.params['writeautomaticsub'] = True
|
self.DL.params['writeautomaticsub'] = True
|
||||||
DL.params['subtitleslangs'] = ['it']
|
self.DL.params['subtitleslangs'] = ['it']
|
||||||
IE = YoutubeIE(DL)
|
subtitles = self.getSubtitles()
|
||||||
info_dict = IE.extract('8YoUxe5ncPo')
|
self.assertTrue(subtitles['it'] is not None)
|
||||||
sub = info_dict[0]['subtitles']['it']
|
def test_youtube_nosubtitles(self):
|
||||||
self.assertTrue(sub is not None)
|
self.DL.expect_warning(u'video doesn\'t have subtitles')
|
||||||
|
self.url = 'sAjKT8FhjI8'
|
||||||
|
self.DL.params['writesubtitles'] = True
|
||||||
|
self.DL.params['allsubtitles'] = True
|
||||||
|
subtitles = self.getSubtitles()
|
||||||
|
self.assertEqual(len(subtitles), 0)
|
||||||
def test_youtube_multiple_langs(self):
|
def test_youtube_multiple_langs(self):
|
||||||
DL = FakeYDL()
|
self.url = 'QRS8MkLhQmM'
|
||||||
DL.params['writesubtitles'] = True
|
self.DL.params['writesubtitles'] = True
|
||||||
langs = ['it', 'fr', 'de']
|
langs = ['it', 'fr', 'de']
|
||||||
DL.params['subtitleslangs'] = langs
|
self.DL.params['subtitleslangs'] = langs
|
||||||
IE = YoutubeIE(DL)
|
subtitles = self.getSubtitles()
|
||||||
subtitles = IE.extract('QRS8MkLhQmM')[0]['subtitles']
|
|
||||||
for lang in langs:
|
for lang in langs:
|
||||||
self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
|
self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
|
||||||
|
|
||||||
|
5
tox.ini
Normal file
5
tox.ini
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
[tox]
|
||||||
|
envlist = py26,py27,py33
|
||||||
|
[testenv]
|
||||||
|
deps = nose
|
||||||
|
commands = nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose test
|
@ -66,7 +66,7 @@ class FileDownloader(object):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def format_seconds(seconds):
|
def format_seconds(seconds):
|
||||||
(mins, secs) = divmod(seconds, 60)
|
(mins, secs) = divmod(seconds, 60)
|
||||||
(hours, eta_mins) = divmod(mins, 60)
|
(hours, mins) = divmod(mins, 60)
|
||||||
if hours > 99:
|
if hours > 99:
|
||||||
return '--:--:--'
|
return '--:--:--'
|
||||||
if hours == 0:
|
if hours == 0:
|
||||||
@ -77,26 +77,43 @@ class FileDownloader(object):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def calc_percent(byte_counter, data_len):
|
def calc_percent(byte_counter, data_len):
|
||||||
if data_len is None:
|
if data_len is None:
|
||||||
|
return None
|
||||||
|
return float(byte_counter) / float(data_len) * 100.0
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def format_percent(percent):
|
||||||
|
if percent is None:
|
||||||
return '---.-%'
|
return '---.-%'
|
||||||
return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
|
return '%6s' % ('%3.1f%%' % percent)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def calc_eta(start, now, total, current):
|
def calc_eta(start, now, total, current):
|
||||||
if total is None:
|
if total is None:
|
||||||
return '--:--'
|
return None
|
||||||
dif = now - start
|
dif = now - start
|
||||||
if current == 0 or dif < 0.001: # One millisecond
|
if current == 0 or dif < 0.001: # One millisecond
|
||||||
return '--:--'
|
return None
|
||||||
rate = float(current) / dif
|
rate = float(current) / dif
|
||||||
eta = int((float(total) - float(current)) / rate)
|
return int((float(total) - float(current)) / rate)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def format_eta(eta):
|
||||||
|
if eta is None:
|
||||||
|
return '--:--'
|
||||||
return FileDownloader.format_seconds(eta)
|
return FileDownloader.format_seconds(eta)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def calc_speed(start, now, bytes):
|
def calc_speed(start, now, bytes):
|
||||||
dif = now - start
|
dif = now - start
|
||||||
if bytes == 0 or dif < 0.001: # One millisecond
|
if bytes == 0 or dif < 0.001: # One millisecond
|
||||||
|
return None
|
||||||
|
return float(bytes) / dif
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def format_speed(speed):
|
||||||
|
if speed is None:
|
||||||
return '%10s' % '---b/s'
|
return '%10s' % '---b/s'
|
||||||
return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
|
return '%10s' % ('%s/s' % FileDownloader.format_bytes(speed))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def best_block_size(elapsed_time, bytes):
|
def best_block_size(elapsed_time, bytes):
|
||||||
@ -205,11 +222,14 @@ class FileDownloader(object):
|
|||||||
"""Report destination filename."""
|
"""Report destination filename."""
|
||||||
self.to_screen(u'[download] Destination: ' + filename)
|
self.to_screen(u'[download] Destination: ' + filename)
|
||||||
|
|
||||||
def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
|
def report_progress(self, percent, data_len_str, speed, eta):
|
||||||
"""Report download progress."""
|
"""Report download progress."""
|
||||||
if self.params.get('noprogress', False):
|
if self.params.get('noprogress', False):
|
||||||
return
|
return
|
||||||
clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
|
clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
|
||||||
|
eta_str = self.format_eta(eta)
|
||||||
|
percent_str = self.format_percent(percent)
|
||||||
|
speed_str = self.format_speed(speed)
|
||||||
if self.params.get('progress_with_newline', False):
|
if self.params.get('progress_with_newline', False):
|
||||||
self.to_screen(u'[download] %s of %s at %s ETA %s' %
|
self.to_screen(u'[download] %s of %s at %s ETA %s' %
|
||||||
(percent_str, data_len_str, speed_str, eta_str))
|
(percent_str, data_len_str, speed_str, eta_str))
|
||||||
@ -378,6 +398,7 @@ class FileDownloader(object):
|
|||||||
self._hook_progress({
|
self._hook_progress({
|
||||||
'filename': filename,
|
'filename': filename,
|
||||||
'status': 'finished',
|
'status': 'finished',
|
||||||
|
'total_bytes': os.path.getsize(encodeFilename(filename)),
|
||||||
})
|
})
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@ -524,13 +545,14 @@ class FileDownloader(object):
|
|||||||
block_size = self.best_block_size(after - before, len(data_block))
|
block_size = self.best_block_size(after - before, len(data_block))
|
||||||
|
|
||||||
# Progress message
|
# Progress message
|
||||||
speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
|
speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
|
||||||
if data_len is None:
|
if data_len is None:
|
||||||
self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
|
self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
|
||||||
|
eta = None
|
||||||
else:
|
else:
|
||||||
percent_str = self.calc_percent(byte_counter, data_len)
|
percent = self.calc_percent(byte_counter, data_len)
|
||||||
eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
|
eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
|
||||||
self.report_progress(percent_str, data_len_str, speed_str, eta_str)
|
self.report_progress(percent, data_len_str, speed, eta)
|
||||||
|
|
||||||
self._hook_progress({
|
self._hook_progress({
|
||||||
'downloaded_bytes': byte_counter,
|
'downloaded_bytes': byte_counter,
|
||||||
@ -538,6 +560,8 @@ class FileDownloader(object):
|
|||||||
'tmpfilename': tmpfilename,
|
'tmpfilename': tmpfilename,
|
||||||
'filename': filename,
|
'filename': filename,
|
||||||
'status': 'downloading',
|
'status': 'downloading',
|
||||||
|
'eta': eta,
|
||||||
|
'speed': speed,
|
||||||
})
|
})
|
||||||
|
|
||||||
# Apply rate limit
|
# Apply rate limit
|
||||||
@ -580,6 +604,8 @@ class FileDownloader(object):
|
|||||||
* downloaded_bytes: Bytes on disks
|
* downloaded_bytes: Bytes on disks
|
||||||
* total_bytes: Total bytes, None if unknown
|
* total_bytes: Total bytes, None if unknown
|
||||||
* tmpfilename: The filename we're currently writing to
|
* tmpfilename: The filename we're currently writing to
|
||||||
|
* eta: The estimated time in seconds, None if unknown
|
||||||
|
* speed: The download speed in bytes/second, None if unknown
|
||||||
|
|
||||||
Hooks are guaranteed to be called at least once (with status "finished")
|
Hooks are guaranteed to be called at least once (with status "finished")
|
||||||
if the download is successful.
|
if the download is successful.
|
||||||
|
@ -444,8 +444,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
|
|||||||
if information['ext'] != u'mp4':
|
if information['ext'] != u'mp4':
|
||||||
self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
|
self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
|
||||||
return True, information
|
return True, information
|
||||||
sub_langs = [key for key in information['subtitles']]
|
if not information.get('subtitles'):
|
||||||
|
self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed')
|
||||||
|
return True, information
|
||||||
|
|
||||||
|
sub_langs = [key for key in information['subtitles']]
|
||||||
filename = information['filepath']
|
filename = information['filepath']
|
||||||
input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
|
input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
|
||||||
|
|
||||||
|
@ -74,12 +74,16 @@ class YoutubeDL(object):
|
|||||||
writesubtitles: Write the video subtitles to a file
|
writesubtitles: Write the video subtitles to a file
|
||||||
writeautomaticsub: Write the automatic subtitles to a file
|
writeautomaticsub: Write the automatic subtitles to a file
|
||||||
allsubtitles: Downloads all the subtitles of the video
|
allsubtitles: Downloads all the subtitles of the video
|
||||||
|
(requires writesubtitles or writeautomaticsub)
|
||||||
listsubtitles: Lists all available subtitles for the video
|
listsubtitles: Lists all available subtitles for the video
|
||||||
subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
|
subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
|
||||||
subtitleslangs: List of languages of the subtitles to download
|
subtitleslangs: List of languages of the subtitles to download
|
||||||
keepvideo: Keep the video file after post-processing
|
keepvideo: Keep the video file after post-processing
|
||||||
daterange: A DateRange object, download only if the upload_date is in the range.
|
daterange: A DateRange object, download only if the upload_date is in the range.
|
||||||
skip_download: Skip the actual download of the video file
|
skip_download: Skip the actual download of the video file
|
||||||
|
cachedir: Location of the cache files in the filesystem.
|
||||||
|
None to disable filesystem cache.
|
||||||
|
noplaylist: Download single video instead of a playlist if in doubt.
|
||||||
|
|
||||||
The following parameters are not used by YoutubeDL itself, they are used by
|
The following parameters are not used by YoutubeDL itself, they are used by
|
||||||
the FileDownloader:
|
the FileDownloader:
|
||||||
@ -103,6 +107,17 @@ class YoutubeDL(object):
|
|||||||
self._download_retcode = 0
|
self._download_retcode = 0
|
||||||
self._num_downloads = 0
|
self._num_downloads = 0
|
||||||
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
|
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
|
||||||
|
|
||||||
|
if (sys.version_info >= (3,) and sys.platform != 'win32' and
|
||||||
|
sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
|
||||||
|
and not params['restrictfilenames']):
|
||||||
|
# On Python 3, the Unicode filesystem API will throw errors (#1474)
|
||||||
|
self.report_warning(
|
||||||
|
u'Assuming --restrict-filenames isnce file system encoding '
|
||||||
|
u'cannot encode all charactes. '
|
||||||
|
u'Set the LC_ALL environment variable to fix this.')
|
||||||
|
params['restrictfilenames'] = True
|
||||||
|
|
||||||
self.params = params
|
self.params = params
|
||||||
self.fd = FileDownloader(self, self.params)
|
self.fd = FileDownloader(self, self.params)
|
||||||
|
|
||||||
@ -141,14 +156,10 @@ class YoutubeDL(object):
|
|||||||
|
|
||||||
def to_screen(self, message, skip_eol=False):
|
def to_screen(self, message, skip_eol=False):
|
||||||
"""Print message to stdout if not in quiet mode."""
|
"""Print message to stdout if not in quiet mode."""
|
||||||
assert type(message) == type(u'')
|
|
||||||
if not self.params.get('quiet', False):
|
if not self.params.get('quiet', False):
|
||||||
terminator = [u'\n', u''][skip_eol]
|
terminator = [u'\n', u''][skip_eol]
|
||||||
output = message + terminator
|
output = message + terminator
|
||||||
if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
|
write_string(output, self._screen_file)
|
||||||
output = output.encode(preferredencoding(), 'ignore')
|
|
||||||
self._screen_file.write(output)
|
|
||||||
self._screen_file.flush()
|
|
||||||
|
|
||||||
def to_stderr(self, message):
|
def to_stderr(self, message):
|
||||||
"""Print message to stderr."""
|
"""Print message to stderr."""
|
||||||
@ -492,13 +503,14 @@ class YoutubeDL(object):
|
|||||||
self.report_writedescription(descfn)
|
self.report_writedescription(descfn)
|
||||||
with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
|
with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
|
||||||
descfile.write(info_dict['description'])
|
descfile.write(info_dict['description'])
|
||||||
|
except (KeyError, TypeError):
|
||||||
|
self.report_warning(u'There\'s no description to write.')
|
||||||
except (OSError, IOError):
|
except (OSError, IOError):
|
||||||
self.report_error(u'Cannot write description file ' + descfn)
|
self.report_error(u'Cannot write description file ' + descfn)
|
||||||
return
|
return
|
||||||
|
|
||||||
subtitles_are_requested = any([self.params.get('writesubtitles', False),
|
subtitles_are_requested = any([self.params.get('writesubtitles', False),
|
||||||
self.params.get('writeautomaticsub'),
|
self.params.get('writeautomaticsub')])
|
||||||
self.params.get('allsubtitles', False)])
|
|
||||||
|
|
||||||
if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
|
if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
|
||||||
# subtitles download errors are already managed as troubles in relevant IE
|
# subtitles download errors are already managed as troubles in relevant IE
|
||||||
@ -534,11 +546,15 @@ class YoutubeDL(object):
|
|||||||
thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
|
thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
|
||||||
self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
|
self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
|
||||||
(info_dict['extractor'], info_dict['id']))
|
(info_dict['extractor'], info_dict['id']))
|
||||||
uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
|
try:
|
||||||
with open(thumb_filename, 'wb') as thumbf:
|
uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
|
||||||
shutil.copyfileobj(uf, thumbf)
|
with open(thumb_filename, 'wb') as thumbf:
|
||||||
self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
|
shutil.copyfileobj(uf, thumbf)
|
||||||
(info_dict['extractor'], info_dict['id'], thumb_filename))
|
self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
|
||||||
|
(info_dict['extractor'], info_dict['id'], thumb_filename))
|
||||||
|
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||||
|
self.report_warning(u'Unable to download thumbnail "%s": %s' %
|
||||||
|
(info_dict['thumbnail'], compat_str(err)))
|
||||||
|
|
||||||
if not self.params.get('skip_download', False):
|
if not self.params.get('skip_download', False):
|
||||||
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
|
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
|
||||||
@ -546,11 +562,11 @@ class YoutubeDL(object):
|
|||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
success = self.fd._do_download(filename, info_dict)
|
success = self.fd._do_download(filename, info_dict)
|
||||||
except (OSError, IOError) as err:
|
|
||||||
raise UnavailableVideoError(err)
|
|
||||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||||
self.report_error(u'unable to download video data: %s' % str(err))
|
self.report_error(u'unable to download video data: %s' % str(err))
|
||||||
return
|
return
|
||||||
|
except (OSError, IOError) as err:
|
||||||
|
raise UnavailableVideoError(err)
|
||||||
except (ContentTooShortError, ) as err:
|
except (ContentTooShortError, ) as err:
|
||||||
self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
|
self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
|
||||||
return
|
return
|
||||||
|
@ -29,6 +29,8 @@ __authors__ = (
|
|||||||
'Albert Kim',
|
'Albert Kim',
|
||||||
'Pierre Rudloff',
|
'Pierre Rudloff',
|
||||||
'Huarong Huo',
|
'Huarong Huo',
|
||||||
|
'Ismael Mejía',
|
||||||
|
'Steffan \'Ruirize\' James',
|
||||||
)
|
)
|
||||||
|
|
||||||
__license__ = 'Public Domain'
|
__license__ = 'Public Domain'
|
||||||
@ -148,7 +150,7 @@ def parseOpts(overrideArguments=None):
|
|||||||
general.add_option('-U', '--update',
|
general.add_option('-U', '--update',
|
||||||
action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
|
action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
|
||||||
general.add_option('-i', '--ignore-errors',
|
general.add_option('-i', '--ignore-errors',
|
||||||
action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
|
action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
|
||||||
general.add_option('--dump-user-agent',
|
general.add_option('--dump-user-agent',
|
||||||
action='store_true', dest='dump_user_agent',
|
action='store_true', dest='dump_user_agent',
|
||||||
help='display the current browser identification', default=False)
|
help='display the current browser identification', default=False)
|
||||||
@ -165,6 +167,12 @@ def parseOpts(overrideArguments=None):
|
|||||||
help='Output descriptions of all supported extractors', default=False)
|
help='Output descriptions of all supported extractors', default=False)
|
||||||
general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
|
general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
|
||||||
general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
|
general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
|
||||||
|
general.add_option(
|
||||||
|
'--cache-dir', dest='cachedir', default=get_cachedir(),
|
||||||
|
help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
|
||||||
|
general.add_option(
|
||||||
|
'--no-cache-dir', action='store_const', const=None, dest='cachedir',
|
||||||
|
help='Disable filesystem caching')
|
||||||
|
|
||||||
|
|
||||||
selection.add_option('--playlist-start',
|
selection.add_option('--playlist-start',
|
||||||
@ -179,6 +187,7 @@ def parseOpts(overrideArguments=None):
|
|||||||
selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
|
selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
|
||||||
selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
|
selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
|
||||||
selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
|
selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
|
||||||
|
selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
|
||||||
|
|
||||||
|
|
||||||
authentication.add_option('-u', '--username',
|
authentication.add_option('-u', '--username',
|
||||||
@ -205,13 +214,10 @@ def parseOpts(overrideArguments=None):
|
|||||||
|
|
||||||
subtitles.add_option('--write-sub', '--write-srt',
|
subtitles.add_option('--write-sub', '--write-srt',
|
||||||
action='store_true', dest='writesubtitles',
|
action='store_true', dest='writesubtitles',
|
||||||
help='write subtitle file (currently youtube only)', default=False)
|
help='write subtitle file', default=False)
|
||||||
subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
|
subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
|
||||||
action='store_true', dest='writeautomaticsub',
|
action='store_true', dest='writeautomaticsub',
|
||||||
help='write automatic subtitle file (currently youtube only)', default=False)
|
help='write automatic subtitle file (youtube only)', default=False)
|
||||||
subtitles.add_option('--only-sub',
|
|
||||||
action='store_true', dest='skip_download',
|
|
||||||
help='[deprecated] alias of --skip-download', default=False)
|
|
||||||
subtitles.add_option('--all-subs',
|
subtitles.add_option('--all-subs',
|
||||||
action='store_true', dest='allsubtitles',
|
action='store_true', dest='allsubtitles',
|
||||||
help='downloads all the available subtitles of the video', default=False)
|
help='downloads all the available subtitles of the video', default=False)
|
||||||
@ -222,7 +228,7 @@ def parseOpts(overrideArguments=None):
|
|||||||
action='store', dest='subtitlesformat', metavar='FORMAT',
|
action='store', dest='subtitlesformat', metavar='FORMAT',
|
||||||
help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
|
help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
|
||||||
subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
|
subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
|
||||||
action='callback', dest='subtitleslang', metavar='LANGS', type='str',
|
action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
|
||||||
default=[], callback=_comma_separated_values_options_callback,
|
default=[], callback=_comma_separated_values_options_callback,
|
||||||
help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
|
help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
|
||||||
|
|
||||||
@ -273,6 +279,10 @@ def parseOpts(overrideArguments=None):
|
|||||||
verbosity.add_option('--dump-intermediate-pages',
|
verbosity.add_option('--dump-intermediate-pages',
|
||||||
action='store_true', dest='dump_intermediate_pages', default=False,
|
action='store_true', dest='dump_intermediate_pages', default=False,
|
||||||
help='print downloaded pages to debug problems(very verbose)')
|
help='print downloaded pages to debug problems(very verbose)')
|
||||||
|
verbosity.add_option('--youtube-print-sig-code',
|
||||||
|
action='store_true', dest='youtube_print_sig_code', default=False,
|
||||||
|
help=optparse.SUPPRESS_HELP)
|
||||||
|
|
||||||
|
|
||||||
filesystem.add_option('-t', '--title',
|
filesystem.add_option('-t', '--title',
|
||||||
action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
|
action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
|
||||||
@ -356,22 +366,26 @@ def parseOpts(overrideArguments=None):
|
|||||||
if overrideArguments is not None:
|
if overrideArguments is not None:
|
||||||
opts, args = parser.parse_args(overrideArguments)
|
opts, args = parser.parse_args(overrideArguments)
|
||||||
if opts.verbose:
|
if opts.verbose:
|
||||||
sys.stderr.write(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
|
write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
|
||||||
else:
|
else:
|
||||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
|
xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
|
||||||
if xdg_config_home:
|
if xdg_config_home:
|
||||||
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
|
userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
|
||||||
|
if not os.path.isfile(userConfFile):
|
||||||
|
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
|
||||||
else:
|
else:
|
||||||
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
|
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
|
||||||
|
if not os.path.isfile(userConfFile):
|
||||||
|
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
|
||||||
systemConf = _readOptions('/etc/youtube-dl.conf')
|
systemConf = _readOptions('/etc/youtube-dl.conf')
|
||||||
userConf = _readOptions(userConfFile)
|
userConf = _readOptions(userConfFile)
|
||||||
commandLineConf = sys.argv[1:]
|
commandLineConf = sys.argv[1:]
|
||||||
argv = systemConf + userConf + commandLineConf
|
argv = systemConf + userConf + commandLineConf
|
||||||
opts, args = parser.parse_args(argv)
|
opts, args = parser.parse_args(argv)
|
||||||
if opts.verbose:
|
if opts.verbose:
|
||||||
sys.stderr.write(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
|
write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
|
||||||
sys.stderr.write(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
|
write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
|
||||||
sys.stderr.write(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
|
write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
|
||||||
|
|
||||||
return parser, opts, args
|
return parser, opts, args
|
||||||
|
|
||||||
@ -394,7 +408,7 @@ def _real_main(argv=None):
|
|||||||
except (IOError, OSError) as err:
|
except (IOError, OSError) as err:
|
||||||
if opts.verbose:
|
if opts.verbose:
|
||||||
traceback.print_exc()
|
traceback.print_exc()
|
||||||
sys.stderr.write(u'ERROR: unable to open cookie file\n')
|
write_string(u'ERROR: unable to open cookie file\n')
|
||||||
sys.exit(101)
|
sys.exit(101)
|
||||||
# Set user agent
|
# Set user agent
|
||||||
if opts.user_agent is not None:
|
if opts.user_agent is not None:
|
||||||
@ -421,7 +435,7 @@ def _real_main(argv=None):
|
|||||||
batchurls = [x.strip() for x in batchurls]
|
batchurls = [x.strip() for x in batchurls]
|
||||||
batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
|
batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
|
||||||
if opts.verbose:
|
if opts.verbose:
|
||||||
sys.stderr.write(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
|
write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
|
||||||
except IOError:
|
except IOError:
|
||||||
sys.exit(u'ERROR: batch file could not be read')
|
sys.exit(u'ERROR: batch file could not be read')
|
||||||
all_urls = batchurls + args
|
all_urls = batchurls + args
|
||||||
@ -535,6 +549,11 @@ def _real_main(argv=None):
|
|||||||
else:
|
else:
|
||||||
date = DateRange(opts.dateafter, opts.datebefore)
|
date = DateRange(opts.dateafter, opts.datebefore)
|
||||||
|
|
||||||
|
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
|
||||||
|
# this was the old behaviour if only --all-sub was given.
|
||||||
|
if opts.allsubtitles and (opts.writeautomaticsub == False):
|
||||||
|
opts.writesubtitles = True
|
||||||
|
|
||||||
if sys.version_info < (3,):
|
if sys.version_info < (3,):
|
||||||
# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
|
# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
|
||||||
if opts.outtmpl is not None:
|
if opts.outtmpl is not None:
|
||||||
@ -547,6 +566,10 @@ def _real_main(argv=None):
|
|||||||
or (opts.useid and u'%(id)s.%(ext)s')
|
or (opts.useid and u'%(id)s.%(ext)s')
|
||||||
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
|
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
|
||||||
or u'%(title)s-%(id)s.%(ext)s')
|
or u'%(title)s-%(id)s.%(ext)s')
|
||||||
|
if '%(ext)s' not in outtmpl and opts.extractaudio:
|
||||||
|
parser.error(u'Cannot download a video and extract audio into the same'
|
||||||
|
u' file! Use "%%(ext)s" instead of %r' %
|
||||||
|
determine_ext(outtmpl, u''))
|
||||||
|
|
||||||
# YoutubeDL
|
# YoutubeDL
|
||||||
ydl = YoutubeDL({
|
ydl = YoutubeDL({
|
||||||
@ -581,6 +604,7 @@ def _real_main(argv=None):
|
|||||||
'progress_with_newline': opts.progress_with_newline,
|
'progress_with_newline': opts.progress_with_newline,
|
||||||
'playliststart': opts.playliststart,
|
'playliststart': opts.playliststart,
|
||||||
'playlistend': opts.playlistend,
|
'playlistend': opts.playlistend,
|
||||||
|
'noplaylist': opts.noplaylist,
|
||||||
'logtostderr': opts.outtmpl == '-',
|
'logtostderr': opts.outtmpl == '-',
|
||||||
'consoletitle': opts.consoletitle,
|
'consoletitle': opts.consoletitle,
|
||||||
'nopart': opts.nopart,
|
'nopart': opts.nopart,
|
||||||
@ -593,7 +617,7 @@ def _real_main(argv=None):
|
|||||||
'allsubtitles': opts.allsubtitles,
|
'allsubtitles': opts.allsubtitles,
|
||||||
'listsubtitles': opts.listsubtitles,
|
'listsubtitles': opts.listsubtitles,
|
||||||
'subtitlesformat': opts.subtitlesformat,
|
'subtitlesformat': opts.subtitlesformat,
|
||||||
'subtitleslangs': opts.subtitleslang,
|
'subtitleslangs': opts.subtitleslangs,
|
||||||
'matchtitle': decodeOption(opts.matchtitle),
|
'matchtitle': decodeOption(opts.matchtitle),
|
||||||
'rejecttitle': decodeOption(opts.rejecttitle),
|
'rejecttitle': decodeOption(opts.rejecttitle),
|
||||||
'max_downloads': opts.max_downloads,
|
'max_downloads': opts.max_downloads,
|
||||||
@ -605,10 +629,12 @@ def _real_main(argv=None):
|
|||||||
'min_filesize': opts.min_filesize,
|
'min_filesize': opts.min_filesize,
|
||||||
'max_filesize': opts.max_filesize,
|
'max_filesize': opts.max_filesize,
|
||||||
'daterange': date,
|
'daterange': date,
|
||||||
|
'cachedir': opts.cachedir,
|
||||||
|
'youtube_print_sig_code': opts.youtube_print_sig_code,
|
||||||
})
|
})
|
||||||
|
|
||||||
if opts.verbose:
|
if opts.verbose:
|
||||||
sys.stderr.write(u'[debug] youtube-dl version ' + __version__ + u'\n')
|
write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
|
||||||
try:
|
try:
|
||||||
sp = subprocess.Popen(
|
sp = subprocess.Popen(
|
||||||
['git', 'rev-parse', '--short', 'HEAD'],
|
['git', 'rev-parse', '--short', 'HEAD'],
|
||||||
@ -617,14 +643,14 @@ def _real_main(argv=None):
|
|||||||
out, err = sp.communicate()
|
out, err = sp.communicate()
|
||||||
out = out.decode().strip()
|
out = out.decode().strip()
|
||||||
if re.match('[0-9a-f]+', out):
|
if re.match('[0-9a-f]+', out):
|
||||||
sys.stderr.write(u'[debug] Git HEAD: ' + out + u'\n')
|
write_string(u'[debug] Git HEAD: ' + out + u'\n')
|
||||||
except:
|
except:
|
||||||
try:
|
try:
|
||||||
sys.exc_clear()
|
sys.exc_clear()
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
sys.stderr.write(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
|
write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
|
||||||
sys.stderr.write(u'[debug] Proxy map: ' + str(proxy_handler.proxies) + u'\n')
|
write_string(u'[debug] Proxy map: ' + str(proxy_handler.proxies) + u'\n')
|
||||||
|
|
||||||
ydl.add_default_info_extractors()
|
ydl.add_default_info_extractors()
|
||||||
|
|
||||||
@ -638,7 +664,7 @@ def _real_main(argv=None):
|
|||||||
|
|
||||||
# Update version
|
# Update version
|
||||||
if opts.update_self:
|
if opts.update_self:
|
||||||
update_self(ydl.to_screen, opts.verbose, sys.argv[0])
|
update_self(ydl.to_screen, opts.verbose)
|
||||||
|
|
||||||
# Maybe do nothing
|
# Maybe do nothing
|
||||||
if len(all_urls) < 1:
|
if len(all_urls) < 1:
|
||||||
|
@ -6,6 +6,7 @@ from .arte import ArteTvIE
|
|||||||
from .auengine import AUEngineIE
|
from .auengine import AUEngineIE
|
||||||
from .bandcamp import BandcampIE
|
from .bandcamp import BandcampIE
|
||||||
from .bliptv import BlipTVIE, BlipTVUserIE
|
from .bliptv import BlipTVIE, BlipTVUserIE
|
||||||
|
from .bloomberg import BloombergIE
|
||||||
from .breakcom import BreakIE
|
from .breakcom import BreakIE
|
||||||
from .brightcove import BrightcoveIE
|
from .brightcove import BrightcoveIE
|
||||||
from .c56 import C56IE
|
from .c56 import C56IE
|
||||||
@ -17,18 +18,33 @@ from .comedycentral import ComedyCentralIE
|
|||||||
from .condenast import CondeNastIE
|
from .condenast import CondeNastIE
|
||||||
from .criterion import CriterionIE
|
from .criterion import CriterionIE
|
||||||
from .cspan import CSpanIE
|
from .cspan import CSpanIE
|
||||||
from .dailymotion import DailymotionIE, DailymotionPlaylistIE
|
from .dailymotion import (
|
||||||
|
DailymotionIE,
|
||||||
|
DailymotionPlaylistIE,
|
||||||
|
DailymotionUserIE,
|
||||||
|
)
|
||||||
from .daum import DaumIE
|
from .daum import DaumIE
|
||||||
from .depositfiles import DepositFilesIE
|
from .depositfiles import DepositFilesIE
|
||||||
from .dotsub import DotsubIE
|
from .dotsub import DotsubIE
|
||||||
from .dreisat import DreiSatIE
|
from .dreisat import DreiSatIE
|
||||||
from .defense import DefenseGouvFrIE
|
from .defense import DefenseGouvFrIE
|
||||||
|
from .ebaumsworld import EbaumsWorldIE
|
||||||
from .ehow import EHowIE
|
from .ehow import EHowIE
|
||||||
from .eighttracks import EightTracksIE
|
from .eighttracks import EightTracksIE
|
||||||
from .escapist import EscapistIE
|
from .escapist import EscapistIE
|
||||||
from .exfm import ExfmIE
|
from .exfm import ExfmIE
|
||||||
from .facebook import FacebookIE
|
from .facebook import FacebookIE
|
||||||
|
from .fktv import (
|
||||||
|
FKTVIE,
|
||||||
|
FKTVPosteckeIE,
|
||||||
|
)
|
||||||
from .flickr import FlickrIE
|
from .flickr import FlickrIE
|
||||||
|
from .francetv import (
|
||||||
|
PluzzIE,
|
||||||
|
FranceTvInfoIE,
|
||||||
|
France2IE,
|
||||||
|
GenerationQuoiIE
|
||||||
|
)
|
||||||
from .freesound import FreesoundIE
|
from .freesound import FreesoundIE
|
||||||
from .funnyordie import FunnyOrDieIE
|
from .funnyordie import FunnyOrDieIE
|
||||||
from .gamespot import GameSpotIE
|
from .gamespot import GameSpotIE
|
||||||
@ -48,10 +64,12 @@ from .jeuxvideo import JeuxVideoIE
|
|||||||
from .jukebox import JukeboxIE
|
from .jukebox import JukeboxIE
|
||||||
from .justintv import JustinTVIE
|
from .justintv import JustinTVIE
|
||||||
from .kankan import KankanIE
|
from .kankan import KankanIE
|
||||||
|
from .kickstarter import KickStarterIE
|
||||||
from .keek import KeekIE
|
from .keek import KeekIE
|
||||||
from .liveleak import LiveLeakIE
|
from .liveleak import LiveLeakIE
|
||||||
from .livestream import LivestreamIE
|
from .livestream import LivestreamIE
|
||||||
from .metacafe import MetacafeIE
|
from .metacafe import MetacafeIE
|
||||||
|
from .metacritic import MetacriticIE
|
||||||
from .mit import TechTVMITIE, MITIE
|
from .mit import TechTVMITIE, MITIE
|
||||||
from .mixcloud import MixcloudIE
|
from .mixcloud import MixcloudIE
|
||||||
from .mtv import MTVIE
|
from .mtv import MTVIE
|
||||||
@ -61,6 +79,7 @@ from .myvideo import MyVideoIE
|
|||||||
from .naver import NaverIE
|
from .naver import NaverIE
|
||||||
from .nba import NBAIE
|
from .nba import NBAIE
|
||||||
from .nbc import NBCNewsIE
|
from .nbc import NBCNewsIE
|
||||||
|
from .newgrounds import NewgroundsIE
|
||||||
from .ooyala import OoyalaIE
|
from .ooyala import OoyalaIE
|
||||||
from .orf import ORFIE
|
from .orf import ORFIE
|
||||||
from .pbs import PBSIE
|
from .pbs import PBSIE
|
||||||
@ -74,8 +93,10 @@ from .roxwel import RoxwelIE
|
|||||||
from .rtlnow import RTLnowIE
|
from .rtlnow import RTLnowIE
|
||||||
from .sina import SinaIE
|
from .sina import SinaIE
|
||||||
from .slashdot import SlashdotIE
|
from .slashdot import SlashdotIE
|
||||||
|
from .slideshare import SlideshareIE
|
||||||
from .sohu import SohuIE
|
from .sohu import SohuIE
|
||||||
from .soundcloud import SoundcloudIE, SoundcloudSetIE
|
from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE
|
||||||
|
from .southparkstudios import SouthParkStudiosIE
|
||||||
from .spiegel import SpiegelIE
|
from .spiegel import SpiegelIE
|
||||||
from .stanfordoc import StanfordOpenClassroomIE
|
from .stanfordoc import StanfordOpenClassroomIE
|
||||||
from .statigram import StatigramIE
|
from .statigram import StatigramIE
|
||||||
@ -90,11 +111,13 @@ from .tudou import TudouIE
|
|||||||
from .tumblr import TumblrIE
|
from .tumblr import TumblrIE
|
||||||
from .tutv import TutvIE
|
from .tutv import TutvIE
|
||||||
from .unistra import UnistraIE
|
from .unistra import UnistraIE
|
||||||
from .ustream import UstreamIE
|
from .ustream import UstreamIE, UstreamChannelIE
|
||||||
from .vbox7 import Vbox7IE
|
from .vbox7 import Vbox7IE
|
||||||
from .veehd import VeeHDIE
|
from .veehd import VeeHDIE
|
||||||
from .veoh import VeohIE
|
from .veoh import VeohIE
|
||||||
from .vevo import VevoIE
|
from .vevo import VevoIE
|
||||||
|
from .vice import ViceIE
|
||||||
|
from .viddler import ViddlerIE
|
||||||
from .videofyme import VideofyMeIE
|
from .videofyme import VideofyMeIE
|
||||||
from .vimeo import VimeoIE, VimeoChannelIE
|
from .vimeo import VimeoIE, VimeoChannelIE
|
||||||
from .vine import VineIE
|
from .vine import VineIE
|
||||||
|
@ -1,8 +1,10 @@
|
|||||||
import re
|
import re
|
||||||
import xml.etree.ElementTree
|
import xml.etree.ElementTree
|
||||||
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
compat_urlparse,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -14,10 +16,9 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
u"playlist": [
|
u"playlist": [
|
||||||
{
|
{
|
||||||
u"file": u"manofsteel-trailer4.mov",
|
u"file": u"manofsteel-trailer4.mov",
|
||||||
u"md5": u"11874af099d480cc09e103b189805d5f",
|
u"md5": u"d97a8e575432dbcb81b7c3acb741f8a8",
|
||||||
u"info_dict": {
|
u"info_dict": {
|
||||||
u"duration": 111,
|
u"duration": 111,
|
||||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_11624.jpg",
|
|
||||||
u"title": u"Trailer 4",
|
u"title": u"Trailer 4",
|
||||||
u"upload_date": u"20130523",
|
u"upload_date": u"20130523",
|
||||||
u"uploader_id": u"wb",
|
u"uploader_id": u"wb",
|
||||||
@ -25,10 +26,9 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
u"file": u"manofsteel-trailer3.mov",
|
u"file": u"manofsteel-trailer3.mov",
|
||||||
u"md5": u"07a0a262aae5afe68120eed61137ab34",
|
u"md5": u"b8017b7131b721fb4e8d6f49e1df908c",
|
||||||
u"info_dict": {
|
u"info_dict": {
|
||||||
u"duration": 182,
|
u"duration": 182,
|
||||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_10793.jpg",
|
|
||||||
u"title": u"Trailer 3",
|
u"title": u"Trailer 3",
|
||||||
u"upload_date": u"20130417",
|
u"upload_date": u"20130417",
|
||||||
u"uploader_id": u"wb",
|
u"uploader_id": u"wb",
|
||||||
@ -36,10 +36,9 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
u"file": u"manofsteel-trailer.mov",
|
u"file": u"manofsteel-trailer.mov",
|
||||||
u"md5": u"e401fde0813008e3307e54b6f384cff1",
|
u"md5": u"d0f1e1150989b9924679b441f3404d48",
|
||||||
u"info_dict": {
|
u"info_dict": {
|
||||||
u"duration": 148,
|
u"duration": 148,
|
||||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_8703.jpg",
|
|
||||||
u"title": u"Trailer",
|
u"title": u"Trailer",
|
||||||
u"upload_date": u"20121212",
|
u"upload_date": u"20121212",
|
||||||
u"uploader_id": u"wb",
|
u"uploader_id": u"wb",
|
||||||
@ -47,10 +46,9 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
u"file": u"manofsteel-teaser.mov",
|
u"file": u"manofsteel-teaser.mov",
|
||||||
u"md5": u"76b392f2ae9e7c98b22913c10a639c97",
|
u"md5": u"5fe08795b943eb2e757fa95cb6def1cb",
|
||||||
u"info_dict": {
|
u"info_dict": {
|
||||||
u"duration": 93,
|
u"duration": 93,
|
||||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_6899.jpg",
|
|
||||||
u"title": u"Teaser",
|
u"title": u"Teaser",
|
||||||
u"upload_date": u"20120721",
|
u"upload_date": u"20120721",
|
||||||
u"uploader_id": u"wb",
|
u"uploader_id": u"wb",
|
||||||
@ -59,87 +57,61 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_JSON_RE = r'iTunes.playURL\((.*?)\);'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
movie = mobj.group('movie')
|
movie = mobj.group('movie')
|
||||||
uploader_id = mobj.group('company')
|
uploader_id = mobj.group('company')
|
||||||
|
|
||||||
playlist_url = url.partition(u'?')[0] + u'/includes/playlists/web.inc'
|
playlist_url = compat_urlparse.urljoin(url, u'includes/playlists/itunes.inc')
|
||||||
playlist_snippet = self._download_webpage(playlist_url, movie)
|
playlist_snippet = self._download_webpage(playlist_url, movie)
|
||||||
playlist_cleaned = re.sub(r'(?s)<script>.*?</script>', u'', playlist_snippet)
|
playlist_cleaned = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', playlist_snippet)
|
||||||
|
playlist_cleaned = re.sub(r'<img ([^<]*?)>', r'<img \1/>', playlist_cleaned)
|
||||||
|
# The ' in the onClick attributes are not escaped, it couldn't be parsed
|
||||||
|
# with xml.etree.ElementTree.fromstring
|
||||||
|
# like: http://trailers.apple.com/trailers/wb/gravity/
|
||||||
|
def _clean_json(m):
|
||||||
|
return u'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')
|
||||||
|
playlist_cleaned = re.sub(self._JSON_RE, _clean_json, playlist_cleaned)
|
||||||
playlist_html = u'<html>' + playlist_cleaned + u'</html>'
|
playlist_html = u'<html>' + playlist_cleaned + u'</html>'
|
||||||
|
|
||||||
size_cache = {}
|
|
||||||
|
|
||||||
doc = xml.etree.ElementTree.fromstring(playlist_html)
|
doc = xml.etree.ElementTree.fromstring(playlist_html)
|
||||||
playlist = []
|
playlist = []
|
||||||
for li in doc.findall('./div/ul/li'):
|
for li in doc.findall('./div/ul/li'):
|
||||||
title = li.find('.//h3').text
|
on_click = li.find('.//a').attrib['onClick']
|
||||||
|
trailer_info_json = self._search_regex(self._JSON_RE,
|
||||||
|
on_click, u'trailer info')
|
||||||
|
trailer_info = json.loads(trailer_info_json)
|
||||||
|
title = trailer_info['title']
|
||||||
video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
|
video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
|
||||||
thumbnail = li.find('.//img').attrib['src']
|
thumbnail = li.find('.//img').attrib['src']
|
||||||
|
upload_date = trailer_info['posted'].replace('-', '')
|
||||||
|
|
||||||
date_el = li.find('.//p')
|
runtime = trailer_info['runtime']
|
||||||
upload_date = None
|
m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
|
||||||
m = re.search(r':\s?(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<year>[0-9]{2})', date_el.text)
|
|
||||||
if m:
|
|
||||||
upload_date = u'20' + m.group('year') + m.group('month') + m.group('day')
|
|
||||||
runtime_el = date_el.find('./br')
|
|
||||||
m = re.search(r':\s?(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime_el.tail)
|
|
||||||
duration = None
|
duration = None
|
||||||
if m:
|
if m:
|
||||||
duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
|
duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
|
||||||
|
|
||||||
|
first_url = trailer_info['url']
|
||||||
|
trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
|
||||||
|
settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
|
||||||
|
settings_json = self._download_webpage(settings_json_url, trailer_id, u'Downloading settings json')
|
||||||
|
settings = json.loads(settings_json)
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for formats_el in li.findall('.//a'):
|
for format in settings['metadata']['sizes']:
|
||||||
if formats_el.attrib['class'] != 'OverlayPanel':
|
# The src is a file pointing to the real video file
|
||||||
continue
|
format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
|
||||||
target = formats_el.attrib['target']
|
formats.append({
|
||||||
|
'url': format_url,
|
||||||
format_code = formats_el.text
|
'ext': determine_ext(format_url),
|
||||||
if 'Automatic' in format_code:
|
'format': format['type'],
|
||||||
continue
|
'width': format['width'],
|
||||||
|
'height': int(format['height']),
|
||||||
size_q = formats_el.attrib['href']
|
})
|
||||||
size_id = size_q.rpartition('#videos-')[2]
|
formats = sorted(formats, key=lambda f: (f['height'], f['width']))
|
||||||
if size_id not in size_cache:
|
|
||||||
size_url = url + size_q
|
|
||||||
sizepage_html = self._download_webpage(
|
|
||||||
size_url, movie,
|
|
||||||
note=u'Downloading size info %s' % size_id,
|
|
||||||
errnote=u'Error while downloading size info %s' % size_id,
|
|
||||||
)
|
|
||||||
_doc = xml.etree.ElementTree.fromstring(sizepage_html)
|
|
||||||
size_cache[size_id] = _doc
|
|
||||||
|
|
||||||
sizepage_doc = size_cache[size_id]
|
|
||||||
links = sizepage_doc.findall('.//{http://www.w3.org/1999/xhtml}ul/{http://www.w3.org/1999/xhtml}li/{http://www.w3.org/1999/xhtml}a')
|
|
||||||
for vid_a in links:
|
|
||||||
href = vid_a.get('href')
|
|
||||||
if not href.endswith(target):
|
|
||||||
continue
|
|
||||||
detail_q = href.partition('#')[0]
|
|
||||||
detail_url = url + '/' + detail_q
|
|
||||||
|
|
||||||
m = re.match(r'includes/(?P<detail_id>[^/]+)/', detail_q)
|
|
||||||
detail_id = m.group('detail_id')
|
|
||||||
|
|
||||||
detail_html = self._download_webpage(
|
|
||||||
detail_url, movie,
|
|
||||||
note=u'Downloading detail %s %s' % (detail_id, size_id),
|
|
||||||
errnote=u'Error while downloading detail %s %s' % (detail_id, size_id)
|
|
||||||
)
|
|
||||||
detail_doc = xml.etree.ElementTree.fromstring(detail_html)
|
|
||||||
movie_link_el = detail_doc.find('.//{http://www.w3.org/1999/xhtml}a')
|
|
||||||
assert movie_link_el.get('class') == 'movieLink'
|
|
||||||
movie_link = movie_link_el.get('href').partition('?')[0].replace('_', '_h')
|
|
||||||
ext = determine_ext(movie_link)
|
|
||||||
assert ext == 'mov'
|
|
||||||
|
|
||||||
formats.append({
|
|
||||||
'format': format_code,
|
|
||||||
'ext': ext,
|
|
||||||
'url': movie_link,
|
|
||||||
})
|
|
||||||
|
|
||||||
info = {
|
info = {
|
||||||
'_type': 'video',
|
'_type': 'video',
|
||||||
|
@ -46,6 +46,8 @@ class ArchiveOrgIE(InfoExtractor):
|
|||||||
for fn,fdata in data['files'].items()
|
for fn,fdata in data['files'].items()
|
||||||
if 'Video' in fdata['format']]
|
if 'Video' in fdata['format']]
|
||||||
formats.sort(key=lambda fdata: fdata['file_size'])
|
formats.sort(key=lambda fdata: fdata['file_size'])
|
||||||
|
for f in formats:
|
||||||
|
f['ext'] = determine_ext(f['url'])
|
||||||
|
|
||||||
info = {
|
info = {
|
||||||
'_type': 'video',
|
'_type': 'video',
|
||||||
@ -61,7 +63,6 @@ class ArchiveOrgIE(InfoExtractor):
|
|||||||
info['thumbnail'] = thumbnail
|
info['thumbnail'] = thumbnail
|
||||||
|
|
||||||
# TODO: Remove when #980 has been merged
|
# TODO: Remove when #980 has been merged
|
||||||
info['url'] = formats[-1]['url']
|
info.update(formats[-1])
|
||||||
info['ext'] = determine_ext(formats[-1]['url'])
|
|
||||||
|
|
||||||
return info
|
return info
|
||||||
|
@ -115,7 +115,7 @@ class BlipTVIE(InfoExtractor):
|
|||||||
ext = umobj.group(1)
|
ext = umobj.group(1)
|
||||||
|
|
||||||
info = {
|
info = {
|
||||||
'id': data['item_id'],
|
'id': compat_str(data['item_id']),
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'uploader': data['display_name'],
|
'uploader': data['display_name'],
|
||||||
'upload_date': upload_date,
|
'upload_date': upload_date,
|
||||||
|
27
youtube_dl/extractor/bloomberg.py
Normal file
27
youtube_dl/extractor/bloomberg.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class BloombergIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<name>.+?).html'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html',
|
||||||
|
u'file': u'12bzhqZTqQHmmlA8I-i0NpzJgcG5NNYX.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Shah\'s Presentation on Foreign-Exchange Strategies',
|
||||||
|
u'description': u'md5:abc86e5236f9f0e4866c59ad36736686',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
# Requires ffmpeg (m3u8 manifest)
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
name = mobj.group('name')
|
||||||
|
webpage = self._download_webpage(url, name)
|
||||||
|
ooyala_url = self._og_search_video_url(webpage)
|
||||||
|
return self.url_result(ooyala_url, ie='Ooyala')
|
@ -1,3 +1,5 @@
|
|||||||
|
# encoding: utf-8
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import json
|
import json
|
||||||
import xml.etree.ElementTree
|
import xml.etree.ElementTree
|
||||||
@ -7,15 +9,39 @@ from ..utils import (
|
|||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
find_xpath_attr,
|
find_xpath_attr,
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
|
|
||||||
|
ExtractorError,
|
||||||
)
|
)
|
||||||
|
|
||||||
class BrightcoveIE(InfoExtractor):
|
class BrightcoveIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
|
_VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
|
||||||
_FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
|
_FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
|
||||||
_PLAYLIST_URL_TEMPLATE = 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=%s'
|
_PLAYLIST_URL_TEMPLATE = 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=%s'
|
||||||
|
|
||||||
# There is a test for Brigtcove in GenericIE, that way we test both the download
|
_TESTS = [
|
||||||
# and the detection of videos, and we don't have to find an URL that is always valid
|
{
|
||||||
|
# From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
|
||||||
|
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
|
||||||
|
u'file': u'2371591881001.mp4',
|
||||||
|
u'md5': u'9e80619e0a94663f0bdc849b4566af19',
|
||||||
|
u'note': u'Test Brightcove downloads and detection in GenericIE',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
|
||||||
|
u'uploader': u'8TV',
|
||||||
|
u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# From http://medianetwork.oracle.com/video/player/1785452137001
|
||||||
|
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
|
||||||
|
u'file': u'1785452137001.flv',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
|
||||||
|
u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.',
|
||||||
|
u'uploader': u'Oracle',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _build_brighcove_url(cls, object_str):
|
def _build_brighcove_url(cls, object_str):
|
||||||
@ -23,6 +49,11 @@ class BrightcoveIE(InfoExtractor):
|
|||||||
Build a Brightcove url from a xml string containing
|
Build a Brightcove url from a xml string containing
|
||||||
<object class="BrightcoveExperience">{params}</object>
|
<object class="BrightcoveExperience">{params}</object>
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
|
||||||
|
object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
|
||||||
|
lambda m: m.group(1) + '/>', object_str)
|
||||||
|
|
||||||
object_doc = xml.etree.ElementTree.fromstring(object_str)
|
object_doc = xml.etree.ElementTree.fromstring(object_str)
|
||||||
assert u'BrightcoveExperience' in object_doc.attrib['class']
|
assert u'BrightcoveExperience' in object_doc.attrib['class']
|
||||||
params = {'flashID': object_doc.attrib['id'],
|
params = {'flashID': object_doc.attrib['id'],
|
||||||
@ -72,15 +103,27 @@ class BrightcoveIE(InfoExtractor):
|
|||||||
playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
|
playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
|
||||||
|
|
||||||
def _extract_video_info(self, video_info):
|
def _extract_video_info(self, video_info):
|
||||||
renditions = video_info['renditions']
|
info = {
|
||||||
renditions = sorted(renditions, key=lambda r: r['size'])
|
'id': video_info['id'],
|
||||||
best_format = renditions[-1]
|
'title': video_info['displayName'],
|
||||||
|
'description': video_info.get('shortDescription'),
|
||||||
|
'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
|
||||||
|
'uploader': video_info.get('publisherName'),
|
||||||
|
}
|
||||||
|
|
||||||
return {'id': video_info['id'],
|
renditions = video_info.get('renditions')
|
||||||
'title': video_info['displayName'],
|
if renditions:
|
||||||
'url': best_format['defaultURL'],
|
renditions = sorted(renditions, key=lambda r: r['size'])
|
||||||
|
best_format = renditions[-1]
|
||||||
|
info.update({
|
||||||
|
'url': best_format['defaultURL'],
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'description': video_info.get('shortDescription'),
|
})
|
||||||
'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
|
elif video_info.get('FLVFullLengthURL') is not None:
|
||||||
'uploader': video_info.get('publisherName'),
|
info.update({
|
||||||
}
|
'url': video_info['FLVFullLengthURL'],
|
||||||
|
'ext': 'flv',
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
|
||||||
|
return info
|
||||||
|
@ -5,7 +5,7 @@ from .common import InfoExtractor
|
|||||||
|
|
||||||
|
|
||||||
class Canalc2IE(InfoExtractor):
|
class Canalc2IE(InfoExtractor):
|
||||||
_IE_NAME = 'canalc2.tv'
|
IE_NAME = 'canalc2.tv'
|
||||||
_VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?idVideo=(\d+)&voir=oui'
|
_VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?idVideo=(\d+)&voir=oui'
|
||||||
|
|
||||||
_TEST = {
|
_TEST = {
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
# encoding: utf-8
|
||||||
import re
|
import re
|
||||||
import xml.etree.ElementTree
|
import xml.etree.ElementTree
|
||||||
|
|
||||||
@ -5,24 +6,29 @@ from .common import InfoExtractor
|
|||||||
from ..utils import unified_strdate
|
from ..utils import unified_strdate
|
||||||
|
|
||||||
class CanalplusIE(InfoExtractor):
|
class CanalplusIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(www\.canalplus\.fr/.*?\?vid=|player\.canalplus\.fr/#/)(?P<id>\d+)'
|
_VALID_URL = r'https?://(www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>\d+))'
|
||||||
_VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
|
_VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
|
||||||
IE_NAME = u'canalplus.fr'
|
IE_NAME = u'canalplus.fr'
|
||||||
|
|
||||||
_TEST = {
|
_TEST = {
|
||||||
u'url': u'http://www.canalplus.fr/c-divertissement/pid3351-c-le-petit-journal.html?vid=889861',
|
u'url': u'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
|
||||||
u'file': u'889861.flv',
|
u'file': u'922470.flv',
|
||||||
u'md5': u'590a888158b5f0d6832f84001fbf3e99',
|
|
||||||
u'info_dict': {
|
u'info_dict': {
|
||||||
u'title': u'Le Petit Journal 20/06/13 - La guerre des drone',
|
u'title': u'Zapping - 26/08/13',
|
||||||
u'upload_date': u'20130620',
|
u'description': u'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
|
||||||
|
u'upload_date': u'20130826',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
},
|
},
|
||||||
u'skip': u'Requires rtmpdump'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
|
if video_id is None:
|
||||||
|
webpage = self._download_webpage(url, mobj.group('path'))
|
||||||
|
video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id')
|
||||||
info_url = self._VIDEO_INFO_TEMPLATE % video_id
|
info_url = self._VIDEO_INFO_TEMPLATE % video_id
|
||||||
info_page = self._download_webpage(info_url,video_id,
|
info_page = self._download_webpage(info_url,video_id,
|
||||||
u'Downloading video info')
|
u'Downloading video info')
|
||||||
@ -43,4 +49,6 @@ class CanalplusIE(InfoExtractor):
|
|||||||
'ext': 'flv',
|
'ext': 'flv',
|
||||||
'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
|
'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
|
||||||
'thumbnail': media.find('IMAGES/GRAND').text,
|
'thumbnail': media.find('IMAGES/GRAND').text,
|
||||||
|
'description': infos.find('DESCRIPTION').text,
|
||||||
|
'view_count': int(infos.find('NB_VUES').text),
|
||||||
}
|
}
|
||||||
|
@ -51,12 +51,12 @@ class ComedyCentralIE(InfoExtractor):
|
|||||||
'400': 'mp4',
|
'400': 'mp4',
|
||||||
}
|
}
|
||||||
_video_dimensions = {
|
_video_dimensions = {
|
||||||
'3500': '1280x720',
|
'3500': (1280, 720),
|
||||||
'2200': '960x540',
|
'2200': (960, 540),
|
||||||
'1700': '768x432',
|
'1700': (768, 432),
|
||||||
'1200': '640x360',
|
'1200': (640, 360),
|
||||||
'750': '512x288',
|
'750': (512, 288),
|
||||||
'400': '384x216',
|
'400': (384, 216),
|
||||||
}
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -64,11 +64,13 @@ class ComedyCentralIE(InfoExtractor):
|
|||||||
"""Receives a URL and returns True if suitable for this IE."""
|
"""Receives a URL and returns True if suitable for this IE."""
|
||||||
return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
|
return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
|
||||||
|
|
||||||
def _print_formats(self, formats):
|
@staticmethod
|
||||||
print('Available formats:')
|
def _transform_rtmp_url(rtmp_video_url):
|
||||||
for x in formats:
|
m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
|
||||||
print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
|
if not m:
|
||||||
|
raise ExtractorError(u'Cannot transform RTMP url')
|
||||||
|
base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
|
||||||
|
return base + m.group('finalid')
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url, re.VERBOSE)
|
mobj = re.match(self._VALID_URL, url, re.VERBOSE)
|
||||||
@ -155,40 +157,31 @@ class ComedyCentralIE(InfoExtractor):
|
|||||||
self._downloader.report_error(u'unable to download ' + mediaId + ': No videos found')
|
self._downloader.report_error(u'unable to download ' + mediaId + ': No videos found')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if self._downloader.params.get('listformats', None):
|
formats = []
|
||||||
self._print_formats([i[0] for i in turls])
|
for format, rtmp_video_url in turls:
|
||||||
return
|
w, h = self._video_dimensions.get(format, (None, None))
|
||||||
|
formats.append({
|
||||||
# For now, just pick the highest bitrate
|
'url': self._transform_rtmp_url(rtmp_video_url),
|
||||||
format,rtmp_video_url = turls[-1]
|
'ext': self._video_extensions.get(format, 'mp4'),
|
||||||
|
'format_id': format,
|
||||||
# Get the format arg from the arg stream
|
'height': h,
|
||||||
req_format = self._downloader.params.get('format', None)
|
'width': w,
|
||||||
|
})
|
||||||
# Select format if we can find one
|
|
||||||
for f,v in turls:
|
|
||||||
if f == req_format:
|
|
||||||
format, rtmp_video_url = f, v
|
|
||||||
break
|
|
||||||
|
|
||||||
m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
|
|
||||||
if not m:
|
|
||||||
raise ExtractorError(u'Cannot transform RTMP url')
|
|
||||||
base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
|
|
||||||
video_url = base + m.group('finalid')
|
|
||||||
|
|
||||||
effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1)
|
effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1)
|
||||||
info = {
|
info = {
|
||||||
'id': shortMediaId,
|
'id': shortMediaId,
|
||||||
'url': video_url,
|
'formats': formats,
|
||||||
'uploader': showId,
|
'uploader': showId,
|
||||||
'upload_date': officialDate,
|
'upload_date': officialDate,
|
||||||
'title': effTitle,
|
'title': effTitle,
|
||||||
'ext': 'mp4',
|
|
||||||
'format': format,
|
|
||||||
'thumbnail': None,
|
'thumbnail': None,
|
||||||
'description': compat_str(officialTitle),
|
'description': compat_str(officialTitle),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# TODO: Remove when #980 has been merged
|
||||||
|
info.update(info['formats'][-1])
|
||||||
|
|
||||||
results.append(info)
|
results.append(info)
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
@ -35,6 +35,8 @@ class InfoExtractor(object):
|
|||||||
title: Video title, unescaped.
|
title: Video title, unescaped.
|
||||||
ext: Video filename extension.
|
ext: Video filename extension.
|
||||||
|
|
||||||
|
Instead of url and ext, formats can also specified.
|
||||||
|
|
||||||
The following fields are optional:
|
The following fields are optional:
|
||||||
|
|
||||||
format: The video format, defaults to ext (used for --get-format)
|
format: The video format, defaults to ext (used for --get-format)
|
||||||
@ -52,8 +54,19 @@ class InfoExtractor(object):
|
|||||||
view_count: How many users have watched the video on the platform.
|
view_count: How many users have watched the video on the platform.
|
||||||
urlhandle: [internal] The urlHandle to be used to download the file,
|
urlhandle: [internal] The urlHandle to be used to download the file,
|
||||||
like returned by urllib.request.urlopen
|
like returned by urllib.request.urlopen
|
||||||
|
formats: A list of dictionaries for each format available, it must
|
||||||
|
be ordered from worst to best quality. Potential fields:
|
||||||
|
* url Mandatory. The URL of the video file
|
||||||
|
* ext Will be calculated from url if missing
|
||||||
|
* format A human-readable description of the format
|
||||||
|
("mp4 container with h264/opus").
|
||||||
|
Calculated from width and height if missing.
|
||||||
|
* format_id A short description of the format
|
||||||
|
("mp4_h264_opus" or "19")
|
||||||
|
* width Width of the video, if known
|
||||||
|
* height Height of the video, if known
|
||||||
|
|
||||||
The fields should all be Unicode strings.
|
Unless mentioned otherwise, the fields should be Unicode strings.
|
||||||
|
|
||||||
Subclasses of this one should re-define the _real_initialize() and
|
Subclasses of this one should re-define the _real_initialize() and
|
||||||
_real_extract() methods and define a _VALID_URL regexp.
|
_real_extract() methods and define a _VALID_URL regexp.
|
||||||
|
@ -3,28 +3,56 @@ import json
|
|||||||
import itertools
|
import itertools
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
from .subtitles import SubtitlesInfoExtractor
|
||||||
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
compat_urllib_request,
|
compat_urllib_request,
|
||||||
|
compat_str,
|
||||||
get_element_by_attribute,
|
get_element_by_attribute,
|
||||||
get_element_by_id,
|
get_element_by_id,
|
||||||
|
orderedSet,
|
||||||
|
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
)
|
)
|
||||||
|
|
||||||
class DailymotionIE(InfoExtractor):
|
class DailymotionBaseInfoExtractor(InfoExtractor):
|
||||||
|
@staticmethod
|
||||||
|
def _build_request(url):
|
||||||
|
"""Build a request with the family filter disabled"""
|
||||||
|
request = compat_urllib_request.Request(url)
|
||||||
|
request.add_header('Cookie', 'family_filter=off')
|
||||||
|
return request
|
||||||
|
|
||||||
|
class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||||
"""Information Extractor for Dailymotion"""
|
"""Information Extractor for Dailymotion"""
|
||||||
|
|
||||||
_VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
|
_VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)'
|
||||||
IE_NAME = u'dailymotion'
|
IE_NAME = u'dailymotion'
|
||||||
_TEST = {
|
_TESTS = [
|
||||||
u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
|
{
|
||||||
u'file': u'x33vw9.mp4',
|
u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
|
||||||
u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
|
u'file': u'x33vw9.mp4',
|
||||||
u'info_dict': {
|
u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
|
||||||
u"uploader": u"Amphora Alex and Van .",
|
u'info_dict': {
|
||||||
u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
|
u"uploader": u"Amphora Alex and Van .",
|
||||||
}
|
u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
# Vevo video
|
||||||
|
{
|
||||||
|
u'url': u'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
|
||||||
|
u'file': u'USUV71301934.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Roar (Official)',
|
||||||
|
u'uploader': u'Katy Perry',
|
||||||
|
u'upload_date': u'20130905',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
u'skip': u'VEVO is only available in some countries',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
# Extract id and simplified title from URL
|
# Extract id and simplified title from URL
|
||||||
@ -33,15 +61,24 @@ class DailymotionIE(InfoExtractor):
|
|||||||
video_id = mobj.group(1).split('_')[0].split('?')[0]
|
video_id = mobj.group(1).split('_')[0].split('?')[0]
|
||||||
|
|
||||||
video_extension = 'mp4'
|
video_extension = 'mp4'
|
||||||
|
url = 'http://www.dailymotion.com/video/%s' % video_id
|
||||||
|
|
||||||
# Retrieve video webpage to extract further information
|
# Retrieve video webpage to extract further information
|
||||||
request = compat_urllib_request.Request(url)
|
request = self._build_request(url)
|
||||||
request.add_header('Cookie', 'family_filter=off')
|
|
||||||
webpage = self._download_webpage(request, video_id)
|
webpage = self._download_webpage(request, video_id)
|
||||||
|
|
||||||
# Extract URL, uploader and title from webpage
|
# Extract URL, uploader and title from webpage
|
||||||
self.report_extraction(video_id)
|
self.report_extraction(video_id)
|
||||||
|
|
||||||
|
# It may just embed a vevo video:
|
||||||
|
m_vevo = re.search(
|
||||||
|
r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?videoId=(?P<id>[\w]*)',
|
||||||
|
webpage)
|
||||||
|
if m_vevo is not None:
|
||||||
|
vevo_id = m_vevo.group('id')
|
||||||
|
self.to_screen(u'Vevo video detected: %s' % vevo_id)
|
||||||
|
return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
|
||||||
|
|
||||||
video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
|
video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
|
||||||
# Looking for official user
|
# Looking for official user
|
||||||
r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
|
r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
|
||||||
@ -58,6 +95,9 @@ class DailymotionIE(InfoExtractor):
|
|||||||
info = self._search_regex(r'var info = ({.*?}),$', embed_page,
|
info = self._search_regex(r'var info = ({.*?}),$', embed_page,
|
||||||
'video info', flags=re.MULTILINE)
|
'video info', flags=re.MULTILINE)
|
||||||
info = json.loads(info)
|
info = json.loads(info)
|
||||||
|
if info.get('error') is not None:
|
||||||
|
msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
|
||||||
|
raise ExtractorError(msg, expected=True)
|
||||||
|
|
||||||
# TODO: support choosing qualities
|
# TODO: support choosing qualities
|
||||||
|
|
||||||
@ -72,6 +112,12 @@ class DailymotionIE(InfoExtractor):
|
|||||||
raise ExtractorError(u'Unable to extract video URL')
|
raise ExtractorError(u'Unable to extract video URL')
|
||||||
video_url = info[max_quality]
|
video_url = info[max_quality]
|
||||||
|
|
||||||
|
# subtitles
|
||||||
|
video_subtitles = self.extract_subtitles(video_id)
|
||||||
|
if self._downloader.params.get('listsubtitles', False):
|
||||||
|
self._list_available_subtitles(video_id)
|
||||||
|
return
|
||||||
|
|
||||||
return [{
|
return [{
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
@ -79,33 +125,76 @@ class DailymotionIE(InfoExtractor):
|
|||||||
'upload_date': video_upload_date,
|
'upload_date': video_upload_date,
|
||||||
'title': self._og_search_title(webpage),
|
'title': self._og_search_title(webpage),
|
||||||
'ext': video_extension,
|
'ext': video_extension,
|
||||||
|
'subtitles': video_subtitles,
|
||||||
'thumbnail': info['thumbnail_url']
|
'thumbnail': info['thumbnail_url']
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
def _get_available_subtitles(self, video_id):
|
||||||
|
try:
|
||||||
|
sub_list = self._download_webpage(
|
||||||
|
'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
|
||||||
|
video_id, note=False)
|
||||||
|
except ExtractorError as err:
|
||||||
|
self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
|
||||||
|
return {}
|
||||||
|
info = json.loads(sub_list)
|
||||||
|
if (info['total'] > 0):
|
||||||
|
sub_lang_list = dict((l['language'], l['url']) for l in info['list'])
|
||||||
|
return sub_lang_list
|
||||||
|
self._downloader.report_warning(u'video doesn\'t have subtitles')
|
||||||
|
return {}
|
||||||
|
|
||||||
class DailymotionPlaylistIE(InfoExtractor):
|
|
||||||
|
class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
|
||||||
|
IE_NAME = u'dailymotion:playlist'
|
||||||
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
|
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
|
||||||
_MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/playlist/.+?".*?>.*?</a>.*?</div>'
|
_MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/playlist/.+?".*?>.*?</a>.*?</div>'
|
||||||
|
_PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _extract_entries(self, id):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
|
||||||
playlist_id = mobj.group('id')
|
|
||||||
video_ids = []
|
video_ids = []
|
||||||
|
|
||||||
for pagenum in itertools.count(1):
|
for pagenum in itertools.count(1):
|
||||||
webpage = self._download_webpage('https://www.dailymotion.com/playlist/%s/%s' % (playlist_id, pagenum),
|
request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
|
||||||
playlist_id, u'Downloading page %s' % pagenum)
|
webpage = self._download_webpage(request,
|
||||||
|
id, u'Downloading page %s' % pagenum)
|
||||||
|
|
||||||
playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
|
playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
|
||||||
video_ids.extend(re.findall(r'data-id="(.+?)" data-ext-id', playlist_el))
|
video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))
|
||||||
|
|
||||||
if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
|
if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
|
||||||
break
|
break
|
||||||
|
return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
|
||||||
|
for video_id in orderedSet(video_ids)]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
playlist_id = mobj.group('id')
|
||||||
|
webpage = self._download_webpage(url, playlist_id)
|
||||||
|
|
||||||
entries = [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
|
|
||||||
for video_id in video_ids]
|
|
||||||
return {'_type': 'playlist',
|
return {'_type': 'playlist',
|
||||||
'id': playlist_id,
|
'id': playlist_id,
|
||||||
'title': get_element_by_id(u'playlist_name', webpage),
|
'title': get_element_by_id(u'playlist_name', webpage),
|
||||||
'entries': entries,
|
'entries': self._extract_entries(playlist_id),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class DailymotionUserIE(DailymotionPlaylistIE):
|
||||||
|
IE_NAME = u'dailymotion:user'
|
||||||
|
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
|
||||||
|
_MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/user/.+?".*?>.*?</a>.*?</div>'
|
||||||
|
_PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
user = mobj.group('user')
|
||||||
|
webpage = self._download_webpage(url, user)
|
||||||
|
full_user = self._html_search_regex(
|
||||||
|
r'<a class="label" href="/%s".*?>(.*?)</' % re.escape(user),
|
||||||
|
webpage, u'user', flags=re.DOTALL)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'playlist',
|
||||||
|
'id': user,
|
||||||
|
'title': full_user,
|
||||||
|
'entries': self._extract_entries(user),
|
||||||
|
}
|
||||||
|
@ -54,6 +54,7 @@ class DreiSatIE(InfoExtractor):
|
|||||||
'width': int(fe.find('./width').text),
|
'width': int(fe.find('./width').text),
|
||||||
'height': int(fe.find('./height').text),
|
'height': int(fe.find('./height').text),
|
||||||
'url': fe.find('./url').text,
|
'url': fe.find('./url').text,
|
||||||
|
'ext': determine_ext(fe.find('./url').text),
|
||||||
'filesize': int(fe.find('./filesize').text),
|
'filesize': int(fe.find('./filesize').text),
|
||||||
'video_bitrate': int(fe.find('./videoBitrate').text),
|
'video_bitrate': int(fe.find('./videoBitrate').text),
|
||||||
'3sat_qualityname': fe.find('./quality').text,
|
'3sat_qualityname': fe.find('./quality').text,
|
||||||
@ -79,7 +80,6 @@ class DreiSatIE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# TODO: Remove when #980 has been merged
|
# TODO: Remove when #980 has been merged
|
||||||
info['url'] = formats[-1]['url']
|
info.update(formats[-1])
|
||||||
info['ext'] = determine_ext(formats[-1]['url'])
|
|
||||||
|
|
||||||
return info
|
return info
|
||||||
|
37
youtube_dl/extractor/ebaumsworld.py
Normal file
37
youtube_dl/extractor/ebaumsworld.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
import re
|
||||||
|
import xml.etree.ElementTree
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import determine_ext
|
||||||
|
|
||||||
|
|
||||||
|
class EbaumsWorldIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://www\.ebaumsworld\.com/video/watch/(?P<id>\d+)'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.ebaumsworld.com/video/watch/83367677/',
|
||||||
|
u'file': u'83367677.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'A Giant Python Opens The Door',
|
||||||
|
u'description': u'This is how nightmares start...',
|
||||||
|
u'uploader': u'jihadpizza',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id = mobj.group('id')
|
||||||
|
config_xml = self._download_webpage(
|
||||||
|
'http://www.ebaumsworld.com/video/player/%s' % video_id, video_id)
|
||||||
|
config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8'))
|
||||||
|
video_url = config.find('file').text
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': config.find('title').text,
|
||||||
|
'url': video_url,
|
||||||
|
'ext': determine_ext(video_url),
|
||||||
|
'description': config.find('description').text,
|
||||||
|
'thumbnail': config.find('image').text,
|
||||||
|
'uploader': config.find('username').text,
|
||||||
|
}
|
@ -106,8 +106,8 @@ class FacebookIE(InfoExtractor):
|
|||||||
video_duration = int(video_data['video_duration'])
|
video_duration = int(video_data['video_duration'])
|
||||||
thumbnail = video_data['thumbnail_src']
|
thumbnail = video_data['thumbnail_src']
|
||||||
|
|
||||||
video_title = self._html_search_regex('<h2 class="uiHeaderTitle">([^<]+)</h2>',
|
video_title = self._html_search_regex(
|
||||||
webpage, u'title')
|
r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, u'title')
|
||||||
|
|
||||||
info = {
|
info = {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
|
79
youtube_dl/extractor/fktv.py
Normal file
79
youtube_dl/extractor/fktv.py
Normal file
@ -0,0 +1,79 @@
|
|||||||
|
import re
|
||||||
|
import random
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
determine_ext,
|
||||||
|
get_element_by_id,
|
||||||
|
clean_html,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FKTVIE(InfoExtractor):
|
||||||
|
IE_NAME = u'fernsehkritik.tv'
|
||||||
|
_VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://fernsehkritik.tv/folge-1',
|
||||||
|
u'file': u'00011.flv',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Folge 1 vom 10. April 2007',
|
||||||
|
u'description': u'md5:fb4818139c7cfe6907d4b83412a6864f',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
episode = int(mobj.group('ep'))
|
||||||
|
|
||||||
|
server = random.randint(2, 4)
|
||||||
|
video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%d.jpg' % episode
|
||||||
|
start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
|
||||||
|
episode)
|
||||||
|
playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
|
||||||
|
u'playlist', flags=re.DOTALL)
|
||||||
|
files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
|
||||||
|
# TODO: return a single multipart video
|
||||||
|
videos = []
|
||||||
|
for i, _ in enumerate(files, 1):
|
||||||
|
video_id = '%04d%d' % (episode, i)
|
||||||
|
video_url = 'http://dl%d.fernsehkritik.tv/fernsehkritik%d%s.flv' % (server, episode, '' if i == 1 else '-%d' % i)
|
||||||
|
video_title = 'Fernsehkritik %d.%d' % (episode, i)
|
||||||
|
videos.append({
|
||||||
|
'id': video_id,
|
||||||
|
'url': video_url,
|
||||||
|
'ext': determine_ext(video_url),
|
||||||
|
'title': clean_html(get_element_by_id('eptitle', start_webpage)),
|
||||||
|
'description': clean_html(get_element_by_id('contentlist', start_webpage)),
|
||||||
|
'thumbnail': video_thumbnail
|
||||||
|
})
|
||||||
|
return videos
|
||||||
|
|
||||||
|
|
||||||
|
class FKTVPosteckeIE(InfoExtractor):
|
||||||
|
IE_NAME = u'fernsehkritik.tv:postecke'
|
||||||
|
_VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik.tv/inline-video/postecke.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
|
||||||
|
u'file': u'0120.flv',
|
||||||
|
u'md5': u'262f0adbac80317412f7e57b4808e5c4',
|
||||||
|
u'info_dict': {
|
||||||
|
u"title": u"Postecke 120"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
episode = int(mobj.group('ep'))
|
||||||
|
|
||||||
|
server = random.randint(2, 4)
|
||||||
|
video_id = '%04d' % episode
|
||||||
|
video_url = 'http://dl%d.fernsehkritik.tv/postecke/postecke%d.flv' % (server, episode)
|
||||||
|
video_title = 'Postecke %d' % episode
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'url': video_url,
|
||||||
|
'ext': determine_ext(video_url),
|
||||||
|
'title': video_title,
|
||||||
|
}
|
@ -9,7 +9,7 @@ from ..utils import (
|
|||||||
|
|
||||||
class FlickrIE(InfoExtractor):
|
class FlickrIE(InfoExtractor):
|
||||||
"""Information Extractor for Flickr videos"""
|
"""Information Extractor for Flickr videos"""
|
||||||
_VALID_URL = r'(?:https?://)?(?:www\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
|
_VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
|
||||||
_TEST = {
|
_TEST = {
|
||||||
u'url': u'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
|
u'url': u'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
|
||||||
u'file': u'5645318632.mp4',
|
u'file': u'5645318632.mp4',
|
||||||
|
129
youtube_dl/extractor/francetv.py
Normal file
129
youtube_dl/extractor/francetv.py
Normal file
@ -0,0 +1,129 @@
|
|||||||
|
# encoding: utf-8
|
||||||
|
import re
|
||||||
|
import xml.etree.ElementTree
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
compat_urlparse,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FranceTVBaseInfoExtractor(InfoExtractor):
|
||||||
|
def _extract_video(self, video_id):
|
||||||
|
xml_desc = self._download_webpage(
|
||||||
|
'http://www.francetvinfo.fr/appftv/webservices/video/'
|
||||||
|
'getInfosOeuvre.php?id-diffusion='
|
||||||
|
+ video_id, video_id, 'Downloading XML config')
|
||||||
|
info = xml.etree.ElementTree.fromstring(xml_desc.encode('utf-8'))
|
||||||
|
|
||||||
|
manifest_url = info.find('videos/video/url').text
|
||||||
|
video_url = manifest_url.replace('manifest.f4m', 'index_2_av.m3u8')
|
||||||
|
video_url = video_url.replace('/z/', '/i/')
|
||||||
|
thumbnail_path = info.find('image').text
|
||||||
|
|
||||||
|
return {'id': video_id,
|
||||||
|
'ext': 'mp4',
|
||||||
|
'url': video_url,
|
||||||
|
'title': info.find('titre').text,
|
||||||
|
'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path),
|
||||||
|
'description': info.find('synopsis').text,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class PluzzIE(FranceTVBaseInfoExtractor):
|
||||||
|
IE_NAME = u'pluzz.francetv.fr'
|
||||||
|
_VALID_URL = r'https?://pluzz\.francetv\.fr/videos/(.*?)\.html'
|
||||||
|
|
||||||
|
# Can't use tests, videos expire in 7 days
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
title = re.match(self._VALID_URL, url).group(1)
|
||||||
|
webpage = self._download_webpage(url, title)
|
||||||
|
video_id = self._search_regex(
|
||||||
|
r'data-diffusion="(\d+)"', webpage, 'ID')
|
||||||
|
return self._extract_video(video_id)
|
||||||
|
|
||||||
|
|
||||||
|
class FranceTvInfoIE(FranceTVBaseInfoExtractor):
|
||||||
|
IE_NAME = u'francetvinfo.fr'
|
||||||
|
_VALID_URL = r'https?://www\.francetvinfo\.fr/replay.*/(?P<title>.+).html'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
|
||||||
|
u'file': u'84981923.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Soir 3',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
page_title = mobj.group('title')
|
||||||
|
webpage = self._download_webpage(url, page_title)
|
||||||
|
video_id = self._search_regex(r'id-video=(\d+?)"', webpage, u'video id')
|
||||||
|
return self._extract_video(video_id)
|
||||||
|
|
||||||
|
|
||||||
|
class France2IE(FranceTVBaseInfoExtractor):
|
||||||
|
IE_NAME = u'france2.fr'
|
||||||
|
_VALID_URL = r'''(?x)https?://www\.france2\.fr/
|
||||||
|
(?:
|
||||||
|
emissions/.*?/videos/(?P<id>\d+)
|
||||||
|
| emission/(?P<key>[^/?]+)
|
||||||
|
)'''
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104',
|
||||||
|
u'file': u'75540104.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'13h15, le samedi...',
|
||||||
|
u'description': u'md5:2e5b58ba7a2d3692b35c792be081a03d',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
if mobj.group('key'):
|
||||||
|
webpage = self._download_webpage(url, mobj.group('key'))
|
||||||
|
video_id = self._html_search_regex(
|
||||||
|
r'''(?x)<div\s+class="video-player">\s*
|
||||||
|
<a\s+href="http://videos.francetv.fr/video/([0-9]+)"\s+
|
||||||
|
class="francetv-video-player">''',
|
||||||
|
webpage, u'video ID')
|
||||||
|
else:
|
||||||
|
video_id = mobj.group('id')
|
||||||
|
return self._extract_video(video_id)
|
||||||
|
|
||||||
|
|
||||||
|
class GenerationQuoiIE(InfoExtractor):
|
||||||
|
IE_NAME = u'france2.fr:generation-quoi'
|
||||||
|
_VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<name>.*)(\?|$)'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://generation-quoi.france2.fr/portrait/garde-a-vous',
|
||||||
|
u'file': u'k7FJX8VBcvvLmX4wA5Q.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Génération Quoi - Garde à Vous',
|
||||||
|
u'uploader': u'Génération Quoi',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
# It uses Dailymotion
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
name = mobj.group('name')
|
||||||
|
info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % name)
|
||||||
|
info_json = self._download_webpage(info_url, name)
|
||||||
|
info = json.loads(info_json)
|
||||||
|
return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
|
||||||
|
ie='Dailymotion')
|
@ -21,7 +21,8 @@ class FunnyOrDieIE(InfoExtractor):
|
|||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
video_url = self._search_regex(r'type: "video/mp4", src: "(.*?)"',
|
video_url = self._search_regex(
|
||||||
|
[r'type="video/mp4" src="(.*?)"', r'src="([^>]*?)" type=\'video/mp4\''],
|
||||||
webpage, u'video URL', flags=re.DOTALL)
|
webpage, u'video URL', flags=re.DOTALL)
|
||||||
|
|
||||||
info = {
|
info = {
|
||||||
|
@ -14,7 +14,7 @@ class GameSpotIE(InfoExtractor):
|
|||||||
u"file": u"6410818.mp4",
|
u"file": u"6410818.mp4",
|
||||||
u"md5": u"b2a30deaa8654fcccd43713a6b6a4825",
|
u"md5": u"b2a30deaa8654fcccd43713a6b6a4825",
|
||||||
u"info_dict": {
|
u"info_dict": {
|
||||||
u"title": u"Arma III - Community Guide: SITREP I",
|
u"title": u"Arma 3 - Community Guide: SITREP I",
|
||||||
u"upload_date": u"20130627",
|
u"upload_date": u"20130627",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,17 +29,6 @@ class GenericIE(InfoExtractor):
|
|||||||
u"title": u"R\u00e9gis plante sa Jeep"
|
u"title": u"R\u00e9gis plante sa Jeep"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
|
||||||
u'url': u'http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/',
|
|
||||||
u'file': u'2371591881001.mp4',
|
|
||||||
u'md5': u'9e80619e0a94663f0bdc849b4566af19',
|
|
||||||
u'note': u'Test Brightcove downloads and detection in GenericIE',
|
|
||||||
u'info_dict': {
|
|
||||||
u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
|
|
||||||
u'uploader': u'8TV',
|
|
||||||
u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
|
|
||||||
}
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
|
|
||||||
def report_download_webpage(self, video_id):
|
def report_download_webpage(self, video_id):
|
||||||
@ -109,6 +98,11 @@ class GenericIE(InfoExtractor):
|
|||||||
return new_url
|
return new_url
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
|
parsed_url = compat_urlparse.urlparse(url)
|
||||||
|
if not parsed_url.scheme:
|
||||||
|
self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
|
||||||
|
return self.url_result('http://' + url)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
new_url = self._test_redirect(url)
|
new_url = self._test_redirect(url)
|
||||||
if new_url:
|
if new_url:
|
||||||
|
@ -40,7 +40,9 @@ class GooglePlusIE(InfoExtractor):
|
|||||||
self.report_extraction(video_id)
|
self.report_extraction(video_id)
|
||||||
|
|
||||||
# Extract update date
|
# Extract update date
|
||||||
upload_date = self._html_search_regex('title="Timestamp">(.*?)</a>',
|
upload_date = self._html_search_regex(
|
||||||
|
r'''(?x)<a.+?class="o-T-s\s[^"]+"\s+style="display:\s*none"\s*>
|
||||||
|
([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
|
||||||
webpage, u'upload date', fatal=False)
|
webpage, u'upload date', fatal=False)
|
||||||
if upload_date:
|
if upload_date:
|
||||||
# Convert timestring to a format suitable for filename
|
# Convert timestring to a format suitable for filename
|
||||||
|
@ -7,11 +7,11 @@ from .common import InfoExtractor
|
|||||||
class HotNewHipHopIE(InfoExtractor):
|
class HotNewHipHopIE(InfoExtractor):
|
||||||
_VALID_URL = r'http://www\.hotnewhiphop.com/.*\.(?P<id>.*)\.html'
|
_VALID_URL = r'http://www\.hotnewhiphop.com/.*\.(?P<id>.*)\.html'
|
||||||
_TEST = {
|
_TEST = {
|
||||||
u'url': u"http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html'",
|
u'url': u"http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html",
|
||||||
u'file': u'1435540.mp3',
|
u'file': u'1435540.mp3',
|
||||||
u'md5': u'2c2cd2f76ef11a9b3b581e8b232f3d96',
|
u'md5': u'2c2cd2f76ef11a9b3b581e8b232f3d96',
|
||||||
u'info_dict': {
|
u'info_dict': {
|
||||||
u"title": u"Freddie Gibbs Songs - Lay It Down"
|
u"title": u"Freddie Gibbs - Lay It Down"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -19,8 +19,7 @@ class HowcastIE(InfoExtractor):
|
|||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
webpage_url = 'http://www.howcast.com/videos/' + video_id
|
webpage = self._download_webpage(url, video_id)
|
||||||
webpage = self._download_webpage(webpage_url, video_id)
|
|
||||||
|
|
||||||
self.report_extraction(video_id)
|
self.report_extraction(video_id)
|
||||||
|
|
||||||
|
@ -13,7 +13,7 @@ class IGNIE(InfoExtractor):
|
|||||||
Some videos of it.ign.com are also supported
|
Some videos of it.ign.com are also supported
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles)(/.+)?/(?P<name_or_id>.+)'
|
_VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles|(?:[^/]*/feature))(/.+)?/(?P<name_or_id>.+)'
|
||||||
IE_NAME = u'ign.com'
|
IE_NAME = u'ign.com'
|
||||||
|
|
||||||
_CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config'
|
_CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config'
|
||||||
@ -21,15 +21,39 @@ class IGNIE(InfoExtractor):
|
|||||||
r'id="my_show_video">.*?<p>(.*?)</p>',
|
r'id="my_show_video">.*?<p>(.*?)</p>',
|
||||||
]
|
]
|
||||||
|
|
||||||
_TEST = {
|
_TESTS = [
|
||||||
u'url': u'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
|
{
|
||||||
u'file': u'8f862beef863986b2785559b9e1aa599.mp4',
|
u'url': u'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
|
||||||
u'md5': u'eac8bdc1890980122c3b66f14bdd02e9',
|
u'file': u'8f862beef863986b2785559b9e1aa599.mp4',
|
||||||
u'info_dict': {
|
u'md5': u'eac8bdc1890980122c3b66f14bdd02e9',
|
||||||
u'title': u'The Last of Us Review',
|
u'info_dict': {
|
||||||
u'description': u'md5:c8946d4260a4d43a00d5ae8ed998870c',
|
u'title': u'The Last of Us Review',
|
||||||
}
|
u'description': u'md5:c8946d4260a4d43a00d5ae8ed998870c',
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
u'url': u'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
|
||||||
|
u'playlist': [
|
||||||
|
{
|
||||||
|
u'file': u'5ebbd138523268b93c9141af17bec937.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'GTA 5 Video Review',
|
||||||
|
u'description': u'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
u'file': u'638672ee848ae4ff108df2a296418ee2.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'GTA 5\'s Twisted Beauty in Super Slow Motion',
|
||||||
|
u'description': u'The twisted beauty of GTA 5 in stunning slow motion.',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
def _find_video_id(self, webpage):
|
def _find_video_id(self, webpage):
|
||||||
res_id = [r'data-video-id="(.+?)"',
|
res_id = [r'data-video-id="(.+?)"',
|
||||||
@ -46,6 +70,13 @@ class IGNIE(InfoExtractor):
|
|||||||
if page_type == 'articles':
|
if page_type == 'articles':
|
||||||
video_url = self._search_regex(r'var videoUrl = "(.+?)"', webpage, u'video url')
|
video_url = self._search_regex(r'var videoUrl = "(.+?)"', webpage, u'video url')
|
||||||
return self.url_result(video_url, ie='IGN')
|
return self.url_result(video_url, ie='IGN')
|
||||||
|
elif page_type != 'video':
|
||||||
|
multiple_urls = re.findall(
|
||||||
|
'<param name="flashvars" value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
|
||||||
|
webpage)
|
||||||
|
if multiple_urls:
|
||||||
|
return [self.url_result(u, ie='IGN') for u in multiple_urls]
|
||||||
|
|
||||||
video_id = self._find_video_id(webpage)
|
video_id = self._find_video_id(webpage)
|
||||||
result = self._get_video_info(video_id)
|
result = self._get_video_info(video_id)
|
||||||
description = self._html_search_regex(self._DESCRIPTION_RE,
|
description = self._html_search_regex(self._DESCRIPTION_RE,
|
||||||
@ -87,6 +118,9 @@ class OneUPIE(IGNIE):
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Override IGN tests
|
||||||
|
_TESTS = []
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
id = mobj.group('name_or_id')
|
id = mobj.group('name_or_id')
|
||||||
|
@ -12,7 +12,7 @@ class JeuxVideoIE(InfoExtractor):
|
|||||||
_TEST = {
|
_TEST = {
|
||||||
u'url': u'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
|
u'url': u'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
|
||||||
u'file': u'5182.mp4',
|
u'file': u'5182.mp4',
|
||||||
u'md5': u'e0fdb0cd3ce98713ef9c1e1e025779d0',
|
u'md5': u'046e491afb32a8aaac1f44dd4ddd54ee',
|
||||||
u'info_dict': {
|
u'info_dict': {
|
||||||
u'title': u'GC 2013 : Tearaway nous présente ses papiers d\'identité',
|
u'title': u'GC 2013 : Tearaway nous présente ses papiers d\'identité',
|
||||||
u'description': u'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n',
|
u'description': u'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n',
|
||||||
|
37
youtube_dl/extractor/kickstarter.py
Normal file
37
youtube_dl/extractor/kickstarter.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class KickStarterIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>\d*)/.*'
|
||||||
|
_TEST = {
|
||||||
|
u"url": u"https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location",
|
||||||
|
u"file": u"1404461844.mp4",
|
||||||
|
u"md5": u"c81addca81327ffa66c642b5d8b08cab",
|
||||||
|
u"info_dict": {
|
||||||
|
u"title": u"Intersection: The Story of Josh Grant by Kyle Cowling",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
m = re.match(self._VALID_URL, url)
|
||||||
|
video_id = m.group('id')
|
||||||
|
webpage_src = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
video_url = self._search_regex(r'data-video="(.*?)">',
|
||||||
|
webpage_src, u'video URL')
|
||||||
|
if 'mp4' in video_url:
|
||||||
|
ext = 'mp4'
|
||||||
|
else:
|
||||||
|
ext = 'flv'
|
||||||
|
video_title = self._html_search_regex(r"<title>(.*?)</title>",
|
||||||
|
webpage_src, u'title').rpartition(u'\u2014 Kickstarter')[0].strip()
|
||||||
|
|
||||||
|
results = [{
|
||||||
|
'id': video_id,
|
||||||
|
'url': video_url,
|
||||||
|
'title': video_title,
|
||||||
|
'ext': ext,
|
||||||
|
}]
|
||||||
|
return results
|
@ -2,7 +2,12 @@ import re
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import compat_urllib_parse_urlparse, compat_urlparse
|
from ..utils import (
|
||||||
|
compat_urllib_parse_urlparse,
|
||||||
|
compat_urlparse,
|
||||||
|
get_meta_content,
|
||||||
|
ExtractorError,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class LivestreamIE(InfoExtractor):
|
class LivestreamIE(InfoExtractor):
|
||||||
@ -35,8 +40,11 @@ class LivestreamIE(InfoExtractor):
|
|||||||
|
|
||||||
if video_id is None:
|
if video_id is None:
|
||||||
# This is an event page:
|
# This is an event page:
|
||||||
api_url = self._search_regex(r'event_design_eventId: \'(.+?)\'',
|
player = get_meta_content('twitter:player', webpage)
|
||||||
webpage, 'api url')
|
if player is None:
|
||||||
|
raise ExtractorError('Couldn\'t extract event api url')
|
||||||
|
api_url = player.replace('/player', '')
|
||||||
|
api_url = re.sub(r'^(https?://)(new\.)', r'\1api.\2', api_url)
|
||||||
info = json.loads(self._download_webpage(api_url, event_name,
|
info = json.loads(self._download_webpage(api_url, event_name,
|
||||||
u'Downloading event info'))
|
u'Downloading event info'))
|
||||||
videos = [self._extract_video_info(video_data['data'])
|
videos = [self._extract_video_info(video_data['data'])
|
||||||
|
55
youtube_dl/extractor/metacritic.py
Normal file
55
youtube_dl/extractor/metacritic.py
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
import re
|
||||||
|
import xml.etree.ElementTree
|
||||||
|
import operator
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class MetacriticIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://www\.metacritic\.com/.+?/trailers/(?P<id>\d+)'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.metacritic.com/game/playstation-4/infamous-second-son/trailers/3698222',
|
||||||
|
u'file': u'3698222.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'inFamous: Second Son - inSide Sucker Punch: Smoke & Mirrors',
|
||||||
|
u'description': u'Take a peak behind-the-scenes to see how Sucker Punch brings smoke into the universe of inFAMOUS Second Son on the PS4.',
|
||||||
|
u'duration': 221,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id = mobj.group('id')
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
# The xml is not well formatted, there are raw '&'
|
||||||
|
info_xml = self._download_webpage('http://www.metacritic.com/video_data?video=' + video_id,
|
||||||
|
video_id, u'Downloading info xml').replace('&', '&')
|
||||||
|
info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8'))
|
||||||
|
|
||||||
|
clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id)
|
||||||
|
formats = []
|
||||||
|
for videoFile in clip.findall('httpURI/videoFile'):
|
||||||
|
rate_str = videoFile.find('rate').text
|
||||||
|
video_url = videoFile.find('filePath').text
|
||||||
|
formats.append({
|
||||||
|
'url': video_url,
|
||||||
|
'ext': 'mp4',
|
||||||
|
'format_id': rate_str,
|
||||||
|
'rate': int(rate_str),
|
||||||
|
})
|
||||||
|
formats.sort(key=operator.itemgetter('rate'))
|
||||||
|
|
||||||
|
description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>',
|
||||||
|
webpage, u'description', flags=re.DOTALL)
|
||||||
|
|
||||||
|
info = {
|
||||||
|
'id': video_id,
|
||||||
|
'title': clip.find('title').text,
|
||||||
|
'formats': formats,
|
||||||
|
'description': description,
|
||||||
|
'duration': int(clip.find('duration').text),
|
||||||
|
}
|
||||||
|
# TODO: Remove when #980 has been merged
|
||||||
|
info.update(formats[-1])
|
||||||
|
return info
|
@ -5,34 +5,27 @@ import socket
|
|||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
compat_http_client,
|
compat_http_client,
|
||||||
compat_str,
|
|
||||||
compat_urllib_error,
|
compat_urllib_error,
|
||||||
compat_urllib_request,
|
compat_urllib_request,
|
||||||
|
unified_strdate,
|
||||||
ExtractorError,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class MixcloudIE(InfoExtractor):
|
class MixcloudIE(InfoExtractor):
|
||||||
_WORKING = False # New API, but it seems good http://www.mixcloud.com/developers/documentation/
|
|
||||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
|
_VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
|
||||||
IE_NAME = u'mixcloud'
|
IE_NAME = u'mixcloud'
|
||||||
|
|
||||||
def report_download_json(self, file_id):
|
_TEST = {
|
||||||
"""Report JSON download."""
|
u'url': u'http://www.mixcloud.com/dholbach/cryptkeeper/',
|
||||||
self.to_screen(u'Downloading json')
|
u'file': u'dholbach-cryptkeeper.mp3',
|
||||||
|
u'info_dict': {
|
||||||
def get_urls(self, jsonData, fmt, bitrate='best'):
|
u'title': u'Cryptkeeper',
|
||||||
"""Get urls from 'audio_formats' section in json"""
|
u'description': u'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
|
||||||
try:
|
u'uploader': u'Daniel Holbach',
|
||||||
bitrate_list = jsonData[fmt]
|
u'uploader_id': u'dholbach',
|
||||||
if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
|
u'upload_date': u'20111115',
|
||||||
bitrate = max(bitrate_list) # select highest
|
},
|
||||||
|
}
|
||||||
url_list = jsonData[fmt][bitrate]
|
|
||||||
except TypeError: # we have no bitrate info.
|
|
||||||
url_list = jsonData[fmt]
|
|
||||||
return url_list
|
|
||||||
|
|
||||||
def check_urls(self, url_list):
|
def check_urls(self, url_list):
|
||||||
"""Returns 1st active url from list"""
|
"""Returns 1st active url from list"""
|
||||||
@ -45,71 +38,32 @@ class MixcloudIE(InfoExtractor):
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _print_formats(self, formats):
|
|
||||||
print('Available formats:')
|
|
||||||
for fmt in formats.keys():
|
|
||||||
for b in formats[fmt]:
|
|
||||||
try:
|
|
||||||
ext = formats[fmt][b][0]
|
|
||||||
print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
|
|
||||||
except TypeError: # we have no bitrate info
|
|
||||||
ext = formats[fmt][0]
|
|
||||||
print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
|
|
||||||
break
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
if mobj is None:
|
|
||||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
|
||||||
# extract uploader & filename from url
|
|
||||||
uploader = mobj.group(1).decode('utf-8')
|
|
||||||
file_id = uploader + "-" + mobj.group(2).decode('utf-8')
|
|
||||||
|
|
||||||
# construct API request
|
uploader = mobj.group(1)
|
||||||
file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
|
cloudcast_name = mobj.group(2)
|
||||||
# retrieve .json file with links to files
|
track_id = '-'.join((uploader, cloudcast_name))
|
||||||
request = compat_urllib_request.Request(file_url)
|
api_url = 'http://api.mixcloud.com/%s/%s/' % (uploader, cloudcast_name)
|
||||||
try:
|
webpage = self._download_webpage(url, track_id)
|
||||||
self.report_download_json(file_url)
|
json_data = self._download_webpage(api_url, track_id,
|
||||||
jsonData = compat_urllib_request.urlopen(request).read()
|
u'Downloading cloudcast info')
|
||||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
info = json.loads(json_data)
|
||||||
raise ExtractorError(u'Unable to retrieve file: %s' % compat_str(err))
|
|
||||||
|
|
||||||
# parse JSON
|
preview_url = self._search_regex(r'data-preview-url="(.+?)"', webpage, u'preview url')
|
||||||
json_data = json.loads(jsonData)
|
song_url = preview_url.replace('/previews/', '/cloudcasts/originals/')
|
||||||
player_url = json_data['player_swf_url']
|
template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
|
||||||
formats = dict(json_data['audio_formats'])
|
final_song_url = self.check_urls(template_url % i for i in range(30))
|
||||||
|
|
||||||
req_format = self._downloader.params.get('format', None)
|
return {
|
||||||
|
'id': track_id,
|
||||||
if self._downloader.params.get('listformats', None):
|
'title': info['name'],
|
||||||
self._print_formats(formats)
|
'url': final_song_url,
|
||||||
return
|
'ext': 'mp3',
|
||||||
|
'description': info['description'],
|
||||||
if req_format is None or req_format == 'best':
|
'thumbnail': info['pictures'].get('extra_large'),
|
||||||
for format_param in formats.keys():
|
'uploader': info['user']['name'],
|
||||||
url_list = self.get_urls(formats, format_param)
|
'uploader_id': info['user']['username'],
|
||||||
# check urls
|
'upload_date': unified_strdate(info['created_time']),
|
||||||
file_url = self.check_urls(url_list)
|
'view_count': info['play_count'],
|
||||||
if file_url is not None:
|
}
|
||||||
break # got it!
|
|
||||||
else:
|
|
||||||
if req_format not in formats:
|
|
||||||
raise ExtractorError(u'Format is not available')
|
|
||||||
|
|
||||||
url_list = self.get_urls(formats, req_format)
|
|
||||||
file_url = self.check_urls(url_list)
|
|
||||||
format_param = req_format
|
|
||||||
|
|
||||||
return [{
|
|
||||||
'id': file_id.decode('utf-8'),
|
|
||||||
'url': file_url.decode('utf-8'),
|
|
||||||
'uploader': uploader.decode('utf-8'),
|
|
||||||
'upload_date': None,
|
|
||||||
'title': json_data['name'],
|
|
||||||
'ext': file_url.split('.')[-1].decode('utf-8'),
|
|
||||||
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
|
|
||||||
'thumbnail': json_data['thumbnail_url'],
|
|
||||||
'description': json_data['description'],
|
|
||||||
'player_url': player_url.decode('utf-8'),
|
|
||||||
}]
|
|
||||||
|
@ -54,23 +54,26 @@ class MTVIE(InfoExtractor):
|
|||||||
def _get_thumbnail_url(self, uri, itemdoc):
|
def _get_thumbnail_url(self, uri, itemdoc):
|
||||||
return 'http://mtv.mtvnimages.com/uri/' + uri
|
return 'http://mtv.mtvnimages.com/uri/' + uri
|
||||||
|
|
||||||
def _extract_video_url(self, metadataXml):
|
def _extract_video_formats(self, metadataXml):
|
||||||
if '/error_country_block.swf' in metadataXml:
|
if '/error_country_block.swf' in metadataXml:
|
||||||
raise ExtractorError(u'This video is not available from your country.', expected=True)
|
raise ExtractorError(u'This video is not available from your country.', expected=True)
|
||||||
mdoc = xml.etree.ElementTree.fromstring(metadataXml.encode('utf-8'))
|
mdoc = xml.etree.ElementTree.fromstring(metadataXml.encode('utf-8'))
|
||||||
renditions = mdoc.findall('.//rendition')
|
renditions = mdoc.findall('.//rendition')
|
||||||
|
|
||||||
# For now, always pick the highest quality.
|
formats = []
|
||||||
rendition = renditions[-1]
|
for rendition in mdoc.findall('.//rendition'):
|
||||||
|
try:
|
||||||
try:
|
_, _, ext = rendition.attrib['type'].partition('/')
|
||||||
_,_,ext = rendition.attrib['type'].partition('/')
|
rtmp_video_url = rendition.find('./src').text
|
||||||
format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
|
formats.append({'ext': ext,
|
||||||
rtmp_video_url = rendition.find('./src').text
|
'url': self._transform_rtmp_url(rtmp_video_url),
|
||||||
except KeyError:
|
'format_id': rendition.get('bitrate'),
|
||||||
raise ExtractorError('Invalid rendition field.')
|
'width': int(rendition.get('width')),
|
||||||
video_url = self._transform_rtmp_url(rtmp_video_url)
|
'height': int(rendition.get('height')),
|
||||||
return {'ext': ext, 'url': video_url, 'format': format}
|
})
|
||||||
|
except (KeyError, TypeError):
|
||||||
|
raise ExtractorError('Invalid rendition field.')
|
||||||
|
return formats
|
||||||
|
|
||||||
def _get_video_info(self, itemdoc):
|
def _get_video_info(self, itemdoc):
|
||||||
uri = itemdoc.find('guid').text
|
uri = itemdoc.find('guid').text
|
||||||
@ -81,19 +84,25 @@ class MTVIE(InfoExtractor):
|
|||||||
mediagen_url += '&acceptMethods=fms'
|
mediagen_url += '&acceptMethods=fms'
|
||||||
mediagen_page = self._download_webpage(mediagen_url, video_id,
|
mediagen_page = self._download_webpage(mediagen_url, video_id,
|
||||||
u'Downloading video urls')
|
u'Downloading video urls')
|
||||||
video_info = self._extract_video_url(mediagen_page)
|
|
||||||
|
|
||||||
description_node = itemdoc.find('description')
|
description_node = itemdoc.find('description')
|
||||||
if description_node is not None:
|
if description_node is not None:
|
||||||
description = description_node.text
|
description = description_node.text
|
||||||
else:
|
else:
|
||||||
description = None
|
description = None
|
||||||
video_info.update({'title': itemdoc.find('title').text,
|
|
||||||
'id': video_id,
|
info = {
|
||||||
'thumbnail': self._get_thumbnail_url(uri, itemdoc),
|
'title': itemdoc.find('title').text,
|
||||||
'description': description,
|
'formats': self._extract_video_formats(mediagen_page),
|
||||||
})
|
'id': video_id,
|
||||||
return video_info
|
'thumbnail': self._get_thumbnail_url(uri, itemdoc),
|
||||||
|
'description': description,
|
||||||
|
}
|
||||||
|
|
||||||
|
# TODO: Remove when #980 has been merged
|
||||||
|
info.update(info['formats'][-1])
|
||||||
|
|
||||||
|
return info
|
||||||
|
|
||||||
def _get_videos_info(self, uri):
|
def _get_videos_info(self, uri):
|
||||||
video_id = self._id_from_uri(uri)
|
video_id = self._id_from_uri(uri)
|
||||||
|
38
youtube_dl/extractor/newgrounds.py
Normal file
38
youtube_dl/extractor/newgrounds.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import determine_ext
|
||||||
|
|
||||||
|
|
||||||
|
class NewgroundsIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'(?:https?://)?(?:www\.)?newgrounds\.com/audio/listen/(?P<id>\d+)'
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.newgrounds.com/audio/listen/549479',
|
||||||
|
u'file': u'549479.mp3',
|
||||||
|
u'md5': u'fe6033d297591288fa1c1f780386f07a',
|
||||||
|
u'info_dict': {
|
||||||
|
u"title": u"B7 - BusMode",
|
||||||
|
u"uploader": u"Burn7",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
music_id = mobj.group('id')
|
||||||
|
webpage = self._download_webpage(url, music_id)
|
||||||
|
|
||||||
|
title = self._html_search_regex(r',"name":"([^"]+)",', webpage, u'music title')
|
||||||
|
uploader = self._html_search_regex(r',"artist":"([^"]+)",', webpage, u'music uploader')
|
||||||
|
|
||||||
|
music_url_json_string = self._html_search_regex(r'({"url":"[^"]+"),', webpage, u'music url') + '}'
|
||||||
|
music_url_json = json.loads(music_url_json_string)
|
||||||
|
music_url = music_url_json['url']
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': music_id,
|
||||||
|
'title': title,
|
||||||
|
'url': music_url,
|
||||||
|
'uploader': uploader,
|
||||||
|
'ext': determine_ext(music_url),
|
||||||
|
}
|
@ -18,11 +18,15 @@ class OoyalaIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _url_for_embed_code(embed_code):
|
||||||
|
return 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code
|
||||||
|
|
||||||
def _extract_result(self, info, more_info):
|
def _extract_result(self, info, more_info):
|
||||||
return {'id': info['embedCode'],
|
return {'id': info['embedCode'],
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': unescapeHTML(info['title']),
|
'title': unescapeHTML(info['title']),
|
||||||
'url': info['url'],
|
'url': info.get('ipad_url') or info['url'],
|
||||||
'description': unescapeHTML(more_info['description']),
|
'description': unescapeHTML(more_info['description']),
|
||||||
'thumbnail': more_info['promo'],
|
'thumbnail': more_info['promo'],
|
||||||
}
|
}
|
||||||
@ -35,7 +39,9 @@ class OoyalaIE(InfoExtractor):
|
|||||||
mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
|
mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="',
|
||||||
player, u'mobile player url')
|
player, u'mobile player url')
|
||||||
mobile_player = self._download_webpage(mobile_url, embedCode)
|
mobile_player = self._download_webpage(mobile_url, embedCode)
|
||||||
videos_info = self._search_regex(r'eval\("\((\[{.*?stream_redirect.*?}\])\)"\);', mobile_player, u'info').replace('\\"','"')
|
videos_info = self._search_regex(
|
||||||
|
r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);',
|
||||||
|
mobile_player, u'info').replace('\\"','"')
|
||||||
videos_more_info = self._search_regex(r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, u'more info').replace('\\"','"')
|
videos_more_info = self._search_regex(r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, u'more info').replace('\\"','"')
|
||||||
videos_info = json.loads(videos_info)
|
videos_info = json.loads(videos_info)
|
||||||
videos_more_info =json.loads(videos_more_info)
|
videos_more_info =json.loads(videos_more_info)
|
||||||
|
@ -14,24 +14,25 @@ class RedTubeIE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def _real_extract(self,url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
video_extension = 'mp4'
|
video_extension = 'mp4'
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
self.report_extraction(video_id)
|
self.report_extraction(video_id)
|
||||||
|
|
||||||
video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">',
|
video_url = self._html_search_regex(
|
||||||
webpage, u'video URL')
|
r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')
|
||||||
|
|
||||||
video_title = self._html_search_regex('<h1 class="videoTitle slidePanelMovable">(.+?)</h1>',
|
video_title = self._html_search_regex(
|
||||||
|
r'<h1 class="videoTitle slidePanelMovable">(.+?)</h1>',
|
||||||
webpage, u'title')
|
webpage, u'title')
|
||||||
|
|
||||||
return [{
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'ext': video_extension,
|
'ext': video_extension,
|
||||||
'title': video_title,
|
'title': video_title,
|
||||||
}]
|
}
|
||||||
|
@ -8,8 +8,8 @@ from ..utils import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
class RTLnowIE(InfoExtractor):
|
class RTLnowIE(InfoExtractor):
|
||||||
"""Information Extractor for RTL NOW, RTL2 NOW, SUPER RTL NOW and VOX NOW"""
|
"""Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
|
||||||
_VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?superrtlnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
|
_VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?rtlnitronow\.de/|(?:www\.)?superrtlnow\.de/|(?:www\.)?n-tvnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
u'url': u'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
|
u'url': u'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
|
||||||
u'file': u'90419.flv',
|
u'file': u'90419.flv',
|
||||||
@ -61,8 +61,35 @@ class RTLnowIE(InfoExtractor):
|
|||||||
u'params': {
|
u'params': {
|
||||||
u'skip_download': True,
|
u'skip_download': True,
|
||||||
},
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
u'url': u'http://www.rtlnitronow.de/recht-ordnung/lebensmittelkontrolle-erlangenordnungsamt-berlin.php?film_id=127367&player=1&season=1',
|
||||||
|
u'file': u'127367.flv',
|
||||||
|
u'info_dict': {
|
||||||
|
u'upload_date': u'20130926',
|
||||||
|
u'title': u'Recht & Ordnung - Lebensmittelkontrolle Erlangen/Ordnungsamt...',
|
||||||
|
u'description': u'Lebensmittelkontrolle Erlangen/Ordnungsamt Berlin',
|
||||||
|
u'thumbnail': u'http://autoimg.static-fra.de/nitronow/344787/1500x1500/image2.jpg',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
u'url': u'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10',
|
||||||
|
u'file': u'124903.flv',
|
||||||
|
u'info_dict': {
|
||||||
|
u'upload_date': u'20130101',
|
||||||
|
u'title': u'Top Gear vom 01.01.2013',
|
||||||
|
u'description': u'Episode 1',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
u'skip': u'Only works from Germany',
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
|
||||||
def _real_extract(self,url):
|
def _real_extract(self,url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
|
||||||
@ -79,20 +106,23 @@ class RTLnowIE(InfoExtractor):
|
|||||||
msg = clean_html(note_m.group(1))
|
msg = clean_html(note_m.group(1))
|
||||||
raise ExtractorError(msg)
|
raise ExtractorError(msg)
|
||||||
|
|
||||||
video_title = self._html_search_regex(r'<title>(?P<title>[^<]+)</title>',
|
video_title = self._html_search_regex(r'<title>(?P<title>[^<]+?)( \| [^<]*)?</title>',
|
||||||
webpage, u'title')
|
webpage, u'title')
|
||||||
playerdata_url = self._html_search_regex(r'\'playerdata\': \'(?P<playerdata_url>[^\']+)\'',
|
playerdata_url = self._html_search_regex(r'\'playerdata\': \'(?P<playerdata_url>[^\']+)\'',
|
||||||
webpage, u'playerdata_url')
|
webpage, u'playerdata_url')
|
||||||
|
|
||||||
playerdata = self._download_webpage(playerdata_url, video_id)
|
playerdata = self._download_webpage(playerdata_url, video_id)
|
||||||
mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr\]\]></title>', playerdata)
|
mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)(?:\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr)?\]\]></title>', playerdata)
|
||||||
if mobj:
|
if mobj:
|
||||||
video_description = mobj.group(u'description')
|
video_description = mobj.group(u'description')
|
||||||
if mobj.group('upload_date_Y'):
|
if mobj.group('upload_date_Y'):
|
||||||
video_upload_date = mobj.group('upload_date_Y')
|
video_upload_date = mobj.group('upload_date_Y')
|
||||||
else:
|
elif mobj.group('upload_date_y'):
|
||||||
video_upload_date = u'20' + mobj.group('upload_date_y')
|
video_upload_date = u'20' + mobj.group('upload_date_y')
|
||||||
video_upload_date += mobj.group('upload_date_m')+mobj.group('upload_date_d')
|
else:
|
||||||
|
video_upload_date = None
|
||||||
|
if video_upload_date:
|
||||||
|
video_upload_date += mobj.group('upload_date_m')+mobj.group('upload_date_d')
|
||||||
else:
|
else:
|
||||||
video_description = None
|
video_description = None
|
||||||
video_upload_date = None
|
video_upload_date = None
|
||||||
|
47
youtube_dl/extractor/slideshare.py
Normal file
47
youtube_dl/extractor/slideshare.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
import re
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
compat_urlparse,
|
||||||
|
ExtractorError,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SlideshareIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://www\.slideshare\.net/[^/]+?/(?P<title>.+?)($|\?)'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.slideshare.net/Dataversity/keynote-presentation-managing-scale-and-complexity',
|
||||||
|
u'file': u'25665706.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Managing Scale and Complexity',
|
||||||
|
u'description': u'This was a keynote presentation at the NoSQL Now! 2013 Conference & Expo (http://www.nosqlnow.com). This presentation was given by Adrian Cockcroft from Netflix',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
page_title = mobj.group('title')
|
||||||
|
webpage = self._download_webpage(url, page_title)
|
||||||
|
slideshare_obj = self._search_regex(
|
||||||
|
r'var slideshare_object = ({.*?}); var user_info =',
|
||||||
|
webpage, u'slideshare object')
|
||||||
|
info = json.loads(slideshare_obj)
|
||||||
|
if info['slideshow']['type'] != u'video':
|
||||||
|
raise ExtractorError(u'Webpage type is "%s": only video extraction is supported for Slideshare' % info['slideshow']['type'], expected=True)
|
||||||
|
|
||||||
|
doc = info['doc']
|
||||||
|
bucket = info['jsplayer']['video_bucket']
|
||||||
|
ext = info['jsplayer']['video_extension']
|
||||||
|
video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'video',
|
||||||
|
'id': info['slideshow']['id'],
|
||||||
|
'title': info['slideshow']['title'],
|
||||||
|
'ext': ext,
|
||||||
|
'url': video_url,
|
||||||
|
'thumbnail': info['slideshow']['pin_image_url'],
|
||||||
|
'description': self._og_search_description(webpage),
|
||||||
|
}
|
@ -8,7 +8,7 @@ from ..utils import ExtractorError
|
|||||||
|
|
||||||
|
|
||||||
class SohuIE(InfoExtractor):
|
class SohuIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://tv\.sohu\.com/\d+?/n(?P<id>\d+)\.shtml.*?'
|
_VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
|
||||||
|
|
||||||
_TEST = {
|
_TEST = {
|
||||||
u'url': u'http://tv.sohu.com/20130724/n382479172.shtml#super',
|
u'url': u'http://tv.sohu.com/20130724/n382479172.shtml#super',
|
||||||
@ -21,8 +21,11 @@ class SohuIE(InfoExtractor):
|
|||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
|
|
||||||
def _fetch_data(vid_id):
|
def _fetch_data(vid_id, mytv=False):
|
||||||
base_data_url = u'http://hot.vrs.sohu.com/vrs_flash.action?vid='
|
if mytv:
|
||||||
|
base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
|
||||||
|
else:
|
||||||
|
base_data_url = u'http://hot.vrs.sohu.com/vrs_flash.action?vid='
|
||||||
data_url = base_data_url + str(vid_id)
|
data_url = base_data_url + str(vid_id)
|
||||||
data_json = self._download_webpage(
|
data_json = self._download_webpage(
|
||||||
data_url, video_id,
|
data_url, video_id,
|
||||||
@ -31,15 +34,16 @@ class SohuIE(InfoExtractor):
|
|||||||
|
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
|
mytv = mobj.group('mytv') is not None
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
raw_title = self._html_search_regex(r'(?s)<title>(.+?)</title>',
|
raw_title = self._html_search_regex(r'(?s)<title>(.+?)</title>',
|
||||||
webpage, u'video title')
|
webpage, u'video title')
|
||||||
title = raw_title.partition('-')[0].strip()
|
title = raw_title.partition('-')[0].strip()
|
||||||
|
|
||||||
vid = self._html_search_regex(r'var vid="(\d+)"', webpage,
|
vid = self._html_search_regex(r'var vid ?= ?["\'](\d+)["\']', webpage,
|
||||||
u'video path')
|
u'video path')
|
||||||
data = _fetch_data(vid)
|
data = _fetch_data(vid, mytv)
|
||||||
|
|
||||||
QUALITIES = ('ori', 'super', 'high', 'nor')
|
QUALITIES = ('ori', 'super', 'high', 'nor')
|
||||||
vid_ids = [data['data'][q + 'Vid']
|
vid_ids = [data['data'][q + 'Vid']
|
||||||
@ -51,7 +55,7 @@ class SohuIE(InfoExtractor):
|
|||||||
# For now, we just pick the highest available quality
|
# For now, we just pick the highest available quality
|
||||||
vid_id = vid_ids[-1]
|
vid_id = vid_ids[-1]
|
||||||
|
|
||||||
format_data = data if vid == vid_id else _fetch_data(vid_id)
|
format_data = data if vid == vid_id else _fetch_data(vid_id, mytv)
|
||||||
part_count = format_data['data']['totalBlocks']
|
part_count = format_data['data']['totalBlocks']
|
||||||
allot = format_data['allot']
|
allot = format_data['allot']
|
||||||
prot = format_data['prot']
|
prot = format_data['prot']
|
||||||
|
@ -1,10 +1,12 @@
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import itertools
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
|
compat_urllib_parse,
|
||||||
|
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
@ -53,10 +55,11 @@ class SoundcloudIE(InfoExtractor):
|
|||||||
def _resolv_url(cls, url):
|
def _resolv_url(cls, url):
|
||||||
return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
|
return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
|
||||||
|
|
||||||
def _extract_info_dict(self, info, full_title=None):
|
def _extract_info_dict(self, info, full_title=None, quiet=False):
|
||||||
video_id = info['id']
|
video_id = info['id']
|
||||||
name = full_title or video_id
|
name = full_title or video_id
|
||||||
self.report_extraction(name)
|
if quiet == False:
|
||||||
|
self.report_extraction(name)
|
||||||
|
|
||||||
thumbnail = info['artwork_url']
|
thumbnail = info['artwork_url']
|
||||||
if thumbnail is not None:
|
if thumbnail is not None:
|
||||||
@ -198,3 +201,41 @@ class SoundcloudSetIE(SoundcloudIE):
|
|||||||
'id': info['id'],
|
'id': info['id'],
|
||||||
'title': info['title'],
|
'title': info['title'],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class SoundcloudUserIE(SoundcloudIE):
|
||||||
|
_VALID_URL = r'https?://(www\.)?soundcloud.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$'
|
||||||
|
IE_NAME = u'soundcloud:user'
|
||||||
|
|
||||||
|
# it's in tests/test_playlists.py
|
||||||
|
_TEST = None
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
uploader = mobj.group('user')
|
||||||
|
|
||||||
|
url = 'http://soundcloud.com/%s/' % uploader
|
||||||
|
resolv_url = self._resolv_url(url)
|
||||||
|
user_json = self._download_webpage(resolv_url, uploader,
|
||||||
|
u'Downloading user info')
|
||||||
|
user = json.loads(user_json)
|
||||||
|
|
||||||
|
tracks = []
|
||||||
|
for i in itertools.count():
|
||||||
|
data = compat_urllib_parse.urlencode({'offset': i*50,
|
||||||
|
'client_id': self._CLIENT_ID,
|
||||||
|
})
|
||||||
|
tracks_url = 'http://api.soundcloud.com/users/%s/tracks.json?' % user['id'] + data
|
||||||
|
response = self._download_webpage(tracks_url, uploader,
|
||||||
|
u'Downloading tracks page %s' % (i+1))
|
||||||
|
new_tracks = json.loads(response)
|
||||||
|
tracks.extend(self._extract_info_dict(track, quiet=True) for track in new_tracks)
|
||||||
|
if len(new_tracks) < 50:
|
||||||
|
break
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'playlist',
|
||||||
|
'id': compat_str(user['id']),
|
||||||
|
'title': user['username'],
|
||||||
|
'entries': tracks,
|
||||||
|
}
|
||||||
|
38
youtube_dl/extractor/southparkstudios.py
Normal file
38
youtube_dl/extractor/southparkstudios.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from .mtv import MTVIE, _media_xml_tag
|
||||||
|
|
||||||
|
|
||||||
|
class SouthParkStudiosIE(MTVIE):
|
||||||
|
IE_NAME = u'southparkstudios.com'
|
||||||
|
_VALID_URL = r'https?://www\.southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$)'
|
||||||
|
|
||||||
|
_FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured',
|
||||||
|
u'file': u'a7bff6c2-ed00-11e0-aca6-0026b9414f30.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Bat Daded',
|
||||||
|
u'description': u'Randy disqualifies South Park by getting into a fight with Bat Dad.',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Overwrite MTVIE properties we don't want
|
||||||
|
_TESTS = []
|
||||||
|
|
||||||
|
def _get_thumbnail_url(self, uri, itemdoc):
|
||||||
|
search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))
|
||||||
|
thumb_node = itemdoc.find(search_path)
|
||||||
|
if thumb_node is None:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return thumb_node.attrib['url']
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id = mobj.group('id')
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"',
|
||||||
|
webpage, u'mgid')
|
||||||
|
return self._get_videos_info(mgid)
|
91
youtube_dl/extractor/subtitles.py
Normal file
91
youtube_dl/extractor/subtitles.py
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
from ..utils import (
|
||||||
|
compat_str,
|
||||||
|
ExtractorError,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SubtitlesInfoExtractor(InfoExtractor):
|
||||||
|
@property
|
||||||
|
def _have_to_download_any_subtitles(self):
|
||||||
|
return any([self._downloader.params.get('writesubtitles', False),
|
||||||
|
self._downloader.params.get('writeautomaticsub')])
|
||||||
|
|
||||||
|
def _list_available_subtitles(self, video_id, webpage=None):
|
||||||
|
""" outputs the available subtitles for the video """
|
||||||
|
sub_lang_list = self._get_available_subtitles(video_id)
|
||||||
|
auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
|
||||||
|
sub_lang = ",".join(list(sub_lang_list.keys()))
|
||||||
|
self.to_screen(u'%s: Available subtitles for video: %s' %
|
||||||
|
(video_id, sub_lang))
|
||||||
|
auto_lang = ",".join(auto_captions_list.keys())
|
||||||
|
self.to_screen(u'%s: Available automatic captions for video: %s' %
|
||||||
|
(video_id, auto_lang))
|
||||||
|
|
||||||
|
def extract_subtitles(self, video_id, video_webpage=None):
|
||||||
|
"""
|
||||||
|
returns {sub_lang: sub} ,{} if subtitles not found or None if the
|
||||||
|
subtitles aren't requested.
|
||||||
|
"""
|
||||||
|
if not self._have_to_download_any_subtitles:
|
||||||
|
return None
|
||||||
|
available_subs_list = {}
|
||||||
|
if self._downloader.params.get('writeautomaticsub', False):
|
||||||
|
available_subs_list.update(self._get_available_automatic_caption(video_id, video_webpage))
|
||||||
|
if self._downloader.params.get('writesubtitles', False):
|
||||||
|
available_subs_list.update(self._get_available_subtitles(video_id))
|
||||||
|
|
||||||
|
if not available_subs_list: # error, it didn't get the available subtitles
|
||||||
|
return {}
|
||||||
|
if self._downloader.params.get('allsubtitles', False):
|
||||||
|
sub_lang_list = available_subs_list
|
||||||
|
else:
|
||||||
|
if self._downloader.params.get('subtitleslangs', False):
|
||||||
|
requested_langs = self._downloader.params.get('subtitleslangs')
|
||||||
|
elif 'en' in available_subs_list:
|
||||||
|
requested_langs = ['en']
|
||||||
|
else:
|
||||||
|
requested_langs = [list(available_subs_list.keys())[0]]
|
||||||
|
|
||||||
|
sub_lang_list = {}
|
||||||
|
for sub_lang in requested_langs:
|
||||||
|
if not sub_lang in available_subs_list:
|
||||||
|
self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
|
||||||
|
continue
|
||||||
|
sub_lang_list[sub_lang] = available_subs_list[sub_lang]
|
||||||
|
|
||||||
|
subtitles = {}
|
||||||
|
for sub_lang, url in sub_lang_list.items():
|
||||||
|
subtitle = self._request_subtitle_url(sub_lang, url)
|
||||||
|
if subtitle:
|
||||||
|
subtitles[sub_lang] = subtitle
|
||||||
|
return subtitles
|
||||||
|
|
||||||
|
def _request_subtitle_url(self, sub_lang, url):
|
||||||
|
""" makes the http request for the subtitle """
|
||||||
|
try:
|
||||||
|
sub = self._download_webpage(url, None, note=False)
|
||||||
|
except ExtractorError as err:
|
||||||
|
self._downloader.report_warning(u'unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
|
||||||
|
return
|
||||||
|
if not sub:
|
||||||
|
self._downloader.report_warning(u'Did not fetch video subtitles')
|
||||||
|
return
|
||||||
|
return sub
|
||||||
|
|
||||||
|
def _get_available_subtitles(self, video_id):
|
||||||
|
"""
|
||||||
|
returns {sub_lang: url} or {} if not available
|
||||||
|
Must be redefined by the subclasses
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _get_available_automatic_caption(self, video_id, webpage):
|
||||||
|
"""
|
||||||
|
returns {sub_lang: url} or {} if not available
|
||||||
|
Must be redefined by the subclasses that support automatic captions,
|
||||||
|
otherwise it will return {}
|
||||||
|
"""
|
||||||
|
self._downloader.report_warning(u'Automatic Captions not supported by this server')
|
||||||
|
return {}
|
@ -77,12 +77,20 @@ class TEDIE(InfoExtractor):
|
|||||||
|
|
||||||
thumbnail = self._search_regex(r'</span>[\s.]*</div>[\s.]*<img src="(.*?)"',
|
thumbnail = self._search_regex(r'</span>[\s.]*</div>[\s.]*<img src="(.*?)"',
|
||||||
webpage, 'thumbnail')
|
webpage, 'thumbnail')
|
||||||
|
formats = [{
|
||||||
|
'ext': 'mp4',
|
||||||
|
'url': stream['file'],
|
||||||
|
'format': stream['id']
|
||||||
|
} for stream in info['htmlStreams']]
|
||||||
info = {
|
info = {
|
||||||
'id': info['id'],
|
'id': info['id'],
|
||||||
'url': info['htmlStreams'][-1]['file'],
|
'title': title,
|
||||||
'ext': 'mp4',
|
'thumbnail': thumbnail,
|
||||||
'title': title,
|
'description': desc,
|
||||||
'thumbnail': thumbnail,
|
'formats': formats,
|
||||||
'description': desc,
|
}
|
||||||
}
|
|
||||||
|
# TODO: Remove when #980 has been merged
|
||||||
|
info.update(info['formats'][-1])
|
||||||
|
|
||||||
return info
|
return info
|
||||||
|
@ -52,6 +52,7 @@ class TriluliluIE(InfoExtractor):
|
|||||||
{
|
{
|
||||||
'format': fnode.text,
|
'format': fnode.text,
|
||||||
'url': video_url_template % fnode.text,
|
'url': video_url_template % fnode.text,
|
||||||
|
'ext': fnode.text.partition('-')[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
for fnode in format_doc.findall('./formats/format')
|
for fnode in format_doc.findall('./formats/format')
|
||||||
@ -67,7 +68,6 @@ class TriluliluIE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# TODO: Remove when #980 has been merged
|
# TODO: Remove when #980 has been merged
|
||||||
info['url'] = formats[-1]['url']
|
info.update(formats[-1])
|
||||||
info['ext'] = formats[-1]['format'].partition('-')[0]
|
|
||||||
|
|
||||||
return info
|
return info
|
||||||
|
@ -1,6 +1,11 @@
|
|||||||
|
import json
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
compat_urlparse,
|
||||||
|
get_meta_content,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class UstreamIE(InfoExtractor):
|
class UstreamIE(InfoExtractor):
|
||||||
@ -43,3 +48,25 @@ class UstreamIE(InfoExtractor):
|
|||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
}
|
}
|
||||||
return info
|
return info
|
||||||
|
|
||||||
|
class UstreamChannelIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://www\.ustream\.tv/channel/(?P<slug>.+)'
|
||||||
|
IE_NAME = u'ustream:channel'
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
m = re.match(self._VALID_URL, url)
|
||||||
|
slug = m.group('slug')
|
||||||
|
webpage = self._download_webpage(url, slug)
|
||||||
|
channel_id = get_meta_content('ustream:channel_id', webpage)
|
||||||
|
|
||||||
|
BASE = 'http://www.ustream.tv'
|
||||||
|
next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
|
||||||
|
video_ids = []
|
||||||
|
while next_url:
|
||||||
|
reply = json.loads(self._download_webpage(compat_urlparse.urljoin(BASE, next_url), channel_id))
|
||||||
|
video_ids.extend(re.findall(r'data-content-id="(\d.*)"', reply['data']))
|
||||||
|
next_url = reply['nextUrl']
|
||||||
|
|
||||||
|
urls = ['http://www.ustream.tv/recorded/' + vid for vid in video_ids]
|
||||||
|
url_entries = [self.url_result(eurl, 'Ustream') for eurl in urls]
|
||||||
|
return self.playlist_result(url_entries, channel_id)
|
||||||
|
38
youtube_dl/extractor/vice.py
Normal file
38
youtube_dl/extractor/vice.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from .ooyala import OoyalaIE
|
||||||
|
from ..utils import ExtractorError
|
||||||
|
|
||||||
|
|
||||||
|
class ViceIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'http://www.vice.com/.*?/(?P<name>.+)'
|
||||||
|
|
||||||
|
_TEST = {
|
||||||
|
u'url': u'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
|
||||||
|
u'file': u'43cW1mYzpia9IlestBjVpd23Yu3afAfp.mp4',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
# Requires ffmpeg (m3u8 manifest)
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
name = mobj.group('name')
|
||||||
|
webpage = self._download_webpage(url, name)
|
||||||
|
try:
|
||||||
|
ooyala_url = self._og_search_video_url(webpage)
|
||||||
|
except ExtractorError:
|
||||||
|
try:
|
||||||
|
embed_code = self._search_regex(
|
||||||
|
r'OO.Player.create\(\'ooyalaplayer\', \'(.+?)\'', webpage,
|
||||||
|
u'ooyala embed code')
|
||||||
|
ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
|
||||||
|
except ExtractorError:
|
||||||
|
raise ExtractorError(u'The page doesn\'t contain a video', expected=True)
|
||||||
|
return self.url_result(ooyala_url, ie='Ooyala')
|
||||||
|
|
64
youtube_dl/extractor/viddler.py
Normal file
64
youtube_dl/extractor/viddler.py
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
determine_ext,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ViddlerIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler.com)/(?:v|embed|player)/(?P<id>[0-9]+)'
|
||||||
|
_TEST = {
|
||||||
|
u"url": u"http://www.viddler.com/v/43903784",
|
||||||
|
u'file': u'43903784.mp4',
|
||||||
|
u'md5': u'fbbaedf7813e514eb7ca30410f439ac9',
|
||||||
|
u'info_dict': {
|
||||||
|
u"title": u"Video Made Easy",
|
||||||
|
u"uploader": u"viddler",
|
||||||
|
u"duration": 100.89,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id = mobj.group('id')
|
||||||
|
|
||||||
|
embed_url = mobj.group('domain') + u'/embed/' + video_id
|
||||||
|
webpage = self._download_webpage(embed_url, video_id)
|
||||||
|
|
||||||
|
video_sources_code = self._search_regex(
|
||||||
|
r"(?ms)sources\s*:\s*(\{.*?\})", webpage, u'video URLs')
|
||||||
|
video_sources = json.loads(video_sources_code.replace("'", '"'))
|
||||||
|
|
||||||
|
formats = [{
|
||||||
|
'url': video_url,
|
||||||
|
'format': format_id,
|
||||||
|
} for video_url, format_id in video_sources.items()]
|
||||||
|
|
||||||
|
title = self._html_search_regex(
|
||||||
|
r"title\s*:\s*'([^']*)'", webpage, u'title')
|
||||||
|
uploader = self._html_search_regex(
|
||||||
|
r"authorName\s*:\s*'([^']*)'", webpage, u'uploader', fatal=False)
|
||||||
|
duration_s = self._html_search_regex(
|
||||||
|
r"duration\s*:\s*([0-9.]*)", webpage, u'duration', fatal=False)
|
||||||
|
duration = float(duration_s) if duration_s else None
|
||||||
|
thumbnail = self._html_search_regex(
|
||||||
|
r"thumbnail\s*:\s*'([^']*)'",
|
||||||
|
webpage, u'thumbnail', fatal=False)
|
||||||
|
|
||||||
|
info = {
|
||||||
|
'_type': 'video',
|
||||||
|
'id': video_id,
|
||||||
|
'title': title,
|
||||||
|
'thumbnail': thumbnail,
|
||||||
|
'uploader': uploader,
|
||||||
|
'duration': duration,
|
||||||
|
'formats': formats,
|
||||||
|
}
|
||||||
|
|
||||||
|
# TODO: Remove when #980 has been merged
|
||||||
|
info['formats'][-1]['ext'] = determine_ext(info['formats'][-1]['url'])
|
||||||
|
info.update(info['formats'][-1])
|
||||||
|
|
||||||
|
return info
|
@ -11,8 +11,8 @@ from ..utils import (
|
|||||||
|
|
||||||
class XHamsterIE(InfoExtractor):
|
class XHamsterIE(InfoExtractor):
|
||||||
"""Information Extractor for xHamster"""
|
"""Information Extractor for xHamster"""
|
||||||
_VALID_URL = r'(?:http://)?(?:www.)?xhamster\.com/movies/(?P<id>[0-9]+)/.*\.html'
|
_VALID_URL = r'(?:http://)?(?:www\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
u'url': u'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
|
u'url': u'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
|
||||||
u'file': u'1509445.flv',
|
u'file': u'1509445.flv',
|
||||||
u'md5': u'9f48e0e8d58e3076bb236ff412ab62fa',
|
u'md5': u'9f48e0e8d58e3076bb236ff412ab62fa',
|
||||||
@ -21,13 +21,24 @@ class XHamsterIE(InfoExtractor):
|
|||||||
u"uploader_id": u"Ruseful2011",
|
u"uploader_id": u"Ruseful2011",
|
||||||
u"title": u"FemaleAgent Shy beauty takes the bait"
|
u"title": u"FemaleAgent Shy beauty takes the bait"
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
u'url': u'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd',
|
||||||
|
u'file': u'2221348.flv',
|
||||||
|
u'md5': u'e767b9475de189320f691f49c679c4c7',
|
||||||
|
u'info_dict': {
|
||||||
|
u"upload_date": u"20130914",
|
||||||
|
u"uploader_id": u"jojo747400",
|
||||||
|
u"title": u"Britney Spears Sexy Booty"
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self,url):
|
def _real_extract(self,url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
mrss_url = 'http://xhamster.com/movies/%s/.html' % video_id
|
seo = mobj.group('seo')
|
||||||
|
mrss_url = 'http://xhamster.com/movies/%s/%s.html?hd' % (video_id, seo)
|
||||||
webpage = self._download_webpage(mrss_url, video_id)
|
webpage = self._download_webpage(mrss_url, video_id)
|
||||||
|
|
||||||
mobj = re.search(r'\'srv\': \'(?P<server>[^\']*)\',\s*\'file\': \'(?P<file>[^\']+)\',', webpage)
|
mobj = re.search(r'\'srv\': \'(?P<server>[^\']*)\',\s*\'file\': \'(?P<file>[^\']+)\',', webpage)
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
import datetime
|
|
||||||
import itertools
|
import itertools
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
@ -6,86 +5,89 @@ import re
|
|||||||
from .common import InfoExtractor, SearchInfoExtractor
|
from .common import InfoExtractor, SearchInfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
compat_urllib_parse,
|
compat_urllib_parse,
|
||||||
|
compat_urlparse,
|
||||||
ExtractorError,
|
determine_ext,
|
||||||
|
clean_html,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class YahooIE(InfoExtractor):
|
class YahooIE(InfoExtractor):
|
||||||
IE_DESC = u'Yahoo screen'
|
IE_DESC = u'Yahoo screen'
|
||||||
_VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P<id>\d*?)\.html'
|
_VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P<id>\d*?)\.html'
|
||||||
_TEST = {
|
_TESTS = [
|
||||||
u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
|
{
|
||||||
u'file': u'214727115.flv',
|
u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
|
||||||
u'md5': u'2e717f169c1be93d84d3794a00d4a325',
|
u'file': u'214727115.flv',
|
||||||
u'info_dict': {
|
u'info_dict': {
|
||||||
u"title": u"Julian Smith & Travis Legg Watch Julian Smith"
|
u'title': u'Julian Smith & Travis Legg Watch Julian Smith',
|
||||||
|
u'description': u'Julian and Travis watch Julian Smith',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
# Requires rtmpdump
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
u'skip': u'Requires rtmpdump'
|
{
|
||||||
}
|
u'url': u'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
|
||||||
|
u'file': u'103000935.flv',
|
||||||
|
u'info_dict': {
|
||||||
|
u'title': u'Codefellas - The Cougar Lies with Spanish Moss',
|
||||||
|
u'description': u'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
|
||||||
|
},
|
||||||
|
u'params': {
|
||||||
|
# Requires rtmpdump
|
||||||
|
u'skip_download': True,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
if mobj is None:
|
|
||||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
m_id = re.search(r'YUI\.namespace\("Media"\)\.CONTENT_ID = "(?P<new_id>.+?)";', webpage)
|
|
||||||
|
|
||||||
if m_id is None:
|
items_json = self._search_regex(r'YVIDEO_INIT_ITEMS = ({.*?});$',
|
||||||
# TODO: Check which url parameters are required
|
webpage, u'items', flags=re.MULTILINE)
|
||||||
info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
|
items = json.loads(items_json)
|
||||||
webpage = self._download_webpage(info_url, video_id, u'Downloading info webpage')
|
info = items['mediaItems']['query']['results']['mediaObj'][0]
|
||||||
info_re = r'''<title><!\[CDATA\[(?P<title>.*?)\]\]></title>.*
|
meta = info['meta']
|
||||||
<description><!\[CDATA\[(?P<description>.*?)\]\]></description>.*
|
|
||||||
<media:pubStart><!\[CDATA\[(?P<date>.*?)\ .*\]\]></media:pubStart>.*
|
|
||||||
<media:content\ medium="image"\ url="(?P<thumb>.*?)"\ name="LARGETHUMB"
|
|
||||||
'''
|
|
||||||
self.report_extraction(video_id)
|
|
||||||
m_info = re.search(info_re, webpage, re.VERBOSE|re.DOTALL)
|
|
||||||
if m_info is None:
|
|
||||||
raise ExtractorError(u'Unable to extract video info')
|
|
||||||
video_title = m_info.group('title')
|
|
||||||
video_description = m_info.group('description')
|
|
||||||
video_thumb = m_info.group('thumb')
|
|
||||||
video_date = m_info.group('date')
|
|
||||||
video_date = datetime.datetime.strptime(video_date, '%m/%d/%Y').strftime('%Y%m%d')
|
|
||||||
|
|
||||||
# TODO: Find a way to get mp4 videos
|
|
||||||
rest_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;element=stream;outputformat=mrss;id=%s;lmsoverride=1;bw=375;dynamicstream=1;cb=83521105;tech=flv,mp4;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
|
|
||||||
webpage = self._download_webpage(rest_url, video_id, u'Downloading video url webpage')
|
|
||||||
m_rest = re.search(r'<media:content url="(?P<url>.*?)" path="(?P<path>.*?)"', webpage)
|
|
||||||
video_url = m_rest.group('url')
|
|
||||||
video_path = m_rest.group('path')
|
|
||||||
if m_rest is None:
|
|
||||||
raise ExtractorError(u'Unable to extract video url')
|
|
||||||
|
|
||||||
else: # We have to use a different method if another id is defined
|
formats = []
|
||||||
long_id = m_id.group('new_id')
|
for s in info['streams']:
|
||||||
info_url = 'http://video.query.yahoo.com/v1/public/yql?q=SELECT%20*%20FROM%20yahoo.media.video.streams%20WHERE%20id%3D%22' + long_id + '%22%20AND%20format%3D%22mp4%2Cflv%22%20AND%20protocol%3D%22rtmp%2Chttp%22%20AND%20plrs%3D%2286Gj0vCaSzV_Iuf6hNylf2%22%20AND%20acctid%3D%22389%22%20AND%20plidl%3D%22%22%20AND%20pspid%3D%22792700001%22%20AND%20offnetwork%3D%22false%22%20AND%20site%3D%22ivy%22%20AND%20lang%3D%22en-US%22%20AND%20region%3D%22US%22%20AND%20override%3D%22none%22%3B&env=prod&format=json&callback=YUI.Env.JSONP.yui_3_8_1_1_1368368376830_335'
|
format_info = {
|
||||||
webpage = self._download_webpage(info_url, video_id, u'Downloading info json')
|
'width': s.get('width'),
|
||||||
json_str = re.search(r'YUI.Env.JSONP.yui.*?\((.*?)\);', webpage).group(1)
|
'height': s.get('height'),
|
||||||
info = json.loads(json_str)
|
'bitrate': s.get('bitrate'),
|
||||||
res = info[u'query'][u'results'][u'mediaObj'][0]
|
}
|
||||||
stream = res[u'streams'][0]
|
|
||||||
video_path = stream[u'path']
|
host = s['host']
|
||||||
video_url = stream[u'host']
|
path = s['path']
|
||||||
meta = res[u'meta']
|
if host.startswith('rtmp'):
|
||||||
video_title = meta[u'title']
|
format_info.update({
|
||||||
video_description = meta[u'description']
|
'url': host,
|
||||||
video_thumb = meta[u'thumbnail']
|
'play_path': path,
|
||||||
video_date = None # I can't find it
|
'ext': 'flv',
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
format_url = compat_urlparse.urljoin(host, path)
|
||||||
|
format_info['url'] = format_url
|
||||||
|
format_info['ext'] = determine_ext(format_url)
|
||||||
|
|
||||||
|
formats.append(format_info)
|
||||||
|
formats = sorted(formats, key=lambda f:(f['height'], f['width']))
|
||||||
|
|
||||||
|
info = {
|
||||||
|
'id': video_id,
|
||||||
|
'title': meta['title'],
|
||||||
|
'formats': formats,
|
||||||
|
'description': clean_html(meta['description']),
|
||||||
|
'thumbnail': meta['thumbnail'],
|
||||||
|
}
|
||||||
|
# TODO: Remove when #980 has been merged
|
||||||
|
info.update(formats[-1])
|
||||||
|
|
||||||
|
return info
|
||||||
|
|
||||||
info_dict = {
|
|
||||||
'id': video_id,
|
|
||||||
'url': video_url,
|
|
||||||
'play_path': video_path,
|
|
||||||
'title':video_title,
|
|
||||||
'description': video_description,
|
|
||||||
'thumbnail': video_thumb,
|
|
||||||
'upload_date': video_date,
|
|
||||||
'ext': 'flv',
|
|
||||||
}
|
|
||||||
return info_dict
|
|
||||||
|
|
||||||
class YahooSearchIE(SearchInfoExtractor):
|
class YahooSearchIE(SearchInfoExtractor):
|
||||||
IE_DESC = u'Yahoo screen search'
|
IE_DESC = u'Yahoo screen search'
|
||||||
|
@ -66,6 +66,12 @@ class YoukuIE(InfoExtractor):
|
|||||||
self.report_extraction(video_id)
|
self.report_extraction(video_id)
|
||||||
try:
|
try:
|
||||||
config = json.loads(jsondata)
|
config = json.loads(jsondata)
|
||||||
|
error_code = config['data'][0].get('error_code')
|
||||||
|
if error_code:
|
||||||
|
# -8 means blocked outside China.
|
||||||
|
error = config['data'][0].get('error') # Chinese and English, separated by newline.
|
||||||
|
raise ExtractorError(error or u'Server reported error %i' % error_code,
|
||||||
|
expected=True)
|
||||||
|
|
||||||
video_title = config['data'][0]['title']
|
video_title = config['data'][0]['title']
|
||||||
seed = config['data'][0]['seed']
|
seed = config['data'][0]['seed']
|
||||||
@ -89,6 +95,7 @@ class YoukuIE(InfoExtractor):
|
|||||||
|
|
||||||
fileid = config['data'][0]['streamfileids'][format]
|
fileid = config['data'][0]['streamfileids'][format]
|
||||||
keys = [s['k'] for s in config['data'][0]['segs'][format]]
|
keys = [s['k'] for s in config['data'][0]['segs'][format]]
|
||||||
|
# segs is usually a dictionary, but an empty *list* if an error occured.
|
||||||
except (UnicodeDecodeError, ValueError, KeyError):
|
except (UnicodeDecodeError, ValueError, KeyError):
|
||||||
raise ExtractorError(u'Unable to extract info section')
|
raise ExtractorError(u'Unable to extract info section')
|
||||||
|
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -2,16 +2,14 @@ import re
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
unescapeHTML,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class ZDFIE(InfoExtractor):
|
class ZDFIE(InfoExtractor):
|
||||||
_VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?'
|
_VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek(?P<hash>#)?\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?'
|
||||||
_TITLE = r'<h1(?: class="beitragHeadline")?>(?P<title>.*)</h1>'
|
|
||||||
_MEDIA_STREAM = r'<a href="(?P<video_url>.+(?P<media_type>.streaming).+/zdf/(?P<quality>[^\/]+)/[^"]*)".+class="play".+>'
|
_MEDIA_STREAM = r'<a href="(?P<video_url>.+(?P<media_type>.streaming).+/zdf/(?P<quality>[^\/]+)/[^"]*)".+class="play".+>'
|
||||||
_MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"'
|
|
||||||
_RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)'
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
@ -19,6 +17,9 @@ class ZDFIE(InfoExtractor):
|
|||||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||||
video_id = mobj.group('video_id')
|
video_id = mobj.group('video_id')
|
||||||
|
|
||||||
|
if mobj.group('hash'):
|
||||||
|
url = url.replace(u'#', u'', 1)
|
||||||
|
|
||||||
html = self._download_webpage(url, video_id)
|
html = self._download_webpage(url, video_id)
|
||||||
streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)]
|
streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)]
|
||||||
if streams is None:
|
if streams is None:
|
||||||
@ -27,39 +28,48 @@ class ZDFIE(InfoExtractor):
|
|||||||
# s['media_type'] == 'wstreaming' -> use 'Windows Media Player' and mms url
|
# s['media_type'] == 'wstreaming' -> use 'Windows Media Player' and mms url
|
||||||
# s['media_type'] == 'hstreaming' -> use 'Quicktime' and rtsp url
|
# s['media_type'] == 'hstreaming' -> use 'Quicktime' and rtsp url
|
||||||
# choose first/default media type and highest quality for now
|
# choose first/default media type and highest quality for now
|
||||||
for s in streams: #find 300 - dsl1000mbit
|
def stream_pref(s):
|
||||||
if s['quality'] == '300' and s['media_type'] == 'wstreaming':
|
TYPE_ORDER = ['ostreaming', 'hstreaming', 'wstreaming']
|
||||||
stream_=s
|
try:
|
||||||
break
|
type_pref = TYPE_ORDER.index(s['media_type'])
|
||||||
for s in streams: #find veryhigh - dsl2000mbit
|
except ValueError:
|
||||||
if s['quality'] == 'veryhigh' and s['media_type'] == 'wstreaming': # 'hstreaming' - rtsp is not working
|
type_pref = 999
|
||||||
stream_=s
|
|
||||||
break
|
QUALITY_ORDER = ['veryhigh', '300']
|
||||||
if stream_ is None:
|
try:
|
||||||
|
quality_pref = QUALITY_ORDER.index(s['quality'])
|
||||||
|
except ValueError:
|
||||||
|
quality_pref = 999
|
||||||
|
|
||||||
|
return (type_pref, quality_pref)
|
||||||
|
|
||||||
|
sorted_streams = sorted(streams, key=stream_pref)
|
||||||
|
if not sorted_streams:
|
||||||
raise ExtractorError(u'No stream found.')
|
raise ExtractorError(u'No stream found.')
|
||||||
|
stream = sorted_streams[0]
|
||||||
|
|
||||||
media_link = self._download_webpage(stream_['video_url'], video_id,'Get stream URL')
|
media_link = self._download_webpage(
|
||||||
|
stream['video_url'],
|
||||||
|
video_id,
|
||||||
|
u'Get stream URL')
|
||||||
|
|
||||||
self.report_extraction(video_id)
|
MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"'
|
||||||
mobj = re.search(self._TITLE, html)
|
RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)'
|
||||||
|
|
||||||
|
mobj = re.search(self._MEDIA_STREAM, media_link)
|
||||||
if mobj is None:
|
if mobj is None:
|
||||||
raise ExtractorError(u'Cannot extract title')
|
mobj = re.search(RTSP_STREAM, media_link)
|
||||||
title = unescapeHTML(mobj.group('title'))
|
|
||||||
|
|
||||||
mobj = re.search(self._MMS_STREAM, media_link)
|
|
||||||
if mobj is None:
|
|
||||||
mobj = re.search(self._RTSP_STREAM, media_link)
|
|
||||||
if mobj is None:
|
if mobj is None:
|
||||||
raise ExtractorError(u'Cannot extract mms:// or rtsp:// URL')
|
raise ExtractorError(u'Cannot extract mms:// or rtsp:// URL')
|
||||||
mms_url = mobj.group('video_url')
|
video_url = mobj.group('video_url')
|
||||||
|
|
||||||
mobj = re.search('(.*)[.](?P<ext>[^.]+)', mms_url)
|
title = self._html_search_regex(
|
||||||
if mobj is None:
|
r'<h1(?: class="beitragHeadline")?>(.*?)</h1>',
|
||||||
raise ExtractorError(u'Cannot extract extention')
|
html, u'title')
|
||||||
ext = mobj.group('ext')
|
|
||||||
|
|
||||||
return [{'id': video_id,
|
return {
|
||||||
'url': mms_url,
|
'id': video_id,
|
||||||
'title': title,
|
'url': video_url,
|
||||||
'ext': ext
|
'title': title,
|
||||||
}]
|
'ext': determine_ext(video_url)
|
||||||
|
}
|
||||||
|
@ -1,6 +1,9 @@
|
|||||||
|
import io
|
||||||
import json
|
import json
|
||||||
import traceback
|
import traceback
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
from zipimport import zipimporter
|
from zipimport import zipimporter
|
||||||
|
|
||||||
from .utils import *
|
from .utils import *
|
||||||
@ -34,7 +37,7 @@ def rsa_verify(message, signature, key):
|
|||||||
if signature != sha256(message).digest(): return False
|
if signature != sha256(message).digest(): return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def update_self(to_screen, verbose, filename):
|
def update_self(to_screen, verbose):
|
||||||
"""Update the program file with the latest version from the repository"""
|
"""Update the program file with the latest version from the repository"""
|
||||||
|
|
||||||
UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
|
UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
|
||||||
@ -42,7 +45,6 @@ def update_self(to_screen, verbose, filename):
|
|||||||
JSON_URL = UPDATE_URL + 'versions.json'
|
JSON_URL = UPDATE_URL + 'versions.json'
|
||||||
UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
|
UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
|
||||||
|
|
||||||
|
|
||||||
if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
|
if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
|
||||||
to_screen(u'It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
|
to_screen(u'It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
|
||||||
return
|
return
|
||||||
@ -75,11 +77,18 @@ def update_self(to_screen, verbose, filename):
|
|||||||
to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
|
to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
|
||||||
return
|
return
|
||||||
|
|
||||||
to_screen(u'Updating to version ' + versions_info['latest'] + '...')
|
version_id = versions_info['latest']
|
||||||
version = versions_info['versions'][versions_info['latest']]
|
to_screen(u'Updating to version ' + version_id + '...')
|
||||||
|
version = versions_info['versions'][version_id]
|
||||||
|
|
||||||
print_notes(to_screen, versions_info['versions'])
|
print_notes(to_screen, versions_info['versions'])
|
||||||
|
|
||||||
|
filename = sys.argv[0]
|
||||||
|
# Py2EXE: Filename could be different
|
||||||
|
if hasattr(sys, "frozen") and not os.path.isfile(filename):
|
||||||
|
if os.path.isfile(filename + u'.exe'):
|
||||||
|
filename += u'.exe'
|
||||||
|
|
||||||
if not os.access(filename, os.W_OK):
|
if not os.access(filename, os.W_OK):
|
||||||
to_screen(u'ERROR: no write permissions on %s' % filename)
|
to_screen(u'ERROR: no write permissions on %s' % filename)
|
||||||
return
|
return
|
||||||
@ -116,16 +125,18 @@ def update_self(to_screen, verbose, filename):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
bat = os.path.join(directory, 'youtube-dl-updater.bat')
|
bat = os.path.join(directory, 'youtube-dl-updater.bat')
|
||||||
b = open(bat, 'w')
|
with io.open(bat, 'w') as batfile:
|
||||||
b.write("""
|
batfile.write(u"""
|
||||||
echo Updating youtube-dl...
|
@echo off
|
||||||
|
echo Waiting for file handle to be closed ...
|
||||||
ping 127.0.0.1 -n 5 -w 1000 > NUL
|
ping 127.0.0.1 -n 5 -w 1000 > NUL
|
||||||
move /Y "%s.new" "%s"
|
move /Y "%s.new" "%s" > NUL
|
||||||
del "%s"
|
echo Updated youtube-dl to version %s.
|
||||||
\n""" %(exe, exe, bat))
|
start /b "" cmd /c del "%%~f0"&exit /b"
|
||||||
b.close()
|
\n""" % (exe, exe, version_id))
|
||||||
|
|
||||||
os.startfile(bat)
|
subprocess.Popen([bat]) # Continues to run in the background
|
||||||
|
return # Do not show premature success messages
|
||||||
except (IOError, OSError) as err:
|
except (IOError, OSError) as err:
|
||||||
if verbose: to_screen(compat_str(traceback.format_exc()))
|
if verbose: to_screen(compat_str(traceback.format_exc()))
|
||||||
to_screen(u'ERROR: unable to overwrite current version')
|
to_screen(u'ERROR: unable to overwrite current version')
|
||||||
|
@ -66,6 +66,12 @@ try:
|
|||||||
except ImportError: # Python 2
|
except ImportError: # Python 2
|
||||||
from urllib2 import HTTPError as compat_HTTPError
|
from urllib2 import HTTPError as compat_HTTPError
|
||||||
|
|
||||||
|
try:
|
||||||
|
from urllib.request import urlretrieve as compat_urlretrieve
|
||||||
|
except ImportError: # Python 2
|
||||||
|
from urllib import urlretrieve as compat_urlretrieve
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from subprocess import DEVNULL
|
from subprocess import DEVNULL
|
||||||
compat_subprocess_get_DEVNULL = lambda: DEVNULL
|
compat_subprocess_get_DEVNULL = lambda: DEVNULL
|
||||||
@ -169,7 +175,7 @@ def compat_ord(c):
|
|||||||
compiled_regex_type = type(re.compile(''))
|
compiled_regex_type = type(re.compile(''))
|
||||||
|
|
||||||
std_headers = {
|
std_headers = {
|
||||||
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
|
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
|
||||||
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
|
||||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||||
'Accept-Encoding': 'gzip, deflate',
|
'Accept-Encoding': 'gzip, deflate',
|
||||||
@ -249,7 +255,17 @@ def htmlentity_transform(matchobj):
|
|||||||
return (u'&%s;' % entity)
|
return (u'&%s;' % entity)
|
||||||
|
|
||||||
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
|
compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
|
||||||
class AttrParser(compat_html_parser.HTMLParser):
|
class BaseHTMLParser(compat_html_parser.HTMLParser):
|
||||||
|
def __init(self):
|
||||||
|
compat_html_parser.HTMLParser.__init__(self)
|
||||||
|
self.html = None
|
||||||
|
|
||||||
|
def loads(self, html):
|
||||||
|
self.html = html
|
||||||
|
self.feed(html)
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
class AttrParser(BaseHTMLParser):
|
||||||
"""Modified HTMLParser that isolates a tag with the specified attribute"""
|
"""Modified HTMLParser that isolates a tag with the specified attribute"""
|
||||||
def __init__(self, attribute, value):
|
def __init__(self, attribute, value):
|
||||||
self.attribute = attribute
|
self.attribute = attribute
|
||||||
@ -257,10 +273,9 @@ class AttrParser(compat_html_parser.HTMLParser):
|
|||||||
self.result = None
|
self.result = None
|
||||||
self.started = False
|
self.started = False
|
||||||
self.depth = {}
|
self.depth = {}
|
||||||
self.html = None
|
|
||||||
self.watch_startpos = False
|
self.watch_startpos = False
|
||||||
self.error_count = 0
|
self.error_count = 0
|
||||||
compat_html_parser.HTMLParser.__init__(self)
|
BaseHTMLParser.__init__(self)
|
||||||
|
|
||||||
def error(self, message):
|
def error(self, message):
|
||||||
if self.error_count > 10 or self.started:
|
if self.error_count > 10 or self.started:
|
||||||
@ -269,11 +284,6 @@ class AttrParser(compat_html_parser.HTMLParser):
|
|||||||
self.error_count += 1
|
self.error_count += 1
|
||||||
self.goahead(1)
|
self.goahead(1)
|
||||||
|
|
||||||
def loads(self, html):
|
|
||||||
self.html = html
|
|
||||||
self.feed(html)
|
|
||||||
self.close()
|
|
||||||
|
|
||||||
def handle_starttag(self, tag, attrs):
|
def handle_starttag(self, tag, attrs):
|
||||||
attrs = dict(attrs)
|
attrs = dict(attrs)
|
||||||
if self.started:
|
if self.started:
|
||||||
@ -334,6 +344,38 @@ def get_element_by_attribute(attribute, value, html):
|
|||||||
pass
|
pass
|
||||||
return parser.get_result()
|
return parser.get_result()
|
||||||
|
|
||||||
|
class MetaParser(BaseHTMLParser):
|
||||||
|
"""
|
||||||
|
Modified HTMLParser that isolates a meta tag with the specified name
|
||||||
|
attribute.
|
||||||
|
"""
|
||||||
|
def __init__(self, name):
|
||||||
|
BaseHTMLParser.__init__(self)
|
||||||
|
self.name = name
|
||||||
|
self.content = None
|
||||||
|
self.result = None
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
if tag != 'meta':
|
||||||
|
return
|
||||||
|
attrs = dict(attrs)
|
||||||
|
if attrs.get('name') == self.name:
|
||||||
|
self.result = attrs.get('content')
|
||||||
|
|
||||||
|
def get_result(self):
|
||||||
|
return self.result
|
||||||
|
|
||||||
|
def get_meta_content(name, html):
|
||||||
|
"""
|
||||||
|
Return the content attribute from the meta tag with the given name attribute.
|
||||||
|
"""
|
||||||
|
parser = MetaParser(name)
|
||||||
|
try:
|
||||||
|
parser.loads(html)
|
||||||
|
except compat_html_parser.HTMLParseError:
|
||||||
|
pass
|
||||||
|
return parser.get_result()
|
||||||
|
|
||||||
|
|
||||||
def clean_html(html):
|
def clean_html(html):
|
||||||
"""Clean an HTML snippet into a readable string"""
|
"""Clean an HTML snippet into a readable string"""
|
||||||
@ -664,7 +706,16 @@ def unified_strdate(date_str):
|
|||||||
date_str = date_str.replace(',',' ')
|
date_str = date_str.replace(',',' ')
|
||||||
# %z (UTC offset) is only supported in python>=3.2
|
# %z (UTC offset) is only supported in python>=3.2
|
||||||
date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
|
date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
|
||||||
format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S', '%d.%m.%Y %H:%M']
|
format_expressions = [
|
||||||
|
'%d %B %Y',
|
||||||
|
'%B %d %Y',
|
||||||
|
'%b %d %Y',
|
||||||
|
'%Y-%m-%d',
|
||||||
|
'%d/%m/%Y',
|
||||||
|
'%Y/%m/%d %H:%M:%S',
|
||||||
|
'%d.%m.%Y %H:%M',
|
||||||
|
'%Y-%m-%dT%H:%M:%SZ',
|
||||||
|
]
|
||||||
for expression in format_expressions:
|
for expression in format_expressions:
|
||||||
try:
|
try:
|
||||||
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
|
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
|
||||||
@ -745,6 +796,18 @@ def platform_name():
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def write_string(s, out=None):
|
||||||
|
if out is None:
|
||||||
|
out = sys.stderr
|
||||||
|
assert type(s) == type(u'')
|
||||||
|
|
||||||
|
if ('b' in getattr(out, 'mode', '') or
|
||||||
|
sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
|
||||||
|
s = s.encode(preferredencoding(), 'ignore')
|
||||||
|
out.write(s)
|
||||||
|
out.flush()
|
||||||
|
|
||||||
|
|
||||||
def bytes_to_intlist(bs):
|
def bytes_to_intlist(bs):
|
||||||
if not bs:
|
if not bs:
|
||||||
return []
|
return []
|
||||||
@ -761,3 +824,9 @@ def intlist_to_bytes(xs):
|
|||||||
return ''.join([chr(x) for x in xs])
|
return ''.join([chr(x) for x in xs])
|
||||||
else:
|
else:
|
||||||
return bytes(xs)
|
return bytes(xs)
|
||||||
|
|
||||||
|
|
||||||
|
def get_cachedir(params={}):
|
||||||
|
cache_root = os.environ.get('XDG_CACHE_HOME',
|
||||||
|
os.path.expanduser('~/.cache'))
|
||||||
|
return params.get('cachedir', os.path.join(cache_root, 'youtube-dl'))
|
||||||
|
@ -1,2 +1,2 @@
|
|||||||
|
|
||||||
__version__ = '2013.09.06.1'
|
__version__ = '2013.10.06'
|
||||||
|
Reference in New Issue
Block a user