Compare commits
397 Commits
2013.09.20
...
2013.11.07
Author | SHA1 | Date | |
---|---|---|---|
|
6161d17579 | ||
|
4ac5306ae7 | ||
|
b1a80ec1a9 | ||
|
672fe94dcb | ||
|
51040b72ed | ||
|
4f045eef8f | ||
|
5d7b253ea0 | ||
|
b0759f0c19 | ||
|
065472936a | ||
|
fc4a0c2aec | ||
|
eeb165e674 | ||
|
9ee2b5f6f2 | ||
|
da54be877a | ||
|
50a886b7ab | ||
|
76e67c2cb6 | ||
|
7f34001d57 | ||
|
2dcf7d8f99 | ||
|
19b0668251 | ||
|
e7e6b54d8a | ||
|
2a1a8ffe41 | ||
|
08fb86c49b | ||
|
3633d77c0f | ||
|
165e179764 | ||
|
12ebdd1506 | ||
|
1baf9a5938 | ||
|
a56f9de156 | ||
|
fa5d47af4b | ||
|
d607038753 | ||
|
9ac6a01aaf | ||
|
be97abc247 | ||
|
9103bbc5cd | ||
|
b6c45014ae | ||
|
a3dd924871 | ||
|
137bbb3e37 | ||
|
86ad94bb2e | ||
|
3e56add7c9 | ||
|
f52f01b5d2 | ||
|
98d7efb537 | ||
|
cf51923545 | ||
|
165e3bb67a | ||
|
72a5b4f702 | ||
|
aa2484e390 | ||
|
8eddf3e91d | ||
|
60d142aa8d | ||
|
66cf3ac342 | ||
|
ab4e151347 | ||
|
ac2547f5ff | ||
|
5f1ea943ab | ||
|
0ef7ad5cd4 | ||
|
9f1109a564 | ||
|
33b1d9595d | ||
|
7193498811 | ||
|
72321ead7b | ||
|
b5d0d817bc | ||
|
94badb2599 | ||
|
b9a836515f | ||
|
21c924f406 | ||
|
e54fd4b23b | ||
|
57dd9a8f2f | ||
|
912cbf5d4e | ||
|
43d7895ea0 | ||
|
f7ff55aa78 | ||
|
795f28f871 | ||
|
f6cc16f5d8 | ||
|
321a01f971 | ||
|
646e17a53d | ||
|
dd508b7c4f | ||
|
2563bcc85c | ||
|
702665c085 | ||
|
dcc2a706ef | ||
|
2bc67c35ac | ||
|
77ae65877e | ||
|
32a35e4418 | ||
|
369a759acc | ||
|
79b3f61228 | ||
|
216d71d001 | ||
|
78a3a9f89e | ||
|
a7685f3bf4 | ||
|
f088ea5486 | ||
|
1003d108d5 | ||
|
8abeeb9449 | ||
|
c1002e96e9 | ||
|
77d0a82fef | ||
|
ebc14f251c | ||
|
d41e6efc85 | ||
|
8ffa13e03e | ||
|
db477d3a37 | ||
|
750e9833b8 | ||
|
82f0ac657c | ||
|
eb6a2277a2 | ||
|
f8778fb0fa | ||
|
e2f9de207c | ||
|
a93cc0d943 | ||
|
7d8c2e07f2 | ||
|
efb4c36b18 | ||
|
29526d0d2b | ||
|
198e370f23 | ||
|
c19f7764a5 | ||
|
bc63d9d329 | ||
|
aa929c37d5 | ||
|
af4d506eb3 | ||
|
5da0549581 | ||
|
749a4fd2fd | ||
|
6f71ef580c | ||
|
67874aeffa | ||
|
3e6a330d38 | ||
|
aee5e18c8f | ||
|
5b11143d05 | ||
|
7b2212e954 | ||
|
71865091ab | ||
|
125cfd78e8 | ||
|
8cb57d9b91 | ||
|
14e10b2b6e | ||
|
6e76104d66 | ||
|
1d45a23b74 | ||
|
7df286540f | ||
|
5d0c97541a | ||
|
49a25557b0 | ||
|
b5936c0059 | ||
|
600cc1a4f0 | ||
|
ea32fbacc8 | ||
|
00fe14fc75 | ||
|
fcc28edb2f | ||
|
fac6be2dd5 | ||
|
1cf64ee468 | ||
|
cdec0190c4 | ||
|
2450bcb28b | ||
|
3126050c0f | ||
|
93b22c7828 | ||
|
0a89b2852e | ||
|
55b3e45bba | ||
|
365bcf6d97 | ||
|
71907db3ba | ||
|
6803655ced | ||
|
df1c39ec5c | ||
|
80f55a9511 | ||
|
7853cc5ae1 | ||
|
586a91b67f | ||
|
b028e96144 | ||
|
ce68b5907c | ||
|
fe7e0c9825 | ||
|
12893efe01 | ||
|
a6387bfd3c | ||
|
f6a54188c2 | ||
|
cbbd9a9c69 | ||
|
685a9cd2f1 | ||
|
182a107877 | ||
|
8c51aa6506 | ||
|
3fd39e37f2 | ||
|
49e86983e7 | ||
|
a9c58ad945 | ||
|
f8b45beacc | ||
|
9d92015d43 | ||
|
50a6150ed9 | ||
|
b0505eb611 | ||
|
284acd57d6 | ||
|
8ed6b34477 | ||
|
f6f1fc9286 | ||
|
8e590a117f | ||
|
d5594202aa | ||
|
b186d949cf | ||
|
3d2986063c | ||
|
41fd7c7e60 | ||
|
fdefe96bf2 | ||
|
16f36a6fc9 | ||
|
f44415360e | ||
|
cce722b79c | ||
|
82697fb2ab | ||
|
53c1d3ef49 | ||
|
8e55e9abfc | ||
|
7c58ef3275 | ||
|
416a5efce7 | ||
|
f4d96df0f1 | ||
|
5d254f776a | ||
|
1c1218fefc | ||
|
d21ab29200 | ||
|
54ed626cf8 | ||
|
a733eb6c53 | ||
|
591454798d | ||
|
38604f1a4f | ||
|
2d0efe70a6 | ||
|
bfd14b1b2f | ||
|
76965512da | ||
|
996d1c3242 | ||
|
8abbf43f21 | ||
|
10eaae48ff | ||
|
9d4660cab1 | ||
|
9d74e308f7 | ||
|
e772692ffd | ||
|
8381a92120 | ||
|
cd054fc491 | ||
|
f219743e33 | ||
|
4f41664de8 | ||
|
a4fd04158e | ||
|
44a5f1718a | ||
|
a623df4c7b | ||
|
7cf67fbe29 | ||
|
3ddf1a6d01 | ||
|
850555c484 | ||
|
9ed3bdc64d | ||
|
c45aa56080 | ||
|
7394b8db3b | ||
|
f9b3d7af47 | ||
|
ea62a2da46 | ||
|
7468b6b71d | ||
|
1fb07d10a3 | ||
|
9378ae6e1d | ||
|
06723d47c4 | ||
|
69a0c470b5 | ||
|
c40f5cf45c | ||
|
4b7b839f24 | ||
|
3d60d33773 | ||
|
d7e66d39a0 | ||
|
d3f46b9aa5 | ||
|
f5e54a1fda | ||
|
4eb7f1d12e | ||
|
0f6d12e43c | ||
|
b4cdc245cf | ||
|
3283533149 | ||
|
8032e31f2d | ||
|
d2f9cdb205 | ||
|
8016c92297 | ||
|
e028d0d1e3 | ||
|
79819f58f2 | ||
|
6ff000b888 | ||
|
99e206d508 | ||
|
dd82ffea0c | ||
|
3823342d9d | ||
|
91dbaef406 | ||
|
9026dd3858 | ||
|
81d7f1928c | ||
|
bc4f29170f | ||
|
cb354c8f62 | ||
|
1cbb27b151 | ||
|
0ab4ff6378 | ||
|
63da13e829 | ||
|
4193a453c2 | ||
|
2e1fa03bf5 | ||
|
8f1ae18a18 | ||
|
57da92b7df | ||
|
df4f632dbc | ||
|
a34c2faae4 | ||
|
1d368c7589 | ||
|
88bd97e34c | ||
|
2ae3edb1cf | ||
|
b2ad967e45 | ||
|
a27b9e8bd5 | ||
|
4481a754e4 | ||
|
faa6ef6bc8 | ||
|
15870e90b0 | ||
|
8e4f824365 | ||
|
387ae5f30b | ||
|
ad7a071ab6 | ||
|
1310bf2474 | ||
|
b24f347190 | ||
|
ee6c9f95e1 | ||
|
2a69c6b879 | ||
|
cfadd183c4 | ||
|
e484c81f0c | ||
|
7e5e8306fd | ||
|
41e8bca4d0 | ||
|
8dbe9899a9 | ||
|
f4aac741d5 | ||
|
c1c9a79c49 | ||
|
226113c880 | ||
|
8932a66e49 | ||
|
79cfb46d42 | ||
|
00fcc17aee | ||
|
e94b783c74 | ||
|
97dae9ae07 | ||
|
ca215e0a4f | ||
|
91a26ca559 | ||
|
1ece880d7c | ||
|
400afddaf4 | ||
|
c3fef636b5 | ||
|
46e28a84ca | ||
|
17ad2b3fb1 | ||
|
5e2a60db4a | ||
|
cd214418f6 | ||
|
ba2d9f213e | ||
|
7f8ae73a5d | ||
|
466880f531 | ||
|
9f1f6d2437 | ||
|
9e0f897f6b | ||
|
c0f6aa876f | ||
|
d93bdee9a6 | ||
|
f13d09332d | ||
|
2f5865cc6d | ||
|
deefc05b88 | ||
|
0d8cb1cc14 | ||
|
a90b9fd209 | ||
|
829493439a | ||
|
73b4fafd82 | ||
|
b039775057 | ||
|
5c1d63b737 | ||
|
3cd022f6e6 | ||
|
abefd1f7c4 | ||
|
c21315f273 | ||
|
9ab1018b1a | ||
|
da0a5d2d6e | ||
|
ee6adb166c | ||
|
be8fe32c92 | ||
|
c38b1e776d | ||
|
4f8bf17f23 | ||
|
ca40186c75 | ||
|
a8c6b24155 | ||
|
bd8e5c7ca2 | ||
|
7c61bd36bb | ||
|
c54283824c | ||
|
52f15da2ca | ||
|
44d466559e | ||
|
05751eb047 | ||
|
f10503db67 | ||
|
adfeafe9e1 | ||
|
4c62a16f4f | ||
|
c0de39e6d4 | ||
|
fa55675593 | ||
|
d4d9920a26 | ||
|
47192f92d8 | ||
|
722076a123 | ||
|
bb4aa62cf7 | ||
|
843530568f | ||
|
138a5454b5 | ||
|
d279037036 | ||
|
46353f6783 | ||
|
70922df8b5 | ||
|
9c15e9de84 | ||
|
123c10608d | ||
|
0b7c2485b6 | ||
|
9abb32045a | ||
|
f490e77e77 | ||
|
2dc592991a | ||
|
0a60edcfa9 | ||
|
c53f9d30c8 | ||
|
509f398292 | ||
|
74bab3f0a4 | ||
|
8574862991 | ||
|
2de957c7e1 | ||
|
920de7a27d | ||
|
63efc427cd | ||
|
ce65fb6c76 | ||
|
4de1994b6e | ||
|
592882aa9f | ||
|
b98d6a1e19 | ||
|
29c7a63df8 | ||
|
8b25323ae2 | ||
|
f426de8460 | ||
|
695dc094ab | ||
|
e80d861064 | ||
|
2cdeb20135 | ||
|
7f74773254 | ||
|
f2c327fd39 | ||
|
e35e4ddc9a | ||
|
c3c88a2664 | ||
|
bb0eee71e7 | ||
|
6f56389b88 | ||
|
5b333c1ce6 | ||
|
a825f33030 | ||
|
92f618f2e2 | ||
|
81ec7c7901 | ||
|
dd5d2eb03c | ||
|
4ae720042c | ||
|
c705320f48 | ||
|
d2d8f89531 | ||
|
bdde940e90 | ||
|
45f4a76dbc | ||
|
13dc64ce74 | ||
|
c35f9e72ce | ||
|
f8061589e6 | ||
|
0ca96d48c7 | ||
|
4ba146f35d | ||
|
edf3e38ebd | ||
|
c4417ddb61 | ||
|
4a2080e407 | ||
|
2f2ffea9ca | ||
|
ba552f542f | ||
|
8379969834 | ||
|
95dbd2f990 | ||
|
a7177865b1 | ||
|
e0df6211cc | ||
|
b00ca882a4 | ||
|
39baacc49f | ||
|
3a1d48d6de | ||
|
34308b30d6 | ||
|
bc1506f8c0 | ||
|
b61067fa4f | ||
|
69b227a9bc | ||
|
0fd49457f5 | ||
|
58f289d013 | ||
|
3d60bb96e1 | ||
|
38d025b3f0 | ||
|
c40c6aaaaa | ||
|
1a810f0d4e | ||
|
cc6943e86a | ||
|
8f77093262 | ||
|
d79a0e233a | ||
|
0025da15cf |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -24,3 +24,5 @@ updates_key.pem
|
||||
*.flv
|
||||
*.mp4
|
||||
*.part
|
||||
test/testdata
|
||||
.tox
|
||||
|
13
Makefile
13
Makefile
@@ -13,13 +13,13 @@ PYTHON=/usr/bin/env python
|
||||
|
||||
# set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
|
||||
ifeq ($(PREFIX),/usr)
|
||||
SYSCONFDIR=/etc
|
||||
SYSCONFDIR=/etc
|
||||
else
|
||||
ifeq ($(PREFIX),/usr/local)
|
||||
SYSCONFDIR=/etc
|
||||
else
|
||||
SYSCONFDIR=$(PREFIX)/etc
|
||||
endif
|
||||
ifeq ($(PREFIX),/usr/local)
|
||||
SYSCONFDIR=/etc
|
||||
else
|
||||
SYSCONFDIR=$(PREFIX)/etc
|
||||
endif
|
||||
endif
|
||||
|
||||
install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
|
||||
@@ -71,6 +71,7 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
|
||||
--exclude '*~' \
|
||||
--exclude '__pycache' \
|
||||
--exclude '.git' \
|
||||
--exclude 'testdata' \
|
||||
-- \
|
||||
bin devscripts test youtube_dl \
|
||||
CHANGELOG LICENSE README.md README.txt \
|
||||
|
25
README.md
25
README.md
@@ -21,6 +21,8 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
sudo if needed)
|
||||
-i, --ignore-errors continue on download errors, for example to to
|
||||
skip unavailable videos in a playlist
|
||||
--abort-on-error Abort downloading of further videos (in the
|
||||
playlist or the command line) if an error occurs
|
||||
--dump-user-agent display the current browser identification
|
||||
--user-agent UA specify a custom user agent
|
||||
--referer REF specify a custom referer, use if the video access
|
||||
@@ -30,6 +32,11 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
--extractor-descriptions Output descriptions of all supported extractors
|
||||
--proxy URL Use the specified HTTP/HTTPS proxy
|
||||
--no-check-certificate Suppress HTTPS certificate validation.
|
||||
--cache-dir DIR Location in the filesystem where youtube-dl can
|
||||
store downloaded information permanently. By
|
||||
default $XDG_CACHE_HOME/youtube-dl or ~/.cache
|
||||
/youtube-dl .
|
||||
--no-cache-dir Disable filesystem caching
|
||||
|
||||
## Video Selection:
|
||||
--playlist-start NUMBER playlist video to start at (default is 1)
|
||||
@@ -46,11 +53,16 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
--date DATE download only videos uploaded in this date
|
||||
--datebefore DATE download only videos uploaded before this date
|
||||
--dateafter DATE download only videos uploaded after this date
|
||||
--no-playlist download only the currently playing video
|
||||
--age-limit YEARS download only videos suitable for the given age
|
||||
--download-archive FILE Download only videos not present in the archive
|
||||
file. Record all downloaded videos in it.
|
||||
|
||||
## Download Options:
|
||||
-r, --rate-limit LIMIT maximum download rate (e.g. 50k or 44.6m)
|
||||
-r, --rate-limit LIMIT maximum download rate in bytes per second (e.g.
|
||||
50K or 4.2M)
|
||||
-R, --retries RETRIES number of retries (default is 10)
|
||||
--buffer-size SIZE size of download buffer (e.g. 1024 or 16k)
|
||||
--buffer-size SIZE size of download buffer (e.g. 1024 or 16K)
|
||||
(default is 1024)
|
||||
--no-resize-buffer do not automatically adjust the buffer size. By
|
||||
default, the buffer size is automatically resized
|
||||
@@ -66,7 +78,10 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
%(uploader_id)s for the uploader nickname if
|
||||
different, %(autonumber)s to get an automatically
|
||||
incremented number, %(ext)s for the filename
|
||||
extension, %(upload_date)s for the upload date
|
||||
extension, %(format)s for the format description
|
||||
(like "22 - 1280x720" or "HD"),%(format_id)s for
|
||||
the unique id of the format (like Youtube's
|
||||
itags: "137"),%(upload_date)s for the upload date
|
||||
(YYYYMMDD), %(extractor)s for the provider
|
||||
(youtube, metacafe, etc), %(id)s for the video id
|
||||
, %(playlist)s for the playlist the video is in,
|
||||
@@ -91,6 +106,7 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
file modification time
|
||||
--write-description write video description to a .description file
|
||||
--write-info-json write video metadata to a .info.json file
|
||||
--write-annotations write video annotations to a .annotation file
|
||||
--write-thumbnail write thumbnail image to disk
|
||||
|
||||
## Verbosity / Simulation Options:
|
||||
@@ -111,6 +127,8 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
-v, --verbose print various debugging information
|
||||
--dump-intermediate-pages print downloaded pages to debug problems(very
|
||||
verbose)
|
||||
--write-pages Write downloaded pages to files in the current
|
||||
directory
|
||||
|
||||
## Video Format Options:
|
||||
-f, --format FORMAT video format code, specifiy the order of
|
||||
@@ -157,6 +175,7 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
processed files are overwritten by default
|
||||
--embed-subs embed subtitles in the video (only for mp4
|
||||
videos)
|
||||
--add-metadata add metadata to the files
|
||||
|
||||
# CONFIGURATION
|
||||
|
||||
|
@@ -1,4 +1,4 @@
|
||||
__youtube-dl()
|
||||
__youtube_dl()
|
||||
{
|
||||
local cur prev opts
|
||||
COMPREPLY=()
|
||||
@@ -15,4 +15,4 @@ __youtube-dl()
|
||||
fi
|
||||
}
|
||||
|
||||
complete -F __youtube-dl youtube-dl
|
||||
complete -F __youtube_dl youtube-dl
|
||||
|
39
devscripts/check-porn.py
Normal file
39
devscripts/check-porn.py
Normal file
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
|
||||
if we are not 'age_limit' tagging some porn site
|
||||
"""
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import get_testcases
|
||||
from youtube_dl.utils import compat_urllib_request
|
||||
|
||||
for test in get_testcases():
|
||||
try:
|
||||
webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
|
||||
except:
|
||||
print('\nFail: {0}'.format(test['name']))
|
||||
continue
|
||||
|
||||
webpage = webpage.decode('utf8', 'replace')
|
||||
|
||||
if 'porn' in webpage.lower() and ('info_dict' not in test
|
||||
or 'age_limit' not in test['info_dict']
|
||||
or test['info_dict']['age_limit'] != 18):
|
||||
print('\nPotential missing age_limit check: {0}'.format(test['name']))
|
||||
|
||||
elif 'porn' not in webpage.lower() and ('info_dict' in test and
|
||||
'age_limit' in test['info_dict'] and
|
||||
test['info_dict']['age_limit'] == 18):
|
||||
print('\nPotential false negative: {0}'.format(test['name']))
|
||||
|
||||
else:
|
||||
sys.stdout.write('.')
|
||||
sys.stdout.flush()
|
||||
|
||||
print()
|
@@ -16,10 +16,11 @@ def main():
|
||||
ie_htmls = []
|
||||
for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()):
|
||||
ie_html = '<b>{}</b>'.format(ie.IE_NAME)
|
||||
try:
|
||||
ie_desc = getattr(ie, 'IE_DESC', None)
|
||||
if ie_desc is False:
|
||||
continue
|
||||
elif ie_desc is not None:
|
||||
ie_html += ': {}'.format(ie.IE_DESC)
|
||||
except AttributeError:
|
||||
pass
|
||||
if ie.working() == False:
|
||||
ie_html += ' (Currently broken)'
|
||||
ie_htmls.append('<li>{}</li>'.format(ie_html))
|
||||
|
@@ -88,10 +88,6 @@ ROOT=$(pwd)
|
||||
"$ROOT/devscripts/gh-pages/update-sites.py"
|
||||
git add *.html *.html.in update
|
||||
git commit -m "release $version"
|
||||
git show HEAD
|
||||
read -p "Is it good, can I push? (y/n) " -n 1
|
||||
if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi
|
||||
echo
|
||||
git push "$ROOT" gh-pages
|
||||
git push "$ORIGIN_URL" gh-pages
|
||||
)
|
||||
|
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# Generate youtube signature algorithm from test cases
|
||||
|
||||
import sys
|
||||
|
||||
tests = [
|
||||
# 93 - vfl79wBKW 2013/07/20
|
||||
(u"qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`~\"€",
|
||||
u".>/?;:|}][{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWQ098765'321mnbvcxzasdfghjklpoiu"),
|
||||
# 92 - vflQw-fB4 2013/07/17
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`~\"",
|
||||
"mrtyuioplkjhgfdsazxcvbnq1234567890QWERTY}IOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]\"|:;"),
|
||||
# 90
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'`",
|
||||
"mrtyuioplkjhgfdsazxcvbne1234567890QWER[YUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={`]}|"),
|
||||
# 89
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<'",
|
||||
"/?;:|}<[{=+-_)(*&^%$#@!MqBVCXZASDFGHJKLPOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuyt"),
|
||||
# 88 - vflapUV9V 2013/08/28
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[]}|:;?/>.<",
|
||||
"ioplkjhgfdsazxcvbnm12<4567890QWERTYUIOZLKJHGFDSAeXCVBNM!@#$%^&*()_-+={[]}|:;?/>.3"),
|
||||
# 87
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$^&*()_-+={[]}|:;?/>.<",
|
||||
"uioplkjhgfdsazxcvbnm1t34567890QWE2TYUIOPLKJHGFDSAZXCVeNM!@#$^&*()_-+={[]}|:;?/>.<"),
|
||||
# 86 - vfluy6kdb 2013/09/06
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
|
||||
"yuioplkjhgfdsazxcvbnm12345678q0QWrRTYUIOELKJHGFD-AZXCVBNM!@#$%^&*()_<+={[|};?/>.S"),
|
||||
# 85 - vflkuzxcs 2013/09/11
|
||||
('0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[',
|
||||
'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@'),
|
||||
# 84 - vflg0g8PQ 2013/08/29 (sporadic)
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[};?>.<",
|
||||
">?;}[{=+-_)(*&^%$#@!MNBVCXZASDFGHJKLPOIUYTREWq0987654321mnbvcxzasdfghjklpoiuytr"),
|
||||
# 83
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!#$%^&*()_+={[};?/>.<",
|
||||
".>/?;}[{=+_)(*&^%<#!MNBVCXZASPFGHJKLwOIUYTREWQ0987654321mnbvcxzasdfghjklpoiuytreq"),
|
||||
# 82 - vflGNjMhJ 2013/09/12
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.<",
|
||||
".>/?;}[<=+-(*&^%$#@!MNBVCXeASDFGHKLPOqUYTREWQ0987654321mnbvcxzasdfghjklpoiuytrIwZ"),
|
||||
# 81 - vflLC8JvQ 2013/07/25
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>.",
|
||||
"C>/?;}[{=+-(*&^%$#@!MNBVYXZASDFGHKLPOIU.TREWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
|
||||
# 80 - vflZK4ZYR 2013/08/23 (sporadic)
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/>",
|
||||
"wertyuioplkjhgfdsaqxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&z(-+={[};?/>"),
|
||||
# 79 - vflLC8JvQ 2013/07/25 (sporadic)
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKHGFDSAZXCVBNM!@#$%^&*(-+={[};?/",
|
||||
"Z?;}[{=+-(*&^%$#@!MNBVCXRASDFGHKLPOIUYT/EWQ0q87659321mnbvcxzasdfghjkl4oiuytrewp"),
|
||||
]
|
||||
|
||||
tests_age_gate = [
|
||||
# 86 - vflqinMWD
|
||||
("qwertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!@#$%^&*()_-+={[|};?/>.<",
|
||||
"ertyuioplkjhgfdsazxcvbnm1234567890QWERTYUIOPLKJHGFDSAZXCVBNM!/#$%^&*()_-+={[|};?@"),
|
||||
]
|
||||
|
||||
def find_matching(wrong, right):
|
||||
idxs = [wrong.index(c) for c in right]
|
||||
return compress(idxs)
|
||||
return ('s[%d]' % i for i in idxs)
|
||||
|
||||
def compress(idxs):
|
||||
def _genslice(start, end, step):
|
||||
starts = '' if start == 0 else str(start)
|
||||
ends = ':%d' % (end+step)
|
||||
steps = '' if step == 1 else (':%d' % step)
|
||||
return 's[%s%s%s]' % (starts, ends, steps)
|
||||
|
||||
step = None
|
||||
for i, prev in zip(idxs[1:], idxs[:-1]):
|
||||
if step is not None:
|
||||
if i - prev == step:
|
||||
continue
|
||||
yield _genslice(start, prev, step)
|
||||
step = None
|
||||
continue
|
||||
if i - prev in [-1, 1]:
|
||||
step = i - prev
|
||||
start = prev
|
||||
continue
|
||||
else:
|
||||
yield 's[%d]' % prev
|
||||
if step is None:
|
||||
yield 's[%d]' % i
|
||||
else:
|
||||
yield _genslice(start, i, step)
|
||||
|
||||
def _assert_compress(inp, exp):
|
||||
res = list(compress(inp))
|
||||
if res != exp:
|
||||
print('Got %r, expected %r' % (res, exp))
|
||||
assert res == exp
|
||||
_assert_compress([0,2,4,6], ['s[0]', 's[2]', 's[4]', 's[6]'])
|
||||
_assert_compress([0,1,2,4,6,7], ['s[:3]', 's[4]', 's[6:8]'])
|
||||
_assert_compress([8,0,1,2,4,7,6,9], ['s[8]', 's[:3]', 's[4]', 's[7:5:-1]', 's[9]'])
|
||||
|
||||
def gen(wrong, right, indent):
|
||||
code = ' + '.join(find_matching(wrong, right))
|
||||
return 'if len(s) == %d:\n%s return %s\n' % (len(wrong), indent, code)
|
||||
|
||||
def genall(tests):
|
||||
indent = ' ' * 8
|
||||
return indent + (indent + 'el').join(gen(wrong, right, indent) for wrong,right in tests)
|
||||
|
||||
def main():
|
||||
print(genall(tests))
|
||||
print(u' Age gate:')
|
||||
print(genall(tests_age_gate))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
8
setup.py
8
setup.py
@@ -8,8 +8,10 @@ import sys
|
||||
|
||||
try:
|
||||
from setuptools import setup
|
||||
setuptools_available = True
|
||||
except ImportError:
|
||||
from distutils.core import setup
|
||||
setuptools_available = False
|
||||
|
||||
try:
|
||||
# This will create an exe that needs Microsoft Visual C++ 2008
|
||||
@@ -43,13 +45,16 @@ if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
|
||||
params = py2exe_params
|
||||
else:
|
||||
params = {
|
||||
'scripts': ['bin/youtube-dl'],
|
||||
'data_files': [ # Installing system-wide would require sudo...
|
||||
('etc/bash_completion.d', ['youtube-dl.bash-completion']),
|
||||
('share/doc/youtube_dl', ['README.txt']),
|
||||
('share/man/man1/', ['youtube-dl.1'])
|
||||
]
|
||||
}
|
||||
if setuptools_available:
|
||||
params['entry_points'] = {'console_scripts': ['youtube-dl = youtube_dl:main']}
|
||||
else:
|
||||
params['scripts'] = ['bin/youtube-dl']
|
||||
|
||||
# Get the version from youtube_dl/version.py without importing the package
|
||||
exec(compile(open('youtube_dl/version.py').read(),
|
||||
@@ -63,6 +68,7 @@ setup(
|
||||
' YouTube.com and other video sites.',
|
||||
url='https://github.com/rg3/youtube-dl',
|
||||
author='Ricardo Garcia',
|
||||
author_email='ytdl@yt-dl.org',
|
||||
maintainer='Philipp Hagemeister',
|
||||
maintainer_email='phihag@phihag.de',
|
||||
packages=['youtube_dl', 'youtube_dl.extractor'],
|
||||
|
0
test/__init__.py
Normal file
0
test/__init__.py
Normal file
@@ -1,38 +1,80 @@
|
||||
import errno
|
||||
import io
|
||||
import hashlib
|
||||
import json
|
||||
import os.path
|
||||
import re
|
||||
import types
|
||||
import sys
|
||||
|
||||
import youtube_dl.extractor
|
||||
from youtube_dl import YoutubeDL, YoutubeDLHandler
|
||||
from youtube_dl.utils import (
|
||||
compat_cookiejar,
|
||||
compat_urllib_request,
|
||||
)
|
||||
from youtube_dl import YoutubeDL
|
||||
from youtube_dl.utils import preferredencoding
|
||||
|
||||
# General configuration (from __init__, not very elegant...)
|
||||
jar = compat_cookiejar.CookieJar()
|
||||
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
|
||||
proxy_handler = compat_urllib_request.ProxyHandler()
|
||||
opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
|
||||
compat_urllib_request.install_opener(opener)
|
||||
|
||||
PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
|
||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||
parameters = json.load(pf)
|
||||
def global_setup():
|
||||
youtube_dl._setup_opener(timeout=10)
|
||||
|
||||
|
||||
def get_params(override=None):
|
||||
PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"parameters.json")
|
||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||
parameters = json.load(pf)
|
||||
if override:
|
||||
parameters.update(override)
|
||||
return parameters
|
||||
|
||||
|
||||
def try_rm(filename):
|
||||
""" Remove a file if it exists """
|
||||
try:
|
||||
os.remove(filename)
|
||||
except OSError as ose:
|
||||
if ose.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
|
||||
def report_warning(message):
|
||||
'''
|
||||
Print the message to stderr, it will be prefixed with 'WARNING:'
|
||||
If stderr is a tty file the 'WARNING:' will be colored
|
||||
'''
|
||||
if sys.stderr.isatty() and os.name != 'nt':
|
||||
_msg_header = u'\033[0;33mWARNING:\033[0m'
|
||||
else:
|
||||
_msg_header = u'WARNING:'
|
||||
output = u'%s %s\n' % (_msg_header, message)
|
||||
if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3:
|
||||
output = output.encode(preferredencoding())
|
||||
sys.stderr.write(output)
|
||||
|
||||
|
||||
class FakeYDL(YoutubeDL):
|
||||
def __init__(self):
|
||||
self.result = []
|
||||
def __init__(self, override=None):
|
||||
# Different instances of the downloader can't share the same dictionary
|
||||
# some test set the "sublang" parameter, which would break the md5 checks.
|
||||
self.params = dict(parameters)
|
||||
def to_screen(self, s):
|
||||
params = get_params(override=override)
|
||||
super(FakeYDL, self).__init__(params)
|
||||
self.result = []
|
||||
|
||||
def to_screen(self, s, skip_eol=None):
|
||||
print(s)
|
||||
|
||||
def trouble(self, s, tb=None):
|
||||
raise Exception(s)
|
||||
|
||||
def download(self, x):
|
||||
self.result.append(x)
|
||||
|
||||
def expect_warning(self, regex):
|
||||
# Silence an expected warning matching a regex
|
||||
old_report_warning = self.report_warning
|
||||
def report_warning(self, message):
|
||||
if re.match(regex, message): return
|
||||
old_report_warning(message)
|
||||
self.report_warning = types.MethodType(report_warning, self)
|
||||
|
||||
def get_testcases():
|
||||
for ie in youtube_dl.extractor.gen_extractors():
|
||||
t = getattr(ie, '_TEST', None)
|
||||
@@ -42,3 +84,6 @@ def get_testcases():
|
||||
for t in getattr(ie, '_TESTS', []):
|
||||
t['name'] = type(ie).__name__[:-len('IE')]
|
||||
yield t
|
||||
|
||||
|
||||
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||
|
145
test/test_YoutubeDL.py
Normal file
145
test/test_YoutubeDL.py
Normal file
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import FakeYDL
|
||||
|
||||
|
||||
class YDL(FakeYDL):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(YDL, self).__init__(*args, **kwargs)
|
||||
self.downloaded_info_dicts = []
|
||||
self.msgs = []
|
||||
|
||||
def process_info(self, info_dict):
|
||||
self.downloaded_info_dicts.append(info_dict)
|
||||
|
||||
def to_screen(self, msg):
|
||||
self.msgs.append(msg)
|
||||
|
||||
|
||||
class TestFormatSelection(unittest.TestCase):
|
||||
def test_prefer_free_formats(self):
|
||||
# Same resolution => download webm
|
||||
ydl = YDL()
|
||||
ydl.params['prefer_free_formats'] = True
|
||||
formats = [
|
||||
{u'ext': u'webm', u'height': 460},
|
||||
{u'ext': u'mp4', u'height': 460},
|
||||
]
|
||||
info_dict = {u'formats': formats, u'extractor': u'test'}
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded[u'ext'], u'webm')
|
||||
|
||||
# Different resolution => download best quality (mp4)
|
||||
ydl = YDL()
|
||||
ydl.params['prefer_free_formats'] = True
|
||||
formats = [
|
||||
{u'ext': u'webm', u'height': 720},
|
||||
{u'ext': u'mp4', u'height': 1080},
|
||||
]
|
||||
info_dict[u'formats'] = formats
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded[u'ext'], u'mp4')
|
||||
|
||||
# No prefer_free_formats => keep original formats order
|
||||
ydl = YDL()
|
||||
ydl.params['prefer_free_formats'] = False
|
||||
formats = [
|
||||
{u'ext': u'webm', u'height': 720},
|
||||
{u'ext': u'flv', u'height': 720},
|
||||
]
|
||||
info_dict[u'formats'] = formats
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded[u'ext'], u'flv')
|
||||
|
||||
def test_format_limit(self):
|
||||
formats = [
|
||||
{u'format_id': u'meh', u'url': u'http://example.com/meh'},
|
||||
{u'format_id': u'good', u'url': u'http://example.com/good'},
|
||||
{u'format_id': u'great', u'url': u'http://example.com/great'},
|
||||
{u'format_id': u'excellent', u'url': u'http://example.com/exc'},
|
||||
]
|
||||
info_dict = {
|
||||
u'formats': formats, u'extractor': u'test', 'id': 'testvid'}
|
||||
|
||||
ydl = YDL()
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded[u'format_id'], u'excellent')
|
||||
|
||||
ydl = YDL({'format_limit': 'good'})
|
||||
assert ydl.params['format_limit'] == 'good'
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded[u'format_id'], u'good')
|
||||
|
||||
ydl = YDL({'format_limit': 'great', 'format': 'all'})
|
||||
ydl.process_ie_result(info_dict)
|
||||
self.assertEqual(ydl.downloaded_info_dicts[0][u'format_id'], u'meh')
|
||||
self.assertEqual(ydl.downloaded_info_dicts[1][u'format_id'], u'good')
|
||||
self.assertEqual(ydl.downloaded_info_dicts[2][u'format_id'], u'great')
|
||||
self.assertTrue('3' in ydl.msgs[0])
|
||||
|
||||
ydl = YDL()
|
||||
ydl.params['format_limit'] = 'excellent'
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded[u'format_id'], u'excellent')
|
||||
|
||||
def test_format_selection(self):
|
||||
formats = [
|
||||
{u'format_id': u'35', u'ext': u'mp4'},
|
||||
{u'format_id': u'45', u'ext': u'webm'},
|
||||
{u'format_id': u'47', u'ext': u'webm'},
|
||||
{u'format_id': u'2', u'ext': u'flv'},
|
||||
]
|
||||
info_dict = {u'formats': formats, u'extractor': u'test'}
|
||||
|
||||
ydl = YDL({'format': u'20/47'})
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded['format_id'], u'47')
|
||||
|
||||
ydl = YDL({'format': u'20/71/worst'})
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded['format_id'], u'35')
|
||||
|
||||
ydl = YDL()
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded['format_id'], u'2')
|
||||
|
||||
ydl = YDL({'format': u'webm/mp4'})
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded['format_id'], u'47')
|
||||
|
||||
ydl = YDL({'format': u'3gp/40/mp4'})
|
||||
ydl.process_ie_result(info_dict)
|
||||
downloaded = ydl.downloaded_info_dicts[0]
|
||||
self.assertEqual(downloaded['format_id'], u'35')
|
||||
|
||||
def test_add_extra_info(self):
|
||||
test_dict = {
|
||||
'extractor': 'Foo',
|
||||
}
|
||||
extra_info = {
|
||||
'extractor': 'Bar',
|
||||
'playlist': 'funny videos',
|
||||
}
|
||||
YDL.add_extra_info(test_dict, extra_info)
|
||||
self.assertEqual(test_dict['extractor'], 'Foo')
|
||||
self.assertEqual(test_dict['playlist'], 'funny videos')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
55
test/test_age_restriction.py
Normal file
55
test/test_age_restriction.py
Normal file
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import global_setup, try_rm
|
||||
global_setup()
|
||||
|
||||
|
||||
from youtube_dl import YoutubeDL
|
||||
|
||||
|
||||
def _download_restricted(url, filename, age):
|
||||
""" Returns true iff the file has been downloaded """
|
||||
|
||||
params = {
|
||||
'age_limit': age,
|
||||
'skip_download': True,
|
||||
'writeinfojson': True,
|
||||
"outtmpl": "%(id)s.%(ext)s",
|
||||
}
|
||||
ydl = YoutubeDL(params)
|
||||
ydl.add_default_info_extractors()
|
||||
json_filename = filename + '.info.json'
|
||||
try_rm(json_filename)
|
||||
ydl.download([url])
|
||||
res = os.path.exists(json_filename)
|
||||
try_rm(json_filename)
|
||||
return res
|
||||
|
||||
|
||||
class TestAgeRestriction(unittest.TestCase):
|
||||
def _assert_restricted(self, url, filename, age, old_age=None):
|
||||
self.assertTrue(_download_restricted(url, filename, old_age))
|
||||
self.assertFalse(_download_restricted(url, filename, age))
|
||||
|
||||
def test_youtube(self):
|
||||
self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)
|
||||
|
||||
def test_youporn(self):
|
||||
self._assert_restricted(
|
||||
'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
|
||||
'505835.mp4', 2, old_age=25)
|
||||
|
||||
def test_pornotube(self):
|
||||
self._assert_restricted(
|
||||
'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
|
||||
'1689755.flv', 13)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
@@ -1,14 +1,20 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
from test.helper import get_testcases
|
||||
|
||||
from youtube_dl.extractor import (
|
||||
gen_extractors,
|
||||
JustinTVIE,
|
||||
YoutubeIE,
|
||||
)
|
||||
|
||||
from youtube_dl.extractor import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE, gen_extractors
|
||||
from helper import get_testcases
|
||||
|
||||
class TestAllURLsMatching(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
@@ -1,20 +1,16 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
import json
|
||||
import io
|
||||
import hashlib
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import FakeYDL, global_setup, md5
|
||||
global_setup()
|
||||
|
||||
|
||||
from youtube_dl.extractor import DailymotionIE
|
||||
from youtube_dl.utils import *
|
||||
from helper import FakeYDL
|
||||
|
||||
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||
|
||||
class TestDailymotionSubtitles(unittest.TestCase):
|
||||
def setUp(self):
|
||||
@@ -26,7 +22,7 @@ class TestDailymotionSubtitles(unittest.TestCase):
|
||||
return info_dict
|
||||
def getSubtitles(self):
|
||||
info_dict = self.getInfoDict()
|
||||
return info_dict[0]['subtitles']
|
||||
return info_dict['subtitles']
|
||||
def test_no_writesubtitles(self):
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(subtitles, None)
|
||||
@@ -45,15 +41,18 @@ class TestDailymotionSubtitles(unittest.TestCase):
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(len(subtitles.keys()), 5)
|
||||
def test_list_subtitles(self):
|
||||
self.DL.expect_warning(u'Automatic Captions not supported by this server')
|
||||
self.DL.params['listsubtitles'] = True
|
||||
info_dict = self.getInfoDict()
|
||||
self.assertEqual(info_dict, None)
|
||||
def test_automatic_captions(self):
|
||||
self.DL.expect_warning(u'Automatic Captions not supported by this server')
|
||||
self.DL.params['writeautomaticsub'] = True
|
||||
self.DL.params['subtitleslang'] = ['en']
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertTrue(len(subtitles.keys()) == 0)
|
||||
def test_nosubtitles(self):
|
||||
self.DL.expect_warning(u'video doesn\'t have subtitles')
|
||||
self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['allsubtitles'] = True
|
||||
|
@@ -1,43 +1,40 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import errno
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import (
|
||||
get_params,
|
||||
get_testcases,
|
||||
global_setup,
|
||||
try_rm,
|
||||
md5,
|
||||
report_warning
|
||||
)
|
||||
global_setup()
|
||||
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import json
|
||||
import unittest
|
||||
import sys
|
||||
import socket
|
||||
import binascii
|
||||
|
||||
# Allow direct execution
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import youtube_dl.YoutubeDL
|
||||
from youtube_dl.utils import *
|
||||
|
||||
PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
|
||||
from youtube_dl.utils import (
|
||||
compat_str,
|
||||
compat_urllib_error,
|
||||
compat_HTTPError,
|
||||
DownloadError,
|
||||
ExtractorError,
|
||||
UnavailableVideoError,
|
||||
)
|
||||
from youtube_dl.extractor import get_info_extractor
|
||||
|
||||
RETRIES = 3
|
||||
|
||||
# General configuration (from __init__, not very elegant...)
|
||||
jar = compat_cookiejar.CookieJar()
|
||||
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
|
||||
proxy_handler = compat_urllib_request.ProxyHandler()
|
||||
opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
|
||||
compat_urllib_request.install_opener(opener)
|
||||
socket.setdefaulttimeout(10)
|
||||
|
||||
def _try_rm(filename):
|
||||
""" Remove a file if it exists """
|
||||
try:
|
||||
os.remove(filename)
|
||||
except OSError as ose:
|
||||
if ose.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||
|
||||
class YoutubeDL(youtube_dl.YoutubeDL):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.to_stderr = self.to_screen
|
||||
@@ -54,17 +51,12 @@ def _file_md5(fn):
|
||||
with open(fn, 'rb') as f:
|
||||
return hashlib.md5(f.read()).hexdigest()
|
||||
|
||||
from helper import get_testcases
|
||||
defs = get_testcases()
|
||||
|
||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||
parameters = json.load(pf)
|
||||
|
||||
|
||||
class TestDownload(unittest.TestCase):
|
||||
maxDiff = None
|
||||
def setUp(self):
|
||||
self.parameters = parameters
|
||||
self.defs = defs
|
||||
|
||||
### Dynamically generate tests
|
||||
@@ -72,20 +64,27 @@ def generator(test_case):
|
||||
|
||||
def test_template(self):
|
||||
ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
|
||||
other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
|
||||
def print_skipping(reason):
|
||||
print('Skipping %s: %s' % (test_case['name'], reason))
|
||||
if not ie._WORKING:
|
||||
if not ie.working():
|
||||
print_skipping('IE marked as not _WORKING')
|
||||
return
|
||||
if 'playlist' not in test_case and not test_case['file']:
|
||||
print_skipping('No output file specified')
|
||||
return
|
||||
if 'playlist' not in test_case:
|
||||
info_dict = test_case.get('info_dict', {})
|
||||
if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
|
||||
print_skipping('The output file cannot be know, the "file" '
|
||||
'key is missing or the info_dict is incomplete')
|
||||
return
|
||||
if 'skip' in test_case:
|
||||
print_skipping(test_case['skip'])
|
||||
return
|
||||
for other_ie in other_ies:
|
||||
if not other_ie.working():
|
||||
print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
|
||||
return
|
||||
|
||||
params = self.parameters.copy()
|
||||
params.update(test_case.get('params', {}))
|
||||
params = get_params(test_case.get('params', {}))
|
||||
|
||||
ydl = YoutubeDL(params)
|
||||
ydl.add_default_info_extractors()
|
||||
@@ -95,35 +94,47 @@ def generator(test_case):
|
||||
finished_hook_called.add(status['filename'])
|
||||
ydl.fd.add_progress_hook(_hook)
|
||||
|
||||
def get_tc_filename(tc):
|
||||
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
|
||||
|
||||
test_cases = test_case.get('playlist', [test_case])
|
||||
for tc in test_cases:
|
||||
_try_rm(tc['file'])
|
||||
_try_rm(tc['file'] + '.part')
|
||||
_try_rm(tc['file'] + '.info.json')
|
||||
def try_rm_tcs_files():
|
||||
for tc in test_cases:
|
||||
tc_filename = get_tc_filename(tc)
|
||||
try_rm(tc_filename)
|
||||
try_rm(tc_filename + '.part')
|
||||
try_rm(tc_filename + '.info.json')
|
||||
try_rm_tcs_files()
|
||||
try:
|
||||
for retry in range(1, RETRIES + 1):
|
||||
try_num = 1
|
||||
while True:
|
||||
try:
|
||||
ydl.download([test_case['url']])
|
||||
except (DownloadError, ExtractorError) as err:
|
||||
if retry == RETRIES: raise
|
||||
|
||||
# Check if the exception is not a network related one
|
||||
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
|
||||
if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
|
||||
raise
|
||||
|
||||
print('Retrying: {0} failed tries\n\n##########\n\n'.format(retry))
|
||||
if try_num == RETRIES:
|
||||
report_warning(u'Failed due to network errors, skipping...')
|
||||
return
|
||||
|
||||
print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
|
||||
|
||||
try_num += 1
|
||||
else:
|
||||
break
|
||||
|
||||
for tc in test_cases:
|
||||
tc_filename = get_tc_filename(tc)
|
||||
if not test_case.get('params', {}).get('skip_download', False):
|
||||
self.assertTrue(os.path.exists(tc['file']), msg='Missing file ' + tc['file'])
|
||||
self.assertTrue(tc['file'] in finished_hook_called)
|
||||
self.assertTrue(os.path.exists(tc['file'] + '.info.json'))
|
||||
self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
|
||||
self.assertTrue(tc_filename in finished_hook_called)
|
||||
self.assertTrue(os.path.exists(tc_filename + '.info.json'))
|
||||
if 'md5' in tc:
|
||||
md5_for_file = _file_md5(tc['file'])
|
||||
md5_for_file = _file_md5(tc_filename)
|
||||
self.assertEqual(md5_for_file, tc['md5'])
|
||||
with io.open(tc['file'] + '.info.json', encoding='utf-8') as infof:
|
||||
with io.open(tc_filename + '.info.json', encoding='utf-8') as infof:
|
||||
info_dict = json.load(infof)
|
||||
for (info_field, expected) in tc.get('info_dict', {}).items():
|
||||
if isinstance(expected, compat_str) and expected.startswith('md5:'):
|
||||
@@ -143,11 +154,11 @@ def generator(test_case):
|
||||
# Check for the presence of mandatory fields
|
||||
for key in ('id', 'url', 'title', 'ext'):
|
||||
self.assertTrue(key in info_dict.keys() and info_dict[key])
|
||||
# Check for mandatory fields that are automatically set by YoutubeDL
|
||||
for key in ['webpage_url', 'extractor', 'extractor_key']:
|
||||
self.assertTrue(info_dict.get(key), u'Missing field: %s' % key)
|
||||
finally:
|
||||
for tc in test_cases:
|
||||
_try_rm(tc['file'])
|
||||
_try_rm(tc['file'] + '.part')
|
||||
_try_rm(tc['file'] + '.info.json')
|
||||
try_rm_tcs_files()
|
||||
|
||||
return test_template
|
||||
|
||||
|
@@ -1,17 +1,28 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
import json
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from youtube_dl.extractor import DailymotionPlaylistIE, VimeoChannelIE, UstreamChannelIE, SoundcloudUserIE
|
||||
from youtube_dl.utils import *
|
||||
from test.helper import FakeYDL, global_setup
|
||||
global_setup()
|
||||
|
||||
|
||||
from youtube_dl.extractor import (
|
||||
DailymotionPlaylistIE,
|
||||
DailymotionUserIE,
|
||||
VimeoChannelIE,
|
||||
UstreamChannelIE,
|
||||
SoundcloudUserIE,
|
||||
LivestreamIE,
|
||||
NHLVideocenterIE,
|
||||
BambuserChannelIE,
|
||||
)
|
||||
|
||||
from helper import FakeYDL
|
||||
|
||||
class TestPlaylists(unittest.TestCase):
|
||||
def assertIsPlaylist(self, info):
|
||||
@@ -26,6 +37,14 @@ class TestPlaylists(unittest.TestCase):
|
||||
self.assertEqual(result['title'], u'SPORT')
|
||||
self.assertTrue(len(result['entries']) > 20)
|
||||
|
||||
def test_dailymotion_user(self):
|
||||
dl = FakeYDL()
|
||||
ie = DailymotionUserIE(dl)
|
||||
result = ie.extract('http://www.dailymotion.com/user/generation-quoi/')
|
||||
self.assertIsPlaylist(result)
|
||||
self.assertEqual(result['title'], u'Génération Quoi')
|
||||
self.assertTrue(len(result['entries']) >= 26)
|
||||
|
||||
def test_vimeo_channel(self):
|
||||
dl = FakeYDL()
|
||||
ie = VimeoChannelIE(dl)
|
||||
@@ -50,5 +69,30 @@ class TestPlaylists(unittest.TestCase):
|
||||
self.assertEqual(result['id'], u'9615865')
|
||||
self.assertTrue(len(result['entries']) >= 12)
|
||||
|
||||
def test_livestream_event(self):
|
||||
dl = FakeYDL()
|
||||
ie = LivestreamIE(dl)
|
||||
result = ie.extract('http://new.livestream.com/tedx/cityenglish')
|
||||
self.assertIsPlaylist(result)
|
||||
self.assertEqual(result['title'], u'TEDCity2.0 (English)')
|
||||
self.assertTrue(len(result['entries']) >= 4)
|
||||
|
||||
def test_nhl_videocenter(self):
|
||||
dl = FakeYDL()
|
||||
ie = NHLVideocenterIE(dl)
|
||||
result = ie.extract('http://video.canucks.nhl.com/videocenter/console?catid=999')
|
||||
self.assertIsPlaylist(result)
|
||||
self.assertEqual(result['id'], u'999')
|
||||
self.assertEqual(result['title'], u'Highlights')
|
||||
self.assertEqual(len(result['entries']), 12)
|
||||
|
||||
def test_bambuser_channel(self):
|
||||
dl = FakeYDL()
|
||||
ie = BambuserChannelIE(dl)
|
||||
result = ie.extract('http://bambuser.com/channel/pixelversity')
|
||||
self.assertIsPlaylist(result)
|
||||
self.assertEqual(result['title'], u'pixelversity')
|
||||
self.assertTrue(len(result['entries']) >= 66)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
@@ -1,14 +1,15 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Various small unit tests
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
import xml.etree.ElementTree
|
||||
# coding: utf-8
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
# Various small unit tests
|
||||
import xml.etree.ElementTree
|
||||
|
||||
#from youtube_dl.utils import htmlentity_transform
|
||||
from youtube_dl.utils import (
|
||||
@@ -20,6 +21,9 @@ from youtube_dl.utils import (
|
||||
unified_strdate,
|
||||
find_xpath_attr,
|
||||
get_meta_content,
|
||||
xpath_with_ns,
|
||||
smuggle_url,
|
||||
unsmuggle_url,
|
||||
)
|
||||
|
||||
if sys.version_info < (3, 0):
|
||||
@@ -141,5 +145,31 @@ class TestUtil(unittest.TestCase):
|
||||
self.assertEqual(get_meta('description'), u'foo & bar')
|
||||
self.assertEqual(get_meta('author'), 'Plato')
|
||||
|
||||
def test_xpath_with_ns(self):
|
||||
testxml = u'''<root xmlns:media="http://example.com/">
|
||||
<media:song>
|
||||
<media:author>The Author</media:author>
|
||||
<url>http://server.com/download.mp3</url>
|
||||
</media:song>
|
||||
</root>'''
|
||||
doc = xml.etree.ElementTree.fromstring(testxml)
|
||||
find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'}))
|
||||
self.assertTrue(find('media:song') is not None)
|
||||
self.assertEqual(find('media:song/media:author').text, u'The Author')
|
||||
self.assertEqual(find('media:song/url').text, u'http://server.com/download.mp3')
|
||||
|
||||
def test_smuggle_url(self):
|
||||
data = {u"ö": u"ö", u"abc": [3]}
|
||||
url = 'https://foo.bar/baz?x=y#a'
|
||||
smug_url = smuggle_url(url, data)
|
||||
unsmug_url, unsmug_data = unsmuggle_url(smug_url)
|
||||
self.assertEqual(url, unsmug_url)
|
||||
self.assertEqual(data, unsmug_data)
|
||||
|
||||
res_url, res_data = unsmuggle_url(url)
|
||||
self.assertEqual(res_url, url)
|
||||
self.assertEqual(res_data, None)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
80
test/test_write_annotations.py
Normal file
80
test/test_write_annotations.py
Normal file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import get_params, global_setup, try_rm
|
||||
global_setup()
|
||||
|
||||
|
||||
import io
|
||||
|
||||
import xml.etree.ElementTree
|
||||
|
||||
import youtube_dl.YoutubeDL
|
||||
import youtube_dl.extractor
|
||||
|
||||
|
||||
class YoutubeDL(youtube_dl.YoutubeDL):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(YoutubeDL, self).__init__(*args, **kwargs)
|
||||
self.to_stderr = self.to_screen
|
||||
|
||||
params = get_params({
|
||||
'writeannotations': True,
|
||||
'skip_download': True,
|
||||
'writeinfojson': False,
|
||||
'format': 'flv',
|
||||
})
|
||||
|
||||
|
||||
|
||||
TEST_ID = 'gr51aVj-mLg'
|
||||
ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'
|
||||
EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']
|
||||
|
||||
class TestAnnotations(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Clear old files
|
||||
self.tearDown()
|
||||
|
||||
|
||||
def test_info_json(self):
|
||||
expected = list(EXPECTED_ANNOTATIONS) #Two annotations could have the same text.
|
||||
ie = youtube_dl.extractor.YoutubeIE()
|
||||
ydl = YoutubeDL(params)
|
||||
ydl.add_info_extractor(ie)
|
||||
ydl.download([TEST_ID])
|
||||
self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
|
||||
annoxml = None
|
||||
with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
|
||||
annoxml = xml.etree.ElementTree.parse(annof)
|
||||
self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
|
||||
root = annoxml.getroot()
|
||||
self.assertEqual(root.tag, 'document')
|
||||
annotationsTag = root.find('annotations')
|
||||
self.assertEqual(annotationsTag.tag, 'annotations')
|
||||
annotations = annotationsTag.findall('annotation')
|
||||
|
||||
#Not all the annotations have TEXT children and the annotations are returned unsorted.
|
||||
for a in annotations:
|
||||
self.assertEqual(a.tag, 'annotation')
|
||||
if a.get('type') == 'text':
|
||||
textTag = a.find('TEXT')
|
||||
text = textTag.text
|
||||
self.assertTrue(text in expected) #assertIn only added in python 2.7
|
||||
#remove the first occurance, there could be more than one annotation with the same text
|
||||
expected.remove(text)
|
||||
#We should have seen (and removed) all the expected annotation texts.
|
||||
self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
|
||||
|
||||
|
||||
def tearDown(self):
|
||||
try_rm(ANNOTATIONS_FILE)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
@@ -1,37 +1,34 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
import json
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
# Allow direct execution
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from test.helper import get_params, global_setup
|
||||
global_setup()
|
||||
|
||||
|
||||
import io
|
||||
import json
|
||||
|
||||
import youtube_dl.YoutubeDL
|
||||
import youtube_dl.extractor
|
||||
from youtube_dl.utils import *
|
||||
|
||||
PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
|
||||
|
||||
# General configuration (from __init__, not very elegant...)
|
||||
jar = compat_cookiejar.CookieJar()
|
||||
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
|
||||
proxy_handler = compat_urllib_request.ProxyHandler()
|
||||
opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
|
||||
compat_urllib_request.install_opener(opener)
|
||||
|
||||
class YoutubeDL(youtube_dl.YoutubeDL):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(YoutubeDL, self).__init__(*args, **kwargs)
|
||||
self.to_stderr = self.to_screen
|
||||
|
||||
with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
|
||||
params = json.load(pf)
|
||||
params['writeinfojson'] = True
|
||||
params['skip_download'] = True
|
||||
params['writedescription'] = True
|
||||
params = get_params({
|
||||
'writeinfojson': True,
|
||||
'skip_download': True,
|
||||
'writedescription': True,
|
||||
})
|
||||
|
||||
|
||||
TEST_ID = 'BaW_jenozKc'
|
||||
INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
|
||||
@@ -42,6 +39,7 @@ This is a test video for youtube-dl.
|
||||
|
||||
For more information, contact phihag@phihag.de .'''
|
||||
|
||||
|
||||
class TestInfoJSON(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Clear old files
|
||||
|
@@ -1,20 +1,26 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
import json
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from youtube_dl.extractor import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE, YoutubeShowIE
|
||||
from youtube_dl.utils import *
|
||||
from test.helper import FakeYDL, global_setup
|
||||
global_setup()
|
||||
|
||||
|
||||
from youtube_dl.extractor import (
|
||||
YoutubeUserIE,
|
||||
YoutubePlaylistIE,
|
||||
YoutubeIE,
|
||||
YoutubeChannelIE,
|
||||
YoutubeShowIE,
|
||||
)
|
||||
|
||||
from helper import FakeYDL
|
||||
|
||||
class TestYoutubeLists(unittest.TestCase):
|
||||
def assertIsPlaylist(self,info):
|
||||
def assertIsPlaylist(self, info):
|
||||
"""Make sure the info has '_type' set to 'playlist'"""
|
||||
self.assertEqual(info['_type'], 'playlist')
|
||||
|
||||
@@ -27,6 +33,14 @@ class TestYoutubeLists(unittest.TestCase):
|
||||
ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']]
|
||||
self.assertEqual(ytie_results, [ 'bV9L5Ht9LgY', 'FXxLjLQi3Fg', 'tU3Bgo5qJZE'])
|
||||
|
||||
def test_youtube_playlist_noplaylist(self):
|
||||
dl = FakeYDL()
|
||||
dl.params['noplaylist'] = True
|
||||
ie = YoutubePlaylistIE(dl)
|
||||
result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
|
||||
self.assertEqual(result['_type'], 'url')
|
||||
self.assertEqual(YoutubeIE()._extract_id(result['url']), 'FXxLjLQi3Fg')
|
||||
|
||||
def test_issue_673(self):
|
||||
dl = FakeYDL()
|
||||
ie = YoutubePlaylistIE(dl)
|
||||
@@ -92,7 +106,7 @@ class TestYoutubeLists(unittest.TestCase):
|
||||
dl = FakeYDL()
|
||||
ie = YoutubeShowIE(dl)
|
||||
result = ie.extract('http://www.youtube.com/show/airdisasters')
|
||||
self.assertTrue(len(result) >= 4)
|
||||
self.assertTrue(len(result) >= 3)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
84
test/test_youtube_signature.py
Normal file
84
test/test_youtube_signature.py
Normal file
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import global_setup
|
||||
global_setup()
|
||||
|
||||
|
||||
import io
|
||||
import re
|
||||
import string
|
||||
|
||||
from youtube_dl.extractor import YoutubeIE
|
||||
from youtube_dl.utils import compat_str, compat_urlretrieve
|
||||
|
||||
_TESTS = [
|
||||
(
|
||||
u'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
|
||||
u'js',
|
||||
86,
|
||||
u'>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
|
||||
),
|
||||
(
|
||||
u'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
|
||||
u'js',
|
||||
85,
|
||||
u'3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
|
||||
),
|
||||
(
|
||||
u'https://s.ytimg.com/yts/swfbin/watch_as3-vflg5GhxU.swf',
|
||||
u'swf',
|
||||
82,
|
||||
u':/.-,+*)=\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBAzyxw>utsrqponmlkjihgfedcba987654321'
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TestSignature(unittest.TestCase):
|
||||
def setUp(self):
|
||||
TEST_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
|
||||
if not os.path.exists(self.TESTDATA_DIR):
|
||||
os.mkdir(self.TESTDATA_DIR)
|
||||
|
||||
|
||||
def make_tfunc(url, stype, sig_length, expected_sig):
|
||||
basename = url.rpartition('/')[2]
|
||||
m = re.match(r'.*-([a-zA-Z0-9_-]+)\.[a-z]+$', basename)
|
||||
assert m, '%r should follow URL format' % basename
|
||||
test_id = m.group(1)
|
||||
|
||||
def test_func(self):
|
||||
fn = os.path.join(self.TESTDATA_DIR, basename)
|
||||
|
||||
if not os.path.exists(fn):
|
||||
compat_urlretrieve(url, fn)
|
||||
|
||||
ie = YoutubeIE()
|
||||
if stype == 'js':
|
||||
with io.open(fn, encoding='utf-8') as testf:
|
||||
jscode = testf.read()
|
||||
func = ie._parse_sig_js(jscode)
|
||||
else:
|
||||
assert stype == 'swf'
|
||||
with open(fn, 'rb') as testf:
|
||||
swfcode = testf.read()
|
||||
func = ie._parse_sig_swf(swfcode)
|
||||
src_sig = compat_str(string.printable[:sig_length])
|
||||
got_sig = func(src_sig)
|
||||
self.assertEqual(got_sig, expected_sig)
|
||||
|
||||
test_func.__name__ = str('test_signature_' + stype + '_' + test_id)
|
||||
setattr(TestSignature, test_func.__name__, test_func)
|
||||
|
||||
for test_spec in _TESTS:
|
||||
make_tfunc(*test_spec)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
@@ -1,76 +1,87 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
import json
|
||||
import io
|
||||
import hashlib
|
||||
|
||||
# Allow direct execution
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import sys
|
||||
import unittest
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import FakeYDL, global_setup, md5
|
||||
global_setup()
|
||||
|
||||
|
||||
from youtube_dl.extractor import YoutubeIE
|
||||
from youtube_dl.utils import *
|
||||
from helper import FakeYDL
|
||||
|
||||
md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||
|
||||
class TestYoutubeSubtitles(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.DL = FakeYDL()
|
||||
self.url = 'QRS8MkLhQmM'
|
||||
|
||||
def getInfoDict(self):
|
||||
IE = YoutubeIE(self.DL)
|
||||
info_dict = IE.extract(self.url)
|
||||
return info_dict
|
||||
|
||||
def getSubtitles(self):
|
||||
info_dict = self.getInfoDict()
|
||||
return info_dict[0]['subtitles']
|
||||
return info_dict[0]['subtitles']
|
||||
|
||||
def test_youtube_no_writesubtitles(self):
|
||||
self.DL.params['writesubtitles'] = False
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(subtitles, None)
|
||||
|
||||
def test_youtube_subtitles(self):
|
||||
self.DL.params['writesubtitles'] = True
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
|
||||
|
||||
def test_youtube_subtitles_lang(self):
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['subtitleslangs'] = ['it']
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
|
||||
|
||||
def test_youtube_allsubtitles(self):
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['allsubtitles'] = True
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(len(subtitles.keys()), 13)
|
||||
|
||||
def test_youtube_subtitles_sbv_format(self):
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['subtitlesformat'] = 'sbv'
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
|
||||
|
||||
def test_youtube_subtitles_vtt_format(self):
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['subtitlesformat'] = 'vtt'
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
|
||||
|
||||
def test_youtube_list_subtitles(self):
|
||||
self.DL.expect_warning(u'Video doesn\'t have automatic captions')
|
||||
self.DL.params['listsubtitles'] = True
|
||||
info_dict = self.getInfoDict()
|
||||
self.assertEqual(info_dict, None)
|
||||
|
||||
def test_youtube_automatic_captions(self):
|
||||
self.url = '8YoUxe5ncPo'
|
||||
self.DL.params['writeautomaticsub'] = True
|
||||
self.DL.params['subtitleslangs'] = ['it']
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertTrue(subtitles['it'] is not None)
|
||||
|
||||
def test_youtube_nosubtitles(self):
|
||||
self.DL.expect_warning(u'video doesn\'t have subtitles')
|
||||
self.url = 'sAjKT8FhjI8'
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['allsubtitles'] = True
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(len(subtitles), 0)
|
||||
|
||||
def test_youtube_multiple_langs(self):
|
||||
self.url = 'QRS8MkLhQmM'
|
||||
self.DL.params['writesubtitles'] = True
|
||||
|
8
tox.ini
Normal file
8
tox.ini
Normal file
@@ -0,0 +1,8 @@
|
||||
[tox]
|
||||
envlist = py26,py27,py33
|
||||
[testenv]
|
||||
deps =
|
||||
nose
|
||||
coverage
|
||||
commands = nosetests --verbose {posargs:test} # --with-coverage --cover-package=youtube_dl --cover-html
|
||||
# test.test_download:TestDownload.test_NowVideo
|
@@ -4,12 +4,19 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
if os.name == 'nt':
|
||||
import ctypes
|
||||
|
||||
from .utils import *
|
||||
from .utils import (
|
||||
compat_urllib_error,
|
||||
compat_urllib_request,
|
||||
ContentTooShortError,
|
||||
determine_ext,
|
||||
encodeFilename,
|
||||
sanitize_open,
|
||||
timeconvert,
|
||||
)
|
||||
|
||||
|
||||
class FileDownloader(object):
|
||||
@@ -77,26 +84,43 @@ class FileDownloader(object):
|
||||
@staticmethod
|
||||
def calc_percent(byte_counter, data_len):
|
||||
if data_len is None:
|
||||
return None
|
||||
return float(byte_counter) / float(data_len) * 100.0
|
||||
|
||||
@staticmethod
|
||||
def format_percent(percent):
|
||||
if percent is None:
|
||||
return '---.-%'
|
||||
return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
|
||||
return '%6s' % ('%3.1f%%' % percent)
|
||||
|
||||
@staticmethod
|
||||
def calc_eta(start, now, total, current):
|
||||
if total is None:
|
||||
return '--:--'
|
||||
return None
|
||||
dif = now - start
|
||||
if current == 0 or dif < 0.001: # One millisecond
|
||||
return '--:--'
|
||||
return None
|
||||
rate = float(current) / dif
|
||||
eta = int((float(total) - float(current)) / rate)
|
||||
return int((float(total) - float(current)) / rate)
|
||||
|
||||
@staticmethod
|
||||
def format_eta(eta):
|
||||
if eta is None:
|
||||
return '--:--'
|
||||
return FileDownloader.format_seconds(eta)
|
||||
|
||||
@staticmethod
|
||||
def calc_speed(start, now, bytes):
|
||||
dif = now - start
|
||||
if bytes == 0 or dif < 0.001: # One millisecond
|
||||
return None
|
||||
return float(bytes) / dif
|
||||
|
||||
@staticmethod
|
||||
def format_speed(speed):
|
||||
if speed is None:
|
||||
return '%10s' % '---b/s'
|
||||
return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
|
||||
return '%10s' % ('%s/s' % FileDownloader.format_bytes(speed))
|
||||
|
||||
@staticmethod
|
||||
def best_block_size(elapsed_time, bytes):
|
||||
@@ -177,7 +201,7 @@ class FileDownloader(object):
|
||||
if old_filename == new_filename:
|
||||
return
|
||||
os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
|
||||
except (IOError, OSError) as err:
|
||||
except (IOError, OSError):
|
||||
self.report_error(u'unable to rename file')
|
||||
|
||||
def try_utime(self, filename, last_modified_hdr):
|
||||
@@ -205,11 +229,20 @@ class FileDownloader(object):
|
||||
"""Report destination filename."""
|
||||
self.to_screen(u'[download] Destination: ' + filename)
|
||||
|
||||
def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
|
||||
def report_progress(self, percent, data_len_str, speed, eta):
|
||||
"""Report download progress."""
|
||||
if self.params.get('noprogress', False):
|
||||
return
|
||||
clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
|
||||
if eta is not None:
|
||||
eta_str = self.format_eta(eta)
|
||||
else:
|
||||
eta_str = 'Unknown ETA'
|
||||
if percent is not None:
|
||||
percent_str = self.format_percent(percent)
|
||||
else:
|
||||
percent_str = 'Unknown %'
|
||||
speed_str = self.format_speed(speed)
|
||||
if self.params.get('progress_with_newline', False):
|
||||
self.to_screen(u'[download] %s of %s at %s ETA %s' %
|
||||
(percent_str, data_len_str, speed_str, eta_str))
|
||||
@@ -231,7 +264,7 @@ class FileDownloader(object):
|
||||
"""Report file has already been fully downloaded."""
|
||||
try:
|
||||
self.to_screen(u'[download] %s has already been downloaded' % file_name)
|
||||
except (UnicodeEncodeError) as err:
|
||||
except UnicodeEncodeError:
|
||||
self.to_screen(u'[download] The file has already been downloaded')
|
||||
|
||||
def report_unable_to_resume(self):
|
||||
@@ -250,6 +283,7 @@ class FileDownloader(object):
|
||||
def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
|
||||
self.report_destination(filename)
|
||||
tmpfilename = self.temp_name(filename)
|
||||
test = self.params.get('test', False)
|
||||
|
||||
# Check for rtmpdump first
|
||||
try:
|
||||
@@ -271,6 +305,8 @@ class FileDownloader(object):
|
||||
basic_args += ['--playpath', play_path]
|
||||
if tc_url is not None:
|
||||
basic_args += ['--tcUrl', url]
|
||||
if test:
|
||||
basic_args += ['--stop', '1']
|
||||
args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
|
||||
if self.params.get('verbose', False):
|
||||
try:
|
||||
@@ -280,7 +316,7 @@ class FileDownloader(object):
|
||||
shell_quote = repr
|
||||
self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
|
||||
retval = subprocess.call(args)
|
||||
while retval == 2 or retval == 1:
|
||||
while (retval == 2 or retval == 1) and not test:
|
||||
prevsize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
|
||||
time.sleep(5.0) # This seems to be needed
|
||||
@@ -293,7 +329,7 @@ class FileDownloader(object):
|
||||
self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
|
||||
retval = 0
|
||||
break
|
||||
if retval == 0:
|
||||
if retval == 0 or (test and retval == 2):
|
||||
fsize = os.path.getsize(encodeFilename(tmpfilename))
|
||||
self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
|
||||
self.try_rename(tmpfilename, filename)
|
||||
@@ -378,6 +414,7 @@ class FileDownloader(object):
|
||||
self._hook_progress({
|
||||
'filename': filename,
|
||||
'status': 'finished',
|
||||
'total_bytes': os.path.getsize(encodeFilename(filename)),
|
||||
})
|
||||
return True
|
||||
|
||||
@@ -524,13 +561,13 @@ class FileDownloader(object):
|
||||
block_size = self.best_block_size(after - before, len(data_block))
|
||||
|
||||
# Progress message
|
||||
speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
|
||||
speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
|
||||
if data_len is None:
|
||||
self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
|
||||
eta = percent = None
|
||||
else:
|
||||
percent_str = self.calc_percent(byte_counter, data_len)
|
||||
eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
|
||||
self.report_progress(percent_str, data_len_str, speed_str, eta_str)
|
||||
percent = self.calc_percent(byte_counter, data_len)
|
||||
eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
|
||||
self.report_progress(percent, data_len_str, speed, eta)
|
||||
|
||||
self._hook_progress({
|
||||
'downloaded_bytes': byte_counter,
|
||||
@@ -538,6 +575,8 @@ class FileDownloader(object):
|
||||
'tmpfilename': tmpfilename,
|
||||
'filename': filename,
|
||||
'status': 'downloading',
|
||||
'eta': eta,
|
||||
'speed': speed,
|
||||
})
|
||||
|
||||
# Apply rate limit
|
||||
@@ -580,6 +619,8 @@ class FileDownloader(object):
|
||||
* downloaded_bytes: Bytes on disks
|
||||
* total_bytes: Total bytes, None if unknown
|
||||
* tmpfilename: The filename we're currently writing to
|
||||
* eta: The estimated time in seconds, None if unknown
|
||||
* speed: The download speed in bytes/second, None if unknown
|
||||
|
||||
Hooks are guaranteed to be called at least once (with status "finished")
|
||||
if the download is successful.
|
||||
|
@@ -3,7 +3,14 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from .utils import *
|
||||
|
||||
from .utils import (
|
||||
compat_subprocess_get_DEVNULL,
|
||||
encodeFilename,
|
||||
PostProcessingError,
|
||||
shell_quote,
|
||||
subtitles_filename,
|
||||
)
|
||||
|
||||
|
||||
class PostProcessor(object):
|
||||
@@ -82,6 +89,8 @@ class FFmpegPostProcessor(PostProcessor):
|
||||
+ opts +
|
||||
[encodeFilename(self._ffmpeg_filename_argument(out_path))])
|
||||
|
||||
if self._downloader.params.get('verbose', False):
|
||||
self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd))
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout,stderr = p.communicate()
|
||||
if p.returncode != 0:
|
||||
@@ -177,7 +186,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
|
||||
extension = self._preferredcodec
|
||||
more_opts = []
|
||||
if self._preferredquality is not None:
|
||||
if int(self._preferredquality) < 10:
|
||||
# The opus codec doesn't support the -aq option
|
||||
if int(self._preferredquality) < 10 and extension != 'opus':
|
||||
more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
|
||||
else:
|
||||
more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
|
||||
@@ -444,8 +454,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
|
||||
if information['ext'] != u'mp4':
|
||||
self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
|
||||
return True, information
|
||||
sub_langs = [key for key in information['subtitles']]
|
||||
if not information.get('subtitles'):
|
||||
self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed')
|
||||
return True, information
|
||||
|
||||
sub_langs = [key for key in information['subtitles']]
|
||||
filename = information['filepath']
|
||||
input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
|
||||
|
||||
@@ -464,3 +477,35 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
|
||||
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
|
||||
return True, information
|
||||
|
||||
|
||||
class FFmpegMetadataPP(FFmpegPostProcessor):
|
||||
def run(self, info):
|
||||
metadata = {}
|
||||
if info.get('title') is not None:
|
||||
metadata['title'] = info['title']
|
||||
if info.get('upload_date') is not None:
|
||||
metadata['date'] = info['upload_date']
|
||||
if info.get('uploader') is not None:
|
||||
metadata['artist'] = info['uploader']
|
||||
elif info.get('uploader_id') is not None:
|
||||
metadata['artist'] = info['uploader_id']
|
||||
|
||||
if not metadata:
|
||||
self._downloader.to_screen(u'[ffmpeg] There isn\'t any metadata to add')
|
||||
return True, info
|
||||
|
||||
filename = info['filepath']
|
||||
ext = os.path.splitext(filename)[1][1:]
|
||||
temp_filename = filename + u'.temp'
|
||||
|
||||
options = ['-c', 'copy']
|
||||
for (name, value) in metadata.items():
|
||||
options.extend(['-metadata', '%s="%s"' % (name, value)])
|
||||
options.extend(['-f', ext])
|
||||
|
||||
self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename)
|
||||
self.run_ffmpeg(filename, temp_filename, options)
|
||||
os.remove(encodeFilename(filename))
|
||||
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
|
||||
return True, info
|
||||
|
@@ -3,6 +3,7 @@
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import errno
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
@@ -70,6 +71,7 @@ class YoutubeDL(object):
|
||||
logtostderr: Log messages to stderr instead of stdout.
|
||||
writedescription: Write the video description to a .description file
|
||||
writeinfojson: Write the video description to a .info.json file
|
||||
writeannotations: Write the video annotations to a .annotations.xml file
|
||||
writethumbnail: Write the thumbnail image to a file
|
||||
writesubtitles: Write the video subtitles to a file
|
||||
writeautomaticsub: Write the automatic subtitles to a file
|
||||
@@ -81,7 +83,15 @@ class YoutubeDL(object):
|
||||
keepvideo: Keep the video file after post-processing
|
||||
daterange: A DateRange object, download only if the upload_date is in the range.
|
||||
skip_download: Skip the actual download of the video file
|
||||
|
||||
cachedir: Location of the cache files in the filesystem.
|
||||
None to disable filesystem cache.
|
||||
noplaylist: Download single video instead of a playlist if in doubt.
|
||||
age_limit: An integer representing the user's age in years.
|
||||
Unsuitable videos for the given age are skipped.
|
||||
downloadarchive: File name of a file where all downloads are recorded.
|
||||
Videos already present in the file are not downloaded
|
||||
again.
|
||||
|
||||
The following parameters are not used by YoutubeDL itself, they are used by
|
||||
the FileDownloader:
|
||||
nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
|
||||
@@ -104,6 +114,17 @@ class YoutubeDL(object):
|
||||
self._download_retcode = 0
|
||||
self._num_downloads = 0
|
||||
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
|
||||
|
||||
if (sys.version_info >= (3,) and sys.platform != 'win32' and
|
||||
sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
|
||||
and not params['restrictfilenames']):
|
||||
# On Python 3, the Unicode filesystem API will throw errors (#1474)
|
||||
self.report_warning(
|
||||
u'Assuming --restrict-filenames since file system encoding '
|
||||
u'cannot encode all charactes. '
|
||||
u'Set the LC_ALL environment variable to fix this.')
|
||||
params['restrictfilenames'] = True
|
||||
|
||||
self.params = params
|
||||
self.fd = FileDownloader(self, self.params)
|
||||
|
||||
@@ -195,10 +216,10 @@ class YoutubeDL(object):
|
||||
If stderr is a tty file the 'WARNING:' will be colored
|
||||
'''
|
||||
if sys.stderr.isatty() and os.name != 'nt':
|
||||
_msg_header=u'\033[0;33mWARNING:\033[0m'
|
||||
_msg_header = u'\033[0;33mWARNING:\033[0m'
|
||||
else:
|
||||
_msg_header=u'WARNING:'
|
||||
warning_message=u'%s %s' % (_msg_header,message)
|
||||
_msg_header = u'WARNING:'
|
||||
warning_message = u'%s %s' % (_msg_header, message)
|
||||
self.to_stderr(warning_message)
|
||||
|
||||
def report_error(self, message, tb=None):
|
||||
@@ -213,19 +234,6 @@ class YoutubeDL(object):
|
||||
error_message = u'%s %s' % (_msg_header, message)
|
||||
self.trouble(error_message, tb)
|
||||
|
||||
def slow_down(self, start_time, byte_counter):
|
||||
"""Sleep if the download speed is over the rate limit."""
|
||||
rate_limit = self.params.get('ratelimit', None)
|
||||
if rate_limit is None or byte_counter == 0:
|
||||
return
|
||||
now = time.time()
|
||||
elapsed = now - start_time
|
||||
if elapsed <= 0.0:
|
||||
return
|
||||
speed = float(byte_counter) / elapsed
|
||||
if speed > rate_limit:
|
||||
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
|
||||
|
||||
def report_writedescription(self, descfn):
|
||||
""" Report that the description file is being written """
|
||||
self.to_screen(u'[info] Writing video description to: ' + descfn)
|
||||
@@ -238,6 +246,10 @@ class YoutubeDL(object):
|
||||
""" Report that the metadata file has been written """
|
||||
self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
|
||||
|
||||
def report_writeannotations(self, annofn):
|
||||
""" Report that the annotations file has been written. """
|
||||
self.to_screen(u'[info] Writing video annotations to: ' + annofn)
|
||||
|
||||
def report_file_already_downloaded(self, file_name):
|
||||
"""Report file has already been fully downloaded."""
|
||||
try:
|
||||
@@ -260,16 +272,18 @@ class YoutubeDL(object):
|
||||
autonumber_size = 5
|
||||
autonumber_templ = u'%0' + str(autonumber_size) + u'd'
|
||||
template_dict['autonumber'] = autonumber_templ % self._num_downloads
|
||||
if template_dict['playlist_index'] is not None:
|
||||
if template_dict.get('playlist_index') is not None:
|
||||
template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
|
||||
|
||||
sanitize = lambda k,v: sanitize_filename(
|
||||
sanitize = lambda k, v: sanitize_filename(
|
||||
u'NA' if v is None else compat_str(v),
|
||||
restricted=self.params.get('restrictfilenames'),
|
||||
is_id=(k==u'id'))
|
||||
template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
|
||||
is_id=(k == u'id'))
|
||||
template_dict = dict((k, sanitize(k, v))
|
||||
for k, v in template_dict.items())
|
||||
|
||||
filename = self.params['outtmpl'] % template_dict
|
||||
tmpl = os.path.expanduser(self.params['outtmpl'])
|
||||
filename = tmpl % template_dict
|
||||
return filename
|
||||
except KeyError as err:
|
||||
self.report_error(u'Erroneous output template')
|
||||
@@ -295,15 +309,28 @@ class YoutubeDL(object):
|
||||
dateRange = self.params.get('daterange', DateRange())
|
||||
if date not in dateRange:
|
||||
return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
|
||||
age_limit = self.params.get('age_limit')
|
||||
if age_limit is not None:
|
||||
if age_limit < info_dict.get('age_limit', 0):
|
||||
return u'Skipping "' + title + '" because it is age restricted'
|
||||
if self.in_download_archive(info_dict):
|
||||
return (u'%(title)s has already been recorded in archive'
|
||||
% info_dict)
|
||||
return None
|
||||
|
||||
|
||||
@staticmethod
|
||||
def add_extra_info(info_dict, extra_info):
|
||||
'''Set the keys from extra_info in info dict if they are missing'''
|
||||
for key, value in extra_info.items():
|
||||
info_dict.setdefault(key, value)
|
||||
|
||||
def extract_info(self, url, download=True, ie_key=None, extra_info={}):
|
||||
'''
|
||||
Returns a list with a dictionary for each video we find.
|
||||
If 'download', also downloads the videos.
|
||||
extra_info is a dict containing the extra values to add to each result
|
||||
'''
|
||||
|
||||
|
||||
if ie_key:
|
||||
ies = [self.get_info_extractor(ie_key)]
|
||||
else:
|
||||
@@ -323,17 +350,17 @@ class YoutubeDL(object):
|
||||
break
|
||||
if isinstance(ie_result, list):
|
||||
# Backwards compatibility: old IE result format
|
||||
for result in ie_result:
|
||||
result.update(extra_info)
|
||||
ie_result = {
|
||||
'_type': 'compat_list',
|
||||
'entries': ie_result,
|
||||
}
|
||||
else:
|
||||
ie_result.update(extra_info)
|
||||
if 'extractor' not in ie_result:
|
||||
ie_result['extractor'] = ie.IE_NAME
|
||||
return self.process_ie_result(ie_result, download=download)
|
||||
self.add_extra_info(ie_result,
|
||||
{
|
||||
'extractor': ie.IE_NAME,
|
||||
'webpage_url': url,
|
||||
'extractor_key': ie.ie_key(),
|
||||
})
|
||||
return self.process_ie_result(ie_result, download, extra_info)
|
||||
except ExtractorError as de: # An error we somewhat expected
|
||||
self.report_error(compat_str(de), de.format_traceback())
|
||||
break
|
||||
@@ -345,7 +372,7 @@ class YoutubeDL(object):
|
||||
raise
|
||||
else:
|
||||
self.report_error(u'no suitable InfoExtractor: %s' % url)
|
||||
|
||||
|
||||
def process_ie_result(self, ie_result, download=True, extra_info={}):
|
||||
"""
|
||||
Take the result of the ie(may be modified) and resolve all unresolved
|
||||
@@ -357,14 +384,8 @@ class YoutubeDL(object):
|
||||
|
||||
result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
|
||||
if result_type == 'video':
|
||||
ie_result.update(extra_info)
|
||||
if 'playlist' not in ie_result:
|
||||
# It isn't part of a playlist
|
||||
ie_result['playlist'] = None
|
||||
ie_result['playlist_index'] = None
|
||||
if download:
|
||||
self.process_info(ie_result)
|
||||
return ie_result
|
||||
self.add_extra_info(ie_result, extra_info)
|
||||
return self.process_video_result(ie_result)
|
||||
elif result_type == 'url':
|
||||
# We have to add extra_info to the results because it may be
|
||||
# contained in a playlist
|
||||
@@ -373,9 +394,10 @@ class YoutubeDL(object):
|
||||
ie_key=ie_result.get('ie_key'),
|
||||
extra_info=extra_info)
|
||||
elif result_type == 'playlist':
|
||||
self.add_extra_info(ie_result, extra_info)
|
||||
# We process each entry in the playlist
|
||||
playlist = ie_result.get('title', None) or ie_result.get('id', None)
|
||||
self.to_screen(u'[download] Downloading playlist: %s' % playlist)
|
||||
self.to_screen(u'[download] Downloading playlist: %s' % playlist)
|
||||
|
||||
playlist_results = []
|
||||
|
||||
@@ -393,17 +415,15 @@ class YoutubeDL(object):
|
||||
self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
|
||||
(ie_result['extractor'], playlist, n_all_entries, n_entries))
|
||||
|
||||
for i,entry in enumerate(entries,1):
|
||||
self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
|
||||
for i, entry in enumerate(entries, 1):
|
||||
self.to_screen(u'[download] Downloading video #%s of %s' % (i, n_entries))
|
||||
extra = {
|
||||
'playlist': playlist,
|
||||
'playlist_index': i + playliststart,
|
||||
}
|
||||
if not 'extractor' in entry:
|
||||
# We set the extractor, if it's an url it will be set then to
|
||||
# the new extractor, but if it's already a video we must make
|
||||
# sure it's present: see issue #877
|
||||
entry['extractor'] = ie_result['extractor']
|
||||
'playlist': playlist,
|
||||
'playlist_index': i + playliststart,
|
||||
'extractor': ie_result['extractor'],
|
||||
'webpage_url': ie_result['webpage_url'],
|
||||
'extractor_key': ie_result['extractor_key'],
|
||||
}
|
||||
entry_result = self.process_ie_result(entry,
|
||||
download=download,
|
||||
extra_info=extra)
|
||||
@@ -412,16 +432,122 @@ class YoutubeDL(object):
|
||||
return ie_result
|
||||
elif result_type == 'compat_list':
|
||||
def _fixup(r):
|
||||
r.setdefault('extractor', ie_result['extractor'])
|
||||
self.add_extra_info(r,
|
||||
{
|
||||
'extractor': ie_result['extractor'],
|
||||
'webpage_url': ie_result['webpage_url'],
|
||||
'extractor_key': ie_result['extractor_key'],
|
||||
})
|
||||
return r
|
||||
ie_result['entries'] = [
|
||||
self.process_ie_result(_fixup(r), download=download)
|
||||
self.process_ie_result(_fixup(r), download, extra_info)
|
||||
for r in ie_result['entries']
|
||||
]
|
||||
return ie_result
|
||||
else:
|
||||
raise Exception('Invalid result type: %s' % result_type)
|
||||
|
||||
def select_format(self, format_spec, available_formats):
|
||||
if format_spec == 'best' or format_spec is None:
|
||||
return available_formats[-1]
|
||||
elif format_spec == 'worst':
|
||||
return available_formats[0]
|
||||
else:
|
||||
extensions = [u'mp4', u'flv', u'webm', u'3gp']
|
||||
if format_spec in extensions:
|
||||
filter_f = lambda f: f['ext'] == format_spec
|
||||
else:
|
||||
filter_f = lambda f: f['format_id'] == format_spec
|
||||
matches = list(filter(filter_f, available_formats))
|
||||
if matches:
|
||||
return matches[-1]
|
||||
return None
|
||||
|
||||
def process_video_result(self, info_dict, download=True):
|
||||
assert info_dict.get('_type', 'video') == 'video'
|
||||
|
||||
if 'playlist' not in info_dict:
|
||||
# It isn't part of a playlist
|
||||
info_dict['playlist'] = None
|
||||
info_dict['playlist_index'] = None
|
||||
|
||||
# This extractors handle format selection themselves
|
||||
if info_dict['extractor'] in [u'youtube', u'Youku']:
|
||||
if download:
|
||||
self.process_info(info_dict)
|
||||
return info_dict
|
||||
|
||||
# We now pick which formats have to be downloaded
|
||||
if info_dict.get('formats') is None:
|
||||
# There's only one format available
|
||||
formats = [info_dict]
|
||||
else:
|
||||
formats = info_dict['formats']
|
||||
|
||||
# We check that all the formats have the format and format_id fields
|
||||
for (i, format) in enumerate(formats):
|
||||
if format.get('format_id') is None:
|
||||
format['format_id'] = compat_str(i)
|
||||
if format.get('format') is None:
|
||||
format['format'] = u'{id} - {res}{note}'.format(
|
||||
id=format['format_id'],
|
||||
res=self.format_resolution(format),
|
||||
note=u' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
|
||||
)
|
||||
# Automatically determine file extension if missing
|
||||
if 'ext' not in format:
|
||||
format['ext'] = determine_ext(format['url'])
|
||||
|
||||
if self.params.get('listformats', None):
|
||||
self.list_formats(info_dict)
|
||||
return
|
||||
|
||||
format_limit = self.params.get('format_limit', None)
|
||||
if format_limit:
|
||||
formats = list(takewhile_inclusive(
|
||||
lambda f: f['format_id'] != format_limit, formats
|
||||
))
|
||||
if self.params.get('prefer_free_formats'):
|
||||
def _free_formats_key(f):
|
||||
try:
|
||||
ext_ord = [u'flv', u'mp4', u'webm'].index(f['ext'])
|
||||
except ValueError:
|
||||
ext_ord = -1
|
||||
# We only compare the extension if they have the same height and width
|
||||
return (f.get('height'), f.get('width'), ext_ord)
|
||||
formats = sorted(formats, key=_free_formats_key)
|
||||
|
||||
req_format = self.params.get('format', 'best')
|
||||
if req_format is None:
|
||||
req_format = 'best'
|
||||
formats_to_download = []
|
||||
# The -1 is for supporting YoutubeIE
|
||||
if req_format in ('-1', 'all'):
|
||||
formats_to_download = formats
|
||||
else:
|
||||
# We can accept formats requestd in the format: 34/5/best, we pick
|
||||
# the first that is available, starting from left
|
||||
req_formats = req_format.split('/')
|
||||
for rf in req_formats:
|
||||
selected_format = self.select_format(rf, formats)
|
||||
if selected_format is not None:
|
||||
formats_to_download = [selected_format]
|
||||
break
|
||||
if not formats_to_download:
|
||||
raise ExtractorError(u'requested format not available',
|
||||
expected=True)
|
||||
|
||||
if download:
|
||||
if len(formats_to_download) > 1:
|
||||
self.to_screen(u'[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
|
||||
for format in formats_to_download:
|
||||
new_info = dict(info_dict)
|
||||
new_info.update(format)
|
||||
self.process_info(new_info)
|
||||
# We update the info dict with the best quality format (backwards compatibility)
|
||||
info_dict.update(formats_to_download[-1])
|
||||
return info_dict
|
||||
|
||||
def process_info(self, info_dict):
|
||||
"""Process a single resolved IE result."""
|
||||
|
||||
@@ -459,9 +585,9 @@ class YoutubeDL(object):
|
||||
if self.params.get('forceurl', False):
|
||||
# For RTMP URLs, also include the playpath
|
||||
compat_print(info_dict['url'] + info_dict.get('play_path', u''))
|
||||
if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
|
||||
if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
|
||||
compat_print(info_dict['thumbnail'])
|
||||
if self.params.get('forcedescription', False) and 'description' in info_dict:
|
||||
if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
|
||||
compat_print(info_dict['description'])
|
||||
if self.params.get('forcefilename', False) and filename is not None:
|
||||
compat_print(filename)
|
||||
@@ -495,10 +621,22 @@ class YoutubeDL(object):
|
||||
self.report_error(u'Cannot write description file ' + descfn)
|
||||
return
|
||||
|
||||
if self.params.get('writeannotations', False):
|
||||
try:
|
||||
annofn = filename + u'.annotations.xml'
|
||||
self.report_writeannotations(annofn)
|
||||
with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
|
||||
annofile.write(info_dict['annotations'])
|
||||
except (KeyError, TypeError):
|
||||
self.report_warning(u'There are no annotations to write.')
|
||||
except (OSError, IOError):
|
||||
self.report_error(u'Cannot write annotations file: ' + annofn)
|
||||
return
|
||||
|
||||
subtitles_are_requested = any([self.params.get('writesubtitles', False),
|
||||
self.params.get('writeautomaticsub')])
|
||||
|
||||
if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
|
||||
if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
|
||||
# subtitles download errors are already managed as troubles in relevant IE
|
||||
# that way it will silently go on when used with unsupporting IE
|
||||
subtitles = info_dict['subtitles']
|
||||
@@ -520,7 +658,7 @@ class YoutubeDL(object):
|
||||
infofn = filename + u'.info.json'
|
||||
self.report_writeinfojson(infofn)
|
||||
try:
|
||||
json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
|
||||
json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle'])
|
||||
write_json_file(json_info_dict, encodeFilename(infofn))
|
||||
except (OSError, IOError):
|
||||
self.report_error(u'Cannot write metadata to JSON file ' + infofn)
|
||||
@@ -532,11 +670,15 @@ class YoutubeDL(object):
|
||||
thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
|
||||
self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
|
||||
(info_dict['extractor'], info_dict['id']))
|
||||
uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
|
||||
with open(thumb_filename, 'wb') as thumbf:
|
||||
shutil.copyfileobj(uf, thumbf)
|
||||
self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
|
||||
(info_dict['extractor'], info_dict['id'], thumb_filename))
|
||||
try:
|
||||
uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
|
||||
with open(thumb_filename, 'wb') as thumbf:
|
||||
shutil.copyfileobj(uf, thumbf)
|
||||
self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
|
||||
(info_dict['extractor'], info_dict['id'], thumb_filename))
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self.report_warning(u'Unable to download thumbnail "%s": %s' %
|
||||
(info_dict['thumbnail'], compat_str(err)))
|
||||
|
||||
if not self.params.get('skip_download', False):
|
||||
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
|
||||
@@ -544,11 +686,11 @@ class YoutubeDL(object):
|
||||
else:
|
||||
try:
|
||||
success = self.fd._do_download(filename, info_dict)
|
||||
except (OSError, IOError) as err:
|
||||
raise UnavailableVideoError(err)
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self.report_error(u'unable to download video data: %s' % str(err))
|
||||
return
|
||||
except (OSError, IOError) as err:
|
||||
raise UnavailableVideoError(err)
|
||||
except (ContentTooShortError, ) as err:
|
||||
self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
|
||||
return
|
||||
@@ -560,6 +702,8 @@ class YoutubeDL(object):
|
||||
self.report_error(u'postprocessing: %s' % str(err))
|
||||
return
|
||||
|
||||
self.record_download_archive(info_dict)
|
||||
|
||||
def download(self, url_list):
|
||||
"""Download a given list of URLs."""
|
||||
if len(url_list) > 1 and self.fixed_template():
|
||||
@@ -584,7 +728,7 @@ class YoutubeDL(object):
|
||||
keep_video = None
|
||||
for pp in self._pps:
|
||||
try:
|
||||
keep_video_wish,new_info = pp.run(info)
|
||||
keep_video_wish, new_info = pp.run(info)
|
||||
if keep_video_wish is not None:
|
||||
if keep_video_wish:
|
||||
keep_video = keep_video_wish
|
||||
@@ -599,3 +743,61 @@ class YoutubeDL(object):
|
||||
os.remove(encodeFilename(filename))
|
||||
except (IOError, OSError):
|
||||
self.report_warning(u'Unable to remove downloaded video file')
|
||||
|
||||
def in_download_archive(self, info_dict):
|
||||
fn = self.params.get('download_archive')
|
||||
if fn is None:
|
||||
return False
|
||||
vid_id = info_dict['extractor'] + u' ' + info_dict['id']
|
||||
try:
|
||||
with locked_file(fn, 'r', encoding='utf-8') as archive_file:
|
||||
for line in archive_file:
|
||||
if line.strip() == vid_id:
|
||||
return True
|
||||
except IOError as ioe:
|
||||
if ioe.errno != errno.ENOENT:
|
||||
raise
|
||||
return False
|
||||
|
||||
def record_download_archive(self, info_dict):
|
||||
fn = self.params.get('download_archive')
|
||||
if fn is None:
|
||||
return
|
||||
vid_id = info_dict['extractor'] + u' ' + info_dict['id']
|
||||
with locked_file(fn, 'a', encoding='utf-8') as archive_file:
|
||||
archive_file.write(vid_id + u'\n')
|
||||
|
||||
@staticmethod
|
||||
def format_resolution(format, default='unknown'):
|
||||
if format.get('_resolution') is not None:
|
||||
return format['_resolution']
|
||||
if format.get('height') is not None:
|
||||
if format.get('width') is not None:
|
||||
res = u'%sx%s' % (format['width'], format['height'])
|
||||
else:
|
||||
res = u'%sp' % format['height']
|
||||
else:
|
||||
res = default
|
||||
return res
|
||||
|
||||
def list_formats(self, info_dict):
|
||||
def line(format):
|
||||
return (u'%-15s%-10s%-12s%s' % (
|
||||
format['format_id'],
|
||||
format['ext'],
|
||||
self.format_resolution(format),
|
||||
format.get('format_note', ''),
|
||||
)
|
||||
)
|
||||
|
||||
formats = info_dict.get('formats', [info_dict])
|
||||
formats_s = list(map(line, formats))
|
||||
if len(formats) > 1:
|
||||
formats_s[0] += (' ' if formats[0].get('format_note') else '') + '(worst)'
|
||||
formats_s[-1] += (' ' if formats[-1].get('format_note') else '') + '(best)'
|
||||
|
||||
header_line = line({
|
||||
'format_id': u'format code', 'ext': u'extension',
|
||||
'_resolution': u'resolution', 'format_note': u'note'})
|
||||
self.to_screen(u'[info] Available formats for %s:\n%s\n%s' %
|
||||
(info_dict['id'], header_line, u"\n".join(formats_s)))
|
||||
|
@@ -31,11 +31,13 @@ __authors__ = (
|
||||
'Huarong Huo',
|
||||
'Ismael Mejía',
|
||||
'Steffan \'Ruirize\' James',
|
||||
'Andras Elso',
|
||||
)
|
||||
|
||||
__license__ = 'Public Domain'
|
||||
|
||||
import codecs
|
||||
import collections
|
||||
import getpass
|
||||
import optparse
|
||||
import os
|
||||
@@ -45,17 +47,43 @@ import shlex
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import warnings
|
||||
import traceback
|
||||
import platform
|
||||
|
||||
|
||||
from .utils import *
|
||||
from .utils import (
|
||||
compat_cookiejar,
|
||||
compat_print,
|
||||
compat_str,
|
||||
compat_urllib_request,
|
||||
DateRange,
|
||||
decodeOption,
|
||||
determine_ext,
|
||||
DownloadError,
|
||||
get_cachedir,
|
||||
make_HTTPS_handler,
|
||||
MaxDownloadsReached,
|
||||
platform_name,
|
||||
preferredencoding,
|
||||
SameFileError,
|
||||
std_headers,
|
||||
write_string,
|
||||
YoutubeDLHandler,
|
||||
)
|
||||
from .update import update_self
|
||||
from .version import __version__
|
||||
from .FileDownloader import *
|
||||
from .FileDownloader import (
|
||||
FileDownloader,
|
||||
)
|
||||
from .extractor import gen_extractors
|
||||
from .YoutubeDL import YoutubeDL
|
||||
from .PostProcessor import *
|
||||
from .PostProcessor import (
|
||||
FFmpegMetadataPP,
|
||||
FFmpegVideoConvertor,
|
||||
FFmpegExtractAudioPP,
|
||||
FFmpegEmbedSubtitlePP,
|
||||
)
|
||||
|
||||
|
||||
def parseOpts(overrideArguments=None):
|
||||
def _readOptions(filename_bytes):
|
||||
@@ -105,7 +133,7 @@ def parseOpts(overrideArguments=None):
|
||||
|
||||
def _hide_login_info(opts):
|
||||
opts = list(opts)
|
||||
for private_opt in ['-p', '--password', '-u', '--username']:
|
||||
for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
|
||||
try:
|
||||
i = opts.index(private_opt)
|
||||
opts[i+1] = '<PRIVATE>'
|
||||
@@ -151,6 +179,9 @@ def parseOpts(overrideArguments=None):
|
||||
action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
|
||||
general.add_option('-i', '--ignore-errors',
|
||||
action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
|
||||
general.add_option('--abort-on-error',
|
||||
action='store_false', dest='ignoreerrors',
|
||||
help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
|
||||
general.add_option('--dump-user-agent',
|
||||
action='store_true', dest='dump_user_agent',
|
||||
help='display the current browser identification', default=False)
|
||||
@@ -167,6 +198,12 @@ def parseOpts(overrideArguments=None):
|
||||
help='Output descriptions of all supported extractors', default=False)
|
||||
general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
|
||||
general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
|
||||
general.add_option(
|
||||
'--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
|
||||
help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
|
||||
general.add_option(
|
||||
'--no-cache-dir', action='store_const', const=None, dest='cachedir',
|
||||
help='Disable filesystem caching')
|
||||
|
||||
|
||||
selection.add_option('--playlist-start',
|
||||
@@ -181,6 +218,13 @@ def parseOpts(overrideArguments=None):
|
||||
selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
|
||||
selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
|
||||
selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
|
||||
selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
|
||||
selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
|
||||
help='download only videos suitable for the given age',
|
||||
default=None, type=int)
|
||||
selection.add_option('--download-archive', metavar='FILE',
|
||||
dest='download_archive',
|
||||
help='Download only videos not present in the archive file. Record all downloaded videos in it.')
|
||||
|
||||
|
||||
authentication.add_option('-u', '--username',
|
||||
@@ -194,7 +238,7 @@ def parseOpts(overrideArguments=None):
|
||||
|
||||
|
||||
video_format.add_option('-f', '--format',
|
||||
action='store', dest='format', metavar='FORMAT',
|
||||
action='store', dest='format', metavar='FORMAT', default='best',
|
||||
help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
|
||||
video_format.add_option('--all-formats',
|
||||
action='store_const', dest='format', help='download all available video formats', const='all')
|
||||
@@ -226,11 +270,11 @@ def parseOpts(overrideArguments=None):
|
||||
help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
|
||||
|
||||
downloader.add_option('-r', '--rate-limit',
|
||||
dest='ratelimit', metavar='LIMIT', help='maximum download rate (e.g. 50k or 44.6m)')
|
||||
dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
|
||||
downloader.add_option('-R', '--retries',
|
||||
dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
|
||||
downloader.add_option('--buffer-size',
|
||||
dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
|
||||
dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
|
||||
downloader.add_option('--no-resize-buffer',
|
||||
action='store_true', dest='noresizebuffer',
|
||||
help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
|
||||
@@ -272,6 +316,13 @@ def parseOpts(overrideArguments=None):
|
||||
verbosity.add_option('--dump-intermediate-pages',
|
||||
action='store_true', dest='dump_intermediate_pages', default=False,
|
||||
help='print downloaded pages to debug problems(very verbose)')
|
||||
verbosity.add_option('--write-pages',
|
||||
action='store_true', dest='write_pages', default=False,
|
||||
help='Write downloaded pages to files in the current directory')
|
||||
verbosity.add_option('--youtube-print-sig-code',
|
||||
action='store_true', dest='youtube_print_sig_code', default=False,
|
||||
help=optparse.SUPPRESS_HELP)
|
||||
|
||||
|
||||
filesystem.add_option('-t', '--title',
|
||||
action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
|
||||
@@ -287,7 +338,10 @@ def parseOpts(overrideArguments=None):
|
||||
help=('output filename template. Use %(title)s to get the title, '
|
||||
'%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
|
||||
'%(autonumber)s to get an automatically incremented number, '
|
||||
'%(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), '
|
||||
'%(ext)s for the filename extension, '
|
||||
'%(format)s for the format description (like "22 - 1280x720" or "HD"),'
|
||||
'%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"),'
|
||||
'%(upload_date)s for the upload date (YYYYMMDD), '
|
||||
'%(extractor)s for the provider (youtube, metacafe, etc), '
|
||||
'%(id)s for the video id , %(playlist)s for the playlist the video is in, '
|
||||
'%(playlist_index)s for the position in the playlist and %% for a literal percent. '
|
||||
@@ -321,6 +375,9 @@ def parseOpts(overrideArguments=None):
|
||||
filesystem.add_option('--write-info-json',
|
||||
action='store_true', dest='writeinfojson',
|
||||
help='write video metadata to a .info.json file', default=False)
|
||||
filesystem.add_option('--write-annotations',
|
||||
action='store_true', dest='writeannotations',
|
||||
help='write video annotations to a .annotation file', default=False)
|
||||
filesystem.add_option('--write-thumbnail',
|
||||
action='store_true', dest='writethumbnail',
|
||||
help='write thumbnail image to disk', default=False)
|
||||
@@ -340,6 +397,8 @@ def parseOpts(overrideArguments=None):
|
||||
help='do not overwrite post-processed files; the post-processed files are overwritten by default')
|
||||
postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
|
||||
help='embed subtitles in the video (only for mp4 videos)')
|
||||
postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
|
||||
help='add metadata to the files')
|
||||
|
||||
|
||||
parser.add_option_group(general)
|
||||
@@ -359,9 +418,13 @@ def parseOpts(overrideArguments=None):
|
||||
else:
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
|
||||
if xdg_config_home:
|
||||
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
|
||||
userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
|
||||
if not os.path.isfile(userConfFile):
|
||||
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
|
||||
else:
|
||||
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
|
||||
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
|
||||
if not os.path.isfile(userConfFile):
|
||||
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
|
||||
systemConf = _readOptions('/etc/youtube-dl.conf')
|
||||
userConf = _readOptions(userConfFile)
|
||||
commandLineConf = sys.argv[1:]
|
||||
@@ -426,27 +489,7 @@ def _real_main(argv=None):
|
||||
all_urls = batchurls + args
|
||||
all_urls = [url.strip() for url in all_urls]
|
||||
|
||||
# General configuration
|
||||
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
|
||||
if opts.proxy is not None:
|
||||
if opts.proxy == '':
|
||||
proxies = {}
|
||||
else:
|
||||
proxies = {'http': opts.proxy, 'https': opts.proxy}
|
||||
else:
|
||||
proxies = compat_urllib_request.getproxies()
|
||||
# Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
|
||||
if 'http' in proxies and 'https' not in proxies:
|
||||
proxies['https'] = proxies['http']
|
||||
proxy_handler = compat_urllib_request.ProxyHandler(proxies)
|
||||
https_handler = make_HTTPS_handler(opts)
|
||||
opener = compat_urllib_request.build_opener(https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
|
||||
# Delete the default user-agent header, which would otherwise apply in
|
||||
# cases where our custom HTTP handler doesn't come into play
|
||||
# (See https://github.com/rg3/youtube-dl/issues/1309 for details)
|
||||
opener.addheaders =[]
|
||||
compat_urllib_request.install_opener(opener)
|
||||
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
|
||||
opener = _setup_opener(jar=jar, opts=opts)
|
||||
|
||||
extractors = gen_extractors()
|
||||
|
||||
@@ -463,6 +506,8 @@ def _real_main(argv=None):
|
||||
if not ie._WORKING:
|
||||
continue
|
||||
desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
|
||||
if desc is False:
|
||||
continue
|
||||
if hasattr(ie, 'SEARCH_KEY'):
|
||||
_SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
|
||||
_COUNTS = (u'', u'5', u'10', u'all')
|
||||
@@ -551,6 +596,10 @@ def _real_main(argv=None):
|
||||
or (opts.useid and u'%(id)s.%(ext)s')
|
||||
or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
|
||||
or u'%(title)s-%(id)s.%(ext)s')
|
||||
if '%(ext)s' not in outtmpl and opts.extractaudio:
|
||||
parser.error(u'Cannot download a video and extract audio into the same'
|
||||
u' file! Use "%%(ext)s" instead of %r' %
|
||||
determine_ext(outtmpl, u''))
|
||||
|
||||
# YoutubeDL
|
||||
ydl = YoutubeDL({
|
||||
@@ -585,11 +634,13 @@ def _real_main(argv=None):
|
||||
'progress_with_newline': opts.progress_with_newline,
|
||||
'playliststart': opts.playliststart,
|
||||
'playlistend': opts.playlistend,
|
||||
'noplaylist': opts.noplaylist,
|
||||
'logtostderr': opts.outtmpl == '-',
|
||||
'consoletitle': opts.consoletitle,
|
||||
'nopart': opts.nopart,
|
||||
'updatetime': opts.updatetime,
|
||||
'writedescription': opts.writedescription,
|
||||
'writeannotations': opts.writeannotations,
|
||||
'writeinfojson': opts.writeinfojson,
|
||||
'writethumbnail': opts.writethumbnail,
|
||||
'writesubtitles': opts.writesubtitles,
|
||||
@@ -604,11 +655,16 @@ def _real_main(argv=None):
|
||||
'prefer_free_formats': opts.prefer_free_formats,
|
||||
'verbose': opts.verbose,
|
||||
'dump_intermediate_pages': opts.dump_intermediate_pages,
|
||||
'write_pages': opts.write_pages,
|
||||
'test': opts.test,
|
||||
'keepvideo': opts.keepvideo,
|
||||
'min_filesize': opts.min_filesize,
|
||||
'max_filesize': opts.max_filesize,
|
||||
'daterange': date,
|
||||
'cachedir': opts.cachedir,
|
||||
'youtube_print_sig_code': opts.youtube_print_sig_code,
|
||||
'age_limit': opts.age_limit,
|
||||
'download_archive': opts.download_archive,
|
||||
})
|
||||
|
||||
if opts.verbose:
|
||||
@@ -628,11 +684,19 @@ def _real_main(argv=None):
|
||||
except:
|
||||
pass
|
||||
write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
|
||||
write_string(u'[debug] Proxy map: ' + str(proxy_handler.proxies) + u'\n')
|
||||
|
||||
proxy_map = {}
|
||||
for handler in opener.handlers:
|
||||
if hasattr(handler, 'proxies'):
|
||||
proxy_map.update(handler.proxies)
|
||||
write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
|
||||
|
||||
ydl.add_default_info_extractors()
|
||||
|
||||
# PostProcessors
|
||||
# Add the metadata pp first, the other pps will copy it
|
||||
if opts.addmetadata:
|
||||
ydl.add_post_processor(FFmpegMetadataPP())
|
||||
if opts.extractaudio:
|
||||
ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
|
||||
if opts.recodevideo:
|
||||
@@ -642,7 +706,7 @@ def _real_main(argv=None):
|
||||
|
||||
# Update version
|
||||
if opts.update_self:
|
||||
update_self(ydl.to_screen, opts.verbose, sys.argv[0])
|
||||
update_self(ydl.to_screen, opts.verbose)
|
||||
|
||||
# Maybe do nothing
|
||||
if len(all_urls) < 1:
|
||||
@@ -661,11 +725,42 @@ def _real_main(argv=None):
|
||||
if opts.cookiefile is not None:
|
||||
try:
|
||||
jar.save()
|
||||
except (IOError, OSError) as err:
|
||||
except (IOError, OSError):
|
||||
sys.exit(u'ERROR: unable to save cookie jar')
|
||||
|
||||
sys.exit(retcode)
|
||||
|
||||
|
||||
def _setup_opener(jar=None, opts=None, timeout=300):
|
||||
if opts is None:
|
||||
FakeOptions = collections.namedtuple(
|
||||
'FakeOptions', ['proxy', 'no_check_certificate'])
|
||||
opts = FakeOptions(proxy=None, no_check_certificate=False)
|
||||
|
||||
cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
|
||||
if opts.proxy is not None:
|
||||
if opts.proxy == '':
|
||||
proxies = {}
|
||||
else:
|
||||
proxies = {'http': opts.proxy, 'https': opts.proxy}
|
||||
else:
|
||||
proxies = compat_urllib_request.getproxies()
|
||||
# Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
|
||||
if 'http' in proxies and 'https' not in proxies:
|
||||
proxies['https'] = proxies['http']
|
||||
proxy_handler = compat_urllib_request.ProxyHandler(proxies)
|
||||
https_handler = make_HTTPS_handler(opts)
|
||||
opener = compat_urllib_request.build_opener(
|
||||
https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
|
||||
# Delete the default user-agent header, which would otherwise apply in
|
||||
# cases where our custom HTTP handler doesn't come into play
|
||||
# (See https://github.com/rg3/youtube-dl/issues/1309 for details)
|
||||
opener.addheaders = []
|
||||
compat_urllib_request.install_opener(opener)
|
||||
socket.setdefaulttimeout(timeout)
|
||||
return opener
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
try:
|
||||
_real_main(argv)
|
||||
|
@@ -2,8 +2,14 @@ from .appletrailers import AppleTrailersIE
|
||||
from .addanime import AddAnimeIE
|
||||
from .archiveorg import ArchiveOrgIE
|
||||
from .ard import ARDIE
|
||||
from .arte import ArteTvIE
|
||||
from .arte import (
|
||||
ArteTvIE,
|
||||
ArteTVPlus7IE,
|
||||
ArteTVCreativeIE,
|
||||
ArteTVFutureIE,
|
||||
)
|
||||
from .auengine import AUEngineIE
|
||||
from .bambuser import BambuserIE, BambuserChannelIE
|
||||
from .bandcamp import BandcampIE
|
||||
from .bliptv import BlipTVIE, BlipTVUserIE
|
||||
from .bloomberg import BloombergIE
|
||||
@@ -12,23 +18,32 @@ from .brightcove import BrightcoveIE
|
||||
from .c56 import C56IE
|
||||
from .canalplus import CanalplusIE
|
||||
from .canalc2 import Canalc2IE
|
||||
from .cinemassacre import CinemassacreIE
|
||||
from .cnn import CNNIE
|
||||
from .collegehumor import CollegeHumorIE
|
||||
from .comedycentral import ComedyCentralIE
|
||||
from .condenast import CondeNastIE
|
||||
from .criterion import CriterionIE
|
||||
from .cspan import CSpanIE
|
||||
from .dailymotion import DailymotionIE, DailymotionPlaylistIE
|
||||
from .dailymotion import (
|
||||
DailymotionIE,
|
||||
DailymotionPlaylistIE,
|
||||
DailymotionUserIE,
|
||||
)
|
||||
from .daum import DaumIE
|
||||
from .depositfiles import DepositFilesIE
|
||||
from .dotsub import DotsubIE
|
||||
from .dreisat import DreiSatIE
|
||||
from .defense import DefenseGouvFrIE
|
||||
from .ebaumsworld import EbaumsWorldIE
|
||||
from .ehow import EHowIE
|
||||
from .eighttracks import EightTracksIE
|
||||
from .eitb import EitbIE
|
||||
from .escapist import EscapistIE
|
||||
from .exfm import ExfmIE
|
||||
from .extremetube import ExtremeTubeIE
|
||||
from .facebook import FacebookIE
|
||||
from .faz import FazIE
|
||||
from .fktv import (
|
||||
FKTVIE,
|
||||
FKTVPosteckeIE,
|
||||
@@ -37,6 +52,8 @@ from .flickr import FlickrIE
|
||||
from .francetv import (
|
||||
PluzzIE,
|
||||
FranceTvInfoIE,
|
||||
France2IE,
|
||||
GenerationQuoiIE
|
||||
)
|
||||
from .freesound import FreesoundIE
|
||||
from .funnyordie import FunnyOrDieIE
|
||||
@@ -53,10 +70,12 @@ from .ign import IGNIE, OneUPIE
|
||||
from .ina import InaIE
|
||||
from .infoq import InfoQIE
|
||||
from .instagram import InstagramIE
|
||||
from .internetvideoarchive import InternetVideoArchiveIE
|
||||
from .jeuxvideo import JeuxVideoIE
|
||||
from .jukebox import JukeboxIE
|
||||
from .justintv import JustinTVIE
|
||||
from .kankan import KankanIE
|
||||
from .keezmovies import KeezMoviesIE
|
||||
from .kickstarter import KickStarterIE
|
||||
from .keek import KeekIE
|
||||
from .liveleak import LiveLeakIE
|
||||
@@ -65,41 +84,53 @@ from .metacafe import MetacafeIE
|
||||
from .metacritic import MetacriticIE
|
||||
from .mit import TechTVMITIE, MITIE
|
||||
from .mixcloud import MixcloudIE
|
||||
from .mofosex import MofosexIE
|
||||
from .mtv import MTVIE
|
||||
from .muzu import MuzuTVIE
|
||||
from .myspace import MySpaceIE
|
||||
from .myspass import MySpassIE
|
||||
from .myvideo import MyVideoIE
|
||||
from .naver import NaverIE
|
||||
from .nba import NBAIE
|
||||
from .nbc import NBCNewsIE
|
||||
from .newgrounds import NewgroundsIE
|
||||
from .nhl import NHLIE, NHLVideocenterIE
|
||||
from .nowvideo import NowVideoIE
|
||||
from .ooyala import OoyalaIE
|
||||
from .orf import ORFIE
|
||||
from .pbs import PBSIE
|
||||
from .photobucket import PhotobucketIE
|
||||
from .pornhub import PornHubIE
|
||||
from .pornotube import PornotubeIE
|
||||
from .rbmaradio import RBMARadioIE
|
||||
from .redtube import RedTubeIE
|
||||
from .ringtv import RingTVIE
|
||||
from .ro220 import Ro220IE
|
||||
from .rottentomatoes import RottenTomatoesIE
|
||||
from .roxwel import RoxwelIE
|
||||
from .rtlnow import RTLnowIE
|
||||
from .rutube import RutubeIE
|
||||
from .sina import SinaIE
|
||||
from .slashdot import SlashdotIE
|
||||
from .slideshare import SlideshareIE
|
||||
from .sohu import SohuIE
|
||||
from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE
|
||||
from .southparkstudios import SouthParkStudiosIE
|
||||
from .space import SpaceIE
|
||||
from .spankwire import SpankwireIE
|
||||
from .spiegel import SpiegelIE
|
||||
from .stanfordoc import StanfordOpenClassroomIE
|
||||
from .statigram import StatigramIE
|
||||
from .steam import SteamIE
|
||||
from .sztvhu import SztvHuIE
|
||||
from .teamcoco import TeamcocoIE
|
||||
from .techtalks import TechTalksIE
|
||||
from .ted import TEDIE
|
||||
from .tf1 import TF1IE
|
||||
from .thisav import ThisAVIE
|
||||
from .traileraddict import TrailerAddictIE
|
||||
from .trilulilu import TriluliluIE
|
||||
from .tube8 import Tube8IE
|
||||
from .tudou import TudouIE
|
||||
from .tumblr import TumblrIE
|
||||
from .tutv import TutvIE
|
||||
@@ -110,16 +141,22 @@ from .veehd import VeeHDIE
|
||||
from .veoh import VeohIE
|
||||
from .vevo import VevoIE
|
||||
from .vice import ViceIE
|
||||
from .viddler import ViddlerIE
|
||||
from .videodetective import VideoDetectiveIE
|
||||
from .videofyme import VideofyMeIE
|
||||
from .videopremium import VideoPremiumIE
|
||||
from .vimeo import VimeoIE, VimeoChannelIE
|
||||
from .vine import VineIE
|
||||
from .vk import VKIE
|
||||
from .wat import WatIE
|
||||
from .websurg import WeBSurgIE
|
||||
from .weibo import WeiboIE
|
||||
from .wimp import WimpIE
|
||||
from .worldstarhiphop import WorldStarHipHopIE
|
||||
from .xhamster import XHamsterIE
|
||||
from .xnxx import XNXXIE
|
||||
from .xvideos import XVideosIE
|
||||
from .xtube import XTubeIE
|
||||
from .yahoo import YahooIE, YahooSearchIE
|
||||
from .youjizz import YouJizzIE
|
||||
from .youku import YoukuIE
|
||||
@@ -128,11 +165,13 @@ from .youtube import (
|
||||
YoutubeIE,
|
||||
YoutubePlaylistIE,
|
||||
YoutubeSearchIE,
|
||||
YoutubeSearchDateIE,
|
||||
YoutubeUserIE,
|
||||
YoutubeChannelIE,
|
||||
YoutubeShowIE,
|
||||
YoutubeSubscriptionsIE,
|
||||
YoutubeRecommendedIE,
|
||||
YoutubeTruncatedURLIE,
|
||||
YoutubeWatchLaterIE,
|
||||
YoutubeFavouritesIE,
|
||||
)
|
||||
|
@@ -17,8 +17,8 @@ class AddAnimeIE(InfoExtractor):
|
||||
IE_NAME = u'AddAnime'
|
||||
_TEST = {
|
||||
u'url': u'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
|
||||
u'file': u'24MR3YO5SAS9.flv',
|
||||
u'md5': u'1036a0e0cd307b95bd8a8c3a5c8cfaf1',
|
||||
u'file': u'24MR3YO5SAS9.mp4',
|
||||
u'md5': u'72954ea10bc979ab5e2eb288b21425a0',
|
||||
u'info_dict': {
|
||||
u"description": u"One Piece 606",
|
||||
u"title": u"One Piece 606"
|
||||
@@ -31,7 +31,8 @@ class AddAnimeIE(InfoExtractor):
|
||||
video_id = mobj.group('video_id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
except ExtractorError as ee:
|
||||
if not isinstance(ee.cause, compat_HTTPError):
|
||||
if not isinstance(ee.cause, compat_HTTPError) or \
|
||||
ee.cause.code != 503:
|
||||
raise
|
||||
|
||||
redir_webpage = ee.cause.read().decode('utf-8')
|
||||
@@ -60,16 +61,26 @@ class AddAnimeIE(InfoExtractor):
|
||||
note=u'Confirming after redirect')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_url = self._search_regex(r"var normal_video_file = '(.*?)';",
|
||||
webpage, u'video file URL')
|
||||
formats = []
|
||||
for format_id in ('normal', 'hq'):
|
||||
rex = r"var %s_video_file = '(.*?)';" % re.escape(format_id)
|
||||
video_url = self._search_regex(rex, webpage, u'video file URLx',
|
||||
fatal=False)
|
||||
if not video_url:
|
||||
continue
|
||||
formats.append({
|
||||
'format_id': format_id,
|
||||
'url': video_url,
|
||||
})
|
||||
if not formats:
|
||||
raise ExtractorError(u'Cannot find any video format!')
|
||||
video_title = self._og_search_title(webpage)
|
||||
video_description = self._og_search_description(webpage)
|
||||
|
||||
return {
|
||||
'_type': 'video',
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'flv',
|
||||
'formats': formats,
|
||||
'title': video_title,
|
||||
'description': video_description
|
||||
}
|
||||
|
@@ -1,8 +1,10 @@
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
determine_ext,
|
||||
)
|
||||
|
||||
@@ -14,10 +16,9 @@ class AppleTrailersIE(InfoExtractor):
|
||||
u"playlist": [
|
||||
{
|
||||
u"file": u"manofsteel-trailer4.mov",
|
||||
u"md5": u"11874af099d480cc09e103b189805d5f",
|
||||
u"md5": u"d97a8e575432dbcb81b7c3acb741f8a8",
|
||||
u"info_dict": {
|
||||
u"duration": 111,
|
||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_11624.jpg",
|
||||
u"title": u"Trailer 4",
|
||||
u"upload_date": u"20130523",
|
||||
u"uploader_id": u"wb",
|
||||
@@ -25,10 +26,9 @@ class AppleTrailersIE(InfoExtractor):
|
||||
},
|
||||
{
|
||||
u"file": u"manofsteel-trailer3.mov",
|
||||
u"md5": u"07a0a262aae5afe68120eed61137ab34",
|
||||
u"md5": u"b8017b7131b721fb4e8d6f49e1df908c",
|
||||
u"info_dict": {
|
||||
u"duration": 182,
|
||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_10793.jpg",
|
||||
u"title": u"Trailer 3",
|
||||
u"upload_date": u"20130417",
|
||||
u"uploader_id": u"wb",
|
||||
@@ -36,10 +36,9 @@ class AppleTrailersIE(InfoExtractor):
|
||||
},
|
||||
{
|
||||
u"file": u"manofsteel-trailer.mov",
|
||||
u"md5": u"e401fde0813008e3307e54b6f384cff1",
|
||||
u"md5": u"d0f1e1150989b9924679b441f3404d48",
|
||||
u"info_dict": {
|
||||
u"duration": 148,
|
||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_8703.jpg",
|
||||
u"title": u"Trailer",
|
||||
u"upload_date": u"20121212",
|
||||
u"uploader_id": u"wb",
|
||||
@@ -47,10 +46,9 @@ class AppleTrailersIE(InfoExtractor):
|
||||
},
|
||||
{
|
||||
u"file": u"manofsteel-teaser.mov",
|
||||
u"md5": u"76b392f2ae9e7c98b22913c10a639c97",
|
||||
u"md5": u"5fe08795b943eb2e757fa95cb6def1cb",
|
||||
u"info_dict": {
|
||||
u"duration": 93,
|
||||
u"thumbnail": u"http://trailers.apple.com/trailers/wb/manofsteel/images/thumbnail_6899.jpg",
|
||||
u"title": u"Teaser",
|
||||
u"upload_date": u"20120721",
|
||||
u"uploader_id": u"wb",
|
||||
@@ -59,87 +57,61 @@ class AppleTrailersIE(InfoExtractor):
|
||||
]
|
||||
}
|
||||
|
||||
_JSON_RE = r'iTunes.playURL\((.*?)\);'
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
movie = mobj.group('movie')
|
||||
uploader_id = mobj.group('company')
|
||||
|
||||
playlist_url = url.partition(u'?')[0] + u'/includes/playlists/web.inc'
|
||||
playlist_url = compat_urlparse.urljoin(url, u'includes/playlists/itunes.inc')
|
||||
playlist_snippet = self._download_webpage(playlist_url, movie)
|
||||
playlist_cleaned = re.sub(r'(?s)<script>.*?</script>', u'', playlist_snippet)
|
||||
playlist_cleaned = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', playlist_snippet)
|
||||
playlist_cleaned = re.sub(r'<img ([^<]*?)>', r'<img \1/>', playlist_cleaned)
|
||||
# The ' in the onClick attributes are not escaped, it couldn't be parsed
|
||||
# with xml.etree.ElementTree.fromstring
|
||||
# like: http://trailers.apple.com/trailers/wb/gravity/
|
||||
def _clean_json(m):
|
||||
return u'iTunes.playURL(%s);' % m.group(1).replace('\'', ''')
|
||||
playlist_cleaned = re.sub(self._JSON_RE, _clean_json, playlist_cleaned)
|
||||
playlist_html = u'<html>' + playlist_cleaned + u'</html>'
|
||||
|
||||
size_cache = {}
|
||||
|
||||
doc = xml.etree.ElementTree.fromstring(playlist_html)
|
||||
playlist = []
|
||||
for li in doc.findall('./div/ul/li'):
|
||||
title = li.find('.//h3').text
|
||||
on_click = li.find('.//a').attrib['onClick']
|
||||
trailer_info_json = self._search_regex(self._JSON_RE,
|
||||
on_click, u'trailer info')
|
||||
trailer_info = json.loads(trailer_info_json)
|
||||
title = trailer_info['title']
|
||||
video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
|
||||
thumbnail = li.find('.//img').attrib['src']
|
||||
upload_date = trailer_info['posted'].replace('-', '')
|
||||
|
||||
date_el = li.find('.//p')
|
||||
upload_date = None
|
||||
m = re.search(r':\s?(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<year>[0-9]{2})', date_el.text)
|
||||
if m:
|
||||
upload_date = u'20' + m.group('year') + m.group('month') + m.group('day')
|
||||
runtime_el = date_el.find('./br')
|
||||
m = re.search(r':\s?(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime_el.tail)
|
||||
runtime = trailer_info['runtime']
|
||||
m = re.search(r'(?P<minutes>[0-9]+):(?P<seconds>[0-9]{1,2})', runtime)
|
||||
duration = None
|
||||
if m:
|
||||
duration = 60 * int(m.group('minutes')) + int(m.group('seconds'))
|
||||
|
||||
first_url = trailer_info['url']
|
||||
trailer_id = first_url.split('/')[-1].rpartition('_')[0].lower()
|
||||
settings_json_url = compat_urlparse.urljoin(url, 'includes/settings/%s.json' % trailer_id)
|
||||
settings_json = self._download_webpage(settings_json_url, trailer_id, u'Downloading settings json')
|
||||
settings = json.loads(settings_json)
|
||||
|
||||
formats = []
|
||||
for formats_el in li.findall('.//a'):
|
||||
if formats_el.attrib['class'] != 'OverlayPanel':
|
||||
continue
|
||||
target = formats_el.attrib['target']
|
||||
|
||||
format_code = formats_el.text
|
||||
if 'Automatic' in format_code:
|
||||
continue
|
||||
|
||||
size_q = formats_el.attrib['href']
|
||||
size_id = size_q.rpartition('#videos-')[2]
|
||||
if size_id not in size_cache:
|
||||
size_url = url + size_q
|
||||
sizepage_html = self._download_webpage(
|
||||
size_url, movie,
|
||||
note=u'Downloading size info %s' % size_id,
|
||||
errnote=u'Error while downloading size info %s' % size_id,
|
||||
)
|
||||
_doc = xml.etree.ElementTree.fromstring(sizepage_html)
|
||||
size_cache[size_id] = _doc
|
||||
|
||||
sizepage_doc = size_cache[size_id]
|
||||
links = sizepage_doc.findall('.//{http://www.w3.org/1999/xhtml}ul/{http://www.w3.org/1999/xhtml}li/{http://www.w3.org/1999/xhtml}a')
|
||||
for vid_a in links:
|
||||
href = vid_a.get('href')
|
||||
if not href.endswith(target):
|
||||
continue
|
||||
detail_q = href.partition('#')[0]
|
||||
detail_url = url + '/' + detail_q
|
||||
|
||||
m = re.match(r'includes/(?P<detail_id>[^/]+)/', detail_q)
|
||||
detail_id = m.group('detail_id')
|
||||
|
||||
detail_html = self._download_webpage(
|
||||
detail_url, movie,
|
||||
note=u'Downloading detail %s %s' % (detail_id, size_id),
|
||||
errnote=u'Error while downloading detail %s %s' % (detail_id, size_id)
|
||||
)
|
||||
detail_doc = xml.etree.ElementTree.fromstring(detail_html)
|
||||
movie_link_el = detail_doc.find('.//{http://www.w3.org/1999/xhtml}a')
|
||||
assert movie_link_el.get('class') == 'movieLink'
|
||||
movie_link = movie_link_el.get('href').partition('?')[0].replace('_', '_h')
|
||||
ext = determine_ext(movie_link)
|
||||
assert ext == 'mov'
|
||||
|
||||
formats.append({
|
||||
'format': format_code,
|
||||
'ext': ext,
|
||||
'url': movie_link,
|
||||
})
|
||||
for format in settings['metadata']['sizes']:
|
||||
# The src is a file pointing to the real video file
|
||||
format_url = re.sub(r'_(\d*p.mov)', r'_h\1', format['src'])
|
||||
formats.append({
|
||||
'url': format_url,
|
||||
'ext': determine_ext(format_url),
|
||||
'format': format['type'],
|
||||
'width': format['width'],
|
||||
'height': int(format['height']),
|
||||
})
|
||||
formats = sorted(formats, key=lambda f: (f['height'], f['width']))
|
||||
|
||||
info = {
|
||||
'_type': 'video',
|
||||
|
@@ -1,3 +1,4 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
import json
|
||||
import xml.etree.ElementTree
|
||||
@@ -7,15 +8,15 @@ from ..utils import (
|
||||
ExtractorError,
|
||||
find_xpath_attr,
|
||||
unified_strdate,
|
||||
determine_ext,
|
||||
get_element_by_id,
|
||||
)
|
||||
|
||||
# There are different sources of video in arte.tv, the extraction process
|
||||
# is different for each one. The videos usually expire in 7 days, so we can't
|
||||
# add tests.
|
||||
|
||||
class ArteTvIE(InfoExtractor):
|
||||
"""
|
||||
There are two sources of video in arte.tv: videos.arte.tv and
|
||||
www.arte.tv/guide, the extraction process is different for each one.
|
||||
The videos expire in 7 days, so we can't add tests.
|
||||
"""
|
||||
_EMISSION_URL = r'(?:http://)?www\.arte.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
|
||||
_VIDEOS_URL = r'(?:http://)?videos.arte.tv/(?P<lang>fr|de)/.*-(?P<id>.*?).html'
|
||||
_LIVEWEB_URL = r'(?:http://)?liveweb.arte.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)'
|
||||
_LIVE_URL = r'index-[0-9]+\.html$'
|
||||
@@ -24,7 +25,7 @@ class ArteTvIE(InfoExtractor):
|
||||
|
||||
@classmethod
|
||||
def suitable(cls, url):
|
||||
return any(re.match(regex, url) for regex in (cls._EMISSION_URL, cls._VIDEOS_URL, cls._LIVEWEB_URL))
|
||||
return any(re.match(regex, url) for regex in (cls._VIDEOS_URL, cls._LIVEWEB_URL))
|
||||
|
||||
# TODO implement Live Stream
|
||||
# from ..utils import compat_urllib_parse
|
||||
@@ -55,14 +56,6 @@ class ArteTvIE(InfoExtractor):
|
||||
# video_url = u'%s/%s' % (info.get('url'), info.get('path'))
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._EMISSION_URL, url)
|
||||
if mobj is not None:
|
||||
lang = mobj.group('lang')
|
||||
# This is not a real id, it can be for example AJT for the news
|
||||
# http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
|
||||
video_id = mobj.group('id')
|
||||
return self._extract_emission(url, video_id, lang)
|
||||
|
||||
mobj = re.match(self._VIDEOS_URL, url)
|
||||
if mobj is not None:
|
||||
id = mobj.group('id')
|
||||
@@ -80,49 +73,6 @@ class ArteTvIE(InfoExtractor):
|
||||
# self.extractLiveStream(url)
|
||||
# return
|
||||
|
||||
def _extract_emission(self, url, video_id, lang):
|
||||
"""Extract from www.arte.tv/guide"""
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
json_url = self._html_search_regex(r'arte_vp_url="(.*?)"', webpage, 'json url')
|
||||
|
||||
json_info = self._download_webpage(json_url, video_id, 'Downloading info json')
|
||||
self.report_extraction(video_id)
|
||||
info = json.loads(json_info)
|
||||
player_info = info['videoJsonPlayer']
|
||||
|
||||
info_dict = {'id': player_info['VID'],
|
||||
'title': player_info['VTI'],
|
||||
'description': player_info.get('VDE'),
|
||||
'upload_date': unified_strdate(player_info['VDA'].split(' ')[0]),
|
||||
'thumbnail': player_info['programImage'],
|
||||
'ext': 'flv',
|
||||
}
|
||||
|
||||
formats = player_info['VSR'].values()
|
||||
def _match_lang(f):
|
||||
# Return true if that format is in the language of the url
|
||||
if lang == 'fr':
|
||||
l = 'F'
|
||||
elif lang == 'de':
|
||||
l = 'A'
|
||||
regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l]
|
||||
return any(re.match(r, f['versionCode']) for r in regexes)
|
||||
# Some formats may not be in the same language as the url
|
||||
formats = filter(_match_lang, formats)
|
||||
# We order the formats by quality
|
||||
formats = sorted(formats, key=lambda f: int(f['height']))
|
||||
# Prefer videos without subtitles in the same language
|
||||
formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f['versionCode']) is None)
|
||||
# Pick the best quality
|
||||
format_info = formats[-1]
|
||||
if format_info['mediaType'] == u'rtmp':
|
||||
info_dict['url'] = format_info['streamer']
|
||||
info_dict['play_path'] = 'mp4:' + format_info['url']
|
||||
else:
|
||||
info_dict['url'] = format_info['url']
|
||||
|
||||
return info_dict
|
||||
|
||||
def _extract_video(self, url, video_id, lang):
|
||||
"""Extract from videos.arte.tv"""
|
||||
ref_xml_url = url.replace('/videos/', '/do_delegate/videos/')
|
||||
@@ -172,3 +122,130 @@ class ArteTvIE(InfoExtractor):
|
||||
'ext': 'flv',
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
}
|
||||
|
||||
|
||||
class ArteTVPlus7IE(InfoExtractor):
|
||||
IE_NAME = u'arte.tv:+7'
|
||||
_VALID_URL = r'https?://www\.arte.tv/guide/(?P<lang>fr|de)/(?:(?:sendungen|emissions)/)?(?P<id>.*?)/(?P<name>.*?)(\?.*)?'
|
||||
|
||||
@classmethod
|
||||
def _extract_url_info(cls, url):
|
||||
mobj = re.match(cls._VALID_URL, url)
|
||||
lang = mobj.group('lang')
|
||||
# This is not a real id, it can be for example AJT for the news
|
||||
# http://www.arte.tv/guide/fr/emissions/AJT/arte-journal
|
||||
video_id = mobj.group('id')
|
||||
return video_id, lang
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id, lang = self._extract_url_info(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
return self._extract_from_webpage(webpage, video_id, lang)
|
||||
|
||||
def _extract_from_webpage(self, webpage, video_id, lang):
|
||||
json_url = self._html_search_regex(r'arte_vp_url="(.*?)"', webpage, 'json url')
|
||||
|
||||
json_info = self._download_webpage(json_url, video_id, 'Downloading info json')
|
||||
self.report_extraction(video_id)
|
||||
info = json.loads(json_info)
|
||||
player_info = info['videoJsonPlayer']
|
||||
|
||||
info_dict = {
|
||||
'id': player_info['VID'],
|
||||
'title': player_info['VTI'],
|
||||
'description': player_info.get('VDE'),
|
||||
'upload_date': unified_strdate(player_info.get('VDA', '').split(' ')[0]),
|
||||
'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
|
||||
}
|
||||
|
||||
all_formats = player_info['VSR'].values()
|
||||
# Some formats use the m3u8 protocol
|
||||
all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
|
||||
def _match_lang(f):
|
||||
if f.get('versionCode') is None:
|
||||
return True
|
||||
# Return true if that format is in the language of the url
|
||||
if lang == 'fr':
|
||||
l = 'F'
|
||||
elif lang == 'de':
|
||||
l = 'A'
|
||||
regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l]
|
||||
return any(re.match(r, f['versionCode']) for r in regexes)
|
||||
# Some formats may not be in the same language as the url
|
||||
formats = filter(_match_lang, all_formats)
|
||||
formats = list(formats) # in python3 filter returns an iterator
|
||||
if not formats:
|
||||
# Some videos are only available in the 'Originalversion'
|
||||
# they aren't tagged as being in French or German
|
||||
if all(f['versionCode'] == 'VO' for f in all_formats):
|
||||
formats = all_formats
|
||||
else:
|
||||
raise ExtractorError(u'The formats list is empty')
|
||||
# We order the formats by quality
|
||||
if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
|
||||
sort_key = lambda f: ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
|
||||
else:
|
||||
sort_key = lambda f: int(f.get('height',-1))
|
||||
formats = sorted(formats, key=sort_key)
|
||||
# Prefer videos without subtitles in the same language
|
||||
formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f.get('versionCode', '')) is None)
|
||||
# Pick the best quality
|
||||
def _format(format_info):
|
||||
quality = format_info['quality']
|
||||
m_quality = re.match(r'\w*? - (\d*)p', quality)
|
||||
if m_quality is not None:
|
||||
quality = m_quality.group(1)
|
||||
if format_info.get('versionCode') is not None:
|
||||
format_id = u'%s-%s' % (quality, format_info['versionCode'])
|
||||
else:
|
||||
format_id = quality
|
||||
info = {
|
||||
'format_id': format_id,
|
||||
'format_note': format_info.get('versionLibelle'),
|
||||
'width': format_info.get('width'),
|
||||
'height': format_info.get('height'),
|
||||
}
|
||||
if format_info['mediaType'] == u'rtmp':
|
||||
info['url'] = format_info['streamer']
|
||||
info['play_path'] = 'mp4:' + format_info['url']
|
||||
info['ext'] = 'flv'
|
||||
else:
|
||||
info['url'] = format_info['url']
|
||||
info['ext'] = determine_ext(info['url'])
|
||||
return info
|
||||
info_dict['formats'] = [_format(f) for f in formats]
|
||||
|
||||
return info_dict
|
||||
|
||||
|
||||
# It also uses the arte_vp_url url from the webpage to extract the information
|
||||
class ArteTVCreativeIE(ArteTVPlus7IE):
|
||||
IE_NAME = u'arte.tv:creative'
|
||||
_VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/magazine?/(?P<id>.+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
|
||||
u'file': u'050489-002.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Agentur Amateur / Agence Amateur #2 : Corporate Design',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ArteTVFutureIE(ArteTVPlus7IE):
|
||||
IE_NAME = u'arte.tv:future'
|
||||
_VALID_URL = r'https?://future\.arte\.tv/(?P<lang>fr|de)/(thema|sujet)/.*?#article-anchor-(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://future.arte.tv/fr/sujet/info-sciences#article-anchor-7081',
|
||||
u'file': u'050940-003.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Les champignons au secours de la planète',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
anchor_id, lang = self._extract_url_info(url)
|
||||
webpage = self._download_webpage(url, anchor_id)
|
||||
row = get_element_by_id(anchor_id, webpage)
|
||||
return self._extract_from_webpage(row, anchor_id, lang)
|
||||
|
80
youtube_dl/extractor/bambuser.py
Normal file
80
youtube_dl/extractor/bambuser.py
Normal file
@@ -0,0 +1,80 @@
|
||||
import re
|
||||
import json
|
||||
import itertools
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_request,
|
||||
)
|
||||
|
||||
|
||||
class BambuserIE(InfoExtractor):
|
||||
IE_NAME = u'bambuser'
|
||||
_VALID_URL = r'https?://bambuser\.com/v/(?P<id>\d+)'
|
||||
_API_KEY = '005f64509e19a868399060af746a00aa'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://bambuser.com/v/4050584',
|
||||
u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
|
||||
u'info_dict': {
|
||||
u'id': u'4050584',
|
||||
u'ext': u'flv',
|
||||
u'title': u'Education engineering days - lightning talks',
|
||||
u'duration': 3741,
|
||||
u'uploader': u'pixelversity',
|
||||
u'uploader_id': u'344706',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
info_url = ('http://player-c.api.bambuser.com/getVideo.json?'
|
||||
'&api_key=%s&vid=%s' % (self._API_KEY, video_id))
|
||||
info_json = self._download_webpage(info_url, video_id)
|
||||
info = json.loads(info_json)['result']
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': info['title'],
|
||||
'url': info['url'],
|
||||
'thumbnail': info.get('preview'),
|
||||
'duration': int(info['length']),
|
||||
'view_count': int(info['views_total']),
|
||||
'uploader': info['username'],
|
||||
'uploader_id': info['uid'],
|
||||
}
|
||||
|
||||
|
||||
class BambuserChannelIE(InfoExtractor):
|
||||
IE_NAME = u'bambuser:channel'
|
||||
_VALID_URL = r'http://bambuser.com/channel/(?P<user>.*?)(?:/|#|\?|$)'
|
||||
# The maximum number we can get with each request
|
||||
_STEP = 50
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
user = mobj.group('user')
|
||||
urls = []
|
||||
last_id = ''
|
||||
for i in itertools.count(1):
|
||||
req_url = ('http://bambuser.com/xhr-api/index.php?username={user}'
|
||||
'&sort=created&access_mode=0%2C1%2C2&limit={count}'
|
||||
'&method=broadcast&format=json&vid_older_than={last}'
|
||||
).format(user=user, count=self._STEP, last=last_id)
|
||||
req = compat_urllib_request.Request(req_url)
|
||||
# Without setting this header, we wouldn't get any result
|
||||
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
|
||||
info_json = self._download_webpage(req, user,
|
||||
u'Downloading page %d' % i)
|
||||
results = json.loads(info_json)['result']
|
||||
if len(results) == 0:
|
||||
break
|
||||
last_id = results[-1]['vid']
|
||||
urls.extend(self.url_result(v['page'], 'Bambuser') for v in results)
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'title': user,
|
||||
'entries': urls,
|
||||
}
|
@@ -115,7 +115,7 @@ class BlipTVIE(InfoExtractor):
|
||||
ext = umobj.group(1)
|
||||
|
||||
info = {
|
||||
'id': data['item_id'],
|
||||
'id': compat_str(data['item_id']),
|
||||
'url': video_url,
|
||||
'uploader': data['display_name'],
|
||||
'upload_date': upload_date,
|
||||
|
@@ -1,3 +1,5 @@
|
||||
# encoding: utf-8
|
||||
|
||||
import re
|
||||
import json
|
||||
import xml.etree.ElementTree
|
||||
@@ -7,15 +9,51 @@ from ..utils import (
|
||||
compat_urllib_parse,
|
||||
find_xpath_attr,
|
||||
compat_urlparse,
|
||||
compat_str,
|
||||
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
class BrightcoveIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
|
||||
_FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
|
||||
_PLAYLIST_URL_TEMPLATE = 'http://c.brightcove.com/services/json/experience/runtime/?command=get_programming_for_experience&playerKey=%s'
|
||||
|
||||
# There is a test for Brigtcove in GenericIE, that way we test both the download
|
||||
# and the detection of videos, and we don't have to find an URL that is always valid
|
||||
|
||||
_TESTS = [
|
||||
{
|
||||
# From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/
|
||||
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001',
|
||||
u'file': u'2371591881001.mp4',
|
||||
u'md5': u'8eccab865181d29ec2958f32a6a754f5',
|
||||
u'note': u'Test Brightcove downloads and detection in GenericIE',
|
||||
u'info_dict': {
|
||||
u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
|
||||
u'uploader': u'8TV',
|
||||
u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
|
||||
}
|
||||
},
|
||||
{
|
||||
# From http://medianetwork.oracle.com/video/player/1785452137001
|
||||
u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1217746023001&flashID=myPlayer&%40videoPlayer=1785452137001',
|
||||
u'file': u'1785452137001.flv',
|
||||
u'info_dict': {
|
||||
u'title': u'JVMLS 2012: Arrays 2.0 - Opportunities and Challenges',
|
||||
u'description': u'John Rose speaks at the JVM Language Summit, August 1, 2012.',
|
||||
u'uploader': u'Oracle',
|
||||
},
|
||||
},
|
||||
{
|
||||
# From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
|
||||
u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
|
||||
u'info_dict': {
|
||||
u'id': u'2750934548001',
|
||||
u'ext': u'mp4',
|
||||
u'title': u'This Bracelet Acts as a Personal Thermostat',
|
||||
u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0',
|
||||
u'uploader': u'Mashable',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def _build_brighcove_url(cls, object_str):
|
||||
@@ -23,6 +61,13 @@ class BrightcoveIE(InfoExtractor):
|
||||
Build a Brightcove url from a xml string containing
|
||||
<object class="BrightcoveExperience">{params}</object>
|
||||
"""
|
||||
|
||||
# Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
|
||||
object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
|
||||
lambda m: m.group(1) + '/>', object_str)
|
||||
# Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
|
||||
object_str = object_str.replace(u'<--', u'<!--')
|
||||
|
||||
object_doc = xml.etree.ElementTree.fromstring(object_str)
|
||||
assert u'BrightcoveExperience' in object_doc.attrib['class']
|
||||
params = {'flashID': object_doc.attrib['id'],
|
||||
@@ -38,7 +83,24 @@ class BrightcoveIE(InfoExtractor):
|
||||
data = compat_urllib_parse.urlencode(params)
|
||||
return cls._FEDERATED_URL_TEMPLATE % data
|
||||
|
||||
@classmethod
|
||||
def _extract_brightcove_url(cls, webpage):
|
||||
"""Try to extract the brightcove url from the wepbage, returns None
|
||||
if it can't be found
|
||||
"""
|
||||
m_brightcove = re.search(
|
||||
r'<object[^>]+?class=([\'"])[^>]*?BrightcoveExperience.*?\1.+?</object>',
|
||||
webpage, re.DOTALL)
|
||||
if m_brightcove is not None:
|
||||
return cls._build_brighcove_url(m_brightcove.group())
|
||||
else:
|
||||
return None
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Change the 'videoId' and others field to '@videoPlayer'
|
||||
url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
|
||||
# Change bckey (used by bcove.me urls) to playerKey
|
||||
url = re.sub(r'(?<=[?&])bckey', 'playerKey', url)
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
query_str = mobj.group('query')
|
||||
query = compat_urlparse.parse_qs(query_str)
|
||||
@@ -65,22 +127,36 @@ class BrightcoveIE(InfoExtractor):
|
||||
playlist_info = self._download_webpage(self._PLAYLIST_URL_TEMPLATE % player_key,
|
||||
player_key, u'Downloading playlist information')
|
||||
|
||||
playlist_info = json.loads(playlist_info)['videoList']
|
||||
json_data = json.loads(playlist_info)
|
||||
if 'videoList' not in json_data:
|
||||
raise ExtractorError(u'Empty playlist')
|
||||
playlist_info = json_data['videoList']
|
||||
videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
|
||||
|
||||
return self.playlist_result(videos, playlist_id=playlist_info['id'],
|
||||
playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
|
||||
|
||||
def _extract_video_info(self, video_info):
|
||||
renditions = video_info['renditions']
|
||||
renditions = sorted(renditions, key=lambda r: r['size'])
|
||||
best_format = renditions[-1]
|
||||
info = {
|
||||
'id': compat_str(video_info['id']),
|
||||
'title': video_info['displayName'],
|
||||
'description': video_info.get('shortDescription'),
|
||||
'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
|
||||
'uploader': video_info.get('publisherName'),
|
||||
}
|
||||
|
||||
return {'id': video_info['id'],
|
||||
'title': video_info['displayName'],
|
||||
'url': best_format['defaultURL'],
|
||||
'ext': 'mp4',
|
||||
'description': video_info.get('shortDescription'),
|
||||
'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
|
||||
'uploader': video_info.get('publisherName'),
|
||||
}
|
||||
renditions = video_info.get('renditions')
|
||||
if renditions:
|
||||
renditions = sorted(renditions, key=lambda r: r['size'])
|
||||
info['formats'] = [{
|
||||
'url': rend['defaultURL'],
|
||||
'height': rend.get('frameHeight'),
|
||||
'width': rend.get('frameWidth'),
|
||||
} for rend in renditions]
|
||||
elif video_info.get('FLVFullLengthURL') is not None:
|
||||
info.update({
|
||||
'url': video_info['FLVFullLengthURL'],
|
||||
})
|
||||
else:
|
||||
raise ExtractorError(u'Unable to extract video url for %s' % info['id'])
|
||||
return info
|
||||
|
@@ -6,7 +6,7 @@ from .common import InfoExtractor
|
||||
|
||||
class Canalc2IE(InfoExtractor):
|
||||
IE_NAME = 'canalc2.tv'
|
||||
_VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?idVideo=(\d+)&voir=oui'
|
||||
_VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?.*?idVideo=(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.canalc2.tv/video.asp?idVideo=12163&voir=oui',
|
||||
@@ -18,7 +18,9 @@ class Canalc2IE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = re.match(self._VALID_URL, url).group(1)
|
||||
video_id = re.match(self._VALID_URL, url).group('id')
|
||||
# We need to set the voir field for getting the file name
|
||||
url = 'http://www.canalc2.tv/video.asp?idVideo=%s&voir=oui' % video_id
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
file_name = self._search_regex(
|
||||
r"so\.addVariable\('file','(.*?)'\);",
|
||||
|
88
youtube_dl/extractor/cinemassacre.py
Normal file
88
youtube_dl/extractor/cinemassacre.py
Normal file
@@ -0,0 +1,88 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
|
||||
class CinemassacreIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:http://)?(?:www\.)?(?P<url>cinemassacre\.com/(?P<date_Y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/.+?)(?:[/?].*)?'
|
||||
_TESTS = [{
|
||||
u'url': u'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
|
||||
u'file': u'19911.flv',
|
||||
u'info_dict': {
|
||||
u'upload_date': u'20121110',
|
||||
u'title': u'“Angry Video Game Nerd: The Movie” – Trailer',
|
||||
u'description': u'md5:fb87405fcb42a331742a0dce2708560b',
|
||||
},
|
||||
u'params': {
|
||||
# rtmp download
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
{
|
||||
u'url': u'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
|
||||
u'file': u'521be8ef82b16.flv',
|
||||
u'info_dict': {
|
||||
u'upload_date': u'20131002',
|
||||
u'title': u'The Mummy’s Hand (1940)',
|
||||
},
|
||||
u'params': {
|
||||
# rtmp download
|
||||
u'skip_download': True,
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
webpage_url = u'http://' + mobj.group('url')
|
||||
webpage = self._download_webpage(webpage_url, None) # Don't know video id yet
|
||||
video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
|
||||
mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?id=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
|
||||
if not mobj:
|
||||
raise ExtractorError(u'Can\'t extract embed url and video id')
|
||||
playerdata_url = mobj.group(u'embed_url')
|
||||
video_id = mobj.group(u'video_id')
|
||||
|
||||
video_title = self._html_search_regex(r'<title>(?P<title>.+?)\|',
|
||||
webpage, u'title')
|
||||
video_description = self._html_search_regex(r'<div class="entry-content">(?P<description>.+?)</div>',
|
||||
webpage, u'description', flags=re.DOTALL, fatal=False)
|
||||
if len(video_description) == 0:
|
||||
video_description = None
|
||||
|
||||
playerdata = self._download_webpage(playerdata_url, video_id)
|
||||
url = self._html_search_regex(r'\'streamer\': \'(?P<url>[^\']+)\'', playerdata, u'url')
|
||||
|
||||
sd_file = self._html_search_regex(r'\'file\': \'(?P<sd_file>[^\']+)\'', playerdata, u'sd_file')
|
||||
hd_file = self._html_search_regex(r'\'?file\'?: "(?P<hd_file>[^"]+)"', playerdata, u'hd_file')
|
||||
video_thumbnail = self._html_search_regex(r'\'image\': \'(?P<thumbnail>[^\']+)\'', playerdata, u'thumbnail', fatal=False)
|
||||
|
||||
formats = [
|
||||
{
|
||||
'url': url,
|
||||
'play_path': 'mp4:' + sd_file,
|
||||
'ext': 'flv',
|
||||
'format': 'sd',
|
||||
'format_id': 'sd',
|
||||
},
|
||||
{
|
||||
'url': url,
|
||||
'play_path': 'mp4:' + hd_file,
|
||||
'ext': 'flv',
|
||||
'format': 'hd',
|
||||
'format_id': 'hd',
|
||||
},
|
||||
]
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'formats': formats,
|
||||
'description': video_description,
|
||||
'upload_date': video_date,
|
||||
'thumbnail': video_thumbnail,
|
||||
}
|
@@ -51,12 +51,12 @@ class ComedyCentralIE(InfoExtractor):
|
||||
'400': 'mp4',
|
||||
}
|
||||
_video_dimensions = {
|
||||
'3500': '1280x720',
|
||||
'2200': '960x540',
|
||||
'1700': '768x432',
|
||||
'1200': '640x360',
|
||||
'750': '512x288',
|
||||
'400': '384x216',
|
||||
'3500': (1280, 720),
|
||||
'2200': (960, 540),
|
||||
'1700': (768, 432),
|
||||
'1200': (640, 360),
|
||||
'750': (512, 288),
|
||||
'400': (384, 216),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -64,11 +64,13 @@ class ComedyCentralIE(InfoExtractor):
|
||||
"""Receives a URL and returns True if suitable for this IE."""
|
||||
return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
|
||||
|
||||
def _print_formats(self, formats):
|
||||
print('Available formats:')
|
||||
for x in formats:
|
||||
print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
|
||||
|
||||
@staticmethod
|
||||
def _transform_rtmp_url(rtmp_video_url):
|
||||
m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
|
||||
if not m:
|
||||
raise ExtractorError(u'Cannot transform RTMP url')
|
||||
base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
|
||||
return base + m.group('finalid')
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url, re.VERBOSE)
|
||||
@@ -155,40 +157,31 @@ class ComedyCentralIE(InfoExtractor):
|
||||
self._downloader.report_error(u'unable to download ' + mediaId + ': No videos found')
|
||||
continue
|
||||
|
||||
if self._downloader.params.get('listformats', None):
|
||||
self._print_formats([i[0] for i in turls])
|
||||
return
|
||||
|
||||
# For now, just pick the highest bitrate
|
||||
format,rtmp_video_url = turls[-1]
|
||||
|
||||
# Get the format arg from the arg stream
|
||||
req_format = self._downloader.params.get('format', None)
|
||||
|
||||
# Select format if we can find one
|
||||
for f,v in turls:
|
||||
if f == req_format:
|
||||
format, rtmp_video_url = f, v
|
||||
break
|
||||
|
||||
m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp.comedystor/.*)$', rtmp_video_url)
|
||||
if not m:
|
||||
raise ExtractorError(u'Cannot transform RTMP url')
|
||||
base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
|
||||
video_url = base + m.group('finalid')
|
||||
formats = []
|
||||
for format, rtmp_video_url in turls:
|
||||
w, h = self._video_dimensions.get(format, (None, None))
|
||||
formats.append({
|
||||
'url': self._transform_rtmp_url(rtmp_video_url),
|
||||
'ext': self._video_extensions.get(format, 'mp4'),
|
||||
'format_id': format,
|
||||
'height': h,
|
||||
'width': w,
|
||||
})
|
||||
|
||||
effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1)
|
||||
info = {
|
||||
'id': shortMediaId,
|
||||
'url': video_url,
|
||||
'formats': formats,
|
||||
'uploader': showId,
|
||||
'upload_date': officialDate,
|
||||
'title': effTitle,
|
||||
'ext': 'mp4',
|
||||
'format': format,
|
||||
'thumbnail': None,
|
||||
'description': compat_str(officialTitle),
|
||||
}
|
||||
|
||||
# TODO: Remove when #980 has been merged
|
||||
info.update(info['formats'][-1])
|
||||
|
||||
results.append(info)
|
||||
|
||||
return results
|
||||
|
@@ -14,6 +14,8 @@ from ..utils import (
|
||||
clean_html,
|
||||
compiled_regex_type,
|
||||
ExtractorError,
|
||||
RegexNotFoundError,
|
||||
sanitize_filename,
|
||||
unescapeHTML,
|
||||
)
|
||||
|
||||
@@ -35,6 +37,8 @@ class InfoExtractor(object):
|
||||
title: Video title, unescaped.
|
||||
ext: Video filename extension.
|
||||
|
||||
Instead of url and ext, formats can also specified.
|
||||
|
||||
The following fields are optional:
|
||||
|
||||
format: The video format, defaults to ext (used for --get-format)
|
||||
@@ -52,8 +56,26 @@ class InfoExtractor(object):
|
||||
view_count: How many users have watched the video on the platform.
|
||||
urlhandle: [internal] The urlHandle to be used to download the file,
|
||||
like returned by urllib.request.urlopen
|
||||
age_limit: Age restriction for the video, as an integer (years)
|
||||
formats: A list of dictionaries for each format available, it must
|
||||
be ordered from worst to best quality. Potential fields:
|
||||
* url Mandatory. The URL of the video file
|
||||
* ext Will be calculated from url if missing
|
||||
* format A human-readable description of the format
|
||||
("mp4 container with h264/opus").
|
||||
Calculated from the format_id, width, height.
|
||||
and format_note fields if missing.
|
||||
* format_id A short description of the format
|
||||
("mp4_h264_opus" or "19")
|
||||
* format_note Additional info about the format
|
||||
("3D" or "DASH video")
|
||||
* width Width of the video, if known
|
||||
* height Height of the video, if known
|
||||
webpage_url: The url to the video webpage, if given to youtube-dl it
|
||||
should allow to get the same result again. (It will be set
|
||||
by YoutubeDL if it's missing)
|
||||
|
||||
The fields should all be Unicode strings.
|
||||
Unless mentioned otherwise, the fields should be Unicode strings.
|
||||
|
||||
Subclasses of this one should re-define the _real_initialize() and
|
||||
_real_extract() methods and define a _VALID_URL regexp.
|
||||
@@ -164,6 +186,17 @@ class InfoExtractor(object):
|
||||
self.to_screen(u'Dumping request to ' + url)
|
||||
dump = base64.b64encode(webpage_bytes).decode('ascii')
|
||||
self._downloader.to_screen(dump)
|
||||
if self._downloader.params.get('write_pages', False):
|
||||
try:
|
||||
url = url_or_request.get_full_url()
|
||||
except AttributeError:
|
||||
url = url_or_request
|
||||
raw_filename = ('%s_%s.dump' % (video_id, url))
|
||||
filename = sanitize_filename(raw_filename, restricted=True)
|
||||
self.to_screen(u'Saving request to ' + filename)
|
||||
with open(filename, 'wb') as outf:
|
||||
outf.write(webpage_bytes)
|
||||
|
||||
content = webpage_bytes.decode(encoding, 'replace')
|
||||
return (content, urlh)
|
||||
|
||||
@@ -214,7 +247,7 @@ class InfoExtractor(object):
|
||||
Perform a regex search on the given string, using a single or a list of
|
||||
patterns returning the first matching group.
|
||||
In case of failure return a default value or raise a WARNING or a
|
||||
ExtractorError, depending on fatal, specifying the field name.
|
||||
RegexNotFoundError, depending on fatal, specifying the field name.
|
||||
"""
|
||||
if isinstance(pattern, (str, compat_str, compiled_regex_type)):
|
||||
mobj = re.search(pattern, string, flags)
|
||||
@@ -234,7 +267,7 @@ class InfoExtractor(object):
|
||||
elif default is not None:
|
||||
return default
|
||||
elif fatal:
|
||||
raise ExtractorError(u'Unable to extract %s' % _name)
|
||||
raise RegexNotFoundError(u'Unable to extract %s' % _name)
|
||||
else:
|
||||
self._downloader.report_warning(u'unable to extract %s; '
|
||||
u'please report this issue on http://yt-dl.org/bug' % _name)
|
||||
@@ -300,10 +333,19 @@ class InfoExtractor(object):
|
||||
def _og_search_title(self, html, **kargs):
|
||||
return self._og_search_property('title', html, **kargs)
|
||||
|
||||
def _og_search_video_url(self, html, name='video url', **kargs):
|
||||
return self._html_search_regex([self._og_regex('video:secure_url'),
|
||||
self._og_regex('video')],
|
||||
html, name, **kargs)
|
||||
def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
|
||||
regexes = [self._og_regex('video')]
|
||||
if secure: regexes.insert(0, self._og_regex('video:secure_url'))
|
||||
return self._html_search_regex(regexes, html, name, **kargs)
|
||||
|
||||
def _rta_search(self, html):
|
||||
# See http://www.rtalabel.org/index.php?content=howtofaq#single
|
||||
if re.search(r'(?ix)<meta\s+name="rating"\s+'
|
||||
r' content="RTA-5042-1996-1400-1577-RTA"',
|
||||
html):
|
||||
return 18
|
||||
return 0
|
||||
|
||||
|
||||
class SearchInfoExtractor(InfoExtractor):
|
||||
"""
|
||||
@@ -342,7 +384,7 @@ class SearchInfoExtractor(InfoExtractor):
|
||||
|
||||
def _get_n_results(self, query, n):
|
||||
"""Get a specified number of results for a query"""
|
||||
raise NotImplementedError("This method must be implemented by sublclasses")
|
||||
raise NotImplementedError("This method must be implemented by subclasses")
|
||||
|
||||
@property
|
||||
def SEARCH_KEY(self):
|
||||
|
@@ -10,25 +10,71 @@ from ..utils import (
|
||||
compat_str,
|
||||
get_element_by_attribute,
|
||||
get_element_by_id,
|
||||
orderedSet,
|
||||
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
class DailymotionBaseInfoExtractor(InfoExtractor):
|
||||
@staticmethod
|
||||
def _build_request(url):
|
||||
"""Build a request with the family filter disabled"""
|
||||
request = compat_urllib_request.Request(url)
|
||||
request.add_header('Cookie', 'family_filter=off')
|
||||
request.add_header('Cookie', 'ff=off')
|
||||
return request
|
||||
|
||||
class DailymotionIE(SubtitlesInfoExtractor):
|
||||
class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||
"""Information Extractor for Dailymotion"""
|
||||
|
||||
_VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)'
|
||||
IE_NAME = u'dailymotion'
|
||||
_TEST = {
|
||||
u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
|
||||
u'file': u'x33vw9.mp4',
|
||||
u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
|
||||
u'info_dict': {
|
||||
u"uploader": u"Amphora Alex and Van .",
|
||||
u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
|
||||
|
||||
_FORMATS = [
|
||||
(u'stream_h264_ld_url', u'ld'),
|
||||
(u'stream_h264_url', u'standard'),
|
||||
(u'stream_h264_hq_url', u'hq'),
|
||||
(u'stream_h264_hd_url', u'hd'),
|
||||
(u'stream_h264_hd1080_url', u'hd180'),
|
||||
]
|
||||
|
||||
_TESTS = [
|
||||
{
|
||||
u'url': u'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
|
||||
u'file': u'x33vw9.mp4',
|
||||
u'md5': u'392c4b85a60a90dc4792da41ce3144eb',
|
||||
u'info_dict': {
|
||||
u"uploader": u"Amphora Alex and Van .",
|
||||
u"title": u"Tutoriel de Youtubeur\"DL DES VIDEO DE YOUTUBE\""
|
||||
}
|
||||
},
|
||||
# Vevo video
|
||||
{
|
||||
u'url': u'http://www.dailymotion.com/video/x149uew_katy-perry-roar-official_musi',
|
||||
u'file': u'USUV71301934.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Roar (Official)',
|
||||
u'uploader': u'Katy Perry',
|
||||
u'upload_date': u'20130905',
|
||||
},
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
u'skip': u'VEVO is only available in some countries',
|
||||
},
|
||||
# age-restricted video
|
||||
{
|
||||
u'url': u'http://www.dailymotion.com/video/xyh2zz_leanna-decker-cyber-girl-of-the-year-desires-nude-playboy-plus_redband',
|
||||
u'file': u'xyh2zz.mp4',
|
||||
u'md5': u'0d667a7b9cebecc3c89ee93099c4159d',
|
||||
u'info_dict': {
|
||||
u'title': 'Leanna Decker - Cyber Girl Of The Year Desires Nude [Playboy Plus]',
|
||||
u'uploader': 'HotWaves1012',
|
||||
u'age_limit': 18,
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Extract id and simplified title from URL
|
||||
@@ -36,21 +82,29 @@ class DailymotionIE(SubtitlesInfoExtractor):
|
||||
|
||||
video_id = mobj.group(1).split('_')[0].split('?')[0]
|
||||
|
||||
video_extension = 'mp4'
|
||||
url = 'http://www.dailymotion.com/video/%s' % video_id
|
||||
|
||||
# Retrieve video webpage to extract further information
|
||||
request = compat_urllib_request.Request(url)
|
||||
request.add_header('Cookie', 'family_filter=off')
|
||||
request = self._build_request(url)
|
||||
webpage = self._download_webpage(request, video_id)
|
||||
|
||||
# Extract URL, uploader and title from webpage
|
||||
self.report_extraction(video_id)
|
||||
|
||||
# It may just embed a vevo video:
|
||||
m_vevo = re.search(
|
||||
r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?videoId=(?P<id>[\w]*)',
|
||||
webpage)
|
||||
if m_vevo is not None:
|
||||
vevo_id = m_vevo.group('id')
|
||||
self.to_screen(u'Vevo video detected: %s' % vevo_id)
|
||||
return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo')
|
||||
|
||||
video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>',
|
||||
# Looking for official user
|
||||
r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'],
|
||||
webpage, 'video uploader')
|
||||
webpage, 'video uploader', fatal=False)
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
video_upload_date = None
|
||||
mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
|
||||
@@ -63,19 +117,28 @@ class DailymotionIE(SubtitlesInfoExtractor):
|
||||
info = self._search_regex(r'var info = ({.*?}),$', embed_page,
|
||||
'video info', flags=re.MULTILINE)
|
||||
info = json.loads(info)
|
||||
if info.get('error') is not None:
|
||||
msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
|
||||
raise ExtractorError(msg, expected=True)
|
||||
|
||||
# TODO: support choosing qualities
|
||||
|
||||
for key in ['stream_h264_hd1080_url','stream_h264_hd_url',
|
||||
'stream_h264_hq_url','stream_h264_url',
|
||||
'stream_h264_ld_url']:
|
||||
if info.get(key):#key in info and info[key]:
|
||||
max_quality = key
|
||||
self.to_screen(u'Using %s' % key)
|
||||
break
|
||||
else:
|
||||
formats = []
|
||||
for (key, format_id) in self._FORMATS:
|
||||
video_url = info.get(key)
|
||||
if video_url is not None:
|
||||
m_size = re.search(r'H264-(\d+)x(\d+)', video_url)
|
||||
if m_size is not None:
|
||||
width, height = m_size.group(1), m_size.group(2)
|
||||
else:
|
||||
width, height = None, None
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'format_id': format_id,
|
||||
'width': width,
|
||||
'height': height,
|
||||
})
|
||||
if not formats:
|
||||
raise ExtractorError(u'Unable to extract video URL')
|
||||
video_url = info[max_quality]
|
||||
|
||||
# subtitles
|
||||
video_subtitles = self.extract_subtitles(video_id)
|
||||
@@ -83,16 +146,16 @@ class DailymotionIE(SubtitlesInfoExtractor):
|
||||
self._list_available_subtitles(video_id)
|
||||
return
|
||||
|
||||
return [{
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'formats': formats,
|
||||
'uploader': video_uploader,
|
||||
'upload_date': video_upload_date,
|
||||
'title': self._og_search_title(webpage),
|
||||
'ext': video_extension,
|
||||
'subtitles': video_subtitles,
|
||||
'thumbnail': info['thumbnail_url']
|
||||
}]
|
||||
'thumbnail': info['thumbnail_url'],
|
||||
'age_limit': age_limit,
|
||||
}
|
||||
|
||||
def _get_available_subtitles(self, video_id):
|
||||
try:
|
||||
@@ -110,29 +173,56 @@ class DailymotionIE(SubtitlesInfoExtractor):
|
||||
return {}
|
||||
|
||||
|
||||
class DailymotionPlaylistIE(InfoExtractor):
|
||||
class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
|
||||
IE_NAME = u'dailymotion:playlist'
|
||||
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
|
||||
_MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/playlist/.+?".*?>.*?</a>.*?</div>'
|
||||
_PAGE_TEMPLATE = 'https://www.dailymotion.com/playlist/%s/%s'
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
playlist_id = mobj.group('id')
|
||||
def _extract_entries(self, id):
|
||||
video_ids = []
|
||||
|
||||
for pagenum in itertools.count(1):
|
||||
webpage = self._download_webpage('https://www.dailymotion.com/playlist/%s/%s' % (playlist_id, pagenum),
|
||||
playlist_id, u'Downloading page %s' % pagenum)
|
||||
request = self._build_request(self._PAGE_TEMPLATE % (id, pagenum))
|
||||
webpage = self._download_webpage(request,
|
||||
id, u'Downloading page %s' % pagenum)
|
||||
|
||||
playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
|
||||
video_ids.extend(re.findall(r'data-id="(.+?)" data-ext-id', playlist_el))
|
||||
video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))
|
||||
|
||||
if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
|
||||
break
|
||||
return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
|
||||
for video_id in orderedSet(video_ids)]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
playlist_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, playlist_id)
|
||||
|
||||
entries = [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
|
||||
for video_id in video_ids]
|
||||
return {'_type': 'playlist',
|
||||
'id': playlist_id,
|
||||
'title': get_element_by_id(u'playlist_name', webpage),
|
||||
'entries': entries,
|
||||
'entries': self._extract_entries(playlist_id),
|
||||
}
|
||||
|
||||
|
||||
class DailymotionUserIE(DailymotionPlaylistIE):
|
||||
IE_NAME = u'dailymotion:user'
|
||||
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/user/(?P<user>[^/]+)'
|
||||
_MORE_PAGES_INDICATOR = r'<div class="next">.*?<a.*?href="/user/.+?".*?>.*?</a>.*?</div>'
|
||||
_PAGE_TEMPLATE = 'http://www.dailymotion.com/user/%s/%s'
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
user = mobj.group('user')
|
||||
webpage = self._download_webpage(url, user)
|
||||
full_user = self._html_search_regex(
|
||||
r'<a class="label" href="/%s".*?>(.*?)</' % re.escape(user),
|
||||
webpage, u'user', flags=re.DOTALL)
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': user,
|
||||
'title': full_user,
|
||||
'entries': self._extract_entries(user),
|
||||
}
|
||||
|
@@ -25,7 +25,7 @@ class DepositFilesIE(InfoExtractor):
|
||||
url = 'http://depositfiles.com/en/files/' + file_id
|
||||
|
||||
# Retrieve file webpage with 'Free download' button pressed
|
||||
free_download_indication = { 'gateway_result' : '1' }
|
||||
free_download_indication = {'gateway_result' : '1'}
|
||||
request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
|
||||
try:
|
||||
self.report_download_webpage(file_id)
|
||||
|
37
youtube_dl/extractor/ebaumsworld.py
Normal file
37
youtube_dl/extractor/ebaumsworld.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import determine_ext
|
||||
|
||||
|
||||
class EbaumsWorldIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://www\.ebaumsworld\.com/video/watch/(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.ebaumsworld.com/video/watch/83367677/',
|
||||
u'file': u'83367677.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'A Giant Python Opens The Door',
|
||||
u'description': u'This is how nightmares start...',
|
||||
u'uploader': u'jihadpizza',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
config_xml = self._download_webpage(
|
||||
'http://www.ebaumsworld.com/video/player/%s' % video_id, video_id)
|
||||
config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8'))
|
||||
video_url = config.find('file').text
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': config.find('title').text,
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'description': config.find('description').text,
|
||||
'thumbnail': config.find('image').text,
|
||||
'uploader': config.find('username').text,
|
||||
}
|
@@ -101,7 +101,7 @@ class EightTracksIE(InfoExtractor):
|
||||
first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
|
||||
next_url = first_url
|
||||
res = []
|
||||
for i in itertools.count():
|
||||
for i in range(track_count):
|
||||
api_json = self._download_webpage(next_url, playlist_id,
|
||||
note=u'Downloading song information %s/%s' % (str(i+1), track_count),
|
||||
errnote=u'Failed to download song information')
|
||||
@@ -116,7 +116,5 @@ class EightTracksIE(InfoExtractor):
|
||||
'ext': 'm4a',
|
||||
}
|
||||
res.append(info)
|
||||
if api_data['set']['at_last_track']:
|
||||
break
|
||||
next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (session, mix_id, track_data['id'])
|
||||
return res
|
||||
|
37
youtube_dl/extractor/eitb.py
Normal file
37
youtube_dl/extractor/eitb.py
Normal file
@@ -0,0 +1,37 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .brightcove import BrightcoveIE
|
||||
from ..utils import ExtractorError
|
||||
|
||||
|
||||
class EitbIE(InfoExtractor):
|
||||
IE_NAME = u'eitb.tv'
|
||||
_VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'add_ie': ['Brightcove'],
|
||||
u'url': u'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
|
||||
u'md5': u'edf4436247185adee3ea18ce64c47998',
|
||||
u'info_dict': {
|
||||
u'id': u'2743577154001',
|
||||
u'ext': u'mp4',
|
||||
u'title': u'60 minutos (Lasa y Zabala, 30 años)',
|
||||
# All videos from eitb has this description in the brightcove info
|
||||
u'description': u'.',
|
||||
u'uploader': u'Euskal Telebista',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
chapter_id = mobj.group('chapter_id')
|
||||
webpage = self._download_webpage(url, chapter_id)
|
||||
bc_url = BrightcoveIE._extract_brightcove_url(webpage)
|
||||
if bc_url is None:
|
||||
raise ExtractorError(u'Could not extract the Brightcove url')
|
||||
# The BrightcoveExperience object doesn't contain the video id, we set
|
||||
# it manually
|
||||
bc_url += '&%40videoPlayer={}'.format(chapter_id)
|
||||
return self.url_result(bc_url, BrightcoveIE.ie_key())
|
@@ -11,16 +11,17 @@ class ExfmIE(InfoExtractor):
|
||||
_SOUNDCLOUD_URL = r'(?:http://)?(?:www\.)?api\.soundcloud.com/tracks/([^/]+)/stream'
|
||||
_TESTS = [
|
||||
{
|
||||
u'url': u'http://ex.fm/song/1bgtzg',
|
||||
u'file': u'95223130.mp3',
|
||||
u'md5': u'8a7967a3fef10e59a1d6f86240fd41cf',
|
||||
u'url': u'http://ex.fm/song/eh359',
|
||||
u'file': u'44216187.mp3',
|
||||
u'md5': u'e45513df5631e6d760970b14cc0c11e7',
|
||||
u'info_dict': {
|
||||
u"title": u"We Can't Stop - Miley Cyrus",
|
||||
u"uploader": u"Miley Cyrus",
|
||||
u'upload_date': u'20130603',
|
||||
u'description': u'Download "We Can\'t Stop" \r\niTunes: http://smarturl.it/WeCantStop?IQid=SC\r\nAmazon: http://smarturl.it/WeCantStopAMZ?IQid=SC',
|
||||
u"title": u"Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive",
|
||||
u"uploader": u"deadjournalist",
|
||||
u'upload_date': u'20120424',
|
||||
u'description': u'Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive',
|
||||
},
|
||||
u'note': u'Soundcloud song',
|
||||
u'skip': u'The site is down too often',
|
||||
},
|
||||
{
|
||||
u'url': u'http://ex.fm/song/wddt8',
|
||||
@@ -30,6 +31,7 @@ class ExfmIE(InfoExtractor):
|
||||
u'title': u'Safe and Sound',
|
||||
u'uploader': u'Capital Cities',
|
||||
},
|
||||
u'skip': u'The site is down too often',
|
||||
},
|
||||
]
|
||||
|
||||
|
50
youtube_dl/extractor/extremetube.py
Normal file
50
youtube_dl/extractor/extremetube.py
Normal file
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
)
|
||||
|
||||
class ExtremeTubeIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
|
||||
_TEST = {
|
||||
u'url': u'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
|
||||
u'file': u'652431.mp4',
|
||||
u'md5': u'1fb9228f5e3332ec8c057d6ac36f33e0',
|
||||
u'info_dict': {
|
||||
u"title": u"Music Video 14 british euro brit european cumshots swallow",
|
||||
u"uploader": u"unknown",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, u'title')
|
||||
uploader = self._html_search_regex(r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
|
||||
video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, u'video_url'))
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[5].split('_')[:2]
|
||||
format = "-".join(format)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'uploader': uploader,
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
'age_limit': 18,
|
||||
}
|
@@ -19,7 +19,8 @@ class FacebookIE(InfoExtractor):
|
||||
"""Information Extractor for Facebook"""
|
||||
|
||||
_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
|
||||
_LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
|
||||
_LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
|
||||
_CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
|
||||
_NETRC_MACHINE = 'facebook'
|
||||
IE_NAME = u'facebook'
|
||||
_TEST = {
|
||||
@@ -36,50 +37,56 @@ class FacebookIE(InfoExtractor):
|
||||
"""Report attempt to log in."""
|
||||
self.to_screen(u'Logging in')
|
||||
|
||||
def _real_initialize(self):
|
||||
if self._downloader is None:
|
||||
return
|
||||
|
||||
useremail = None
|
||||
password = None
|
||||
downloader_params = self._downloader.params
|
||||
|
||||
# Attempt to use provided username and password or .netrc data
|
||||
if downloader_params.get('username', None) is not None:
|
||||
useremail = downloader_params['username']
|
||||
password = downloader_params['password']
|
||||
elif downloader_params.get('usenetrc', False):
|
||||
try:
|
||||
info = netrc.netrc().authenticators(self._NETRC_MACHINE)
|
||||
if info is not None:
|
||||
useremail = info[0]
|
||||
password = info[2]
|
||||
else:
|
||||
raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
|
||||
except (IOError, netrc.NetrcParseError) as err:
|
||||
self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
def _login(self):
|
||||
(useremail, password) = self._get_login_info()
|
||||
if useremail is None:
|
||||
return
|
||||
|
||||
# Log in
|
||||
login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
|
||||
login_page_req.add_header('Cookie', 'locale=en_US')
|
||||
self.report_login()
|
||||
login_page = self._download_webpage(login_page_req, None, note=False,
|
||||
errnote=u'Unable to download login page')
|
||||
lsd = self._search_regex(r'"lsd":"(\w*?)"', login_page, u'lsd')
|
||||
lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, u'lgnrnd')
|
||||
|
||||
login_form = {
|
||||
'email': useremail,
|
||||
'pass': password,
|
||||
'login': 'Log+In'
|
||||
'lsd': lsd,
|
||||
'lgnrnd': lgnrnd,
|
||||
'next': 'http://facebook.com/home.php',
|
||||
'default_persistent': '0',
|
||||
'legacy_return': '1',
|
||||
'timezone': '-60',
|
||||
'trynum': '1',
|
||||
}
|
||||
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
|
||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||
try:
|
||||
self.report_login()
|
||||
login_results = compat_urllib_request.urlopen(request).read()
|
||||
if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
|
||||
self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
|
||||
return
|
||||
|
||||
check_form = {
|
||||
'fb_dtsg': self._search_regex(r'"fb_dtsg":"(.*?)"', login_results, u'fb_dtsg'),
|
||||
'nh': self._search_regex(r'name="nh" value="(\w*?)"', login_results, u'nh'),
|
||||
'name_action_selected': 'dont_save',
|
||||
'submit[Continue]': self._search_regex(r'<input value="(.*?)" name="submit\[Continue\]"', login_results, u'continue'),
|
||||
}
|
||||
check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, compat_urllib_parse.urlencode(check_form))
|
||||
check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||
check_response = compat_urllib_request.urlopen(check_req).read()
|
||||
if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
|
||||
self._downloader.report_warning(u'Unable to confirm login, you have to login in your brower and authorize the login.')
|
||||
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
|
||||
self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
|
||||
return
|
||||
|
||||
def _real_initialize(self):
|
||||
self._login()
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
if mobj is None:
|
||||
@@ -93,7 +100,13 @@ class FacebookIE(InfoExtractor):
|
||||
AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
|
||||
m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
|
||||
if not m:
|
||||
raise ExtractorError(u'Cannot parse data')
|
||||
m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
|
||||
if m_msg is not None:
|
||||
raise ExtractorError(
|
||||
u'The video is not available, Facebook said: "%s"' % m_msg.group(1),
|
||||
expected=True)
|
||||
else:
|
||||
raise ExtractorError(u'Cannot parse data')
|
||||
data = dict(json.loads(m.group(1)))
|
||||
params_raw = compat_urllib_parse.unquote(data['params'])
|
||||
params = json.loads(params_raw)
|
||||
@@ -106,8 +119,8 @@ class FacebookIE(InfoExtractor):
|
||||
video_duration = int(video_data['video_duration'])
|
||||
thumbnail = video_data['thumbnail_src']
|
||||
|
||||
video_title = self._html_search_regex('<h2 class="uiHeaderTitle">([^<]+)</h2>',
|
||||
webpage, u'title')
|
||||
video_title = self._html_search_regex(
|
||||
r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, u'title')
|
||||
|
||||
info = {
|
||||
'id': video_id,
|
||||
|
58
youtube_dl/extractor/faz.py
Normal file
58
youtube_dl/extractor/faz.py
Normal file
@@ -0,0 +1,58 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
)
|
||||
|
||||
|
||||
class FazIE(InfoExtractor):
|
||||
IE_NAME = u'faz.net'
|
||||
_VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+).html'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
|
||||
u'file': u'12610585.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
|
||||
u'description': u'md5:1453fbf9a0d041d985a47306192ea253',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
self.to_screen(video_id)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
config_xml_url = self._search_regex(r'writeFLV\(\'(.+?)\',', webpage,
|
||||
u'config xml url')
|
||||
config_xml = self._download_webpage(config_xml_url, video_id,
|
||||
u'Downloading config xml')
|
||||
config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8'))
|
||||
|
||||
encodings = config.find('ENCODINGS')
|
||||
formats = []
|
||||
for code in ['LOW', 'HIGH', 'HQ']:
|
||||
encoding = encodings.find(code)
|
||||
if encoding is None:
|
||||
continue
|
||||
encoding_url = encoding.find('FILENAME').text
|
||||
formats.append({
|
||||
'url': encoding_url,
|
||||
'ext': determine_ext(encoding_url),
|
||||
'format_id': code.lower(),
|
||||
})
|
||||
|
||||
descr = self._html_search_regex(r'<p class="Content Copy">(.*?)</p>', webpage, u'description')
|
||||
info = {
|
||||
'id': video_id,
|
||||
'title': self._og_search_title(webpage),
|
||||
'formats': formats,
|
||||
'description': descr,
|
||||
'thumbnail': config.find('STILL/STILL_BIG').text,
|
||||
}
|
||||
# TODO: Remove when #980 has been merged
|
||||
info.update(formats[-1])
|
||||
return info
|
@@ -9,7 +9,7 @@ from ..utils import (
|
||||
|
||||
class FlickrIE(InfoExtractor):
|
||||
"""Information Extractor for Flickr videos"""
|
||||
_VALID_URL = r'(?:https?://)?(?:www\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
|
||||
_VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
|
||||
_TEST = {
|
||||
u'url': u'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
|
||||
u'file': u'5645318632.mp4',
|
||||
|
@@ -1,6 +1,7 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
@@ -65,3 +66,64 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
|
||||
webpage = self._download_webpage(url, page_title)
|
||||
video_id = self._search_regex(r'id-video=(\d+?)"', webpage, u'video id')
|
||||
return self._extract_video(video_id)
|
||||
|
||||
|
||||
class France2IE(FranceTVBaseInfoExtractor):
|
||||
IE_NAME = u'france2.fr'
|
||||
_VALID_URL = r'''(?x)https?://www\.france2\.fr/
|
||||
(?:
|
||||
emissions/.*?/videos/(?P<id>\d+)
|
||||
| emission/(?P<key>[^/?]+)
|
||||
)'''
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104',
|
||||
u'file': u'75540104.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'13h15, le samedi...',
|
||||
u'description': u'md5:2e5b58ba7a2d3692b35c792be081a03d',
|
||||
},
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
if mobj.group('key'):
|
||||
webpage = self._download_webpage(url, mobj.group('key'))
|
||||
video_id = self._html_search_regex(
|
||||
r'''(?x)<div\s+class="video-player">\s*
|
||||
<a\s+href="http://videos.francetv.fr/video/([0-9]+)"\s+
|
||||
class="francetv-video-player">''',
|
||||
webpage, u'video ID')
|
||||
else:
|
||||
video_id = mobj.group('id')
|
||||
return self._extract_video(video_id)
|
||||
|
||||
|
||||
class GenerationQuoiIE(InfoExtractor):
|
||||
IE_NAME = u'france2.fr:generation-quoi'
|
||||
_VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<name>.*)(\?|$)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://generation-quoi.france2.fr/portrait/garde-a-vous',
|
||||
u'file': u'k7FJX8VBcvvLmX4wA5Q.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Génération Quoi - Garde à Vous',
|
||||
u'uploader': u'Génération Quoi',
|
||||
},
|
||||
u'params': {
|
||||
# It uses Dailymotion
|
||||
u'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
name = mobj.group('name')
|
||||
info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % name)
|
||||
info_json = self._download_webpage(info_url, name)
|
||||
info = json.loads(info_json)
|
||||
return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
|
||||
ie='Dailymotion')
|
||||
|
@@ -21,7 +21,8 @@ class FunnyOrDieIE(InfoExtractor):
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_url = self._search_regex(r'type="video/mp4" src="(.*?)"',
|
||||
video_url = self._search_regex(
|
||||
[r'type="video/mp4" src="(.*?)"', r'src="([^>]*?)" type=\'video/mp4\''],
|
||||
webpage, u'video URL', flags=re.DOTALL)
|
||||
|
||||
info = {
|
||||
|
@@ -1,55 +1,59 @@
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
unified_strdate,
|
||||
compat_urllib_parse,
|
||||
compat_urlparse,
|
||||
unescapeHTML,
|
||||
get_meta_content,
|
||||
)
|
||||
|
||||
|
||||
class GameSpotIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<page_id>\d+)/?'
|
||||
_TEST = {
|
||||
u"url": u"http://www.gamespot.com/arma-iii/videos/arma-iii-community-guide-sitrep-i-6410818/",
|
||||
u"file": u"6410818.mp4",
|
||||
u"file": u"gs-2300-6410818.mp4",
|
||||
u"md5": u"b2a30deaa8654fcccd43713a6b6a4825",
|
||||
u"info_dict": {
|
||||
u"title": u"Arma 3 - Community Guide: SITREP I",
|
||||
u"upload_date": u"20130627",
|
||||
u'description': u'Check out this video where some of the basics of Arma 3 is explained.',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
page_id = mobj.group('page_id')
|
||||
page_id = video_id = mobj.group('page_id')
|
||||
webpage = self._download_webpage(url, page_id)
|
||||
video_id = self._html_search_regex([r'"og:video" content=".*?\?id=(\d+)"',
|
||||
r'http://www\.gamespot\.com/videoembed/(\d+)'],
|
||||
webpage, 'video id')
|
||||
data = compat_urllib_parse.urlencode({'id': video_id, 'newplayer': '1'})
|
||||
info_url = 'http://www.gamespot.com/pages/video_player/xml.php?' + data
|
||||
info_xml = self._download_webpage(info_url, video_id)
|
||||
doc = xml.etree.ElementTree.fromstring(info_xml)
|
||||
clip_el = doc.find('./playList/clip')
|
||||
data_video_json = self._search_regex(r'data-video=\'(.*?)\'', webpage, u'data video')
|
||||
data_video = json.loads(unescapeHTML(data_video_json))
|
||||
|
||||
http_urls = [{'url': node.find('filePath').text,
|
||||
'rate': int(node.find('rate').text)}
|
||||
for node in clip_el.find('./httpURI')]
|
||||
best_quality = sorted(http_urls, key=lambda f: f['rate'])[-1]
|
||||
video_url = best_quality['url']
|
||||
title = clip_el.find('./title').text
|
||||
ext = video_url.rpartition('.')[2]
|
||||
thumbnail_url = clip_el.find('./screenGrabURI').text
|
||||
view_count = int(clip_el.find('./views').text)
|
||||
upload_date = unified_strdate(clip_el.find('./postDate').text)
|
||||
# Transform the manifest url to a link to the mp4 files
|
||||
# they are used in mobile devices.
|
||||
f4m_url = data_video['videoStreams']['f4m_stream']
|
||||
f4m_path = compat_urlparse.urlparse(f4m_url).path
|
||||
QUALITIES_RE = r'((,\d+)+,?)'
|
||||
qualities = self._search_regex(QUALITIES_RE, f4m_path, u'qualities').strip(',').split(',')
|
||||
http_path = f4m_path[1:].split('/', 1)[1]
|
||||
http_template = re.sub(QUALITIES_RE, r'%s', http_path)
|
||||
http_template = http_template.replace('.csmil/manifest.f4m', '')
|
||||
http_template = compat_urlparse.urljoin('http://video.gamespotcdn.com/', http_template)
|
||||
formats = []
|
||||
for q in qualities:
|
||||
formats.append({
|
||||
'url': http_template % q,
|
||||
'ext': 'mp4',
|
||||
'format_id': q,
|
||||
})
|
||||
|
||||
return [{
|
||||
'id' : video_id,
|
||||
'url' : video_url,
|
||||
'ext' : ext,
|
||||
'title' : title,
|
||||
'thumbnail' : thumbnail_url,
|
||||
'upload_date' : upload_date,
|
||||
'view_count' : view_count,
|
||||
}]
|
||||
info = {
|
||||
'id': data_video['guid'],
|
||||
'title': compat_urllib_parse.unquote(data_video['title']),
|
||||
'formats': formats,
|
||||
'description': get_meta_content('description', webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
}
|
||||
# TODO: Remove when #980 has been merged
|
||||
info.update(formats[-1])
|
||||
return info
|
||||
|
@@ -11,6 +11,8 @@ from ..utils import (
|
||||
compat_urlparse,
|
||||
|
||||
ExtractorError,
|
||||
smuggle_url,
|
||||
unescapeHTML,
|
||||
)
|
||||
from .brightcove import BrightcoveIE
|
||||
|
||||
@@ -23,23 +25,50 @@ class GenericIE(InfoExtractor):
|
||||
{
|
||||
u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
|
||||
u'file': u'13601338388002.mp4',
|
||||
u'md5': u'85b90ccc9d73b4acd9138d3af4c27f89',
|
||||
u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd',
|
||||
u'info_dict': {
|
||||
u"uploader": u"www.hodiho.fr",
|
||||
u"title": u"R\u00e9gis plante sa Jeep"
|
||||
}
|
||||
},
|
||||
# embedded vimeo video
|
||||
{
|
||||
u'url': u'http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/',
|
||||
u'file': u'2371591881001.mp4',
|
||||
u'md5': u'9e80619e0a94663f0bdc849b4566af19',
|
||||
u'note': u'Test Brightcove downloads and detection in GenericIE',
|
||||
u'add_ie': ['Vimeo'],
|
||||
u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
|
||||
u'file': u'22444065.mp4',
|
||||
u'md5': u'2903896e23df39722c33f015af0666e2',
|
||||
u'info_dict': {
|
||||
u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
|
||||
u'uploader': u'8TV',
|
||||
u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
|
||||
u'title': u'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011',
|
||||
u"uploader_id": u"skillsmatter",
|
||||
u"uploader": u"Skills Matter",
|
||||
}
|
||||
},
|
||||
# bandcamp page with custom domain
|
||||
{
|
||||
u'add_ie': ['Bandcamp'],
|
||||
u'url': u'http://bronyrock.com/track/the-pony-mash',
|
||||
u'file': u'3235767654.mp3',
|
||||
u'info_dict': {
|
||||
u'title': u'The Pony Mash',
|
||||
u'uploader': u'M_Pallante',
|
||||
},
|
||||
u'skip': u'There is a limit of 200 free downloads / month for the test song',
|
||||
},
|
||||
# embedded brightcove video
|
||||
{
|
||||
u'add_ie': ['Brightcove'],
|
||||
u'url': u'http://www.scientificamerican.com/article.cfm?id=soap-bubble-physics',
|
||||
u'info_dict': {
|
||||
u'id': u'2365799484001',
|
||||
u'ext': u'mp4',
|
||||
u'title': u'Bubble Simulation',
|
||||
u'description': u'A visualization from a new computer model of foam behavior.',
|
||||
u'uploader': u'Scientific American',
|
||||
},
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def report_download_webpage(self, video_id):
|
||||
@@ -128,16 +157,36 @@ class GenericIE(InfoExtractor):
|
||||
except ValueError:
|
||||
# since this is the last-resort InfoExtractor, if
|
||||
# this error is thrown, it'll be thrown here
|
||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||
raise ExtractorError(u'Failed to download URL: %s' % url)
|
||||
|
||||
self.report_extraction(video_id)
|
||||
# Look for BrightCove:
|
||||
m_brightcove = re.search(r'<object.+?class=([\'"]).*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
|
||||
if m_brightcove is not None:
|
||||
bc_url = BrightcoveIE._extract_brightcove_url(webpage)
|
||||
if bc_url is not None:
|
||||
self.to_screen(u'Brightcove video detected.')
|
||||
bc_url = BrightcoveIE._build_brighcove_url(m_brightcove.group())
|
||||
return self.url_result(bc_url, 'Brightcove')
|
||||
|
||||
# Look for embedded Vimeo player
|
||||
mobj = re.search(
|
||||
r'<iframe[^>]+?src="(https?://player.vimeo.com/video/.+?)"', webpage)
|
||||
if mobj:
|
||||
player_url = unescapeHTML(mobj.group(1))
|
||||
surl = smuggle_url(player_url, {'Referer': url})
|
||||
return self.url_result(surl, 'Vimeo')
|
||||
|
||||
# Look for embedded YouTube player
|
||||
mobj = re.search(
|
||||
r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?youtube.com/embed/.+?)\1', webpage)
|
||||
if mobj:
|
||||
surl = unescapeHTML(mobj.group(u'url'))
|
||||
return self.url_result(surl, 'Youtube')
|
||||
|
||||
# Look for Bandcamp pages with custom domain
|
||||
mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
|
||||
if mobj is not None:
|
||||
burl = unescapeHTML(mobj.group(1))
|
||||
return self.url_result(burl, 'Bandcamp')
|
||||
|
||||
# Start with something easy: JW Player in SWFObject
|
||||
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
|
||||
if mobj is None:
|
||||
@@ -160,12 +209,12 @@ class GenericIE(InfoExtractor):
|
||||
# HTML5 video
|
||||
mobj = re.search(r'<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage, flags=re.DOTALL)
|
||||
if mobj is None:
|
||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||
raise ExtractorError(u'Unsupported URL: %s' % url)
|
||||
|
||||
# It's possible that one of the regexes
|
||||
# matched, but returned an empty group:
|
||||
if mobj.group(1) is None:
|
||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||
raise ExtractorError(u'Did not find a valid video URL at %s' % url)
|
||||
|
||||
video_url = mobj.group(1)
|
||||
video_url = compat_urlparse.urljoin(url, video_url)
|
||||
|
@@ -41,8 +41,9 @@ class GooglePlusIE(InfoExtractor):
|
||||
|
||||
# Extract update date
|
||||
upload_date = self._html_search_regex(
|
||||
['title="Timestamp">(.*?)</a>', r'<a.+?class="g-M.+?>(.+?)</a>'],
|
||||
webpage, u'upload date', fatal=False)
|
||||
r'''(?x)<a.+?class="o-U-s\s[^"]+"\s+style="display:\s*none"\s*>
|
||||
([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
|
||||
webpage, u'upload date', fatal=False, flags=re.VERBOSE)
|
||||
if upload_date:
|
||||
# Convert timestring to a format suitable for filename
|
||||
upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
|
||||
|
@@ -30,7 +30,7 @@ class HypemIE(InfoExtractor):
|
||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||
track_id = mobj.group(1)
|
||||
|
||||
data = { 'ax': 1, 'ts': time.time() }
|
||||
data = {'ax': 1, 'ts': time.time()}
|
||||
data_encoded = compat_urllib_parse.urlencode(data)
|
||||
complete_url = url + "?" + data_encoded
|
||||
request = compat_urllib_request.Request(complete_url)
|
||||
@@ -68,4 +68,4 @@ class HypemIE(InfoExtractor):
|
||||
'ext': "mp3",
|
||||
'title': title,
|
||||
'artist': artist,
|
||||
}]
|
||||
}]
|
||||
|
@@ -13,7 +13,7 @@ class IGNIE(InfoExtractor):
|
||||
Some videos of it.ign.com are also supported
|
||||
"""
|
||||
|
||||
_VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles)(/.+)?/(?P<name_or_id>.+)'
|
||||
_VALID_URL = r'https?://.+?\.ign\.com/(?P<type>videos|show_videos|articles|(?:[^/]*/feature))(/.+)?/(?P<name_or_id>.+)'
|
||||
IE_NAME = u'ign.com'
|
||||
|
||||
_CONFIG_URL_TEMPLATE = 'http://www.ign.com/videos/configs/id/%s.config'
|
||||
@@ -21,15 +21,39 @@ class IGNIE(InfoExtractor):
|
||||
r'id="my_show_video">.*?<p>(.*?)</p>',
|
||||
]
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
|
||||
u'file': u'8f862beef863986b2785559b9e1aa599.mp4',
|
||||
u'md5': u'eac8bdc1890980122c3b66f14bdd02e9',
|
||||
u'info_dict': {
|
||||
u'title': u'The Last of Us Review',
|
||||
u'description': u'md5:c8946d4260a4d43a00d5ae8ed998870c',
|
||||
}
|
||||
}
|
||||
_TESTS = [
|
||||
{
|
||||
u'url': u'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
|
||||
u'file': u'8f862beef863986b2785559b9e1aa599.mp4',
|
||||
u'md5': u'eac8bdc1890980122c3b66f14bdd02e9',
|
||||
u'info_dict': {
|
||||
u'title': u'The Last of Us Review',
|
||||
u'description': u'md5:c8946d4260a4d43a00d5ae8ed998870c',
|
||||
}
|
||||
},
|
||||
{
|
||||
u'url': u'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
|
||||
u'playlist': [
|
||||
{
|
||||
u'file': u'5ebbd138523268b93c9141af17bec937.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'GTA 5 Video Review',
|
||||
u'description': u'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
|
||||
},
|
||||
},
|
||||
{
|
||||
u'file': u'638672ee848ae4ff108df2a296418ee2.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'GTA 5\'s Twisted Beauty in Super Slow Motion',
|
||||
u'description': u'The twisted beauty of GTA 5 in stunning slow motion.',
|
||||
},
|
||||
},
|
||||
],
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _find_video_id(self, webpage):
|
||||
res_id = [r'data-video-id="(.+?)"',
|
||||
@@ -46,6 +70,13 @@ class IGNIE(InfoExtractor):
|
||||
if page_type == 'articles':
|
||||
video_url = self._search_regex(r'var videoUrl = "(.+?)"', webpage, u'video url')
|
||||
return self.url_result(video_url, ie='IGN')
|
||||
elif page_type != 'video':
|
||||
multiple_urls = re.findall(
|
||||
'<param name="flashvars" value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
|
||||
webpage)
|
||||
if multiple_urls:
|
||||
return [self.url_result(u, ie='IGN') for u in multiple_urls]
|
||||
|
||||
video_id = self._find_video_id(webpage)
|
||||
result = self._get_video_info(video_id)
|
||||
description = self._html_search_regex(self._DESCRIPTION_RE,
|
||||
@@ -87,6 +118,9 @@ class OneUPIE(IGNIE):
|
||||
}
|
||||
}
|
||||
|
||||
# Override IGN tests
|
||||
_TESTS = []
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
id = mobj.group('name_or_id')
|
||||
|
@@ -26,7 +26,7 @@ class InstagramIE(InfoExtractor):
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': self._og_search_video_url(webpage),
|
||||
'url': self._og_search_video_url(webpage, secure=False),
|
||||
'ext': 'mp4',
|
||||
'title': u'Video by %s' % uploader_id,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
|
84
youtube_dl/extractor/internetvideoarchive.py
Normal file
84
youtube_dl/extractor/internetvideoarchive.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
compat_urllib_parse,
|
||||
xpath_with_ns,
|
||||
determine_ext,
|
||||
)
|
||||
|
||||
|
||||
class InternetVideoArchiveIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://video\.internetvideoarchive\.net/flash/players/.*?\?.*?publishedid.*?'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?customerid=69249&publishedid=452693&playerid=247',
|
||||
u'file': u'452693.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'SKYFALL',
|
||||
u'description': u'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
|
||||
u'duration': 153,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_url(query):
|
||||
return 'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?' + query
|
||||
|
||||
@staticmethod
|
||||
def _clean_query(query):
|
||||
NEEDED_ARGS = ['publishedid', 'customerid']
|
||||
query_dic = compat_urlparse.parse_qs(query)
|
||||
cleaned_dic = dict((k,v[0]) for (k,v) in query_dic.items() if k in NEEDED_ARGS)
|
||||
# Other player ids return m3u8 urls
|
||||
cleaned_dic['playerid'] = '247'
|
||||
cleaned_dic['videokbrate'] = '100000'
|
||||
return compat_urllib_parse.urlencode(cleaned_dic)
|
||||
|
||||
def _real_extract(self, url):
|
||||
query = compat_urlparse.urlparse(url).query
|
||||
query_dic = compat_urlparse.parse_qs(query)
|
||||
video_id = query_dic['publishedid'][0]
|
||||
url = self._build_url(query)
|
||||
|
||||
flashconfiguration_xml = self._download_webpage(url, video_id,
|
||||
u'Downloading flash configuration')
|
||||
flashconfiguration = xml.etree.ElementTree.fromstring(flashconfiguration_xml.encode('utf-8'))
|
||||
file_url = flashconfiguration.find('file').text
|
||||
file_url = file_url.replace('/playlist.aspx', '/mrssplaylist.aspx')
|
||||
# Replace some of the parameters in the query to get the best quality
|
||||
# and http links (no m3u8 manifests)
|
||||
file_url = re.sub(r'(?<=\?)(.+)$',
|
||||
lambda m: self._clean_query(m.group()),
|
||||
file_url)
|
||||
info_xml = self._download_webpage(file_url, video_id,
|
||||
u'Downloading video info')
|
||||
info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8'))
|
||||
item = info.find('channel/item')
|
||||
|
||||
def _bp(p):
|
||||
return xpath_with_ns(p,
|
||||
{'media': 'http://search.yahoo.com/mrss/',
|
||||
'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'})
|
||||
formats = []
|
||||
for content in item.findall(_bp('media:group/media:content')):
|
||||
attr = content.attrib
|
||||
f_url = attr['url']
|
||||
formats.append({
|
||||
'url': f_url,
|
||||
'ext': determine_ext(f_url),
|
||||
'width': int(attr['width']),
|
||||
'bitrate': int(attr['bitrate']),
|
||||
})
|
||||
formats = sorted(formats, key=lambda f: f['bitrate'])
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': item.find('title').text,
|
||||
'formats': formats,
|
||||
'thumbnail': item.find(_bp('media:thumbnail')).attrib['url'],
|
||||
'description': item.find('description').text,
|
||||
'duration': int(attr['duration']),
|
||||
}
|
@@ -6,13 +6,14 @@ import xml.etree.ElementTree
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class JeuxVideoIE(InfoExtractor):
|
||||
_VALID_URL = r'http://.*?\.jeuxvideo\.com/.*/(.*?)-\d+\.htm'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
|
||||
u'file': u'5182.mp4',
|
||||
u'md5': u'e0fdb0cd3ce98713ef9c1e1e025779d0',
|
||||
u'md5': u'046e491afb32a8aaac1f44dd4ddd54ee',
|
||||
u'info_dict': {
|
||||
u'title': u'GC 2013 : Tearaway nous présente ses papiers d\'identité',
|
||||
u'description': u'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n',
|
||||
@@ -23,25 +24,29 @@ class JeuxVideoIE(InfoExtractor):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
title = re.match(self._VALID_URL, url).group(1)
|
||||
webpage = self._download_webpage(url, title)
|
||||
m_download = re.search(r'<param name="flashvars" value="config=(.*?)" />', webpage)
|
||||
|
||||
xml_link = m_download.group(1)
|
||||
xml_link = self._html_search_regex(
|
||||
r'<param name="flashvars" value="config=(.*?)" />',
|
||||
webpage, u'config URL')
|
||||
|
||||
id = re.search(r'http://www.jeuxvideo.com/config/\w+/0011/(.*?)/\d+_player\.xml', xml_link).group(1)
|
||||
video_id = self._search_regex(
|
||||
r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml',
|
||||
xml_link, u'video ID')
|
||||
|
||||
xml_config = self._download_webpage(xml_link, title,
|
||||
'Downloading XML config')
|
||||
xml_config = self._download_webpage(
|
||||
xml_link, title, u'Downloading XML config')
|
||||
config = xml.etree.ElementTree.fromstring(xml_config.encode('utf-8'))
|
||||
info = re.search(r'<format\.json>(.*?)</format\.json>',
|
||||
xml_config, re.MULTILINE|re.DOTALL).group(1)
|
||||
info = json.loads(info)['versions'][0]
|
||||
info_json = self._search_regex(
|
||||
r'(?sm)<format\.json>(.*?)</format\.json>',
|
||||
xml_config, u'JSON information')
|
||||
info = json.loads(info_json)['versions'][0]
|
||||
|
||||
video_url = 'http://video720.jeuxvideo.com/' + info['file']
|
||||
|
||||
return {'id': id,
|
||||
'title' : config.find('titre_video').text,
|
||||
'ext' : 'mp4',
|
||||
'url' : video_url,
|
||||
'description': self._og_search_description(webpage),
|
||||
'thumbnail': config.find('image').text,
|
||||
}
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': config.find('titre_video').text,
|
||||
'ext': 'mp4',
|
||||
'url': video_url,
|
||||
'description': self._og_search_description(webpage),
|
||||
'thumbnail': config.find('image').text,
|
||||
}
|
||||
|
61
youtube_dl/extractor/keezmovies.py
Normal file
61
youtube_dl/extractor/keezmovies.py
Normal file
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
)
|
||||
from ..aes import (
|
||||
aes_decrypt_text
|
||||
)
|
||||
|
||||
class KeezMoviesIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>keezmovies\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
|
||||
_TEST = {
|
||||
u'url': u'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
|
||||
u'file': u'1214711.mp4',
|
||||
u'md5': u'6e297b7e789329923fcf83abb67c9289',
|
||||
u'info_dict': {
|
||||
u"title": u"Petite Asian Lady Mai Playing In Bathtub",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
# embedded video
|
||||
mobj = re.search(r'href="([^"]+)"></iframe>', webpage)
|
||||
if mobj:
|
||||
embedded_url = mobj.group(1)
|
||||
return self.url_result(embedded_url)
|
||||
|
||||
video_title = self._html_search_regex(r'<h1 [^>]*>([^<]+)', webpage, u'title')
|
||||
video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, u'video_url'))
|
||||
if webpage.find('encrypted=true')!=-1:
|
||||
password = self._html_search_regex(r'video_title=(.+?)&', webpage, u'password')
|
||||
video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[4].split('_')[:2]
|
||||
format = "-".join(format)
|
||||
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
'age_limit': age_limit,
|
||||
}
|
@@ -2,7 +2,12 @@ import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import compat_urllib_parse_urlparse, compat_urlparse
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urlparse,
|
||||
get_meta_content,
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
|
||||
class LivestreamIE(InfoExtractor):
|
||||
@@ -35,10 +40,9 @@ class LivestreamIE(InfoExtractor):
|
||||
|
||||
if video_id is None:
|
||||
# This is an event page:
|
||||
api_url = self._search_regex(r'event_design_eventId: \'(.+?)\'',
|
||||
webpage, 'api url')
|
||||
info = json.loads(self._download_webpage(api_url, event_name,
|
||||
u'Downloading event info'))
|
||||
config_json = self._search_regex(r'window.config = ({.*?});',
|
||||
webpage, u'window config')
|
||||
info = json.loads(config_json)['event']
|
||||
videos = [self._extract_video_info(video_data['data'])
|
||||
for video_data in info['feed']['data'] if video_data['type'] == u'video']
|
||||
return self.playlist_result(videos, info['id'], info['full_name'])
|
||||
|
@@ -20,10 +20,12 @@ class MetacafeIE(InfoExtractor):
|
||||
_DISCLAIMER = 'http://www.metacafe.com/family_filter/'
|
||||
_FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
|
||||
IE_NAME = u'metacafe'
|
||||
_TESTS = [{
|
||||
_TESTS = [
|
||||
# Youtube video
|
||||
{
|
||||
u"add_ie": ["Youtube"],
|
||||
u"url": u"http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
|
||||
u"file": u"_aUehQsCQtM.flv",
|
||||
u"file": u"_aUehQsCQtM.mp4",
|
||||
u"info_dict": {
|
||||
u"upload_date": u"20090102",
|
||||
u"title": u"The Electric Company | \"Short I\" | PBS KIDS GO!",
|
||||
@@ -32,15 +34,42 @@ class MetacafeIE(InfoExtractor):
|
||||
u"uploader_id": u"PBS"
|
||||
}
|
||||
},
|
||||
# Normal metacafe video
|
||||
{
|
||||
u'url': u'http://www.metacafe.com/watch/11121940/news_stuff_you_wont_do_with_your_playstation_4/',
|
||||
u'md5': u'6e0bca200eaad2552e6915ed6fd4d9ad',
|
||||
u'info_dict': {
|
||||
u'id': u'11121940',
|
||||
u'ext': u'mp4',
|
||||
u'title': u'News: Stuff You Won\'t Do with Your PlayStation 4',
|
||||
u'uploader': u'ign',
|
||||
u'description': u'Sony released a massive FAQ on the PlayStation Blog detailing the PS4\'s capabilities and limitations.',
|
||||
},
|
||||
},
|
||||
# AnyClip video
|
||||
{
|
||||
u"url": u"http://www.metacafe.com/watch/an-dVVXnuY7Jh77J/the_andromeda_strain_1971_stop_the_bomb_part_3/",
|
||||
u"file": u"an-dVVXnuY7Jh77J.mp4",
|
||||
u"info_dict": {
|
||||
u"title": u"The Andromeda Strain (1971): Stop the Bomb Part 3",
|
||||
u"uploader": u"anyclip",
|
||||
u"description": u"md5:38c711dd98f5bb87acf973d573442e67"
|
||||
}
|
||||
}]
|
||||
u"description": u"md5:38c711dd98f5bb87acf973d573442e67",
|
||||
},
|
||||
},
|
||||
# age-restricted video
|
||||
{
|
||||
u'url': u'http://www.metacafe.com/watch/5186653/bbc_internal_christmas_tape_79_uncensored_outtakes_etc/',
|
||||
u'md5': u'98dde7c1a35d02178e8ab7560fe8bd09',
|
||||
u'info_dict': {
|
||||
u'id': u'5186653',
|
||||
u'ext': u'mp4',
|
||||
u'title': u'BBC INTERNAL Christmas Tape \'79 - UNCENSORED Outtakes, Etc.',
|
||||
u'uploader': u'Dwayne Pipe',
|
||||
u'description': u'md5:950bf4c581e2c059911fa3ffbe377e4b',
|
||||
u'age_limit': 18,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def report_disclaimer(self):
|
||||
@@ -62,6 +91,7 @@ class MetacafeIE(InfoExtractor):
|
||||
'submit': "Continue - I'm over 18",
|
||||
}
|
||||
request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
|
||||
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||
try:
|
||||
self.report_age_confirmation()
|
||||
compat_urllib_request.urlopen(request).read()
|
||||
@@ -83,7 +113,12 @@ class MetacafeIE(InfoExtractor):
|
||||
|
||||
# Retrieve video webpage to extract further information
|
||||
req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
|
||||
req.headers['Cookie'] = 'flashVersion=0;'
|
||||
|
||||
# AnyClip videos require the flashversion cookie so that we get the link
|
||||
# to the mp4 file
|
||||
mobj_an = re.match(r'^an-(.*?)$', video_id)
|
||||
if mobj_an:
|
||||
req.headers['Cookie'] = 'flashVersion=0;'
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
# Extract URL, uploader and title from webpage
|
||||
@@ -125,6 +160,11 @@ class MetacafeIE(InfoExtractor):
|
||||
r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
|
||||
webpage, u'uploader nickname', fatal=False)
|
||||
|
||||
if re.search(r'"contentRating":"restricted"', webpage) is not None:
|
||||
age_limit = 18
|
||||
else:
|
||||
age_limit = 0
|
||||
|
||||
return {
|
||||
'_type': 'video',
|
||||
'id': video_id,
|
||||
@@ -134,4 +174,5 @@ class MetacafeIE(InfoExtractor):
|
||||
'upload_date': None,
|
||||
'title': video_title,
|
||||
'ext': video_ext,
|
||||
'age_limit': age_limit,
|
||||
}
|
||||
|
49
youtube_dl/extractor/mofosex.py
Normal file
49
youtube_dl/extractor/mofosex.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
)
|
||||
|
||||
class MofosexIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
|
||||
_TEST = {
|
||||
u'url': u'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
|
||||
u'file': u'5018.mp4',
|
||||
u'md5': u'1b2eb47ac33cc75d4a80e3026b613c5a',
|
||||
u'info_dict': {
|
||||
u"title": u"Japanese Teen Music Video",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<h1>(.+?)<', webpage, u'title')
|
||||
video_url = compat_urllib_parse.unquote(self._html_search_regex(r'flashvars.video_url = \'([^\']+)', webpage, u'video_url'))
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[5].split('_')[:2]
|
||||
format = "-".join(format)
|
||||
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
'age_limit': age_limit,
|
||||
}
|
@@ -26,6 +26,7 @@ class MTVIE(InfoExtractor):
|
||||
},
|
||||
},
|
||||
{
|
||||
u'add_ie': ['Vevo'],
|
||||
u'url': u'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
|
||||
u'file': u'USCJY1331283.mp4',
|
||||
u'md5': u'73b4e7fcadd88929292fe52c3ced8caf',
|
||||
@@ -54,46 +55,57 @@ class MTVIE(InfoExtractor):
|
||||
def _get_thumbnail_url(self, uri, itemdoc):
|
||||
return 'http://mtv.mtvnimages.com/uri/' + uri
|
||||
|
||||
def _extract_video_url(self, metadataXml):
|
||||
def _extract_video_formats(self, metadataXml):
|
||||
if '/error_country_block.swf' in metadataXml:
|
||||
raise ExtractorError(u'This video is not available from your country.', expected=True)
|
||||
mdoc = xml.etree.ElementTree.fromstring(metadataXml.encode('utf-8'))
|
||||
renditions = mdoc.findall('.//rendition')
|
||||
|
||||
# For now, always pick the highest quality.
|
||||
rendition = renditions[-1]
|
||||
|
||||
try:
|
||||
_,_,ext = rendition.attrib['type'].partition('/')
|
||||
format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
|
||||
rtmp_video_url = rendition.find('./src').text
|
||||
except KeyError:
|
||||
raise ExtractorError('Invalid rendition field.')
|
||||
video_url = self._transform_rtmp_url(rtmp_video_url)
|
||||
return {'ext': ext, 'url': video_url, 'format': format}
|
||||
formats = []
|
||||
for rendition in mdoc.findall('.//rendition'):
|
||||
try:
|
||||
_, _, ext = rendition.attrib['type'].partition('/')
|
||||
rtmp_video_url = rendition.find('./src').text
|
||||
formats.append({'ext': ext,
|
||||
'url': self._transform_rtmp_url(rtmp_video_url),
|
||||
'format_id': rendition.get('bitrate'),
|
||||
'width': int(rendition.get('width')),
|
||||
'height': int(rendition.get('height')),
|
||||
})
|
||||
except (KeyError, TypeError):
|
||||
raise ExtractorError('Invalid rendition field.')
|
||||
return formats
|
||||
|
||||
def _get_video_info(self, itemdoc):
|
||||
uri = itemdoc.find('guid').text
|
||||
video_id = self._id_from_uri(uri)
|
||||
self.report_extraction(video_id)
|
||||
mediagen_url = itemdoc.find('%s/%s' % (_media_xml_tag('group'), _media_xml_tag('content'))).attrib['url']
|
||||
# Remove the templates, like &device={device}
|
||||
mediagen_url = re.sub(r'&[^=]*?={.*?}(?=(&|$))', u'', mediagen_url)
|
||||
if 'acceptMethods' not in mediagen_url:
|
||||
mediagen_url += '&acceptMethods=fms'
|
||||
mediagen_page = self._download_webpage(mediagen_url, video_id,
|
||||
u'Downloading video urls')
|
||||
video_info = self._extract_video_url(mediagen_page)
|
||||
|
||||
description_node = itemdoc.find('description')
|
||||
if description_node is not None:
|
||||
description = description_node.text
|
||||
description = description_node.text.strip()
|
||||
else:
|
||||
description = None
|
||||
video_info.update({'title': itemdoc.find('title').text,
|
||||
'id': video_id,
|
||||
'thumbnail': self._get_thumbnail_url(uri, itemdoc),
|
||||
'description': description,
|
||||
})
|
||||
return video_info
|
||||
|
||||
info = {
|
||||
'title': itemdoc.find('title').text,
|
||||
'formats': self._extract_video_formats(mediagen_page),
|
||||
'id': video_id,
|
||||
'thumbnail': self._get_thumbnail_url(uri, itemdoc),
|
||||
'description': description,
|
||||
}
|
||||
|
||||
# TODO: Remove when #980 has been merged
|
||||
info.update(info['formats'][-1])
|
||||
|
||||
return info
|
||||
|
||||
def _get_videos_info(self, uri):
|
||||
video_id = self._id_from_uri(uri)
|
||||
|
48
youtube_dl/extractor/myspace.py
Normal file
48
youtube_dl/extractor/myspace.py
Normal file
@@ -0,0 +1,48 @@
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_str,
|
||||
)
|
||||
|
||||
|
||||
class MySpaceIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://myspace\.com/([^/]+)/video/[^/]+/(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'https://myspace.com/coldplay/video/viva-la-vida/100008689',
|
||||
u'info_dict': {
|
||||
u'id': u'100008689',
|
||||
u'ext': u'flv',
|
||||
u'title': u'Viva La Vida',
|
||||
u'description': u'The official Viva La Vida video, directed by Hype Williams',
|
||||
u'uploader': u'Coldplay',
|
||||
u'uploader_id': u'coldplay',
|
||||
},
|
||||
u'params': {
|
||||
# rtmp download
|
||||
u'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
context = json.loads(self._search_regex(r'context = ({.*?});', webpage,
|
||||
u'context'))
|
||||
video = context['video']
|
||||
rtmp_url, play_path = video['streamUrl'].split(';', 1)
|
||||
|
||||
return {
|
||||
'id': compat_str(video['mediaId']),
|
||||
'title': video['title'],
|
||||
'url': rtmp_url,
|
||||
'play_path': play_path,
|
||||
'ext': 'flv',
|
||||
'description': video['description'],
|
||||
'thumbnail': video['imageUrl'],
|
||||
'uploader': video['artistName'],
|
||||
'uploader_id': video['artistUsername'],
|
||||
}
|
120
youtube_dl/extractor/nhl.py
Normal file
120
youtube_dl/extractor/nhl.py
Normal file
@@ -0,0 +1,120 @@
|
||||
import re
|
||||
import json
|
||||
import xml.etree.ElementTree
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
compat_urllib_parse,
|
||||
determine_ext,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class NHLBaseInfoExtractor(InfoExtractor):
|
||||
@staticmethod
|
||||
def _fix_json(json_string):
|
||||
return json_string.replace('\\\'', '\'')
|
||||
|
||||
def _extract_video(self, info):
|
||||
video_id = info['id']
|
||||
self.report_extraction(video_id)
|
||||
|
||||
initial_video_url = info['publishPoint']
|
||||
data = compat_urllib_parse.urlencode({
|
||||
'type': 'fvod',
|
||||
'path': initial_video_url.replace('.mp4', '_sd.mp4'),
|
||||
})
|
||||
path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
|
||||
path_response = self._download_webpage(path_url, video_id,
|
||||
u'Downloading final video url')
|
||||
path_doc = xml.etree.ElementTree.fromstring(path_response)
|
||||
video_url = path_doc.find('path').text
|
||||
|
||||
join = compat_urlparse.urljoin
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': info['name'],
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'description': info['description'],
|
||||
'duration': int(info['duration']),
|
||||
'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
|
||||
'upload_date': unified_strdate(info['releaseDate'].split('.')[0]),
|
||||
}
|
||||
|
||||
|
||||
class NHLIE(NHLBaseInfoExtractor):
|
||||
IE_NAME = u'nhl.com'
|
||||
_VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console\?.*?(?<=[?&])id=(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
|
||||
u'file': u'453614.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Quick clip: Weise 4-3 goal vs Flames',
|
||||
u'description': u'Dale Weise scores his first of the season to put the Canucks up 4-3.',
|
||||
u'duration': 18,
|
||||
u'upload_date': u'20131006',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
|
||||
info_json = self._download_webpage(json_url, video_id,
|
||||
u'Downloading info json')
|
||||
info_json = self._fix_json(info_json)
|
||||
info = json.loads(info_json)[0]
|
||||
return self._extract_video(info)
|
||||
|
||||
|
||||
class NHLVideocenterIE(NHLBaseInfoExtractor):
|
||||
IE_NAME = u'nhl.com:videocenter'
|
||||
IE_DESC = u'Download the first 12 videos from a videocenter category'
|
||||
_VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P<catid>[^&]+))?'
|
||||
|
||||
@classmethod
|
||||
def suitable(cls, url):
|
||||
if NHLIE.suitable(url):
|
||||
return False
|
||||
return super(NHLVideocenterIE, cls).suitable(url)
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
team = mobj.group('team')
|
||||
webpage = self._download_webpage(url, team)
|
||||
cat_id = self._search_regex(
|
||||
[r'var defaultCatId = "(.+?)";',
|
||||
r'{statusIndex:0,index:0,.*?id:(.*?),'],
|
||||
webpage, u'category id')
|
||||
playlist_title = self._html_search_regex(
|
||||
r'tab0"[^>]*?>(.*?)</td>',
|
||||
webpage, u'playlist title', flags=re.DOTALL).lower().capitalize()
|
||||
|
||||
data = compat_urllib_parse.urlencode({
|
||||
'cid': cat_id,
|
||||
# This is the default value
|
||||
'count': 12,
|
||||
'ptrs': 3,
|
||||
'format': 'json',
|
||||
})
|
||||
path = '/videocenter/servlets/browse?' + data
|
||||
request_url = compat_urlparse.urljoin(url, path)
|
||||
response = self._download_webpage(request_url, playlist_title)
|
||||
response = self._fix_json(response)
|
||||
if not response.strip():
|
||||
self._downloader.report_warning(u'Got an empty reponse, trying '
|
||||
u'adding the "newvideos" parameter')
|
||||
response = self._download_webpage(request_url + '&newvideos=true',
|
||||
playlist_title)
|
||||
response = self._fix_json(response)
|
||||
videos = json.loads(response)
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'title': playlist_title,
|
||||
'id': cat_id,
|
||||
'entries': [self._extract_video(i) for i in videos],
|
||||
}
|
46
youtube_dl/extractor/nowvideo.py
Normal file
46
youtube_dl/extractor/nowvideo.py
Normal file
@@ -0,0 +1,46 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import compat_urlparse
|
||||
|
||||
|
||||
class NowVideoIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:https?://)?(?:www\.)?nowvideo\.ch/video/(?P<id>\w+)'
|
||||
_TEST = {
|
||||
u'url': u'http://www.nowvideo.ch/video/0mw0yow7b6dxa',
|
||||
u'file': u'0mw0yow7b6dxa.flv',
|
||||
u'md5': u'f8fbbc8add72bd95b7850c6a02fc8817',
|
||||
u'info_dict': {
|
||||
u"title": u"youtubedl test video _BaW_jenozKc.mp4"
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
webpage_url = 'http://www.nowvideo.ch/video/' + video_id
|
||||
embed_url = 'http://embed.nowvideo.ch/embed.php?v=' + video_id
|
||||
webpage = self._download_webpage(webpage_url, video_id)
|
||||
embed_page = self._download_webpage(embed_url, video_id,
|
||||
u'Downloading embed page')
|
||||
|
||||
self.report_extraction(video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<h4>(.*)</h4>',
|
||||
webpage, u'video title')
|
||||
|
||||
video_key = self._search_regex(r'var fkzd="(.*)";',
|
||||
embed_page, u'video key')
|
||||
|
||||
api_call = "http://www.nowvideo.ch/api/player.api.php?file={0}&numOfErrors=0&cid=1&key={1}".format(video_id, video_key)
|
||||
api_response = self._download_webpage(api_call, video_id,
|
||||
u'Downloading API page')
|
||||
video_url = compat_urlparse.parse_qs(api_response)[u'url'][0]
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'flv',
|
||||
'title': video_title,
|
||||
}]
|
69
youtube_dl/extractor/pornhub.py
Normal file
69
youtube_dl/extractor/pornhub.py
Normal file
@@ -0,0 +1,69 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
unescapeHTML,
|
||||
)
|
||||
from ..aes import (
|
||||
aes_decrypt_text
|
||||
)
|
||||
|
||||
class PornHubIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9]+))'
|
||||
_TEST = {
|
||||
u'url': u'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
||||
u'file': u'648719015.mp4',
|
||||
u'md5': u'882f488fa1f0026f023f33576004a2ed',
|
||||
u'info_dict': {
|
||||
u"uploader": u"BABES-COM",
|
||||
u"title": u"Seductive Indian beauty strips down and fingers her pink pussy",
|
||||
u"age_limit": 18
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, u'title')
|
||||
video_uploader = self._html_search_regex(r'<b>From: </b>(?:\s|<[^>]*>)*(.+?)<', webpage, u'uploader', fatal=False)
|
||||
thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, u'thumbnail', fatal=False)
|
||||
if thumbnail:
|
||||
thumbnail = compat_urllib_parse.unquote(thumbnail)
|
||||
|
||||
video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
|
||||
if webpage.find('"encrypted":true') != -1:
|
||||
password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, u'password').replace('+', ' ')
|
||||
video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
|
||||
|
||||
formats = []
|
||||
for video_url in video_urls:
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[5].split('_')[:2]
|
||||
format = "-".join(format)
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
})
|
||||
formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-'))))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'uploader': video_uploader,
|
||||
'title': video_title,
|
||||
'thumbnail': thumbnail,
|
||||
'formats': formats,
|
||||
'age_limit': 18,
|
||||
}
|
@@ -16,7 +16,8 @@ class PornotubeIE(InfoExtractor):
|
||||
u'md5': u'374dd6dcedd24234453b295209aa69b6',
|
||||
u'info_dict': {
|
||||
u"upload_date": u"20090708",
|
||||
u"title": u"Marilyn-Monroe-Bathing"
|
||||
u"title": u"Marilyn-Monroe-Bathing",
|
||||
u"age_limit": 18
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +39,7 @@ class PornotubeIE(InfoExtractor):
|
||||
VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
|
||||
upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False)
|
||||
if upload_date: upload_date = unified_strdate(upload_date)
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
info = {'id': video_id,
|
||||
'url': video_url,
|
||||
@@ -45,6 +47,7 @@ class PornotubeIE(InfoExtractor):
|
||||
'upload_date': upload_date,
|
||||
'title': video_title,
|
||||
'ext': 'flv',
|
||||
'format': 'flv'}
|
||||
'format': 'flv',
|
||||
'age_limit': age_limit}
|
||||
|
||||
return [info]
|
||||
|
@@ -10,28 +10,35 @@ class RedTubeIE(InfoExtractor):
|
||||
u'file': u'66418.mp4',
|
||||
u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d',
|
||||
u'info_dict': {
|
||||
u"title": u"Sucked on a toilet"
|
||||
u"title": u"Sucked on a toilet",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self,url):
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
video_extension = 'mp4'
|
||||
video_extension = 'mp4'
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
self.report_extraction(video_id)
|
||||
|
||||
video_url = self._html_search_regex(r'<source src="(.+?)" type="video/mp4">',
|
||||
webpage, u'video URL')
|
||||
video_url = self._html_search_regex(
|
||||
r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')
|
||||
|
||||
video_title = self._html_search_regex('<h1 class="videoTitle slidePanelMovable">(.+?)</h1>',
|
||||
video_title = self._html_search_regex(
|
||||
r'<h1 class="videoTitle slidePanelMovable">(.+?)</h1>',
|
||||
webpage, u'title')
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': video_extension,
|
||||
'title': video_title,
|
||||
}]
|
||||
# No self-labeling, but they describe themselves as
|
||||
# "Home of Videos Porno"
|
||||
age_limit = 18
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': video_extension,
|
||||
'title': video_title,
|
||||
'age_limit': age_limit,
|
||||
}
|
||||
|
16
youtube_dl/extractor/rottentomatoes.py
Normal file
16
youtube_dl/extractor/rottentomatoes.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from .videodetective import VideoDetectiveIE
|
||||
|
||||
|
||||
# It just uses the same method as videodetective.com,
|
||||
# the internetvideoarchive.com is extracted from the og:video property
|
||||
class RottenTomatoesIE(VideoDetectiveIE):
|
||||
_VALID_URL = r'https?://www\.rottentomatoes\.com/m/[^/]+/trailers/(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
|
||||
u'file': '613340.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'TOY STORY 3',
|
||||
u'description': u'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
|
||||
},
|
||||
}
|
@@ -8,8 +8,8 @@ from ..utils import (
|
||||
)
|
||||
|
||||
class RTLnowIE(InfoExtractor):
|
||||
"""Information Extractor for RTL NOW, RTL2 NOW, SUPER RTL NOW and VOX NOW"""
|
||||
_VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?superrtlnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
|
||||
"""Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
|
||||
_VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?rtlnitronow\.de/|(?:www\.)?superrtlnow\.de/|(?:www\.)?n-tvnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)'
|
||||
_TESTS = [{
|
||||
u'url': u'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1',
|
||||
u'file': u'90419.flv',
|
||||
@@ -61,8 +61,34 @@ class RTLnowIE(InfoExtractor):
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
{
|
||||
u'url': u'http://www.rtlnitronow.de/recht-ordnung/stadtpolizei-frankfurt-gerichtsvollzieher-leipzig.php?film_id=129679&player=1&season=1',
|
||||
u'file': u'129679.flv',
|
||||
u'info_dict': {
|
||||
u'upload_date': u'20131016',
|
||||
u'title': u'Recht & Ordnung - Stadtpolizei Frankfurt/ Gerichtsvollzieher...',
|
||||
u'description': u'Stadtpolizei Frankfurt/ Gerichtsvollzieher Leipzig',
|
||||
},
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
{
|
||||
u'url': u'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10',
|
||||
u'file': u'124903.flv',
|
||||
u'info_dict': {
|
||||
u'upload_date': u'20130101',
|
||||
u'title': u'Top Gear vom 01.01.2013',
|
||||
u'description': u'Episode 1',
|
||||
},
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
u'skip': u'Only works from Germany',
|
||||
}]
|
||||
|
||||
|
||||
def _real_extract(self,url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
@@ -79,20 +105,23 @@ class RTLnowIE(InfoExtractor):
|
||||
msg = clean_html(note_m.group(1))
|
||||
raise ExtractorError(msg)
|
||||
|
||||
video_title = self._html_search_regex(r'<title>(?P<title>[^<]+)</title>',
|
||||
video_title = self._html_search_regex(r'<title>(?P<title>[^<]+?)( \| [^<]*)?</title>',
|
||||
webpage, u'title')
|
||||
playerdata_url = self._html_search_regex(r'\'playerdata\': \'(?P<playerdata_url>[^\']+)\'',
|
||||
webpage, u'playerdata_url')
|
||||
|
||||
playerdata = self._download_webpage(playerdata_url, video_id)
|
||||
mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr\]\]></title>', playerdata)
|
||||
mobj = re.search(r'<title><!\[CDATA\[(?P<description>.+?)(?:\s+- (?:Sendung )?vom (?P<upload_date_d>[0-9]{2})\.(?P<upload_date_m>[0-9]{2})\.(?:(?P<upload_date_Y>[0-9]{4})|(?P<upload_date_y>[0-9]{2})) [0-9]{2}:[0-9]{2} Uhr)?\]\]></title>', playerdata)
|
||||
if mobj:
|
||||
video_description = mobj.group(u'description')
|
||||
if mobj.group('upload_date_Y'):
|
||||
video_upload_date = mobj.group('upload_date_Y')
|
||||
else:
|
||||
elif mobj.group('upload_date_y'):
|
||||
video_upload_date = u'20' + mobj.group('upload_date_y')
|
||||
video_upload_date += mobj.group('upload_date_m')+mobj.group('upload_date_d')
|
||||
else:
|
||||
video_upload_date = None
|
||||
if video_upload_date:
|
||||
video_upload_date += mobj.group('upload_date_m')+mobj.group('upload_date_d')
|
||||
else:
|
||||
video_description = None
|
||||
video_upload_date = None
|
||||
|
58
youtube_dl/extractor/rutube.py
Normal file
58
youtube_dl/extractor/rutube.py
Normal file
@@ -0,0 +1,58 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
compat_str,
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
|
||||
class RutubeIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://rutube.ru/video/(?P<long_id>\w+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/',
|
||||
u'file': u'3eac3b4561676c17df9132a9a1e62e3e.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Раненный кенгуру забежал в аптеку',
|
||||
u'uploader': u'NTDRussian',
|
||||
u'uploader_id': u'29790',
|
||||
},
|
||||
u'params': {
|
||||
# It requires ffmpeg (m3u8 download)
|
||||
u'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _get_api_response(self, short_id, subpath):
|
||||
api_url = 'http://rutube.ru/api/play/%s/%s/?format=json' % (subpath, short_id)
|
||||
response_json = self._download_webpage(api_url, short_id,
|
||||
u'Downloading %s json' % subpath)
|
||||
return json.loads(response_json)
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
long_id = mobj.group('long_id')
|
||||
webpage = self._download_webpage(url, long_id)
|
||||
og_video = self._og_search_video_url(webpage)
|
||||
short_id = compat_urlparse.urlparse(og_video).path[1:]
|
||||
options = self._get_api_response(short_id, 'options')
|
||||
trackinfo = self._get_api_response(short_id, 'trackinfo')
|
||||
# Some videos don't have the author field
|
||||
author = trackinfo.get('author') or {}
|
||||
m3u8_url = trackinfo['video_balancer'].get('m3u8')
|
||||
if m3u8_url is None:
|
||||
raise ExtractorError(u'Couldn\'t find m3u8 manifest url')
|
||||
|
||||
return {
|
||||
'id': trackinfo['id'],
|
||||
'title': trackinfo['title'],
|
||||
'url': m3u8_url,
|
||||
'ext': 'mp4',
|
||||
'thumbnail': options['thumbnail_url'],
|
||||
'uploader': author.get('name'),
|
||||
'uploader_id': compat_str(author['id']) if author else None,
|
||||
}
|
@@ -7,6 +7,7 @@ class SlashdotIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://tv.slashdot.org/video/\?embed=(?P<id>.*?)(&|$)'
|
||||
|
||||
_TEST = {
|
||||
u'add_ie': ['Ooyala'],
|
||||
u'url': u'http://tv.slashdot.org/video/?embed=JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz',
|
||||
u'file': u'JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz.mp4',
|
||||
u'md5': u'd2222e7a4a4c1541b3e0cf732fb26735',
|
||||
|
@@ -5,7 +5,7 @@ from .mtv import MTVIE, _media_xml_tag
|
||||
|
||||
class SouthParkStudiosIE(MTVIE):
|
||||
IE_NAME = u'southparkstudios.com'
|
||||
_VALID_URL = r'https?://www\.southparkstudios\.com/clips/(?P<id>\d+)'
|
||||
_VALID_URL = r'https?://www\.southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$)'
|
||||
|
||||
_FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
|
||||
|
||||
@@ -23,7 +23,11 @@ class SouthParkStudiosIE(MTVIE):
|
||||
|
||||
def _get_thumbnail_url(self, uri, itemdoc):
|
||||
search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))
|
||||
return itemdoc.find(search_path).attrib['url']
|
||||
thumb_node = itemdoc.find(search_path)
|
||||
if thumb_node is None:
|
||||
return None
|
||||
else:
|
||||
return thumb_node.attrib['url']
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
35
youtube_dl/extractor/space.py
Normal file
35
youtube_dl/extractor/space.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .brightcove import BrightcoveIE
|
||||
from ..utils import RegexNotFoundError, ExtractorError
|
||||
|
||||
|
||||
class SpaceIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://www\.space\.com/\d+-(?P<title>[^/\.\?]*?)-video.html'
|
||||
_TEST = {
|
||||
u'add_ie': ['Brightcove'],
|
||||
u'url': u'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
|
||||
u'info_dict': {
|
||||
u'id': u'2780937028001',
|
||||
u'ext': u'mp4',
|
||||
u'title': u'Huge Martian Landforms\' Detail Revealed By European Probe | Video',
|
||||
u'description': u'md5:db81cf7f3122f95ed234b631a6ea1e61',
|
||||
u'uploader': u'TechMedia Networks',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
title = mobj.group('title')
|
||||
webpage = self._download_webpage(url, title)
|
||||
try:
|
||||
# Some videos require the playerKey field, which isn't define in
|
||||
# the BrightcoveExperience object
|
||||
brightcove_url = self._og_search_video_url(webpage)
|
||||
except RegexNotFoundError:
|
||||
# Other videos works fine with the info from the object
|
||||
brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
|
||||
if brightcove_url is None:
|
||||
raise ExtractorError(u'The webpage does not contain a video', expected=True)
|
||||
return self.url_result(brightcove_url, BrightcoveIE.ie_key())
|
74
youtube_dl/extractor/spankwire.py
Normal file
74
youtube_dl/extractor/spankwire.py
Normal file
@@ -0,0 +1,74 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
unescapeHTML,
|
||||
)
|
||||
from ..aes import (
|
||||
aes_decrypt_text
|
||||
)
|
||||
|
||||
class SpankwireIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>spankwire\.com/[^/]*/video(?P<videoid>[0-9]+)/?)'
|
||||
_TEST = {
|
||||
u'url': u'http://www.spankwire.com/Buckcherry-s-X-Rated-Music-Video-Crazy-Bitch/video103545/',
|
||||
u'file': u'103545.mp4',
|
||||
u'md5': u'1b3f55e345500552dbc252a3e9c1af43',
|
||||
u'info_dict': {
|
||||
u"uploader": u"oreusz",
|
||||
u"title": u"Buckcherry`s X Rated Music Video Crazy Bitch",
|
||||
u"description": u"Crazy Bitch X rated music video.",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<h1>([^<]+)', webpage, u'title')
|
||||
video_uploader = self._html_search_regex(r'by:\s*<a [^>]*>(.+?)</a>', webpage, u'uploader', fatal=False)
|
||||
thumbnail = self._html_search_regex(r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False)
|
||||
description = self._html_search_regex(r'>\s*Description:</div>\s*<[^>]*>([^<]+)', webpage, u'description', fatal=False)
|
||||
if len(description) == 0:
|
||||
description = None
|
||||
|
||||
video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage)))
|
||||
if webpage.find('flashvars\.encrypted = "true"') != -1:
|
||||
password = self._html_search_regex(r'flashvars\.video_title = "([^"]+)', webpage, u'password').replace('+', ' ')
|
||||
video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
|
||||
|
||||
formats = []
|
||||
for video_url in video_urls:
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[4].split('_')[:2]
|
||||
format = "-".join(format)
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
})
|
||||
formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-'))))
|
||||
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'uploader': video_uploader,
|
||||
'title': video_title,
|
||||
'thumbnail': thumbnail,
|
||||
'description': description,
|
||||
'formats': formats,
|
||||
'age_limit': age_limit,
|
||||
}
|
44
youtube_dl/extractor/sztvhu.py
Normal file
44
youtube_dl/extractor/sztvhu.py
Normal file
@@ -0,0 +1,44 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import determine_ext
|
||||
|
||||
|
||||
class SztvHuIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:http://)?(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
|
||||
_TEST = {
|
||||
u'url': u'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
|
||||
u'file': u'20130909.mp4',
|
||||
u'md5': u'a6df607b11fb07d0e9f2ad94613375cb',
|
||||
u'info_dict': {
|
||||
u"title": u"Cserkészek népszerűsítették a környezettudatos életmódot a Savaria téren",
|
||||
u"description": u'A zöld nap játékos ismeretterjesztő programjait a Magyar Cserkész Szövetség szervezte, akik az ország nyolc városában adják át tudásukat az érdeklődőknek. A PET...',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_file = self._search_regex(
|
||||
r'file: "...:(.*?)",', webpage, 'video file')
|
||||
title = self._html_search_regex(
|
||||
r'<meta name="title" content="([^"]*?) - [^-]*? - [^-]*?"',
|
||||
webpage, 'video title')
|
||||
description = self._html_search_regex(
|
||||
r'<meta name="description" content="([^"]*)"/>',
|
||||
webpage, 'video description', fatal=False)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
|
||||
video_url = 'http://media.sztv.hu/vod/' + video_file
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'title': title,
|
||||
'ext': determine_ext(video_url),
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
}
|
@@ -1,4 +1,5 @@
|
||||
import re
|
||||
import xml.etree.ElementTree
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
@@ -11,7 +12,7 @@ class TeamcocoIE(InfoExtractor):
|
||||
_TEST = {
|
||||
u'url': u'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
|
||||
u'file': u'19705.mp4',
|
||||
u'md5': u'27b6f7527da5acf534b15f21b032656e',
|
||||
u'md5': u'cde9ba0fa3506f5f017ce11ead928f9a',
|
||||
u'info_dict': {
|
||||
u"description": u"Louis C.K. got starstruck by George W. Bush, so what? Part one.",
|
||||
u"title": u"Louis C.K. Interview Pt. 1 11/3/11"
|
||||
@@ -31,16 +32,40 @@ class TeamcocoIE(InfoExtractor):
|
||||
self.report_extraction(video_id)
|
||||
|
||||
data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
|
||||
data = self._download_webpage(data_url, video_id, 'Downloading data webpage')
|
||||
data_xml = self._download_webpage(data_url, video_id, 'Downloading data webpage')
|
||||
data = xml.etree.ElementTree.fromstring(data_xml.encode('utf-8'))
|
||||
|
||||
video_url = self._html_search_regex(r'<file [^>]*type="high".*?>(.*?)</file>',
|
||||
data, u'video URL')
|
||||
|
||||
return [{
|
||||
qualities = ['500k', '480p', '1000k', '720p', '1080p']
|
||||
formats = []
|
||||
for file in data.findall('files/file'):
|
||||
if file.attrib.get('playmode') == 'all':
|
||||
# it just duplicates one of the entries
|
||||
break
|
||||
file_url = file.text
|
||||
m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
|
||||
if m_format is not None:
|
||||
format_id = m_format.group(1)
|
||||
else:
|
||||
format_id = file.attrib['bitrate']
|
||||
formats.append({
|
||||
'url': file_url,
|
||||
'ext': 'mp4',
|
||||
'format_id': format_id,
|
||||
})
|
||||
def sort_key(f):
|
||||
try:
|
||||
return qualities.index(f['format_id'])
|
||||
except ValueError:
|
||||
return -1
|
||||
formats.sort(key=sort_key)
|
||||
if not formats:
|
||||
raise RegexNotFoundError(u'Unable to extract video URL')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'formats': formats,
|
||||
'title': self._og_search_title(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
}]
|
||||
}
|
||||
|
65
youtube_dl/extractor/techtalks.py
Normal file
65
youtube_dl/extractor/techtalks.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
get_element_by_attribute,
|
||||
clean_html,
|
||||
)
|
||||
|
||||
|
||||
class TechTalksIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://techtalks\.tv/talks/[^/]*/(?P<id>\d+)/'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://techtalks.tv/talks/learning-topic-models-going-beyond-svd/57758/',
|
||||
u'playlist': [
|
||||
{
|
||||
u'file': u'57758.flv',
|
||||
u'info_dict': {
|
||||
u'title': u'Learning Topic Models --- Going beyond SVD',
|
||||
},
|
||||
},
|
||||
{
|
||||
u'file': u'57758-slides.flv',
|
||||
u'info_dict': {
|
||||
u'title': u'Learning Topic Models --- Going beyond SVD',
|
||||
},
|
||||
},
|
||||
],
|
||||
u'params': {
|
||||
# rtmp download
|
||||
u'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
talk_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, talk_id)
|
||||
rtmp_url = self._search_regex(r'netConnectionUrl: \'(.*?)\'', webpage,
|
||||
u'rtmp url')
|
||||
play_path = self._search_regex(r'href=\'(.*?)\' [^>]*id="flowplayer_presenter"',
|
||||
webpage, u'presenter play path')
|
||||
title = clean_html(get_element_by_attribute('class', 'title', webpage))
|
||||
video_info = {
|
||||
'id': talk_id,
|
||||
'title': title,
|
||||
'url': rtmp_url,
|
||||
'play_path': play_path,
|
||||
'ext': 'flv',
|
||||
}
|
||||
m_slides = re.search(r'<a class="slides" href=\'(.*?)\'', webpage)
|
||||
if m_slides is None:
|
||||
return video_info
|
||||
else:
|
||||
return [
|
||||
video_info,
|
||||
# The slides video
|
||||
{
|
||||
'id': talk_id + '-slides',
|
||||
'title': title,
|
||||
'url': rtmp_url,
|
||||
'play_path': m_slides.group(1),
|
||||
'ext': 'flv',
|
||||
},
|
||||
]
|
@@ -77,12 +77,20 @@ class TEDIE(InfoExtractor):
|
||||
|
||||
thumbnail = self._search_regex(r'</span>[\s.]*</div>[\s.]*<img src="(.*?)"',
|
||||
webpage, 'thumbnail')
|
||||
formats = [{
|
||||
'ext': 'mp4',
|
||||
'url': stream['file'],
|
||||
'format': stream['id']
|
||||
} for stream in info['htmlStreams']]
|
||||
info = {
|
||||
'id': info['id'],
|
||||
'url': info['htmlStreams'][-1]['file'],
|
||||
'ext': 'mp4',
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'description': desc,
|
||||
}
|
||||
'id': info['id'],
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'description': desc,
|
||||
'formats': formats,
|
||||
}
|
||||
|
||||
# TODO: Remove when #980 has been merged
|
||||
info.update(info['formats'][-1])
|
||||
|
||||
return info
|
||||
|
65
youtube_dl/extractor/tube8.py
Normal file
65
youtube_dl/extractor/tube8.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
unescapeHTML,
|
||||
)
|
||||
from ..aes import (
|
||||
aes_decrypt_text
|
||||
)
|
||||
|
||||
class Tube8IE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>tube8\.com/[^/]+/[^/]+/(?P<videoid>[0-9]+)/?)'
|
||||
_TEST = {
|
||||
u'url': u'http://www.tube8.com/teen/kasia-music-video/229795/',
|
||||
u'file': u'229795.mp4',
|
||||
u'md5': u'e9e0b0c86734e5e3766e653509475db0',
|
||||
u'info_dict': {
|
||||
u"description": u"hot teen Kasia grinding",
|
||||
u"uploader": u"unknown",
|
||||
u"title": u"Kasia music video",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'videotitle ="([^"]+)', webpage, u'title')
|
||||
video_description = self._html_search_regex(r'>Description:</strong>(.+?)<', webpage, u'description', fatal=False)
|
||||
video_uploader = self._html_search_regex(r'>Submitted by:</strong>(?:\s|<[^>]*>)*(.+?)<', webpage, u'uploader', fatal=False)
|
||||
thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, u'thumbnail', fatal=False)
|
||||
if thumbnail:
|
||||
thumbnail = thumbnail.replace('\\/', '/')
|
||||
|
||||
video_url = self._html_search_regex(r'"video_url":"([^"]+)', webpage, u'video_url')
|
||||
if webpage.find('"encrypted":true')!=-1:
|
||||
password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, u'password')
|
||||
video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[4].split('_')[:2]
|
||||
format = "-".join(format)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'uploader': video_uploader,
|
||||
'title': video_title,
|
||||
'thumbnail': thumbnail,
|
||||
'description': video_description,
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
'age_limit': 18,
|
||||
}
|
@@ -7,15 +7,25 @@ from .common import InfoExtractor
|
||||
|
||||
|
||||
class TudouIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:http://)?(?:www\.)?tudou\.com/(?:listplay|programs)/(?:view|(.+?))/(?:([^/]+)|([^/]+))(?:\.html)?'
|
||||
_TEST = {
|
||||
_VALID_URL = r'(?:http://)?(?:www\.)?tudou\.com/(?:listplay|programs|albumplay)/(?:view|(.+?))/(?:([^/]+)|([^/]+))(?:\.html)?'
|
||||
_TESTS = [{
|
||||
u'url': u'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
|
||||
u'file': u'159448201.f4v',
|
||||
u'md5': u'140a49ed444bd22f93330985d8475fcb',
|
||||
u'info_dict': {
|
||||
u"title": u"卡马乔国足开大脚长传冲吊集锦"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
u'url': u'http://www.tudou.com/albumplay/TenTw_JgiPM/PzsAs5usU9A.html',
|
||||
u'file': u'todo.mp4',
|
||||
u'md5': u'todo.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'todo.mp4',
|
||||
},
|
||||
u'add_ie': [u'Youku'],
|
||||
u'skip': u'Only works from China'
|
||||
}]
|
||||
|
||||
def _url_for_id(self, id, quality = None):
|
||||
info_url = "http://v2.tudou.com/f?id="+str(id)
|
||||
@@ -29,14 +39,19 @@ class TudouIE(InfoExtractor):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group(2)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
title = re.search(",kw:\"(.+)\"",webpage)
|
||||
if title is None:
|
||||
title = re.search(",kw: \'(.+)\'",webpage)
|
||||
title = title.group(1)
|
||||
thumbnail_url = re.search(",pic: \'(.+?)\'",webpage)
|
||||
if thumbnail_url is None:
|
||||
thumbnail_url = re.search(",pic:\"(.+?)\"",webpage)
|
||||
thumbnail_url = thumbnail_url.group(1)
|
||||
|
||||
m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage)
|
||||
if m and m.group(1):
|
||||
return {
|
||||
'_type': 'url',
|
||||
'url': u'youku:' + m.group(1),
|
||||
'ie_key': 'Youku'
|
||||
}
|
||||
|
||||
title = self._search_regex(
|
||||
r",kw:\s*['\"](.+?)[\"']", webpage, u'title')
|
||||
thumbnail_url = self._search_regex(
|
||||
r",pic:\s*[\"'](.+?)[\"']", webpage, u'thumbnail URL', fatal=False)
|
||||
|
||||
segs_json = self._search_regex(r'segs: \'(.*)\'', webpage, 'segments')
|
||||
segments = json.loads(segs_json)
|
||||
|
@@ -1,53 +1,127 @@
|
||||
import re
|
||||
import json
|
||||
import xml.etree.ElementTree
|
||||
import datetime
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_HTTPError,
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
|
||||
class VevoIE(InfoExtractor):
|
||||
"""
|
||||
Accepts urls from vevo.com or in the format 'vevo:{id}'
|
||||
(currently used by MTVIE)
|
||||
"""
|
||||
_VALID_URL = r'((http://www.vevo.com/watch/.*?/.*?/)|(vevo:))(?P<id>.*?)(\?|$)'
|
||||
_TEST = {
|
||||
_TESTS = [{
|
||||
u'url': u'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
|
||||
u'file': u'GB1101300280.mp4',
|
||||
u'md5': u'06bea460acb744eab74a9d7dcb4bfd61',
|
||||
u"md5": u"06bea460acb744eab74a9d7dcb4bfd61",
|
||||
u'info_dict': {
|
||||
u"upload_date": u"20130624",
|
||||
u"uploader": u"Hurts",
|
||||
u"title": u"Somebody to Die For"
|
||||
u"title": u"Somebody to Die For",
|
||||
u"duration": 230,
|
||||
u"width": 1920,
|
||||
u"height": 1080,
|
||||
}
|
||||
}
|
||||
}]
|
||||
_SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
|
||||
|
||||
def _formats_from_json(self, video_info):
|
||||
last_version = {'version': -1}
|
||||
for version in video_info['videoVersions']:
|
||||
# These are the HTTP downloads, other types are for different manifests
|
||||
if version['sourceType'] == 2:
|
||||
if version['version'] > last_version['version']:
|
||||
last_version = version
|
||||
if last_version['version'] == -1:
|
||||
raise ExtractorError(u'Unable to extract last version of the video')
|
||||
|
||||
renditions = xml.etree.ElementTree.fromstring(last_version['data'])
|
||||
formats = []
|
||||
# Already sorted from worst to best quality
|
||||
for rend in renditions.findall('rendition'):
|
||||
attr = rend.attrib
|
||||
format_note = '%(videoCodec)s@%(videoBitrate)4sk, %(audioCodec)s@%(audioBitrate)3sk' % attr
|
||||
formats.append({
|
||||
'url': attr['url'],
|
||||
'format_id': attr['name'],
|
||||
'format_note': format_note,
|
||||
'height': int(attr['frameheight']),
|
||||
'width': int(attr['frameWidth']),
|
||||
})
|
||||
return formats
|
||||
|
||||
def _formats_from_smil(self, smil_xml):
|
||||
formats = []
|
||||
smil_doc = xml.etree.ElementTree.fromstring(smil_xml.encode('utf-8'))
|
||||
els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
|
||||
for el in els:
|
||||
src = el.attrib['src']
|
||||
m = re.match(r'''(?xi)
|
||||
(?P<ext>[a-z0-9]+):
|
||||
(?P<path>
|
||||
[/a-z0-9]+ # The directory and main part of the URL
|
||||
_(?P<cbr>[0-9]+)k
|
||||
_(?P<width>[0-9]+)x(?P<height>[0-9]+)
|
||||
_(?P<vcodec>[a-z0-9]+)
|
||||
_(?P<vbr>[0-9]+)
|
||||
_(?P<acodec>[a-z0-9]+)
|
||||
_(?P<abr>[0-9]+)
|
||||
\.[a-z0-9]+ # File extension
|
||||
)''', src)
|
||||
if not m:
|
||||
continue
|
||||
|
||||
format_url = self._SMIL_BASE_URL + m.group('path')
|
||||
format_note = ('%(vcodec)s@%(vbr)4sk, %(acodec)s@%(abr)3sk' %
|
||||
m.groupdict())
|
||||
formats.append({
|
||||
'url': format_url,
|
||||
'format_id': u'SMIL_' + m.group('cbr'),
|
||||
'format_note': format_note,
|
||||
'ext': m.group('ext'),
|
||||
'width': int(m.group('width')),
|
||||
'height': int(m.group('height')),
|
||||
})
|
||||
return formats
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
json_url = 'http://www.vevo.com/data/video/%s' % video_id
|
||||
base_url = 'http://smil.lvl3.vevo.com'
|
||||
videos_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (base_url, video_id, video_id.lower())
|
||||
json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
|
||||
info_json = self._download_webpage(json_url, video_id, u'Downloading json info')
|
||||
links_webpage = self._download_webpage(videos_url, video_id, u'Downloading videos urls')
|
||||
video_info = json.loads(info_json)['video']
|
||||
|
||||
self.report_extraction(video_id)
|
||||
video_info = json.loads(info_json)
|
||||
m_urls = list(re.finditer(r'<video src="(?P<ext>.*?):/?(?P<url>.*?)"', links_webpage))
|
||||
if m_urls is None or len(m_urls) == 0:
|
||||
raise ExtractorError(u'Unable to extract video url')
|
||||
# They are sorted from worst to best quality
|
||||
m_url = m_urls[-1]
|
||||
video_url = base_url + '/' + m_url.group('url')
|
||||
ext = m_url.group('ext')
|
||||
formats = self._formats_from_json(video_info)
|
||||
try:
|
||||
smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
|
||||
self._SMIL_BASE_URL, video_id, video_id.lower())
|
||||
smil_xml = self._download_webpage(smil_url, video_id,
|
||||
u'Downloading SMIL info')
|
||||
formats.extend(self._formats_from_smil(smil_xml))
|
||||
except ExtractorError as ee:
|
||||
if not isinstance(ee.cause, compat_HTTPError):
|
||||
raise
|
||||
self._downloader.report_warning(
|
||||
u'Cannot download SMIL information, falling back to JSON ..')
|
||||
|
||||
return {'url': video_url,
|
||||
'ext': ext,
|
||||
'id': video_id,
|
||||
'title': video_info['title'],
|
||||
'thumbnail': video_info['img'],
|
||||
'upload_date': video_info['launchDate'].replace('/',''),
|
||||
'uploader': video_info['Artists'][0]['title'],
|
||||
}
|
||||
timestamp_ms = int(self._search_regex(
|
||||
r'/Date\((\d+)\)/', video_info['launchDate'], u'launch date'))
|
||||
upload_date = datetime.datetime.fromtimestamp(timestamp_ms // 1000)
|
||||
info = {
|
||||
'id': video_id,
|
||||
'title': video_info['title'],
|
||||
'formats': formats,
|
||||
'thumbnail': video_info['imageUrl'],
|
||||
'upload_date': upload_date.strftime('%Y%m%d'),
|
||||
'uploader': video_info['mainArtists'][0]['artistName'],
|
||||
'duration': video_info['duration'],
|
||||
}
|
||||
|
||||
return info
|
||||
|
64
youtube_dl/extractor/viddler.py
Normal file
64
youtube_dl/extractor/viddler.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
)
|
||||
|
||||
|
||||
class ViddlerIE(InfoExtractor):
|
||||
_VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler.com)/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
|
||||
_TEST = {
|
||||
u"url": u"http://www.viddler.com/v/43903784",
|
||||
u'file': u'43903784.mp4',
|
||||
u'md5': u'fbbaedf7813e514eb7ca30410f439ac9',
|
||||
u'info_dict': {
|
||||
u"title": u"Video Made Easy",
|
||||
u"uploader": u"viddler",
|
||||
u"duration": 100.89,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
embed_url = mobj.group('domain') + u'/embed/' + video_id
|
||||
webpage = self._download_webpage(embed_url, video_id)
|
||||
|
||||
video_sources_code = self._search_regex(
|
||||
r"(?ms)sources\s*:\s*(\{.*?\})", webpage, u'video URLs')
|
||||
video_sources = json.loads(video_sources_code.replace("'", '"'))
|
||||
|
||||
formats = [{
|
||||
'url': video_url,
|
||||
'format': format_id,
|
||||
} for video_url, format_id in video_sources.items()]
|
||||
|
||||
title = self._html_search_regex(
|
||||
r"title\s*:\s*'([^']*)'", webpage, u'title')
|
||||
uploader = self._html_search_regex(
|
||||
r"authorName\s*:\s*'([^']*)'", webpage, u'uploader', fatal=False)
|
||||
duration_s = self._html_search_regex(
|
||||
r"duration\s*:\s*([0-9.]*)", webpage, u'duration', fatal=False)
|
||||
duration = float(duration_s) if duration_s else None
|
||||
thumbnail = self._html_search_regex(
|
||||
r"thumbnail\s*:\s*'([^']*)'",
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
|
||||
info = {
|
||||
'_type': 'video',
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'uploader': uploader,
|
||||
'duration': duration,
|
||||
'formats': formats,
|
||||
}
|
||||
|
||||
# TODO: Remove when #980 has been merged
|
||||
info['formats'][-1]['ext'] = determine_ext(info['formats'][-1]['url'])
|
||||
info.update(info['formats'][-1])
|
||||
|
||||
return info
|
30
youtube_dl/extractor/videodetective.py
Normal file
30
youtube_dl/extractor/videodetective.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .internetvideoarchive import InternetVideoArchiveIE
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
)
|
||||
|
||||
|
||||
class VideoDetectiveIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://www\.videodetective\.com/[^/]+/[^/]+/(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.videodetective.com/movies/kick-ass-2/194487',
|
||||
u'file': u'194487.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'KICK-ASS 2',
|
||||
u'description': u'md5:65ba37ad619165afac7d432eaded6013',
|
||||
u'duration': 135,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
og_video = self._og_search_video_url(webpage)
|
||||
query = compat_urlparse.urlparse(og_video).query
|
||||
return self.url_result(InternetVideoArchiveIE._build_url(query),
|
||||
ie=InternetVideoArchiveIE.ie_key())
|
40
youtube_dl/extractor/videopremium.py
Normal file
40
youtube_dl/extractor/videopremium.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import re
|
||||
import random
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class VideoPremiumIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:https?://)?(?:www\.)?videopremium\.tv/(?P<id>\w+)(?:/.*)?'
|
||||
_TEST = {
|
||||
u'url': u'http://videopremium.tv/4w7oadjsf156',
|
||||
u'file': u'4w7oadjsf156.f4v',
|
||||
u'info_dict': {
|
||||
u"title": u"youtube-dl_test_video____a_________-BaW_jenozKc.mp4.mp4"
|
||||
},
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
webpage_url = 'http://videopremium.tv/' + video_id
|
||||
webpage = self._download_webpage(webpage_url, video_id)
|
||||
|
||||
self.report_extraction(video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<h2(?:.*?)>\s*(.+?)\s*<',
|
||||
webpage, u'video title')
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16),
|
||||
'play_path': "mp4:%s.f4v" % video_id,
|
||||
'page_url': "http://videopremium.tv/" + video_id,
|
||||
'player_url': "http://videopremium.tv/uplayer/uppod.swf",
|
||||
'ext': 'f4v',
|
||||
'title': video_title,
|
||||
}]
|
@@ -1,3 +1,4 @@
|
||||
# encoding: utf-8
|
||||
import json
|
||||
import re
|
||||
import itertools
|
||||
@@ -10,19 +11,21 @@ from ..utils import (
|
||||
clean_html,
|
||||
get_element_by_attribute,
|
||||
ExtractorError,
|
||||
RegexNotFoundError,
|
||||
std_headers,
|
||||
unsmuggle_url,
|
||||
)
|
||||
|
||||
class VimeoIE(InfoExtractor):
|
||||
"""Information extractor for vimeo.com."""
|
||||
|
||||
# _VALID_URL matches Vimeo URLs
|
||||
_VALID_URL = r'(?P<proto>https?://)?(?:(?:www|player)\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)(?:[?].*)?$'
|
||||
_VALID_URL = r'(?P<proto>https?://)?(?:(?:www|(?P<player>player))\.)?vimeo(?P<pro>pro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)/?(?:[?].*)?(?:#.*)?$'
|
||||
_NETRC_MACHINE = 'vimeo'
|
||||
IE_NAME = u'vimeo'
|
||||
_TESTS = [
|
||||
{
|
||||
u'url': u'http://vimeo.com/56015672',
|
||||
u'url': u'http://vimeo.com/56015672#at=0',
|
||||
u'file': u'56015672.mp4',
|
||||
u'md5': u'8879b6cc097e987f02484baf890129e5',
|
||||
u'info_dict': {
|
||||
@@ -54,6 +57,21 @@ class VimeoIE(InfoExtractor):
|
||||
u'uploader': u'The BLN & Business of Software',
|
||||
},
|
||||
},
|
||||
{
|
||||
u'url': u'http://vimeo.com/68375962',
|
||||
u'file': u'68375962.mp4',
|
||||
u'md5': u'aaf896bdb7ddd6476df50007a0ac0ae7',
|
||||
u'note': u'Video protected with password',
|
||||
u'info_dict': {
|
||||
u'title': u'youtube-dl password protected test video',
|
||||
u'upload_date': u'20130614',
|
||||
u'uploader_id': u'user18948128',
|
||||
u'uploader': u'Jaime Marquínez Ferrándiz',
|
||||
},
|
||||
u'params': {
|
||||
u'videopassword': u'youtube-dl',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _login(self):
|
||||
@@ -98,21 +116,25 @@ class VimeoIE(InfoExtractor):
|
||||
self._login()
|
||||
|
||||
def _real_extract(self, url, new_video=True):
|
||||
url, data = unsmuggle_url(url)
|
||||
headers = std_headers
|
||||
if data is not None:
|
||||
headers = headers.copy()
|
||||
headers.update(data)
|
||||
|
||||
# Extract ID from URL
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
if mobj is None:
|
||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
if not mobj.group('proto'):
|
||||
url = 'https://' + url
|
||||
elif mobj.group('pro'):
|
||||
if mobj.group('pro') or mobj.group('player'):
|
||||
url = 'http://player.vimeo.com/video/' + video_id
|
||||
elif mobj.group('direct_link'):
|
||||
else:
|
||||
url = 'https://vimeo.com/' + video_id
|
||||
|
||||
# Retrieve video webpage to extract further information
|
||||
request = compat_urllib_request.Request(url, None, std_headers)
|
||||
request = compat_urllib_request.Request(url, None, headers)
|
||||
webpage = self._download_webpage(request, video_id)
|
||||
|
||||
# Now we begin extracting as much information as we can from what we
|
||||
@@ -122,18 +144,26 @@ class VimeoIE(InfoExtractor):
|
||||
|
||||
# Extract the config JSON
|
||||
try:
|
||||
config = self._search_regex([r' = {config:({.+?}),assets:', r'c=({.+?);'],
|
||||
webpage, u'info section', flags=re.DOTALL)
|
||||
config = json.loads(config)
|
||||
except:
|
||||
try:
|
||||
config_url = self._html_search_regex(
|
||||
r' data-config-url="(.+?)"', webpage, u'config URL')
|
||||
config_json = self._download_webpage(config_url, video_id)
|
||||
config = json.loads(config_json)
|
||||
except RegexNotFoundError:
|
||||
# For pro videos or player.vimeo.com urls
|
||||
config = self._search_regex([r' = {config:({.+?}),assets:', r'c=({.+?);'],
|
||||
webpage, u'info section', flags=re.DOTALL)
|
||||
config = json.loads(config)
|
||||
except Exception as e:
|
||||
if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
|
||||
raise ExtractorError(u'The author has restricted the access to this video, try with the "--referer" option')
|
||||
|
||||
if re.search('If so please provide the correct password.', webpage):
|
||||
if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
|
||||
self._verify_video_password(url, video_id, webpage)
|
||||
return self._real_extract(url)
|
||||
else:
|
||||
raise ExtractorError(u'Unable to extract info section')
|
||||
raise ExtractorError(u'Unable to extract info section',
|
||||
cause=e)
|
||||
|
||||
# Extract title
|
||||
video_title = config["video"]["title"]
|
||||
@@ -172,47 +202,47 @@ class VimeoIE(InfoExtractor):
|
||||
|
||||
# Vimeo specific: extract video codec and quality information
|
||||
# First consider quality, then codecs, then take everything
|
||||
# TODO bind to format param
|
||||
codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
|
||||
files = { 'hd': [], 'sd': [], 'other': []}
|
||||
codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
|
||||
files = {'hd': [], 'sd': [], 'other': []}
|
||||
config_files = config["video"].get("files") or config["request"].get("files")
|
||||
for codec_name, codec_extension in codecs:
|
||||
if codec_name in config_files:
|
||||
if 'hd' in config_files[codec_name]:
|
||||
files['hd'].append((codec_name, codec_extension, 'hd'))
|
||||
elif 'sd' in config_files[codec_name]:
|
||||
files['sd'].append((codec_name, codec_extension, 'sd'))
|
||||
for quality in config_files.get(codec_name, []):
|
||||
format_id = '-'.join((codec_name, quality)).lower()
|
||||
key = quality if quality in files else 'other'
|
||||
video_url = None
|
||||
if isinstance(config_files[codec_name], dict):
|
||||
file_info = config_files[codec_name][quality]
|
||||
video_url = file_info.get('url')
|
||||
else:
|
||||
files['other'].append((codec_name, codec_extension, config_files[codec_name][0]))
|
||||
file_info = {}
|
||||
if video_url is None:
|
||||
video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
|
||||
%(video_id, sig, timestamp, quality, codec_name.upper())
|
||||
|
||||
for quality in ('hd', 'sd', 'other'):
|
||||
if len(files[quality]) > 0:
|
||||
video_quality = files[quality][0][2]
|
||||
video_codec = files[quality][0][0]
|
||||
video_extension = files[quality][0][1]
|
||||
self.to_screen(u'%s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
|
||||
break
|
||||
else:
|
||||
files[key].append({
|
||||
'ext': codec_extension,
|
||||
'url': video_url,
|
||||
'format_id': format_id,
|
||||
'width': file_info.get('width'),
|
||||
'height': file_info.get('height'),
|
||||
})
|
||||
formats = []
|
||||
for key in ('other', 'sd', 'hd'):
|
||||
formats += files[key]
|
||||
if len(formats) == 0:
|
||||
raise ExtractorError(u'No known codec found')
|
||||
|
||||
video_url = None
|
||||
if isinstance(config_files[video_codec], dict):
|
||||
video_url = config_files[video_codec][video_quality].get("url")
|
||||
if video_url is None:
|
||||
video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
|
||||
%(video_id, sig, timestamp, video_quality, video_codec.upper())
|
||||
|
||||
return [{
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'uploader': video_uploader,
|
||||
'uploader_id': video_uploader_id,
|
||||
'upload_date': video_upload_date,
|
||||
'title': video_title,
|
||||
'ext': video_extension,
|
||||
'thumbnail': video_thumbnail,
|
||||
'description': video_description,
|
||||
}]
|
||||
'formats': formats,
|
||||
'webpage_url': url,
|
||||
}
|
||||
|
||||
|
||||
class VimeoChannelIE(InfoExtractor):
|
||||
|
45
youtube_dl/extractor/vk.py
Normal file
45
youtube_dl/extractor/vk.py
Normal file
@@ -0,0 +1,45 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_str,
|
||||
unescapeHTML,
|
||||
)
|
||||
|
||||
|
||||
class VKIE(InfoExtractor):
|
||||
IE_NAME = u'vk.com'
|
||||
_VALID_URL = r'https?://vk\.com/(?:videos.*?\?.*?z=)?video(?P<id>.*?)(?:\?|%2F|$)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://vk.com/videos-77521?z=video-77521_162222515%2Fclub77521',
|
||||
u'md5': u'0deae91935c54e00003c2a00646315f0',
|
||||
u'info_dict': {
|
||||
u'id': u'162222515',
|
||||
u'ext': u'flv',
|
||||
u'title': u'ProtivoGunz - Хуёвая песня',
|
||||
u'uploader': u'Noize MC',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
info_url = 'http://vk.com/al_video.php?act=show&al=1&video=%s' % video_id
|
||||
info_page = self._download_webpage(info_url, video_id)
|
||||
m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
|
||||
if m_yt is not None:
|
||||
self.to_screen(u'Youtube video detected')
|
||||
return self.url_result(m_yt.group(1), 'Youtube')
|
||||
vars_json = self._search_regex(r'var vars = ({.*?});', info_page, u'vars')
|
||||
vars = json.loads(vars_json)
|
||||
|
||||
return {
|
||||
'id': compat_str(vars['vid']),
|
||||
'url': vars['url240'],
|
||||
'title': unescapeHTML(vars['md_title']),
|
||||
'thumbnail': vars['jpg'],
|
||||
'uploader': vars['md_author'],
|
||||
}
|
59
youtube_dl/extractor/websurg.py
Normal file
59
youtube_dl/extractor/websurg.py
Normal file
@@ -0,0 +1,59 @@
|
||||
# coding: utf-8
|
||||
|
||||
import re
|
||||
|
||||
from ..utils import (
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse
|
||||
)
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
class WeBSurgIE(InfoExtractor):
|
||||
IE_NAME = u'websurg.com'
|
||||
_VALID_URL = r'http://.*?\.websurg\.com/MEDIA/\?noheader=1&doi=(.*)'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.websurg.com/MEDIA/?noheader=1&doi=vd01en4012',
|
||||
u'file': u'vd01en4012.mp4',
|
||||
u'params': {
|
||||
u'skip_download': True,
|
||||
},
|
||||
u'skip': u'Requires login information',
|
||||
}
|
||||
|
||||
_LOGIN_URL = 'http://www.websurg.com/inc/login/login_div.ajax.php?login=1'
|
||||
|
||||
def _real_initialize(self):
|
||||
|
||||
login_form = {
|
||||
'username': self._downloader.params['username'],
|
||||
'password': self._downloader.params['password'],
|
||||
'Submit': 1
|
||||
}
|
||||
|
||||
request = compat_urllib_request.Request(
|
||||
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
|
||||
request.add_header(
|
||||
'Content-Type', 'application/x-www-form-urlencoded;charset=utf-8')
|
||||
compat_urllib_request.urlopen(request).info()
|
||||
webpage = self._download_webpage(self._LOGIN_URL, '', 'Logging in')
|
||||
|
||||
if webpage != 'OK':
|
||||
self._downloader.report_error(
|
||||
u'Unable to log in: bad username/password')
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = re.match(self._VALID_URL, url).group(1)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
url_info = re.search(r'streamer="(.*?)" src="(.*?)"', webpage)
|
||||
|
||||
return {'id': video_id,
|
||||
'title': self._og_search_title(webpage),
|
||||
'description': self._og_search_description(webpage),
|
||||
'ext' : 'mp4',
|
||||
'url' : url_info.group(1) + '/' + url_info.group(2),
|
||||
'thumbnail': self._og_search_thumbnail(webpage)
|
||||
}
|
@@ -13,6 +13,7 @@ class WeiboIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://video\.weibo\.com/v/weishipin/t_(?P<id>.+?)\.htm'
|
||||
|
||||
_TEST = {
|
||||
u'add_ie': ['Sina'],
|
||||
u'url': u'http://video.weibo.com/v/weishipin/t_zjUw2kZ.htm',
|
||||
u'file': u'98322879.flv',
|
||||
u'info_dict': {
|
||||
|
@@ -19,7 +19,8 @@ class XHamsterIE(InfoExtractor):
|
||||
u'info_dict': {
|
||||
u"upload_date": u"20121014",
|
||||
u"uploader_id": u"Ruseful2011",
|
||||
u"title": u"FemaleAgent Shy beauty takes the bait"
|
||||
u"title": u"FemaleAgent Shy beauty takes the bait",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -27,28 +28,33 @@ class XHamsterIE(InfoExtractor):
|
||||
u'file': u'2221348.flv',
|
||||
u'md5': u'e767b9475de189320f691f49c679c4c7',
|
||||
u'info_dict': {
|
||||
u"upload_date": u"20130914",
|
||||
u"uploader_id": u"jojo747400",
|
||||
u"title": u"Britney Spears Sexy Booty"
|
||||
u"upload_date": u"20130914",
|
||||
u"uploader_id": u"jojo747400",
|
||||
u"title": u"Britney Spears Sexy Booty",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self,url):
|
||||
def extract_video_url(webpage):
|
||||
mobj = re.search(r'\'srv\': \'(?P<server>[^\']*)\',\s*\'file\': \'(?P<file>[^\']+)\',', webpage)
|
||||
if mobj is None:
|
||||
raise ExtractorError(u'Unable to extract media URL')
|
||||
if len(mobj.group('server')) == 0:
|
||||
return compat_urllib_parse.unquote(mobj.group('file'))
|
||||
else:
|
||||
return mobj.group('server')+'/key='+mobj.group('file')
|
||||
|
||||
def is_hd(webpage):
|
||||
return webpage.find('<div class=\'icon iconHD\'>') != -1
|
||||
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
seo = mobj.group('seo')
|
||||
mrss_url = 'http://xhamster.com/movies/%s/%s.html?hd' % (video_id, seo)
|
||||
mrss_url = 'http://xhamster.com/movies/%s/%s.html' % (video_id, seo)
|
||||
webpage = self._download_webpage(mrss_url, video_id)
|
||||
|
||||
mobj = re.search(r'\'srv\': \'(?P<server>[^\']*)\',\s*\'file\': \'(?P<file>[^\']+)\',', webpage)
|
||||
if mobj is None:
|
||||
raise ExtractorError(u'Unable to extract media URL')
|
||||
if len(mobj.group('server')) == 0:
|
||||
video_url = compat_urllib_parse.unquote(mobj.group('file'))
|
||||
else:
|
||||
video_url = mobj.group('server')+'/key='+mobj.group('file')
|
||||
|
||||
video_title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>',
|
||||
webpage, u'title')
|
||||
|
||||
@@ -72,13 +78,34 @@ class XHamsterIE(InfoExtractor):
|
||||
video_thumbnail = self._search_regex(r'\'image\':\'(?P<thumbnail>[^\']+)\'',
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
|
||||
return [{
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'title': video_title,
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
video_url = extract_video_url(webpage)
|
||||
hd = is_hd(webpage)
|
||||
formats = [{
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'format': 'hd' if hd else 'sd',
|
||||
'format_id': 'hd' if hd else 'sd',
|
||||
}]
|
||||
if not hd:
|
||||
webpage = self._download_webpage(mrss_url+'?hd', video_id)
|
||||
if is_hd(webpage):
|
||||
video_url = extract_video_url(webpage)
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'format': 'hd',
|
||||
'format_id': 'hd',
|
||||
})
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'formats': formats,
|
||||
'description': video_description,
|
||||
'upload_date': video_upload_date,
|
||||
'uploader_id': video_uploader_id,
|
||||
'thumbnail': video_thumbnail
|
||||
}]
|
||||
'thumbnail': video_thumbnail,
|
||||
'age_limit': age_limit,
|
||||
}
|
||||
|
@@ -9,7 +9,7 @@ from ..utils import (
|
||||
|
||||
|
||||
class XNXXIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)'
|
||||
_VALID_URL = r'^(?:https?://)?(?:video|www)\.xnxx\.com/video([0-9]+)/(.*)'
|
||||
VIDEO_URL_RE = r'flv_url=(.*?)&'
|
||||
VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
|
||||
VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&'
|
||||
@@ -18,7 +18,8 @@ class XNXXIE(InfoExtractor):
|
||||
u'file': u'1135332.flv',
|
||||
u'md5': u'0831677e2b4761795f68d417e0b7b445',
|
||||
u'info_dict': {
|
||||
u"title": u"lida \u00bb Naked Funny Actress (5)"
|
||||
u"title": u"lida \u00bb Naked Funny Actress (5)",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,4 +51,5 @@ class XNXXIE(InfoExtractor):
|
||||
'ext': 'flv',
|
||||
'thumbnail': video_thumbnail,
|
||||
'description': None,
|
||||
'age_limit': 18,
|
||||
}]
|
||||
|
55
youtube_dl/extractor/xtube.py
Normal file
55
youtube_dl/extractor/xtube.py
Normal file
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse_urlparse,
|
||||
compat_urllib_request,
|
||||
compat_urllib_parse,
|
||||
)
|
||||
|
||||
class XTubeIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<videoid>[^/?&]+))'
|
||||
_TEST = {
|
||||
u'url': u'http://www.xtube.com/watch.php?v=kVTUy_G222_',
|
||||
u'file': u'kVTUy_G222_.mp4',
|
||||
u'md5': u'092fbdd3cbe292c920ef6fc6a8a9cdab',
|
||||
u'info_dict': {
|
||||
u"title": u"strange erotica",
|
||||
u"description": u"surreal gay themed erotica...almost an ET kind of thing",
|
||||
u"uploader": u"greenshowers",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
|
||||
video_title = self._html_search_regex(r'<div class="p_5px[^>]*>([^<]+)', webpage, u'title')
|
||||
video_uploader = self._html_search_regex(r'so_s\.addVariable\("owner_u", "([^"]+)', webpage, u'uploader', fatal=False)
|
||||
video_description = self._html_search_regex(r'<p class="video_description">([^<]+)', webpage, u'description', default=None)
|
||||
video_url= self._html_search_regex(r'var videoMp4 = "([^"]+)', webpage, u'video_url').replace('\\/', '/')
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[5].split('_')[:2]
|
||||
format[0] += 'p'
|
||||
format[1] += 'k'
|
||||
format = "-".join(format)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'uploader': video_uploader,
|
||||
'description': video_description,
|
||||
'url': video_url,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'format_id': format,
|
||||
'age_limit': 18,
|
||||
}
|
@@ -13,7 +13,8 @@ class XVideosIE(InfoExtractor):
|
||||
u'file': u'939581.flv',
|
||||
u'md5': u'1d0c835822f0a71a7bf011855db929d0',
|
||||
u'info_dict': {
|
||||
u"title": u"Funny Porns By >>>>S<<<<<< -1"
|
||||
u"title": u"Funny Porns By >>>>S<<<<<< -1",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +47,7 @@ class XVideosIE(InfoExtractor):
|
||||
'ext': 'flv',
|
||||
'thumbnail': video_thumbnail,
|
||||
'description': None,
|
||||
'age_limit': 18,
|
||||
}
|
||||
|
||||
return [info]
|
||||
|
@@ -1,4 +1,3 @@
|
||||
import datetime
|
||||
import itertools
|
||||
import json
|
||||
import re
|
||||
@@ -6,86 +5,104 @@ import re
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse,
|
||||
|
||||
ExtractorError,
|
||||
compat_urlparse,
|
||||
determine_ext,
|
||||
clean_html,
|
||||
)
|
||||
|
||||
|
||||
class YahooIE(InfoExtractor):
|
||||
IE_DESC = u'Yahoo screen'
|
||||
_VALID_URL = r'http://screen\.yahoo\.com/.*?-(?P<id>\d*?)\.html'
|
||||
_TEST = {
|
||||
u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
|
||||
u'file': u'214727115.flv',
|
||||
u'md5': u'2e717f169c1be93d84d3794a00d4a325',
|
||||
u'info_dict': {
|
||||
u"title": u"Julian Smith & Travis Legg Watch Julian Smith"
|
||||
_TESTS = [
|
||||
{
|
||||
u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
|
||||
u'file': u'214727115.flv',
|
||||
u'info_dict': {
|
||||
u'title': u'Julian Smith & Travis Legg Watch Julian Smith',
|
||||
u'description': u'Julian and Travis watch Julian Smith',
|
||||
},
|
||||
u'params': {
|
||||
# Requires rtmpdump
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
u'skip': u'Requires rtmpdump'
|
||||
}
|
||||
{
|
||||
u'url': u'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html',
|
||||
u'file': u'103000935.flv',
|
||||
u'info_dict': {
|
||||
u'title': u'Codefellas - The Cougar Lies with Spanish Moss',
|
||||
u'description': u'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
|
||||
},
|
||||
u'params': {
|
||||
# Requires rtmpdump
|
||||
u'skip_download': True,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
if mobj is None:
|
||||
raise ExtractorError(u'Invalid URL: %s' % url)
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
m_id = re.search(r'YUI\.namespace\("Media"\)\.CONTENT_ID = "(?P<new_id>.+?)";', webpage)
|
||||
|
||||
if m_id is None:
|
||||
# TODO: Check which url parameters are required
|
||||
info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
|
||||
webpage = self._download_webpage(info_url, video_id, u'Downloading info webpage')
|
||||
info_re = r'''<title><!\[CDATA\[(?P<title>.*?)\]\]></title>.*
|
||||
<description><!\[CDATA\[(?P<description>.*?)\]\]></description>.*
|
||||
<media:pubStart><!\[CDATA\[(?P<date>.*?)\ .*\]\]></media:pubStart>.*
|
||||
<media:content\ medium="image"\ url="(?P<thumb>.*?)"\ name="LARGETHUMB"
|
||||
'''
|
||||
self.report_extraction(video_id)
|
||||
m_info = re.search(info_re, webpage, re.VERBOSE|re.DOTALL)
|
||||
if m_info is None:
|
||||
raise ExtractorError(u'Unable to extract video info')
|
||||
video_title = m_info.group('title')
|
||||
video_description = m_info.group('description')
|
||||
video_thumb = m_info.group('thumb')
|
||||
video_date = m_info.group('date')
|
||||
video_date = datetime.datetime.strptime(video_date, '%m/%d/%Y').strftime('%Y%m%d')
|
||||
|
||||
# TODO: Find a way to get mp4 videos
|
||||
rest_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;element=stream;outputformat=mrss;id=%s;lmsoverride=1;bw=375;dynamicstream=1;cb=83521105;tech=flv,mp4;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
|
||||
webpage = self._download_webpage(rest_url, video_id, u'Downloading video url webpage')
|
||||
m_rest = re.search(r'<media:content url="(?P<url>.*?)" path="(?P<path>.*?)"', webpage)
|
||||
video_url = m_rest.group('url')
|
||||
video_path = m_rest.group('path')
|
||||
if m_rest is None:
|
||||
raise ExtractorError(u'Unable to extract video url')
|
||||
items_json = self._search_regex(r'YVIDEO_INIT_ITEMS = ({.*?});$',
|
||||
webpage, u'items', flags=re.MULTILINE)
|
||||
items = json.loads(items_json)
|
||||
info = items['mediaItems']['query']['results']['mediaObj'][0]
|
||||
# The 'meta' field is not always in the video webpage, we request it
|
||||
# from another page
|
||||
long_id = info['id']
|
||||
query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
|
||||
' AND plrs="86Gj0vCaSzV_Iuf6hNylf2"' % long_id)
|
||||
data = compat_urllib_parse.urlencode({
|
||||
'q': query,
|
||||
'env': 'prod',
|
||||
'format': 'json',
|
||||
})
|
||||
query_result_json = self._download_webpage(
|
||||
'http://video.query.yahoo.com/v1/public/yql?' + data,
|
||||
video_id, u'Downloading video info')
|
||||
query_result = json.loads(query_result_json)
|
||||
info = query_result['query']['results']['mediaObj'][0]
|
||||
meta = info['meta']
|
||||
|
||||
else: # We have to use a different method if another id is defined
|
||||
long_id = m_id.group('new_id')
|
||||
info_url = 'http://video.query.yahoo.com/v1/public/yql?q=SELECT%20*%20FROM%20yahoo.media.video.streams%20WHERE%20id%3D%22' + long_id + '%22%20AND%20format%3D%22mp4%2Cflv%22%20AND%20protocol%3D%22rtmp%2Chttp%22%20AND%20plrs%3D%2286Gj0vCaSzV_Iuf6hNylf2%22%20AND%20acctid%3D%22389%22%20AND%20plidl%3D%22%22%20AND%20pspid%3D%22792700001%22%20AND%20offnetwork%3D%22false%22%20AND%20site%3D%22ivy%22%20AND%20lang%3D%22en-US%22%20AND%20region%3D%22US%22%20AND%20override%3D%22none%22%3B&env=prod&format=json&callback=YUI.Env.JSONP.yui_3_8_1_1_1368368376830_335'
|
||||
webpage = self._download_webpage(info_url, video_id, u'Downloading info json')
|
||||
json_str = re.search(r'YUI.Env.JSONP.yui.*?\((.*?)\);', webpage).group(1)
|
||||
info = json.loads(json_str)
|
||||
res = info[u'query'][u'results'][u'mediaObj'][0]
|
||||
stream = res[u'streams'][0]
|
||||
video_path = stream[u'path']
|
||||
video_url = stream[u'host']
|
||||
meta = res[u'meta']
|
||||
video_title = meta[u'title']
|
||||
video_description = meta[u'description']
|
||||
video_thumb = meta[u'thumbnail']
|
||||
video_date = None # I can't find it
|
||||
formats = []
|
||||
for s in info['streams']:
|
||||
format_info = {
|
||||
'width': s.get('width'),
|
||||
'height': s.get('height'),
|
||||
'bitrate': s.get('bitrate'),
|
||||
}
|
||||
|
||||
host = s['host']
|
||||
path = s['path']
|
||||
if host.startswith('rtmp'):
|
||||
format_info.update({
|
||||
'url': host,
|
||||
'play_path': path,
|
||||
'ext': 'flv',
|
||||
})
|
||||
else:
|
||||
format_url = compat_urlparse.urljoin(host, path)
|
||||
format_info['url'] = format_url
|
||||
format_info['ext'] = determine_ext(format_url)
|
||||
|
||||
formats.append(format_info)
|
||||
formats = sorted(formats, key=lambda f:(f['height'], f['width']))
|
||||
|
||||
info = {
|
||||
'id': video_id,
|
||||
'title': meta['title'],
|
||||
'formats': formats,
|
||||
'description': clean_html(meta['description']),
|
||||
'thumbnail': meta['thumbnail'],
|
||||
}
|
||||
# TODO: Remove when #980 has been merged
|
||||
info.update(formats[-1])
|
||||
|
||||
return info
|
||||
|
||||
info_dict = {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'play_path': video_path,
|
||||
'title':video_title,
|
||||
'description': video_description,
|
||||
'thumbnail': video_thumb,
|
||||
'upload_date': video_date,
|
||||
'ext': 'flv',
|
||||
}
|
||||
return info_dict
|
||||
|
||||
class YahooSearchIE(SearchInfoExtractor):
|
||||
IE_DESC = u'Yahoo screen search'
|
||||
@@ -115,7 +132,7 @@ class YahooSearchIE(SearchInfoExtractor):
|
||||
mobj = re.search(r'(?P<url>screen\.yahoo\.com/.*?-\d*?\.html)"', r)
|
||||
e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
|
||||
res['entries'].append(e)
|
||||
if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1 )):
|
||||
if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1)):
|
||||
break
|
||||
|
||||
return res
|
||||
|
@@ -13,7 +13,8 @@ class YouJizzIE(InfoExtractor):
|
||||
u'file': u'2189178.flv',
|
||||
u'md5': u'07e15fa469ba384c7693fd246905547c',
|
||||
u'info_dict': {
|
||||
u"title": u"Zeichentrick 1"
|
||||
u"title": u"Zeichentrick 1",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +26,8 @@ class YouJizzIE(InfoExtractor):
|
||||
# Get webpage content
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
# Get the video title
|
||||
video_title = self._html_search_regex(r'<title>(?P<title>.*)</title>',
|
||||
webpage, u'title').strip()
|
||||
@@ -60,6 +63,7 @@ class YouJizzIE(InfoExtractor):
|
||||
'title': video_title,
|
||||
'ext': 'flv',
|
||||
'format': 'flv',
|
||||
'player_url': embed_page_url}
|
||||
'player_url': embed_page_url,
|
||||
'age_limit': age_limit}
|
||||
|
||||
return [info]
|
||||
|
@@ -13,12 +13,12 @@ from ..utils import (
|
||||
|
||||
|
||||
class YoukuIE(InfoExtractor):
|
||||
_VALID_URL = r'(?:http://)?(v|player)\.youku\.com/(v_show/id_|player\.php/sid/)(?P<ID>[A-Za-z0-9]+)(\.html|/v.swf)'
|
||||
_VALID_URL = r'(?:(?:http://)?(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|youku:)(?P<ID>[A-Za-z0-9]+)(?:\.html|/v\.swf|)'
|
||||
_TEST = {
|
||||
u"url": u"http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
|
||||
u"file": u"XNDgyMDQ2NTQw_part00.flv",
|
||||
u"md5": u"ffe3f2e435663dc2d1eea34faeff5b5b",
|
||||
u"params": { u"test": False },
|
||||
u"params": {u"test": False},
|
||||
u"info_dict": {
|
||||
u"title": u"youtube-dl test video \"'/\\ä↭𝕐"
|
||||
}
|
||||
@@ -37,8 +37,8 @@ class YoukuIE(InfoExtractor):
|
||||
source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
|
||||
seed = float(seed)
|
||||
for i in range(len(source)):
|
||||
seed = (seed * 211 + 30031 ) % 65536
|
||||
index = math.floor(seed / 65536 * len(source) )
|
||||
seed = (seed * 211 + 30031) % 65536
|
||||
index = math.floor(seed / 65536 * len(source))
|
||||
mixed.append(source[int(index)])
|
||||
source.remove(source[int(index)])
|
||||
#return ''.join(mixed)
|
||||
@@ -66,6 +66,12 @@ class YoukuIE(InfoExtractor):
|
||||
self.report_extraction(video_id)
|
||||
try:
|
||||
config = json.loads(jsondata)
|
||||
error_code = config['data'][0].get('error_code')
|
||||
if error_code:
|
||||
# -8 means blocked outside China.
|
||||
error = config['data'][0].get('error') # Chinese and English, separated by newline.
|
||||
raise ExtractorError(error or u'Server reported error %i' % error_code,
|
||||
expected=True)
|
||||
|
||||
video_title = config['data'][0]['title']
|
||||
seed = config['data'][0]['seed']
|
||||
@@ -89,6 +95,7 @@ class YoukuIE(InfoExtractor):
|
||||
|
||||
fileid = config['data'][0]['streamfileids'][format]
|
||||
keys = [s['k'] for s in config['data'][0]['segs'][format]]
|
||||
# segs is usually a dictionary, but an empty *list* if an error occured.
|
||||
except (UnicodeDecodeError, ValueError, KeyError):
|
||||
raise ExtractorError(u'Unable to extract info section')
|
||||
|
||||
|
@@ -17,7 +17,7 @@ from ..aes import (
|
||||
)
|
||||
|
||||
class YouPornIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:\w+\.)?youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+)'
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+))'
|
||||
_TEST = {
|
||||
u'url': u'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
|
||||
u'file': u'505835.mp4',
|
||||
@@ -26,31 +26,20 @@ class YouPornIE(InfoExtractor):
|
||||
u"upload_date": u"20101221",
|
||||
u"description": u"Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?",
|
||||
u"uploader": u"Ask Dan And Jennifer",
|
||||
u"title": u"Sex Ed: Is It Safe To Masturbate Daily?"
|
||||
u"title": u"Sex Ed: Is It Safe To Masturbate Daily?",
|
||||
u"age_limit": 18,
|
||||
}
|
||||
}
|
||||
|
||||
def _print_formats(self, formats):
|
||||
"""Print all available formats"""
|
||||
print(u'Available formats:')
|
||||
print(u'ext\t\tformat')
|
||||
print(u'---------------------------------')
|
||||
for format in formats:
|
||||
print(u'%s\t\t%s' % (format['ext'], format['format']))
|
||||
|
||||
def _specific(self, req_format, formats):
|
||||
for x in formats:
|
||||
if x["format"] == req_format:
|
||||
return x
|
||||
return None
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
webpage = self._download_webpage(req, video_id)
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
# Get JSON parameters
|
||||
json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, u'JSON parameters')
|
||||
@@ -69,27 +58,22 @@ class YouPornIE(InfoExtractor):
|
||||
except KeyError:
|
||||
raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
|
||||
|
||||
# Get all of the formats available
|
||||
# Get all of the links from the page
|
||||
DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
|
||||
download_list_html = self._search_regex(DOWNLOAD_LIST_RE,
|
||||
webpage, u'download list').strip()
|
||||
|
||||
# Get all of the links from the page
|
||||
LINK_RE = r'(?s)<a href="(?P<url>[^"]+)">'
|
||||
LINK_RE = r'<a href="([^"]+)">'
|
||||
links = re.findall(LINK_RE, download_list_html)
|
||||
|
||||
# Get link of hd video if available
|
||||
mobj = re.search(r'var encryptedQuality720URL = \'(?P<encrypted_video_url>[a-zA-Z0-9+/]+={0,2})\';', webpage)
|
||||
if mobj != None:
|
||||
encrypted_video_url = mobj.group(u'encrypted_video_url')
|
||||
video_url = aes_decrypt_text(encrypted_video_url, video_title, 32).decode('utf-8')
|
||||
links = [video_url] + links
|
||||
|
||||
# Get all encrypted links
|
||||
encrypted_links = re.findall(r'var encryptedQuality[0-9]{3}URL = \'([a-zA-Z0-9+/]+={0,2})\';', webpage)
|
||||
for encrypted_link in encrypted_links:
|
||||
link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
|
||||
links.append(link)
|
||||
|
||||
if not links:
|
||||
raise ExtractorError(u'ERROR: no known formats available for video')
|
||||
|
||||
self.to_screen(u'Links found: %d' % len(links))
|
||||
|
||||
formats = []
|
||||
for link in links:
|
||||
|
||||
@@ -97,42 +81,36 @@ class YouPornIE(InfoExtractor):
|
||||
# http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
|
||||
# A path looks like this:
|
||||
# /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
|
||||
video_url = unescapeHTML( link )
|
||||
path = compat_urllib_parse_urlparse( video_url ).path
|
||||
extension = os.path.splitext( path )[1][1:]
|
||||
video_url = unescapeHTML(link)
|
||||
path = compat_urllib_parse_urlparse(video_url).path
|
||||
extension = os.path.splitext(path)[1][1:]
|
||||
format = path.split('/')[4].split('_')[:2]
|
||||
|
||||
# size = format[0]
|
||||
# bitrate = format[1]
|
||||
format = "-".join( format )
|
||||
format = "-".join(format)
|
||||
# title = u'%s-%s-%s' % (video_title, size, bitrate)
|
||||
|
||||
formats.append({
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'uploader': video_uploader,
|
||||
'upload_date': upload_date,
|
||||
'title': video_title,
|
||||
'ext': extension,
|
||||
'format': format,
|
||||
'thumbnail': thumbnail,
|
||||
'description': video_description
|
||||
'format_id': format,
|
||||
})
|
||||
|
||||
if self._downloader.params.get('listformats', None):
|
||||
self._print_formats(formats)
|
||||
return
|
||||
|
||||
req_format = self._downloader.params.get('format', 'best')
|
||||
self.to_screen(u'Format: %s' % req_format)
|
||||
|
||||
if req_format is None or req_format == 'best':
|
||||
return [formats[0]]
|
||||
elif req_format == 'worst':
|
||||
return [formats[-1]]
|
||||
elif req_format in ('-1', 'all'):
|
||||
return formats
|
||||
else:
|
||||
format = self._specific( req_format, formats )
|
||||
if format is None:
|
||||
raise ExtractorError(u'Requested format not available')
|
||||
return [format]
|
||||
# Sort and remove doubles
|
||||
formats.sort(key=lambda format: list(map(lambda s: s.zfill(6), format['format'].split('-'))))
|
||||
for i in range(len(formats)-1,0,-1):
|
||||
if formats[i]['format_id'] == formats[i-1]['format_id']:
|
||||
del formats[i]
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'uploader': video_uploader,
|
||||
'upload_date': upload_date,
|
||||
'title': video_title,
|
||||
'thumbnail': thumbnail,
|
||||
'description': video_description,
|
||||
'age_limit': age_limit,
|
||||
'formats': formats,
|
||||
}
|
||||
|
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user