Compare commits
250 Commits
2014.10.02
...
2014.11.02
Author | SHA1 | Date | |
---|---|---|---|
c30ae9594c | |||
ffae28ae18 | |||
d9116714f2 | |||
08965906a8 | |||
5263cdfcf9 | |||
b2a68d14cf | |||
6e1cff9c33 | |||
72975729c8 | |||
d319948b6a | |||
9a4bf889f9 | |||
2a834bdb21 | |||
0d2c141865 | |||
5ec39d8b96 | |||
7b6de3728a | |||
a51d3aa001 | |||
2c8e03d937 | |||
fbb21cf528 | |||
b8a618f898 | |||
feb74960eb | |||
d65d628613 | |||
ac645ac7d0 | |||
7d11297f3f | |||
6ad4013d40 | |||
dbd1283d31 | |||
c451d4f553 | |||
8abec2c8bb | |||
a9bad429b3 | |||
50c8266ef0 | |||
00edd4f9be | |||
ee966928af | |||
e5193599ec | |||
01d663bca3 | |||
e0c51cdadc | |||
9334f8f17a | |||
632256d9ec | |||
03df7baa6a | |||
3511266bc3 | |||
9fdece5d34 | |||
bbf1092ad0 | |||
9ef55c5bbc | |||
48a24ab746 | |||
68acdbda9d | |||
27c542c06f | |||
aaa399d2f6 | |||
b2e6a1c14c | |||
8cc3eba79a | |||
b0fb6d4db1 | |||
81515ad9f6 | |||
8112d4b284 | |||
bf7aa6301b | |||
aea856621f | |||
f24a5a2faa | |||
ecfe623422 | |||
4a6c94288a | |||
10e3d73472 | |||
15956b5aa1 | |||
586f7082ef | |||
d6d9186f0d | |||
2e9ff8f362 | |||
6407432333 | |||
f744c0f398 | |||
249efaf44b | |||
8d32abff9e | |||
94f052cbf4 | |||
446a03bd96 | |||
6009b69f81 | |||
3d6047113c | |||
9dec99303d | |||
7706927370 | |||
3adba6fa2a | |||
f46a8702cc | |||
8d11b59bbb | |||
cf501a23d2 | |||
2bcae58d46 | |||
c9f08154a3 | |||
526b276fd7 | |||
77ec444d9a | |||
bfc2bedcfc | |||
83855f3a1f | |||
50b51830fb | |||
3d6eed9b52 | |||
1a253e134c | |||
6194bb1419 | |||
37d66e7f1e | |||
70b7e3fbb6 | |||
579657ad87 | |||
5f82b129e0 | |||
64269e4d01 | |||
d481699a7a | |||
5894a4f4ee | |||
09e5d6a6e5 | |||
274b12b5a8 | |||
23be51d8ce | |||
488447455d | |||
d28b517154 | |||
a7e97f6db1 | |||
639a422d21 | |||
f889cea109 | |||
1bdeb7be2e | |||
699151bcb1 | |||
911344e5ac | |||
03936f6e6d | |||
b13ccb1b87 | |||
f64f8a4662 | |||
681b9caa9c | |||
0eb9fb9f24 | |||
9a76f416ce | |||
603821161f | |||
d3c72db894 | |||
43d9718fb9 | |||
7fc54e5262 | |||
ec9c978481 | |||
d36cae46d8 | |||
fdfefa1b9c | |||
724d031893 | |||
63e0be3415 | |||
c64ed2a310 | |||
cdc5cb7c2b | |||
8efd06aa42 | |||
7f9ced64cb | |||
7608815cc2 | |||
5823eda139 | |||
e82c1e9a6e | |||
1ede5b2481 | |||
964ae0a122 | |||
98e1d28982 | |||
2c26df763c | |||
018e835594 | |||
e65e06fbe2 | |||
95ee84421e | |||
2acfe95f58 | |||
b5a14350b9 | |||
8d81f872fb | |||
36f1c90497 | |||
057a5206cc | |||
9e9bc793f3 | |||
5c565ac9e7 | |||
67500bf939 | |||
b1edd7a48a | |||
2c63ccec78 | |||
f2f2c0c2c6 | |||
4661e243f8 | |||
f3cd403c2b | |||
ad5f53ac72 | |||
75da98e9e1 | |||
281d3f1d68 | |||
6283c10b1c | |||
85d7b76586 | |||
2399535fd1 | |||
52cffcb186 | |||
8f3b5397a7 | |||
9bbec55255 | |||
6b445558ff | |||
bfd91588f3 | |||
6bf6962062 | |||
40bca5f927 | |||
74214d35c5 | |||
1b10a011ec | |||
d24a2b20b4 | |||
8230018c20 | |||
cc98a3f096 | |||
ce519b747e | |||
16efb3695f | |||
4510d14f0a | |||
0f175a932f | |||
849b269273 | |||
95fa5fb569 | |||
77c3c5c5ed | |||
159444a668 | |||
f9befee1f5 | |||
9471c44405 | |||
013bfdd84c | |||
46fd0dd5a5 | |||
4698f0d858 | |||
355d074ff9 | |||
7da224c907 | |||
1723edb1a5 | |||
4740864508 | |||
09a42738fc | |||
df928d500f | |||
a72cbfacf0 | |||
62a164e713 | |||
5f58165def | |||
a86c73cf80 | |||
bd4e40df1a | |||
1419fafd36 | |||
9b36dcbd65 | |||
2aefb886fa | |||
72961c2a8a | |||
4c1ce987b0 | |||
8a2300a597 | |||
1cc887cbf0 | |||
203fb43f36 | |||
4d7b03f1f2 | |||
72ebb5e4b4 | |||
8450c15c25 | |||
b88b45e46c | |||
2417dc1715 | |||
23d83ad4d5 | |||
772ece3571 | |||
2c9f31188b | |||
d18be55533 | |||
ac20fc047a | |||
b4c3c8c172 | |||
3357110a4c | |||
e29fdedb45 | |||
4828703f14 | |||
afe08e0d4a | |||
071420e136 | |||
f4cf848d1d | |||
b7b2ca6e2b | |||
1409704afa | |||
3741302a10 | |||
c8e390c2b0 | |||
823f1e015a | |||
3c06d3715e | |||
762958d5af | |||
53d9009bdb | |||
1b725173a5 | |||
0ca41c3d9c | |||
fc6861b175 | |||
b097b5f246 | |||
385009fc44 | |||
ced659bb4d | |||
842cca7d56 | |||
b3826f6c8d | |||
7bc8780c57 | |||
c59c3c84ed | |||
24f7fb5e1e | |||
3b700f8d43 | |||
31d06400ec | |||
642b76ac15 | |||
4c4de296d4 | |||
b10609d98c | |||
3ae165aa10 | |||
e4b85e35d0 | |||
bb0c206f59 | |||
b81f484b60 | |||
5e69192ef7 | |||
e9be9a6acd | |||
f47754f061 | |||
d838b1bd4a | |||
fe506288bd | |||
d397c0b3dd | |||
146c80e256 | |||
f78c01f68b | |||
8489578df4 | |||
e4d6cca0c1 | |||
fc66e4a0d5 | |||
4644ac5527 |
1
.gitignore
vendored
1
.gitignore
vendored
@ -30,3 +30,4 @@ updates_key.pem
|
||||
*.swp
|
||||
test/testdata
|
||||
.tox
|
||||
youtube-dl.zsh
|
||||
|
82
AUTHORS
Normal file
82
AUTHORS
Normal file
@ -0,0 +1,82 @@
|
||||
Ricardo Garcia Gonzalez
|
||||
Danny Colligan
|
||||
Benjamin Johnson
|
||||
Vasyl' Vavrychuk
|
||||
Witold Baryluk
|
||||
Paweł Paprota
|
||||
Gergely Imreh
|
||||
Rogério Brito
|
||||
Philipp Hagemeister
|
||||
Sören Schulze
|
||||
Kevin Ngo
|
||||
Ori Avtalion
|
||||
shizeeg
|
||||
Filippo Valsorda
|
||||
Christian Albrecht
|
||||
Dave Vasilevsky
|
||||
Jaime Marquínez Ferrándiz
|
||||
Jeff Crouse
|
||||
Osama Khalid
|
||||
Michael Walter
|
||||
M. Yasoob Ullah Khalid
|
||||
Julien Fraichard
|
||||
Johny Mo Swag
|
||||
Axel Noack
|
||||
Albert Kim
|
||||
Pierre Rudloff
|
||||
Huarong Huo
|
||||
Ismael Mejía
|
||||
Steffan 'Ruirize' James
|
||||
Andras Elso
|
||||
Jelle van der Waa
|
||||
Marcin Cieślak
|
||||
Anton Larionov
|
||||
Takuya Tsuchida
|
||||
Sergey M.
|
||||
Michael Orlitzky
|
||||
Chris Gahan
|
||||
Saimadhav Heblikar
|
||||
Mike Col
|
||||
Oleg Prutz
|
||||
pulpe
|
||||
Andreas Schmitz
|
||||
Michael Kaiser
|
||||
Niklas Laxström
|
||||
David Triendl
|
||||
Anthony Weems
|
||||
David Wagner
|
||||
Juan C. Olivares
|
||||
Mattias Harrysson
|
||||
phaer
|
||||
Sainyam Kapoor
|
||||
Nicolas Évrard
|
||||
Jason Normore
|
||||
Hoje Lee
|
||||
Adam Thalhammer
|
||||
Georg Jähnig
|
||||
Ralf Haring
|
||||
Koki Takahashi
|
||||
Ariset Llerena
|
||||
Adam Malcontenti-Wilson
|
||||
Tobias Bell
|
||||
Naglis Jonaitis
|
||||
Charles Chen
|
||||
Hassaan Ali
|
||||
Dobrosław Żybort
|
||||
David Fabijan
|
||||
Sebastian Haas
|
||||
Alexander Kirk
|
||||
Erik Johnson
|
||||
Keith Beckman
|
||||
Ole Ernst
|
||||
Aaron McDaniel (mcd1992)
|
||||
Magnus Kolstad
|
||||
Hari Padmanaban
|
||||
Carlos Ramos
|
||||
5moufl
|
||||
lenaten
|
||||
Dennis Scheiba
|
||||
Damon Timm
|
||||
winwon
|
||||
Xavier Beynon
|
||||
Gabriel Schubiner
|
20
Makefile
20
Makefile
@ -1,7 +1,7 @@
|
||||
all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.fish
|
||||
all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
|
||||
|
||||
clean:
|
||||
rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.fish
|
||||
rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part
|
||||
|
||||
cleanall: clean
|
||||
rm -f youtube-dl youtube-dl.exe
|
||||
@ -9,6 +9,7 @@ cleanall: clean
|
||||
PREFIX ?= /usr/local
|
||||
BINDIR ?= $(PREFIX)/bin
|
||||
MANDIR ?= $(PREFIX)/man
|
||||
SHAREDIR ?= $(PREFIX)/share
|
||||
PYTHON ?= /usr/bin/env python
|
||||
|
||||
# set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
|
||||
@ -22,13 +23,15 @@ else
|
||||
endif
|
||||
endif
|
||||
|
||||
install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
|
||||
install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
|
||||
install -d $(DESTDIR)$(BINDIR)
|
||||
install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
|
||||
install -d $(DESTDIR)$(MANDIR)/man1
|
||||
install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
|
||||
install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
|
||||
install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
|
||||
install -d $(DESTDIR)$(SHAREDIR)/zsh/site-functions
|
||||
install -m 644 youtube-dl.zsh $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_youtube-dl
|
||||
install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions
|
||||
install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
|
||||
|
||||
@ -38,7 +41,7 @@ test:
|
||||
|
||||
tar: youtube-dl.tar.gz
|
||||
|
||||
.PHONY: all clean install test tar bash-completion pypi-files fish-completion
|
||||
.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion
|
||||
|
||||
pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
|
||||
|
||||
@ -66,12 +69,17 @@ youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-co
|
||||
|
||||
bash-completion: youtube-dl.bash-completion
|
||||
|
||||
youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in
|
||||
python devscripts/zsh-completion.py
|
||||
|
||||
zsh-completion: youtube-dl.zsh
|
||||
|
||||
youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
|
||||
python devscripts/fish-completion.py
|
||||
|
||||
fish-completion: youtube-dl.fish
|
||||
|
||||
youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.fish
|
||||
youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
|
||||
@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
|
||||
--exclude '*.DS_Store' \
|
||||
--exclude '*.kate-swp' \
|
||||
@ -86,5 +94,5 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
|
||||
bin devscripts test youtube_dl docs \
|
||||
LICENSE README.md README.txt \
|
||||
Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
|
||||
youtube-dl.fish setup.py \
|
||||
youtube-dl.zsh youtube-dl.fish setup.py \
|
||||
youtube-dl
|
||||
|
35
README.md
35
README.md
@ -69,6 +69,8 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
configuration in ~/.config/youtube-dl.conf
|
||||
(%APPDATA%/youtube-dl/config.txt on
|
||||
Windows)
|
||||
--flat-playlist Do not extract the videos of a playlist,
|
||||
only list them.
|
||||
|
||||
## Video Selection:
|
||||
--playlist-start NUMBER playlist video to start at (default is 1)
|
||||
@ -99,8 +101,6 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
downloaded videos in it.
|
||||
--include-ads Download advertisements as well
|
||||
(experimental)
|
||||
--youtube-include-dash-manifest Try to download the DASH manifest on
|
||||
YouTube videos (experimental)
|
||||
|
||||
## Download Options:
|
||||
-r, --rate-limit LIMIT maximum download rate in bytes per second
|
||||
@ -158,7 +158,8 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
downloads if possible.
|
||||
--no-continue do not resume partially downloaded files
|
||||
(restart from beginning)
|
||||
--no-part do not use .part files
|
||||
--no-part do not use .part files - write directly
|
||||
into output file
|
||||
--no-mtime do not use the Last-modified header to set
|
||||
the file modification time
|
||||
--write-description write video description to a .description
|
||||
@ -198,6 +199,10 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
-j, --dump-json simulate, quiet but print JSON information.
|
||||
See --output for a description of available
|
||||
keys.
|
||||
-J, --dump-single-json simulate, quiet but print JSON information
|
||||
for each command-line argument. If the URL
|
||||
refers to a playlist, dump the whole
|
||||
playlist information in a single line.
|
||||
--newline output progress bar as new lines
|
||||
--no-progress do not print progress bar
|
||||
--console-title display progress in console titlebar
|
||||
@ -216,7 +221,7 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
information about the video. (Currently
|
||||
supported only for YouTube)
|
||||
--user-agent UA specify a custom user agent
|
||||
--referer REF specify a custom referer, use if the video
|
||||
--referer URL specify a custom referer, use if the video
|
||||
access is restricted to one domain
|
||||
--add-header FIELD:VALUE specify a custom HTTP header and its value,
|
||||
separated by a colon ':'. You can use this
|
||||
@ -241,6 +246,8 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
one is requested
|
||||
--max-quality FORMAT highest quality format to download
|
||||
-F, --list-formats list all available formats
|
||||
--youtube-skip-dash-manifest Do not download the DASH manifest on
|
||||
YouTube videos
|
||||
|
||||
## Subtitle Options:
|
||||
--write-sub write subtitle file
|
||||
@ -256,7 +263,7 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
language tags like 'en,pt'
|
||||
|
||||
## Authentication Options:
|
||||
-u, --username USERNAME account username
|
||||
-u, --username USERNAME login with this account ID
|
||||
-p, --password PASSWORD account password
|
||||
-2, --twofactor TWOFACTOR two-factor auth code
|
||||
-n, --netrc use .netrc authentication data
|
||||
@ -267,7 +274,7 @@ which means you can modify it, redistribute it or use it however you like.
|
||||
(requires ffmpeg or avconv and ffprobe or
|
||||
avprobe)
|
||||
--audio-format FORMAT "best", "aac", "vorbis", "mp3", "m4a",
|
||||
"opus", or "wav"; best by default
|
||||
"opus", or "wav"; "best" by default
|
||||
--audio-quality QUALITY ffmpeg/avconv audio quality specification,
|
||||
insert a value between 0 (better) and 9
|
||||
(worse) for VBR or a specific bitrate like
|
||||
@ -374,7 +381,7 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
|
||||
|
||||
YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
|
||||
|
||||
If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version. See above for a way to update.
|
||||
If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
|
||||
|
||||
### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
|
||||
|
||||
@ -504,6 +511,20 @@ If you want to add support for a new site, you can follow this quick list (assum
|
||||
|
||||
In any case, thank you very much for your contributions!
|
||||
|
||||
# EMBEDDING YOUTUBE-DL
|
||||
|
||||
youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
|
||||
|
||||
From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
|
||||
|
||||
import youtube_dl
|
||||
|
||||
ydl_opts = {}
|
||||
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
|
||||
|
||||
Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
|
||||
|
||||
# BUGS
|
||||
|
||||
Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email.
|
||||
|
28
devscripts/zsh-completion.in
Normal file
28
devscripts/zsh-completion.in
Normal file
@ -0,0 +1,28 @@
|
||||
#compdef youtube-dl
|
||||
|
||||
__youtube_dl() {
|
||||
local curcontext="$curcontext" fileopts diropts cur prev
|
||||
typeset -A opt_args
|
||||
fileopts="{{fileopts}}"
|
||||
diropts="{{diropts}}"
|
||||
cur=$words[CURRENT]
|
||||
case $cur in
|
||||
:)
|
||||
_arguments '*: :(::ytfavorites ::ytrecommended ::ytsubscriptions ::ytwatchlater ::ythistory)'
|
||||
;;
|
||||
*)
|
||||
prev=$words[CURRENT-1]
|
||||
if [[ ${prev} =~ ${fileopts} ]]; then
|
||||
_path_files
|
||||
elif [[ ${prev} =~ ${diropts} ]]; then
|
||||
_path_files -/
|
||||
elif [[ ${prev} == "--recode-video" ]]; then
|
||||
_arguments '*: :(mp4 flv ogg webm mkv)'
|
||||
else
|
||||
_arguments '*: :({{flags}})'
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
__youtube_dl
|
46
devscripts/zsh-completion.py
Executable file
46
devscripts/zsh-completion.py
Executable file
@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python
|
||||
import os
|
||||
from os.path import dirname as dirn
|
||||
import sys
|
||||
|
||||
sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
|
||||
import youtube_dl
|
||||
|
||||
ZSH_COMPLETION_FILE = "youtube-dl.zsh"
|
||||
ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"
|
||||
|
||||
|
||||
def build_completion(opt_parser):
|
||||
opts = [opt for group in opt_parser.option_groups
|
||||
for opt in group.option_list]
|
||||
opts_file = [opt for opt in opts if opt.metavar == "FILE"]
|
||||
opts_dir = [opt for opt in opts if opt.metavar == "DIR"]
|
||||
|
||||
fileopts = []
|
||||
for opt in opts_file:
|
||||
if opt._short_opts:
|
||||
fileopts.extend(opt._short_opts)
|
||||
if opt._long_opts:
|
||||
fileopts.extend(opt._long_opts)
|
||||
|
||||
diropts = []
|
||||
for opt in opts_dir:
|
||||
if opt._short_opts:
|
||||
diropts.extend(opt._short_opts)
|
||||
if opt._long_opts:
|
||||
diropts.extend(opt._long_opts)
|
||||
|
||||
flags = [opt.get_opt_string() for opt in opts]
|
||||
|
||||
with open(ZSH_COMPLETION_TEMPLATE) as f:
|
||||
template = f.read()
|
||||
|
||||
template = template.replace("{{fileopts}}", "|".join(fileopts))
|
||||
template = template.replace("{{diropts}}", "|".join(diropts))
|
||||
template = template.replace("{{flags}}", " ".join(flags))
|
||||
|
||||
with open(ZSH_COMPLETION_FILE, "w") as f:
|
||||
f.write(template)
|
||||
|
||||
parser = youtube_dl.parseOpts()[0]
|
||||
build_completion(parser)
|
@ -44,8 +44,8 @@ copyright = u'2014, Ricardo Garcia Gonzalez'
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
import youtube_dl
|
||||
version = youtube_dl.__version__
|
||||
from youtube_dl.version import __version__
|
||||
version = __version__
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = version
|
||||
|
||||
|
@ -145,7 +145,7 @@ def expect_info_dict(self, expected_dict, got_dict):
|
||||
info_dict_str = ''.join(
|
||||
' %s: %s,\n' % (_repr(k), _repr(v))
|
||||
for k, v in test_info_dict.items())
|
||||
write_string('\n"info_dict": {' + info_dict_str + '}\n', out=sys.stderr)
|
||||
write_string('\n"info_dict": {\n' + info_dict_str + '}\n', out=sys.stderr)
|
||||
self.assertFalse(
|
||||
missing_keys,
|
||||
'Missing keys in test definition: %s' % (
|
||||
@ -171,3 +171,13 @@ def assertGreaterEqual(self, got, expected, msg=None):
|
||||
if msg is None:
|
||||
msg = '%r not greater than or equal to %r' % (got, expected)
|
||||
self.assertTrue(got >= expected, msg)
|
||||
|
||||
|
||||
def expect_warnings(ydl, warnings_re):
|
||||
real_warning = ydl.report_warning
|
||||
|
||||
def _report_warning(w):
|
||||
if not any(re.search(w_re, w) for w_re in warnings_re):
|
||||
real_warning(w)
|
||||
|
||||
ydl.report_warning = _report_warning
|
||||
|
@ -14,7 +14,7 @@ from test.helper import gettestcases
|
||||
from youtube_dl.extractor import (
|
||||
FacebookIE,
|
||||
gen_extractors,
|
||||
JustinTVIE,
|
||||
TwitchIE,
|
||||
YoutubeIE,
|
||||
)
|
||||
|
||||
@ -72,21 +72,17 @@ class TestAllURLsMatching(unittest.TestCase):
|
||||
self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
|
||||
self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
|
||||
|
||||
def test_justin_tv_channelid_matching(self):
|
||||
self.assertTrue(JustinTVIE.suitable('justin.tv/vanillatv'))
|
||||
self.assertTrue(JustinTVIE.suitable('twitch.tv/vanillatv'))
|
||||
self.assertTrue(JustinTVIE.suitable('www.justin.tv/vanillatv'))
|
||||
self.assertTrue(JustinTVIE.suitable('www.twitch.tv/vanillatv'))
|
||||
self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv'))
|
||||
self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv'))
|
||||
self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv/'))
|
||||
self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/'))
|
||||
def test_twitch_channelid_matching(self):
|
||||
self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv'))
|
||||
self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv'))
|
||||
self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv'))
|
||||
self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/'))
|
||||
|
||||
def test_justintv_videoid_matching(self):
|
||||
self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
|
||||
def test_twitch_videoid_matching(self):
|
||||
self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
|
||||
|
||||
def test_justin_tv_chapterid_matching(self):
|
||||
self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
|
||||
def test_twitch_chapterid_matching(self):
|
||||
self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
|
||||
|
||||
def test_youtube_extract(self):
|
||||
assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
|
||||
|
@ -8,6 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from test.helper import (
|
||||
assertGreaterEqual,
|
||||
expect_warnings,
|
||||
get_params,
|
||||
gettestcases,
|
||||
expect_info_dict,
|
||||
@ -100,6 +101,7 @@ def generator(test_case):
|
||||
if status['status'] == 'finished':
|
||||
finished_hook_called.add(status['filename'])
|
||||
ydl.add_progress_hook(_hook)
|
||||
expect_warnings(ydl, test_case.get('expected_warnings', []))
|
||||
|
||||
def get_tc_filename(tc):
|
||||
return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
|
||||
@ -183,7 +185,9 @@ def generator(test_case):
|
||||
md5_for_file = _file_md5(tc_filename)
|
||||
self.assertEqual(md5_for_file, tc['md5'])
|
||||
info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
|
||||
self.assertTrue(os.path.exists(info_json_fn))
|
||||
self.assertTrue(
|
||||
os.path.exists(info_json_fn),
|
||||
'Missing info file %s' % info_json_fn)
|
||||
with io.open(info_json_fn, encoding='utf-8') as infof:
|
||||
info_dict = json.load(infof)
|
||||
|
||||
|
@ -15,6 +15,7 @@ from youtube_dl.extractor import (
|
||||
DailymotionIE,
|
||||
TEDIE,
|
||||
VimeoIE,
|
||||
WallaIE,
|
||||
)
|
||||
|
||||
|
||||
@ -279,5 +280,32 @@ class TestVimeoSubtitles(BaseTestSubtitles):
|
||||
self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
|
||||
|
||||
|
||||
class TestWallaSubtitles(BaseTestSubtitles):
|
||||
url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
|
||||
IE = WallaIE
|
||||
|
||||
def test_list_subtitles(self):
|
||||
self.DL.expect_warning(u'Automatic Captions not supported by this server')
|
||||
self.DL.params['listsubtitles'] = True
|
||||
info_dict = self.getInfoDict()
|
||||
self.assertEqual(info_dict, None)
|
||||
|
||||
def test_allsubtitles(self):
|
||||
self.DL.expect_warning(u'Automatic Captions not supported by this server')
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['allsubtitles'] = True
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(set(subtitles.keys()), set(['heb']))
|
||||
self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
|
||||
|
||||
def test_nosubtitles(self):
|
||||
self.DL.expect_warning(u'video doesn\'t have subtitles')
|
||||
self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
|
||||
self.DL.params['writesubtitles'] = True
|
||||
self.DL.params['allsubtitles'] = True
|
||||
subtitles = self.getSubtitles()
|
||||
self.assertEqual(len(subtitles), 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
@ -45,6 +45,9 @@ from youtube_dl.utils import (
|
||||
escape_rfc3986,
|
||||
escape_url,
|
||||
js_to_json,
|
||||
get_filesystem_encoding,
|
||||
compat_getenv,
|
||||
compat_expanduser,
|
||||
)
|
||||
|
||||
|
||||
@ -286,6 +289,7 @@ class TestUtil(unittest.TestCase):
|
||||
self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266)
|
||||
self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
|
||||
self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
|
||||
self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)
|
||||
|
||||
def test_strip_jsonp(self):
|
||||
stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
|
||||
@ -355,5 +359,17 @@ class TestUtil(unittest.TestCase):
|
||||
on = js_to_json('{"abc": true}')
|
||||
self.assertEqual(json.loads(on), {'abc': True})
|
||||
|
||||
def test_compat_getenv(self):
|
||||
test_str = 'тест'
|
||||
os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0)
|
||||
else test_str.encode(get_filesystem_encoding()))
|
||||
self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
|
||||
|
||||
def test_compat_expanduser(self):
|
||||
test_str = 'C:\Documents and Settings\тест\Application Data'
|
||||
os.environ['HOME'] = (test_str if sys.version_info >= (3, 0)
|
||||
else test_str.encode(get_filesystem_encoding()))
|
||||
self.assertEqual(compat_expanduser('~'), test_str)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
@ -24,6 +24,7 @@ if os.name == 'nt':
|
||||
|
||||
from .utils import (
|
||||
compat_cookiejar,
|
||||
compat_expanduser,
|
||||
compat_http_client,
|
||||
compat_str,
|
||||
compat_urllib_error,
|
||||
@ -61,7 +62,7 @@ from .utils import (
|
||||
from .cache import Cache
|
||||
from .extractor import get_info_extractor, gen_extractors
|
||||
from .downloader import get_suitable_downloader
|
||||
from .postprocessor import FFmpegMergerPP
|
||||
from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
|
||||
from .version import __version__
|
||||
|
||||
|
||||
@ -107,6 +108,8 @@ class YoutubeDL(object):
|
||||
forcefilename: Force printing final filename.
|
||||
forceduration: Force printing duration.
|
||||
forcejson: Force printing info_dict as JSON.
|
||||
dump_single_json: Force printing the info_dict of the whole playlist
|
||||
(or video) as a single JSON line.
|
||||
simulate: Do not download the video files.
|
||||
format: Video format code.
|
||||
format_limit: Highest quality format to try.
|
||||
@ -165,6 +168,8 @@ class YoutubeDL(object):
|
||||
'auto' for elaborate guessing
|
||||
encoding: Use this encoding instead of the system-specified.
|
||||
extract_flat: Do not resolve URLs, return the immediate result.
|
||||
Pass in 'in_playlist' to only show this behavior for
|
||||
playlist items.
|
||||
|
||||
The following parameters are not used by YoutubeDL itself, they are used by
|
||||
the FileDownloader:
|
||||
@ -184,7 +189,7 @@ class YoutubeDL(object):
|
||||
_num_downloads = None
|
||||
_screen_file = None
|
||||
|
||||
def __init__(self, params=None):
|
||||
def __init__(self, params=None, auto_init=True):
|
||||
"""Create a FileDownloader object with the given options."""
|
||||
if params is None:
|
||||
params = {}
|
||||
@ -228,11 +233,11 @@ class YoutubeDL(object):
|
||||
|
||||
if (sys.version_info >= (3,) and sys.platform != 'win32' and
|
||||
sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
|
||||
and not params['restrictfilenames']):
|
||||
and not params.get('restrictfilenames', False)):
|
||||
# On Python 3, the Unicode filesystem API will throw errors (#1474)
|
||||
self.report_warning(
|
||||
'Assuming --restrict-filenames since file system encoding '
|
||||
'cannot encode all charactes. '
|
||||
'cannot encode all characters. '
|
||||
'Set the LC_ALL environment variable to fix this.')
|
||||
self.params['restrictfilenames'] = True
|
||||
|
||||
@ -241,6 +246,10 @@ class YoutubeDL(object):
|
||||
|
||||
self._setup_opener()
|
||||
|
||||
if auto_init:
|
||||
self.print_debug_header()
|
||||
self.add_default_info_extractors()
|
||||
|
||||
def add_info_extractor(self, ie):
|
||||
"""Add an InfoExtractor object to the end of the list."""
|
||||
self._ies.append(ie)
|
||||
@ -447,7 +456,7 @@ class YoutubeDL(object):
|
||||
template_dict = collections.defaultdict(lambda: 'NA', template_dict)
|
||||
|
||||
outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
|
||||
tmpl = os.path.expanduser(outtmpl)
|
||||
tmpl = compat_expanduser(outtmpl)
|
||||
filename = tmpl % template_dict
|
||||
return filename
|
||||
except ValueError as err:
|
||||
@ -568,8 +577,12 @@ class YoutubeDL(object):
|
||||
|
||||
result_type = ie_result.get('_type', 'video')
|
||||
|
||||
if self.params.get('extract_flat', False):
|
||||
if result_type in ('url', 'url_transparent'):
|
||||
if result_type in ('url', 'url_transparent'):
|
||||
extract_flat = self.params.get('extract_flat', False)
|
||||
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
|
||||
extract_flat is True):
|
||||
if self.params.get('forcejson', False):
|
||||
self.to_stdout(json.dumps(ie_result))
|
||||
return ie_result
|
||||
|
||||
if result_type == 'video':
|
||||
@ -897,6 +910,8 @@ class YoutubeDL(object):
|
||||
if self.params.get('forcejson', False):
|
||||
info_dict['_filename'] = filename
|
||||
self.to_stdout(json.dumps(info_dict))
|
||||
if self.params.get('dump_single_json', False):
|
||||
info_dict['_filename'] = filename
|
||||
|
||||
# Do nothing else if in simulate mode
|
||||
if self.params.get('simulate', False):
|
||||
@ -1015,7 +1030,7 @@ class YoutubeDL(object):
|
||||
downloaded = []
|
||||
success = True
|
||||
merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
|
||||
if not merger._get_executable():
|
||||
if not merger._executable:
|
||||
postprocessors = []
|
||||
self.report_warning('You have requested multiple '
|
||||
'formats but ffmpeg or avconv are not installed.'
|
||||
@ -1064,12 +1079,15 @@ class YoutubeDL(object):
|
||||
for url in url_list:
|
||||
try:
|
||||
#It also downloads the videos
|
||||
self.extract_info(url)
|
||||
res = self.extract_info(url)
|
||||
except UnavailableVideoError:
|
||||
self.report_error('unable to download video')
|
||||
except MaxDownloadsReached:
|
||||
self.to_screen('[info] Maximum number of downloaded files reached.')
|
||||
raise
|
||||
else:
|
||||
if self.params.get('dump_single_json', False):
|
||||
self.to_stdout(json.dumps(res))
|
||||
|
||||
return self._download_retcode
|
||||
|
||||
@ -1193,6 +1211,8 @@ class YoutubeDL(object):
|
||||
res += 'video@'
|
||||
if fdict.get('vbr') is not None:
|
||||
res += '%4dk' % fdict['vbr']
|
||||
if fdict.get('fps') is not None:
|
||||
res += ', %sfps' % fdict['fps']
|
||||
if fdict.get('acodec') is not None:
|
||||
if res:
|
||||
res += ', '
|
||||
@ -1297,8 +1317,18 @@ class YoutubeDL(object):
|
||||
sys.exc_clear()
|
||||
except:
|
||||
pass
|
||||
self._write_string('[debug] Python version %s - %s' %
|
||||
(platform.python_version(), platform_name()) + '\n')
|
||||
self._write_string('[debug] Python version %s - %s\n' % (
|
||||
platform.python_version(), platform_name()))
|
||||
|
||||
exe_versions = FFmpegPostProcessor.get_versions()
|
||||
exe_str = ', '.join(
|
||||
'%s %s' % (exe, v)
|
||||
for exe, v in sorted(exe_versions.items())
|
||||
if v
|
||||
)
|
||||
if not exe_str:
|
||||
exe_str = 'none'
|
||||
self._write_string('[debug] exe versions: %s\n' % exe_str)
|
||||
|
||||
proxy_map = {}
|
||||
for handler in self._opener.handlers:
|
||||
|
@ -1,86 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__authors__ = (
|
||||
'Ricardo Garcia Gonzalez',
|
||||
'Danny Colligan',
|
||||
'Benjamin Johnson',
|
||||
'Vasyl\' Vavrychuk',
|
||||
'Witold Baryluk',
|
||||
'Paweł Paprota',
|
||||
'Gergely Imreh',
|
||||
'Rogério Brito',
|
||||
'Philipp Hagemeister',
|
||||
'Sören Schulze',
|
||||
'Kevin Ngo',
|
||||
'Ori Avtalion',
|
||||
'shizeeg',
|
||||
'Filippo Valsorda',
|
||||
'Christian Albrecht',
|
||||
'Dave Vasilevsky',
|
||||
'Jaime Marquínez Ferrándiz',
|
||||
'Jeff Crouse',
|
||||
'Osama Khalid',
|
||||
'Michael Walter',
|
||||
'M. Yasoob Ullah Khalid',
|
||||
'Julien Fraichard',
|
||||
'Johny Mo Swag',
|
||||
'Axel Noack',
|
||||
'Albert Kim',
|
||||
'Pierre Rudloff',
|
||||
'Huarong Huo',
|
||||
'Ismael Mejía',
|
||||
'Steffan \'Ruirize\' James',
|
||||
'Andras Elso',
|
||||
'Jelle van der Waa',
|
||||
'Marcin Cieślak',
|
||||
'Anton Larionov',
|
||||
'Takuya Tsuchida',
|
||||
'Sergey M.',
|
||||
'Michael Orlitzky',
|
||||
'Chris Gahan',
|
||||
'Saimadhav Heblikar',
|
||||
'Mike Col',
|
||||
'Oleg Prutz',
|
||||
'pulpe',
|
||||
'Andreas Schmitz',
|
||||
'Michael Kaiser',
|
||||
'Niklas Laxström',
|
||||
'David Triendl',
|
||||
'Anthony Weems',
|
||||
'David Wagner',
|
||||
'Juan C. Olivares',
|
||||
'Mattias Harrysson',
|
||||
'phaer',
|
||||
'Sainyam Kapoor',
|
||||
'Nicolas Évrard',
|
||||
'Jason Normore',
|
||||
'Hoje Lee',
|
||||
'Adam Thalhammer',
|
||||
'Georg Jähnig',
|
||||
'Ralf Haring',
|
||||
'Koki Takahashi',
|
||||
'Ariset Llerena',
|
||||
'Adam Malcontenti-Wilson',
|
||||
'Tobias Bell',
|
||||
'Naglis Jonaitis',
|
||||
'Charles Chen',
|
||||
'Hassaan Ali',
|
||||
'Dobrosław Żybort',
|
||||
'David Fabijan',
|
||||
'Sebastian Haas',
|
||||
'Alexander Kirk',
|
||||
'Erik Johnson',
|
||||
'Keith Beckman',
|
||||
'Ole Ernst',
|
||||
'Aaron McDaniel (mcd1992)',
|
||||
'Magnus Kolstad',
|
||||
'Hari Padmanaban',
|
||||
'Carlos Ramos',
|
||||
'5moufl',
|
||||
'lenaten',
|
||||
)
|
||||
|
||||
__license__ = 'Public Domain'
|
||||
|
||||
import codecs
|
||||
@ -94,6 +14,7 @@ from .options import (
|
||||
parseOpts,
|
||||
)
|
||||
from .utils import (
|
||||
compat_expanduser,
|
||||
compat_getpass,
|
||||
compat_print,
|
||||
DateRange,
|
||||
@ -255,8 +176,6 @@ def _real_main(argv=None):
|
||||
date = DateRange.day(opts.date)
|
||||
else:
|
||||
date = DateRange(opts.dateafter, opts.datebefore)
|
||||
if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
|
||||
parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
|
||||
|
||||
# Do not download videos when there are audio-only formats
|
||||
if opts.extractaudio and not opts.keepvideo and opts.format is None:
|
||||
@ -284,8 +203,8 @@ def _real_main(argv=None):
|
||||
u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
|
||||
u' template'.format(outtmpl))
|
||||
|
||||
any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
|
||||
download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
|
||||
any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
|
||||
download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
|
||||
|
||||
ydl_opts = {
|
||||
'usenetrc': opts.usenetrc,
|
||||
@ -304,8 +223,9 @@ def _real_main(argv=None):
|
||||
'forcefilename': opts.getfilename,
|
||||
'forceformat': opts.getformat,
|
||||
'forcejson': opts.dumpjson,
|
||||
'simulate': opts.simulate,
|
||||
'skip_download': (opts.skip_download or opts.simulate or any_printing),
|
||||
'dump_single_json': opts.dump_single_json,
|
||||
'simulate': opts.simulate or any_printing,
|
||||
'skip_download': opts.skip_download,
|
||||
'format': opts.format,
|
||||
'format_limit': opts.format_limit,
|
||||
'listformats': opts.listformats,
|
||||
@ -369,12 +289,10 @@ def _real_main(argv=None):
|
||||
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
|
||||
'encoding': opts.encoding,
|
||||
'exec_cmd': opts.exec_cmd,
|
||||
'extract_flat': opts.extract_flat,
|
||||
}
|
||||
|
||||
with YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.print_debug_header()
|
||||
ydl.add_default_info_extractors()
|
||||
|
||||
# PostProcessors
|
||||
# Add the metadata pp first, the other pps will copy it
|
||||
if opts.addmetadata:
|
||||
|
@ -9,6 +9,7 @@ import shutil
|
||||
import traceback
|
||||
|
||||
from .utils import (
|
||||
compat_expanduser,
|
||||
write_json_file,
|
||||
)
|
||||
|
||||
@ -22,7 +23,7 @@ class Cache(object):
|
||||
if res is None:
|
||||
cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache')
|
||||
res = os.path.join(cache_root, 'youtube-dl')
|
||||
return os.path.expanduser(res)
|
||||
return compat_expanduser(res)
|
||||
|
||||
def _get_cache_fn(self, section, key, dtype):
|
||||
assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
|
||||
|
@ -244,9 +244,16 @@ class F4mFD(FileDownloader):
|
||||
lambda f: int(f[0]) == requested_bitrate, formats))[0]
|
||||
|
||||
base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
|
||||
bootstrap = base64.b64decode(doc.find(_add_ns('bootstrapInfo')).text)
|
||||
bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
|
||||
if bootstrap_node.text is None:
|
||||
bootstrap_url = compat_urlparse.urljoin(
|
||||
base_url, bootstrap_node.attrib['url'])
|
||||
bootstrap = self.ydl.urlopen(bootstrap_url).read()
|
||||
else:
|
||||
bootstrap = base64.b64decode(bootstrap_node.text)
|
||||
metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
|
||||
boot_info = read_bootstrap_info(bootstrap)
|
||||
|
||||
fragments_list = build_fragments_list(boot_info)
|
||||
if self.params.get('test', False):
|
||||
# We only download the first fragment
|
||||
|
@ -20,12 +20,14 @@ from .arte import (
|
||||
ArteTVDDCIE,
|
||||
ArteTVEmbedIE,
|
||||
)
|
||||
from .audiomack import AudiomackIE
|
||||
from .auengine import AUEngineIE
|
||||
from .bambuser import BambuserIE, BambuserChannelIE
|
||||
from .bandcamp import BandcampIE, BandcampAlbumIE
|
||||
from .bbccouk import BBCCoUkIE
|
||||
from .beeg import BeegIE
|
||||
from .behindkink import BehindKinkIE
|
||||
from .bild import BildIE
|
||||
from .bilibili import BiliBiliIE
|
||||
from .blinkx import BlinkxIE
|
||||
from .bliptv import BlipTVIE, BlipTVUserIE
|
||||
@ -60,9 +62,11 @@ from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
|
||||
from .condenast import CondeNastIE
|
||||
from .cracked import CrackedIE
|
||||
from .criterion import CriterionIE
|
||||
from .crunchyroll import CrunchyrollIE
|
||||
from .crunchyroll import (
|
||||
CrunchyrollIE,
|
||||
CrunchyrollShowPlaylistIE
|
||||
)
|
||||
from .cspan import CSpanIE
|
||||
from .d8 import D8IE
|
||||
from .dailymotion import (
|
||||
DailymotionIE,
|
||||
DailymotionPlaylistIE,
|
||||
@ -134,6 +138,8 @@ from .gamestar import GameStarIE
|
||||
from .gametrailers import GametrailersIE
|
||||
from .gdcvault import GDCVaultIE
|
||||
from .generic import GenericIE
|
||||
from .glide import GlideIE
|
||||
from .globo import GloboIE
|
||||
from .godtube import GodTubeIE
|
||||
from .golem import GolemIE
|
||||
from .googleplus import GooglePlusIE
|
||||
@ -172,7 +178,6 @@ from .jadorecettepub import JadoreCettePubIE
|
||||
from .jeuxvideo import JeuxVideoIE
|
||||
from .jove import JoveIE
|
||||
from .jukebox import JukeboxIE
|
||||
from .justintv import JustinTVIE
|
||||
from .jpopsukitv import JpopsukiIE
|
||||
from .kankan import KankanIE
|
||||
from .keezmovies import KeezMoviesIE
|
||||
@ -183,6 +188,7 @@ from .kontrtube import KontrTubeIE
|
||||
from .krasview import KrasViewIE
|
||||
from .ku6 import Ku6IE
|
||||
from .la7 import LA7IE
|
||||
from .laola1tv import Laola1TvIE
|
||||
from .lifenews import LifeNewsIE
|
||||
from .liveleak import LiveLeakIE
|
||||
from .livestream import (
|
||||
@ -245,7 +251,7 @@ from .newstube import NewstubeIE
|
||||
from .nfb import NFBIE
|
||||
from .nfl import NFLIE
|
||||
from .nhl import NHLIE, NHLVideocenterIE
|
||||
from .niconico import NiconicoIE
|
||||
from .niconico import NiconicoIE, NiconicoPlaylistIE
|
||||
from .ninegag import NineGagIE
|
||||
from .noco import NocoIE
|
||||
from .normalboots import NormalbootsIE
|
||||
@ -274,7 +280,9 @@ from .orf import (
|
||||
from .parliamentliveuk import ParliamentLiveUKIE
|
||||
from .patreon import PatreonIE
|
||||
from .pbs import PBSIE
|
||||
from .phoenix import PhoenixIE
|
||||
from .photobucket import PhotobucketIE
|
||||
from .planetaplay import PlanetaPlayIE
|
||||
from .played import PlayedIE
|
||||
from .playfm import PlayFMIE
|
||||
from .playvid import PlayvidIE
|
||||
@ -286,6 +294,7 @@ from .pornoxo import PornoXOIE
|
||||
from .promptfile import PromptFileIE
|
||||
from .prosiebensat1 import ProSiebenSat1IE
|
||||
from .pyvideo import PyvideoIE
|
||||
from .quickvid import QuickVidIE
|
||||
from .radiofrance import RadioFranceIE
|
||||
from .rai import RaiIE
|
||||
from .rbmaradio import RBMARadioIE
|
||||
@ -314,6 +323,7 @@ from .sbs import SBSIE
|
||||
from .scivee import SciVeeIE
|
||||
from .screencast import ScreencastIE
|
||||
from .servingsys import ServingSysIE
|
||||
from .sexykarma import SexyKarmaIE
|
||||
from .shared import SharedIE
|
||||
from .sharesix import ShareSixIE
|
||||
from .sina import SinaIE
|
||||
@ -345,7 +355,9 @@ from .spiegel import SpiegelIE, SpiegelArticleIE
|
||||
from .spiegeltv import SpiegeltvIE
|
||||
from .spike import SpikeIE
|
||||
from .sport5 import Sport5IE
|
||||
from .sportbox import SportBoxIE
|
||||
from .sportdeutschland import SportDeutschlandIE
|
||||
from .srmediathek import SRMediathekIE
|
||||
from .stanfordoc import StanfordOpenClassroomIE
|
||||
from .steam import SteamIE
|
||||
from .streamcloud import StreamcloudIE
|
||||
@ -364,11 +376,14 @@ from .teachingchannel import TeachingChannelIE
|
||||
from .teamcoco import TeamcocoIE
|
||||
from .techtalks import TechTalksIE
|
||||
from .ted import TEDIE
|
||||
from .telecinco import TelecincoIE
|
||||
from .telemb import TeleMBIE
|
||||
from .tenplay import TenPlayIE
|
||||
from .testurl import TestURLIE
|
||||
from .tf1 import TF1IE
|
||||
from .theonion import TheOnionIE
|
||||
from .theplatform import ThePlatformIE
|
||||
from .thesixtyone import TheSixtyOneIE
|
||||
from .thisav import ThisAVIE
|
||||
from .tinypic import TinyPicIE
|
||||
from .tlc import TlcIE, TlcDeIE
|
||||
@ -390,6 +405,7 @@ from .tutv import TutvIE
|
||||
from .tvigle import TvigleIE
|
||||
from .tvp import TvpIE
|
||||
from .tvplay import TVPlayIE
|
||||
from .twitch import TwitchIE
|
||||
from .ubu import UbuIE
|
||||
from .udemy import (
|
||||
UdemyIE,
|
||||
@ -415,6 +431,7 @@ from .videopremium import VideoPremiumIE
|
||||
from .videott import VideoTtIE
|
||||
from .videoweed import VideoWeedIE
|
||||
from .vidme import VidmeIE
|
||||
from .vidzi import VidziIE
|
||||
from .vimeo import (
|
||||
VimeoIE,
|
||||
VimeoAlbumIE,
|
||||
@ -434,9 +451,11 @@ from .viki import VikiIE
|
||||
from .vk import VKIE
|
||||
from .vodlocker import VodlockerIE
|
||||
from .vporn import VpornIE
|
||||
from .vrt import VRTIE
|
||||
from .vube import VubeIE
|
||||
from .vuclip import VuClipIE
|
||||
from .vulture import VultureIE
|
||||
from .walla import WallaIE
|
||||
from .washingtonpost import WashingtonPostIE
|
||||
from .wat import WatIE
|
||||
from .wayofthemaster import WayOfTheMasterIE
|
||||
@ -458,7 +477,6 @@ from .xvideos import XVideosIE
|
||||
from .xtube import XTubeUserIE, XTubeIE
|
||||
from .yahoo import (
|
||||
YahooIE,
|
||||
YahooNewsIE,
|
||||
YahooSearchIE,
|
||||
)
|
||||
from .ynet import YnetIE
|
||||
@ -483,10 +501,8 @@ from .youtube import (
|
||||
YoutubeUserIE,
|
||||
YoutubeWatchLaterIE,
|
||||
)
|
||||
|
||||
from .zdf import ZDFIE
|
||||
|
||||
|
||||
_ALL_CLASSES = [
|
||||
klass
|
||||
for name, klass in globals().items()
|
||||
|
@ -4,6 +4,7 @@ from __future__ import unicode_literals
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .generic import GenericIE
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
@ -12,6 +13,7 @@ from ..utils import (
|
||||
parse_duration,
|
||||
unified_strdate,
|
||||
xpath_text,
|
||||
parse_xml,
|
||||
)
|
||||
|
||||
|
||||
@ -54,6 +56,11 @@ class ARDMediathekIE(InfoExtractor):
|
||||
if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
|
||||
raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
|
||||
|
||||
if re.search(r'[\?&]rss($|[=&])', url):
|
||||
doc = parse_xml(webpage)
|
||||
if doc.tag == 'rss':
|
||||
return GenericIE()._extract_rss(url, video_id, doc)
|
||||
|
||||
title = self._html_search_regex(
|
||||
[r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
|
||||
r'<meta name="dcterms.title" content="(.*?)"/>',
|
||||
|
@ -10,8 +10,8 @@ from ..utils import (
|
||||
unified_strdate,
|
||||
determine_ext,
|
||||
get_element_by_id,
|
||||
compat_str,
|
||||
get_element_by_attribute,
|
||||
int_or_none,
|
||||
)
|
||||
|
||||
# There are different sources of video in arte.tv, the extraction process
|
||||
@ -90,15 +90,24 @@ class ArteTVPlus7IE(InfoExtractor):
|
||||
if not upload_date_str:
|
||||
upload_date_str = player_info.get('VDA', '').split(' ')[0]
|
||||
|
||||
title = player_info['VTI'].strip()
|
||||
subtitle = player_info.get('VSU', '').strip()
|
||||
if subtitle:
|
||||
title += ' - %s' % subtitle
|
||||
|
||||
info_dict = {
|
||||
'id': player_info['VID'],
|
||||
'title': player_info['VTI'],
|
||||
'title': title,
|
||||
'description': player_info.get('VDE'),
|
||||
'upload_date': unified_strdate(upload_date_str),
|
||||
'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
|
||||
}
|
||||
|
||||
all_formats = player_info['VSR'].values()
|
||||
all_formats = []
|
||||
for format_id, format_dict in player_info['VSR'].items():
|
||||
fmt = dict(format_dict)
|
||||
fmt['format_id'] = format_id
|
||||
all_formats.append(fmt)
|
||||
# Some formats use the m3u8 protocol
|
||||
all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
|
||||
def _match_lang(f):
|
||||
@ -149,22 +158,12 @@ class ArteTVPlus7IE(InfoExtractor):
|
||||
)
|
||||
formats = sorted(formats, key=sort_key)
|
||||
def _format(format_info):
|
||||
quality = ''
|
||||
height = format_info.get('height')
|
||||
if height is not None:
|
||||
quality = compat_str(height)
|
||||
bitrate = format_info.get('bitrate')
|
||||
if bitrate is not None:
|
||||
quality += '-%d' % bitrate
|
||||
if format_info.get('versionCode') is not None:
|
||||
format_id = '%s-%s' % (quality, format_info['versionCode'])
|
||||
else:
|
||||
format_id = quality
|
||||
info = {
|
||||
'format_id': format_id,
|
||||
'format_note': format_info.get('versionLibelle'),
|
||||
'width': format_info.get('width'),
|
||||
'height': height,
|
||||
'format_id': format_info['format_id'],
|
||||
'format_note': '%s, %s' % (format_info.get('versionCode'), format_info.get('versionLibelle')),
|
||||
'width': int_or_none(format_info.get('width')),
|
||||
'height': int_or_none(format_info.get('height')),
|
||||
'tbr': int_or_none(format_info.get('bitrate')),
|
||||
}
|
||||
if format_info['mediaType'] == 'rtmp':
|
||||
info['url'] = format_info['streamer']
|
||||
|
69
youtube_dl/extractor/audiomack.py
Normal file
69
youtube_dl/extractor/audiomack.py
Normal file
@ -0,0 +1,69 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .soundcloud import SoundcloudIE
|
||||
from ..utils import ExtractorError
|
||||
|
||||
import time
|
||||
|
||||
|
||||
class AudiomackIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
|
||||
IE_NAME = 'audiomack'
|
||||
_TESTS = [
|
||||
#hosted on audiomack
|
||||
{
|
||||
'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
|
||||
'info_dict':
|
||||
{
|
||||
'id' : 'roosh-williams/extraordinary',
|
||||
'ext': 'mp3',
|
||||
'title': 'Roosh Williams - Extraordinary'
|
||||
}
|
||||
},
|
||||
#hosted on soundcloud via audiomack
|
||||
{
|
||||
'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
|
||||
'file': '172419696.mp3',
|
||||
'info_dict':
|
||||
{
|
||||
'ext': 'mp3',
|
||||
'title': 'Young Thug ft Lil Wayne - Take Kare',
|
||||
"upload_date": "20141016",
|
||||
"description": "New track produced by London On Da Track called “Take Kare\"\n\nhttp://instagram.com/theyoungthugworld\nhttps://www.facebook.com/ThuggerThuggerCashMoney\n",
|
||||
"uploader": "Young Thug World"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
api_response = self._download_json(
|
||||
"http://www.audiomack.com/api/music/url/song/%s?_=%d" % (
|
||||
video_id, time.time()),
|
||||
video_id)
|
||||
|
||||
if "url" not in api_response:
|
||||
raise ExtractorError("Unable to deduce api url of song")
|
||||
realurl = api_response["url"]
|
||||
|
||||
#Audiomack wraps a lot of soundcloud tracks in their branded wrapper
|
||||
# - if so, pass the work off to the soundcloud extractor
|
||||
if SoundcloudIE.suitable(realurl):
|
||||
return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
artist = self._html_search_regex(
|
||||
r'<span class="artist">(.*?)</span>', webpage, "artist")
|
||||
songtitle = self._html_search_regex(
|
||||
r'<h1 class="profile-title song-title"><span class="artist">.*?</span>(.*?)</h1>',
|
||||
webpage, "title")
|
||||
title = artist + " - " + songtitle
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'url': realurl,
|
||||
}
|
@ -24,8 +24,7 @@ class AUEngineIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', webpage, 'title')
|
||||
|
39
youtube_dl/extractor/bild.py
Normal file
39
youtube_dl/extractor/bild.py
Normal file
@ -0,0 +1,39 @@
|
||||
#coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none
|
||||
|
||||
|
||||
class BildIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+)(?:,auto=true)?\.bild\.html'
|
||||
IE_DESC = 'Bild.de'
|
||||
_TEST = {
|
||||
'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
|
||||
'md5': 'dd495cbd99f2413502a1713a1156ac8a',
|
||||
'info_dict': {
|
||||
'id': '38184146',
|
||||
'ext': 'mp4',
|
||||
'title': 'BILD hat sie getestet',
|
||||
'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
|
||||
'duration': 196,
|
||||
'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
|
||||
doc = self._download_xml(xml_url, video_id)
|
||||
|
||||
duration = int_or_none(doc.attrib.get('duration'), scale=1000)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': doc.attrib['ueberschrift'],
|
||||
'description': doc.attrib.get('text'),
|
||||
'url': doc.attrib['src'],
|
||||
'thumbnail': doc.attrib.get('img'),
|
||||
'duration': duration,
|
||||
}
|
@ -1,8 +1,6 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
|
@ -4,37 +4,61 @@ import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
parse_age_limit,
|
||||
)
|
||||
|
||||
|
||||
class BreakIE(InfoExtractor):
|
||||
_VALID_URL = r'http://(?:www\.)?break\.com/video/([^/]+)'
|
||||
_TEST = {
|
||||
_VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
|
||||
'md5': 'a3513fb1547fba4fb6cfac1bffc6c46b',
|
||||
'md5': '33aa4ff477ecd124d18d7b5d23b87ce5',
|
||||
'info_dict': {
|
||||
'id': '2468056',
|
||||
'ext': 'mp4',
|
||||
'title': 'When Girls Act Like D-Bags',
|
||||
}
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group(1).split("-")[-1]
|
||||
embed_url = 'http://www.break.com/embed/%s' % video_id
|
||||
webpage = self._download_webpage(embed_url, video_id)
|
||||
info_json = self._search_regex(r'var embedVars = ({.*})\s*?</script>',
|
||||
webpage, 'info json', flags=re.DOTALL)
|
||||
info = json.loads(info_json)
|
||||
video_url = info['videoUri']
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(
|
||||
'http://www.break.com/embed/%s' % video_id, video_id)
|
||||
info = json.loads(self._search_regex(
|
||||
r'var embedVars = ({.*})\s*?</script>',
|
||||
webpage, 'info json', flags=re.DOTALL))
|
||||
|
||||
youtube_id = info.get('youtubeId')
|
||||
if youtube_id:
|
||||
return self.url_result(youtube_id, 'Youtube')
|
||||
|
||||
final_url = video_url + '?' + info['AuthToken']
|
||||
formats = [{
|
||||
'url': media['uri'] + '?' + info['AuthToken'],
|
||||
'tbr': media['bitRate'],
|
||||
'width': media['width'],
|
||||
'height': media['height'],
|
||||
} for media in info['media']]
|
||||
|
||||
if not formats:
|
||||
formats.append({
|
||||
'url': info['videoUri']
|
||||
})
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
duration = int_or_none(info.get('videoLengthInSeconds'))
|
||||
age_limit = parse_age_limit(info.get('audienceRating'))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': final_url,
|
||||
'title': info['contentName'],
|
||||
'thumbnail': info['thumbUri'],
|
||||
'duration': duration,
|
||||
'age_limit': age_limit,
|
||||
'formats': formats,
|
||||
}
|
||||
|
@ -87,6 +87,15 @@ class BrightcoveIE(InfoExtractor):
|
||||
'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
|
||||
},
|
||||
},
|
||||
{
|
||||
# playlist test
|
||||
# from http://support.brightcove.com/en/video-cloud/docs/playlist-support-single-video-players
|
||||
'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
|
||||
'info_dict': {
|
||||
'title': 'Sealife',
|
||||
},
|
||||
'playlist_mincount': 7,
|
||||
},
|
||||
]
|
||||
|
||||
@classmethod
|
||||
|
@ -7,15 +7,21 @@ from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
unified_strdate,
|
||||
url_basename,
|
||||
qualities,
|
||||
)
|
||||
|
||||
|
||||
class CanalplusIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
|
||||
_VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
|
||||
IE_NAME = 'canalplus.fr'
|
||||
IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
|
||||
_VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
|
||||
_VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
|
||||
_SITE_ID_MAP = {
|
||||
'canalplus.fr': 'cplus',
|
||||
'piwiplus.fr': 'teletoon',
|
||||
'd8.tv': 'd8',
|
||||
}
|
||||
|
||||
_TEST = {
|
||||
_TESTS = [{
|
||||
'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
|
||||
'md5': '3db39fb48b9685438ecf33a1078023e4',
|
||||
'info_dict': {
|
||||
@ -25,36 +31,73 @@ class CanalplusIE(InfoExtractor):
|
||||
'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
|
||||
'upload_date': '20130826',
|
||||
},
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
|
||||
'info_dict': {
|
||||
'id': '1108190',
|
||||
'ext': 'flv',
|
||||
'title': 'Le labyrinthe - Boing super ranger',
|
||||
'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
|
||||
'upload_date': '20140724',
|
||||
},
|
||||
'skip': 'Only works from France',
|
||||
}, {
|
||||
'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
|
||||
'info_dict': {
|
||||
'id': '966289',
|
||||
'ext': 'flv',
|
||||
'title': 'Campagne intime - Documentaire exceptionnel',
|
||||
'description': 'md5:d2643b799fb190846ae09c61e59a859f',
|
||||
'upload_date': '20131108',
|
||||
},
|
||||
'skip': 'videos get deleted after a while',
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.groupdict().get('id')
|
||||
|
||||
site_id = self._SITE_ID_MAP[mobj.group('site') or 'canal']
|
||||
|
||||
# Beware, some subclasses do not define an id group
|
||||
display_id = url_basename(mobj.group('path'))
|
||||
|
||||
if video_id is None:
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
video_id = self._search_regex(r'<canal:player videoId="(\d+)"', webpage, 'video id')
|
||||
video_id = self._search_regex(
|
||||
r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
|
||||
|
||||
info_url = self._VIDEO_INFO_TEMPLATE % video_id
|
||||
info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
|
||||
doc = self._download_xml(info_url, video_id, 'Downloading video XML')
|
||||
|
||||
video_info = [video for video in doc if video.find('ID').text == video_id][0]
|
||||
media = video_info.find('MEDIA')
|
||||
infos = video_info.find('INFOS')
|
||||
|
||||
preferences = ['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS']
|
||||
preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
|
||||
|
||||
formats = [
|
||||
{
|
||||
'url': fmt.text + '?hdcore=2.11.3' if fmt.tag == 'HDS' else fmt.text,
|
||||
'format_id': fmt.tag,
|
||||
'ext': 'mp4' if fmt.tag == 'HLS' else 'flv',
|
||||
'preference': preferences.index(fmt.tag) if fmt.tag in preferences else -1,
|
||||
} for fmt in media.find('VIDEOS') if fmt.text
|
||||
]
|
||||
formats = []
|
||||
for fmt in media.find('VIDEOS'):
|
||||
format_url = fmt.text
|
||||
if not format_url:
|
||||
continue
|
||||
format_id = fmt.tag
|
||||
if format_id == 'HLS':
|
||||
hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
|
||||
for fmt in hls_formats:
|
||||
fmt['preference'] = preference(format_id)
|
||||
formats.extend(hls_formats)
|
||||
elif format_id == 'HDS':
|
||||
hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
|
||||
for fmt in hds_formats:
|
||||
fmt['preference'] = preference(format_id)
|
||||
formats.extend(hds_formats)
|
||||
else:
|
||||
formats.append({
|
||||
'url': format_url,
|
||||
'format_id': format_id,
|
||||
'preference': preference(format_id),
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
|
@ -42,7 +42,7 @@ class CinemassacreIE(InfoExtractor):
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
|
||||
mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?id=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
|
||||
mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
|
||||
if not mobj:
|
||||
raise ExtractorError('Can\'t extract embed url and video id')
|
||||
playerdata_url = mobj.group('embed_url')
|
||||
@ -53,17 +53,19 @@ class CinemassacreIE(InfoExtractor):
|
||||
video_description = self._html_search_regex(
|
||||
r'<div class="entry-content">(?P<description>.+?)</div>',
|
||||
webpage, 'description', flags=re.DOTALL, fatal=False)
|
||||
video_thumbnail = self._og_search_thumbnail(webpage)
|
||||
|
||||
playerdata = self._download_webpage(playerdata_url, video_id, 'Downloading player webpage')
|
||||
video_thumbnail = self._search_regex(
|
||||
r'image: \'(?P<thumbnail>[^\']+)\'', playerdata, 'thumbnail', fatal=False)
|
||||
sd_url = self._search_regex(r'file: \'([^\']+)\', label: \'SD\'', playerdata, 'sd_file')
|
||||
videolist_url = self._search_regex(r'file: \'([^\']+\.smil)\'}', playerdata, 'videolist_url')
|
||||
|
||||
vidurl = self._search_regex(
|
||||
r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
|
||||
|
||||
videolist_url = self._search_regex(
|
||||
r"file\s*:\s*'(http.+?/jwplayer\.smil)'", playerdata, 'jwplayer.smil')
|
||||
videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
|
||||
|
||||
formats = []
|
||||
baseurl = sd_url[:sd_url.rfind('/')+1]
|
||||
baseurl = vidurl[:vidurl.rfind('/')+1]
|
||||
for video in videolist.findall('.//video'):
|
||||
src = video.get('src')
|
||||
if not src:
|
||||
|
@ -4,7 +4,6 @@ import json
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none
|
||||
|
||||
|
||||
_translation_table = {
|
||||
@ -39,9 +38,7 @@ class CliphunterIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_title = self._search_regex(
|
||||
|
@ -12,7 +12,7 @@ from ..utils import (
|
||||
|
||||
class CNNIE(InfoExtractor):
|
||||
_VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/
|
||||
(?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))'''
|
||||
(?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(-ap)?|(?=&)))'''
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
|
||||
|
@ -72,6 +72,7 @@ class InfoExtractor(object):
|
||||
* acodec Name of the audio codec in use
|
||||
* asr Audio sampling rate in Hertz
|
||||
* vbr Average video bitrate in KBit/s
|
||||
* fps Frame rate
|
||||
* vcodec Name of the video codec in use
|
||||
* container Name of the container format
|
||||
* filesize The number of bytes, if known in advance
|
||||
@ -89,6 +90,10 @@ class InfoExtractor(object):
|
||||
format, irrespective of the file format.
|
||||
-1 for default (order by other properties),
|
||||
-2 or smaller for less than default.
|
||||
* source_preference Order number for this video source
|
||||
(quality takes higher priority)
|
||||
-1 for default (order by other properties),
|
||||
-2 or smaller for less than default.
|
||||
* http_referer HTTP Referer header value to set.
|
||||
* http_method HTTP method to use for the download.
|
||||
* http_headers A dictionary of additional HTTP headers
|
||||
@ -138,6 +143,8 @@ class InfoExtractor(object):
|
||||
|
||||
Unless mentioned otherwise, the fields should be Unicode strings.
|
||||
|
||||
Unless mentioned otherwise, None is equivalent to absence of information.
|
||||
|
||||
Subclasses of this one should re-define the _real_initialize() and
|
||||
_real_extract() methods and define a _VALID_URL regexp.
|
||||
Probably, they should also be added to the list of extractors.
|
||||
@ -236,7 +243,6 @@ class InfoExtractor(object):
|
||||
|
||||
def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
|
||||
""" Returns a tuple (page content as string, URL handle) """
|
||||
|
||||
# Strip hashes from the URL (#1038)
|
||||
if isinstance(url_or_request, (compat_str, str)):
|
||||
url_or_request = url_or_request.partition('#')[0]
|
||||
@ -245,6 +251,10 @@ class InfoExtractor(object):
|
||||
if urlh is False:
|
||||
assert not fatal
|
||||
return False
|
||||
content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
|
||||
return (content, urlh)
|
||||
|
||||
def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True):
|
||||
content_type = urlh.headers.get('Content-Type', '')
|
||||
webpage_bytes = urlh.read()
|
||||
m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
|
||||
@ -279,6 +289,12 @@ class InfoExtractor(object):
|
||||
raw_filename = basen + '.dump'
|
||||
filename = sanitize_filename(raw_filename, restricted=True)
|
||||
self.to_screen('Saving request to ' + filename)
|
||||
# Working around MAX_PATH limitation on Windows (see
|
||||
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
|
||||
if os.name == 'nt':
|
||||
absfilepath = os.path.abspath(filename)
|
||||
if len(absfilepath) > 259:
|
||||
filename = '\\\\?\\' + absfilepath
|
||||
with open(filename, 'wb') as outf:
|
||||
outf.write(webpage_bytes)
|
||||
|
||||
@ -297,7 +313,7 @@ class InfoExtractor(object):
|
||||
msg += ' Visit %s for more details' % blocked_iframe
|
||||
raise ExtractorError(msg, expected=True)
|
||||
|
||||
return (content, urlh)
|
||||
return content
|
||||
|
||||
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
|
||||
""" Returns the data of the page as a string """
|
||||
@ -603,14 +619,16 @@ class InfoExtractor(object):
|
||||
f.get('vbr') if f.get('vbr') is not None else -1,
|
||||
f.get('abr') if f.get('abr') is not None else -1,
|
||||
audio_ext_preference,
|
||||
f.get('fps') if f.get('fps') is not None else -1,
|
||||
f.get('filesize') if f.get('filesize') is not None else -1,
|
||||
f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
|
||||
f.get('source_preference') if f.get('source_preference') is not None else -1,
|
||||
f.get('format_id'),
|
||||
)
|
||||
formats.sort(key=_formats_key)
|
||||
|
||||
def http_scheme(self):
|
||||
""" Either "https:" or "https:", depending on the user's preferences """
|
||||
""" Either "http:" or "https:", depending on the user's preferences """
|
||||
return (
|
||||
'http:'
|
||||
if self._downloader.params.get('prefer_insecure', False)
|
||||
@ -673,7 +691,10 @@ class InfoExtractor(object):
|
||||
if re.match(r'^https?://', u)
|
||||
else compat_urlparse.urljoin(m3u8_url, u))
|
||||
|
||||
m3u8_doc = self._download_webpage(m3u8_url, video_id)
|
||||
m3u8_doc = self._download_webpage(
|
||||
m3u8_url, video_id,
|
||||
note='Downloading m3u8 information',
|
||||
errnote='Failed to download m3u8 information')
|
||||
last_info = None
|
||||
kv_rex = re.compile(
|
||||
r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
|
||||
|
@ -34,6 +34,8 @@ class CondeNastIE(InfoExtractor):
|
||||
_VALID_URL = r'http://(video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
|
||||
IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
|
||||
|
||||
EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed)/.+?' % '|'.join(_SITES.keys())
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
|
||||
'md5': '1921f713ed48aabd715691f774c451f7',
|
||||
|
@ -24,6 +24,7 @@ from ..aes import (
|
||||
aes_cbc_decrypt,
|
||||
inc,
|
||||
)
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class CrunchyrollIE(SubtitlesInfoExtractor):
|
||||
@ -39,6 +40,7 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
|
||||
'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
|
||||
'uploader': 'Yomiuri Telecasting Corporation (YTV)',
|
||||
'upload_date': '20131013',
|
||||
'url': 're:(?!.*&)',
|
||||
},
|
||||
'params': {
|
||||
# rtmp
|
||||
@ -107,19 +109,17 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
|
||||
decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
|
||||
return zlib.decompress(decrypted_data)
|
||||
|
||||
def _convert_subtitles_to_srt(self, subtitles):
|
||||
def _convert_subtitles_to_srt(self, sub_root):
|
||||
output = ''
|
||||
for i, (start, end, text) in enumerate(re.findall(r'<event [^>]*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles), 1):
|
||||
start = start.replace('.', ',')
|
||||
end = end.replace('.', ',')
|
||||
text = clean_html(text)
|
||||
text = text.replace('\\N', '\n')
|
||||
if not text:
|
||||
continue
|
||||
|
||||
for i, event in enumerate(sub_root.findall('./events/event'), 1):
|
||||
start = event.attrib['start'].replace('.', ',')
|
||||
end = event.attrib['end'].replace('.', ',')
|
||||
text = event.attrib['text'].replace('\\N', '\n')
|
||||
output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
|
||||
return output
|
||||
|
||||
def _convert_subtitles_to_ass(self, subtitles):
|
||||
def _convert_subtitles_to_ass(self, sub_root):
|
||||
output = ''
|
||||
|
||||
def ass_bool(strvalue):
|
||||
@ -128,10 +128,6 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
|
||||
assvalue = '-1'
|
||||
return assvalue
|
||||
|
||||
sub_root = xml.etree.ElementTree.fromstring(subtitles)
|
||||
if not sub_root:
|
||||
return output
|
||||
|
||||
output = '[Script Info]\n'
|
||||
output += 'Title: %s\n' % sub_root.attrib["title"]
|
||||
output += 'ScriptType: v4.00+\n'
|
||||
@ -237,12 +233,14 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
||||
streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+'&media%5Fid='+stream_id+'&video%5Fformat='+stream_format
|
||||
streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
||||
streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
|
||||
streamdata = self._download_webpage(streamdata_req, video_id, note='Downloading media info for '+video_format)
|
||||
video_url = self._search_regex(r'<host>([^<]+)', streamdata, 'video_url')
|
||||
video_play_path = self._search_regex(r'<file>([^<]+)', streamdata, 'video_play_path')
|
||||
streamdata = self._download_xml(
|
||||
streamdata_req, video_id,
|
||||
note='Downloading media info for %s' % video_format)
|
||||
video_url = streamdata.find('.//host').text
|
||||
video_play_path = streamdata.find('.//file').text
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'play_path': video_play_path,
|
||||
'play_path': video_play_path,
|
||||
'ext': 'flv',
|
||||
'format': video_format,
|
||||
'format_id': video_format,
|
||||
@ -266,10 +264,13 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
||||
lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
|
||||
if not lang_code:
|
||||
continue
|
||||
sub_root = xml.etree.ElementTree.fromstring(subtitle)
|
||||
if not sub_root:
|
||||
subtitles[lang_code] = ''
|
||||
if sub_format == 'ass':
|
||||
subtitles[lang_code] = self._convert_subtitles_to_ass(subtitle)
|
||||
subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
|
||||
else:
|
||||
subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
|
||||
subtitles[lang_code] = self._convert_subtitles_to_srt(sub_root)
|
||||
|
||||
if self._downloader.params.get('listsubtitles', False):
|
||||
self._list_available_subtitles(video_id, subtitles)
|
||||
@ -285,3 +286,40 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
|
||||
'subtitles': subtitles,
|
||||
'formats': formats,
|
||||
}
|
||||
|
||||
|
||||
class CrunchyrollShowPlaylistIE(InfoExtractor):
|
||||
IE_NAME = "crunchyroll:playlist"
|
||||
_VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
|
||||
'info_dict': {
|
||||
'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
|
||||
'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
|
||||
},
|
||||
'playlist_count': 13,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
show_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, show_id)
|
||||
title = self._html_search_regex(
|
||||
r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
|
||||
webpage, 'title')
|
||||
episode_paths = re.findall(
|
||||
r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"',
|
||||
webpage)
|
||||
entries = [
|
||||
self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll')
|
||||
for ep in episode_paths
|
||||
]
|
||||
entries.reverse()
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': show_id,
|
||||
'title': title,
|
||||
'entries': entries,
|
||||
}
|
||||
|
@ -1,25 +0,0 @@
|
||||
# encoding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .canalplus import CanalplusIE
|
||||
|
||||
|
||||
class D8IE(CanalplusIE):
|
||||
_VALID_URL = r'https?://www\.d8\.tv/.*?/(?P<path>.*)'
|
||||
_VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s'
|
||||
IE_NAME = 'd8.tv'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
|
||||
'file': '966289.flv',
|
||||
'info_dict': {
|
||||
'title': 'Campagne intime - Documentaire exceptionnel',
|
||||
'description': 'md5:d2643b799fb190846ae09c61e59a859f',
|
||||
'upload_date': '20131108',
|
||||
},
|
||||
'params': {
|
||||
# rtmp
|
||||
'skip_download': True,
|
||||
},
|
||||
'skip': 'videos get deleted after a while',
|
||||
}
|
@ -82,11 +82,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Extract id and simplified title from URL
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
|
||||
video_id = self._match_id(url)
|
||||
url = 'http://www.dailymotion.com/video/%s' % video_id
|
||||
|
||||
# Retrieve video webpage to extract further information
|
||||
@ -147,18 +143,23 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||
self._list_available_subtitles(video_id, webpage)
|
||||
return
|
||||
|
||||
view_count = self._search_regex(
|
||||
r'video_views_count[^>]+>\s+([\d\.,]+)', webpage, 'view count', fatal=False)
|
||||
if view_count is not None:
|
||||
view_count = str_to_int(view_count)
|
||||
view_count = str_to_int(self._search_regex(
|
||||
r'video_views_count[^>]+>\s+([\d\.,]+)',
|
||||
webpage, 'view count', fatal=False))
|
||||
|
||||
title = self._og_search_title(webpage, default=None)
|
||||
if title is None:
|
||||
title = self._html_search_regex(
|
||||
r'(?s)<span\s+id="video_title"[^>]*>(.*?)</span>', webpage,
|
||||
'title')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'id': video_id,
|
||||
'formats': formats,
|
||||
'uploader': info['owner.screenname'],
|
||||
'upload_date': video_upload_date,
|
||||
'title': self._og_search_title(webpage),
|
||||
'subtitles': video_subtitles,
|
||||
'upload_date': video_upload_date,
|
||||
'title': title,
|
||||
'subtitles': video_subtitles,
|
||||
'thumbnail': info['thumbnail_url'],
|
||||
'age_limit': age_limit,
|
||||
'view_count': view_count,
|
||||
|
@ -1,7 +1,5 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .subtitles import SubtitlesInfoExtractor
|
||||
from .common import ExtractorError
|
||||
from ..utils import parse_iso8601
|
||||
@ -25,8 +23,7 @@ class DRTVIE(SubtitlesInfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
programcard = self._download_json(
|
||||
'http://www.dr.dk/mu/programcard/expanded/%s' % video_id, video_id, 'Downloading video JSON')
|
||||
@ -35,7 +32,7 @@ class DRTVIE(SubtitlesInfoExtractor):
|
||||
|
||||
title = data['Title']
|
||||
description = data['Description']
|
||||
timestamp = parse_iso8601(data['CreatedTime'][:-5])
|
||||
timestamp = parse_iso8601(data['CreatedTime'])
|
||||
|
||||
thumbnail = None
|
||||
duration = None
|
||||
|
@ -1,49 +1,48 @@
|
||||
# encoding: utf-8
|
||||
import re
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
)
|
||||
|
||||
|
||||
class FazIE(InfoExtractor):
|
||||
IE_NAME = u'faz.net'
|
||||
IE_NAME = 'faz.net'
|
||||
_VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+)\.html'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
|
||||
u'file': u'12610585.mp4',
|
||||
u'info_dict': {
|
||||
u'title': u'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
|
||||
u'description': u'md5:1453fbf9a0d041d985a47306192ea253',
|
||||
'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
|
||||
'info_dict': {
|
||||
'id': '12610585',
|
||||
'ext': 'mp4',
|
||||
'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
|
||||
'description': 'md5:1453fbf9a0d041d985a47306192ea253',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
self.to_screen(video_id)
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
config_xml_url = self._search_regex(r'writeFLV\(\'(.+?)\',', webpage,
|
||||
u'config xml url')
|
||||
config = self._download_xml(config_xml_url, video_id,
|
||||
u'Downloading config xml')
|
||||
config_xml_url = self._search_regex(
|
||||
r'writeFLV\(\'(.+?)\',', webpage, 'config xml url')
|
||||
config = self._download_xml(
|
||||
config_xml_url, video_id, 'Downloading config xml')
|
||||
|
||||
encodings = config.find('ENCODINGS')
|
||||
formats = []
|
||||
for code in ['LOW', 'HIGH', 'HQ']:
|
||||
for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
|
||||
encoding = encodings.find(code)
|
||||
if encoding is None:
|
||||
continue
|
||||
encoding_url = encoding.find('FILENAME').text
|
||||
formats.append({
|
||||
'url': encoding_url,
|
||||
'ext': determine_ext(encoding_url),
|
||||
'format_id': code.lower(),
|
||||
'quality': pref,
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
|
||||
descr = self._html_search_regex(r'<p class="Content Copy">(.*?)</p>', webpage, u'description')
|
||||
descr = self._html_search_regex(
|
||||
r'<p class="Content Copy">(.*?)</p>', webpage, 'description', fatal=False)
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._og_search_title(webpage),
|
||||
|
@ -1,25 +1,27 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import random
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
get_element_by_id,
|
||||
clean_html,
|
||||
)
|
||||
|
||||
|
||||
class FKTVIE(InfoExtractor):
|
||||
IE_NAME = u'fernsehkritik.tv'
|
||||
_VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
|
||||
IE_NAME = 'fernsehkritik.tv'
|
||||
_VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
|
||||
|
||||
_TEST = {
|
||||
u'url': u'http://fernsehkritik.tv/folge-1',
|
||||
u'file': u'00011.flv',
|
||||
u'info_dict': {
|
||||
u'title': u'Folge 1 vom 10. April 2007',
|
||||
u'description': u'md5:fb4818139c7cfe6907d4b83412a6864f',
|
||||
'url': 'http://fernsehkritik.tv/folge-1',
|
||||
'info_dict': {
|
||||
'id': '00011',
|
||||
'ext': 'flv',
|
||||
'title': 'Folge 1 vom 10. April 2007',
|
||||
'description': 'md5:fb4818139c7cfe6907d4b83412a6864f',
|
||||
},
|
||||
}
|
||||
|
||||
@ -32,7 +34,7 @@ class FKTVIE(InfoExtractor):
|
||||
start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
|
||||
episode)
|
||||
playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
|
||||
u'playlist', flags=re.DOTALL)
|
||||
'playlist', flags=re.DOTALL)
|
||||
files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
|
||||
# TODO: return a single multipart video
|
||||
videos = []
|
||||
@ -42,7 +44,6 @@ class FKTVIE(InfoExtractor):
|
||||
videos.append({
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'title': clean_html(get_element_by_id('eptitle', start_webpage)),
|
||||
'description': clean_html(get_element_by_id('contentlist', start_webpage)),
|
||||
'thumbnail': video_thumbnail
|
||||
@ -51,14 +52,15 @@ class FKTVIE(InfoExtractor):
|
||||
|
||||
|
||||
class FKTVPosteckeIE(InfoExtractor):
|
||||
IE_NAME = u'fernsehkritik.tv:postecke'
|
||||
_VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
|
||||
IE_NAME = 'fernsehkritik.tv:postecke'
|
||||
_VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
|
||||
_TEST = {
|
||||
u'url': u'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
|
||||
u'file': u'0120.flv',
|
||||
u'md5': u'262f0adbac80317412f7e57b4808e5c4',
|
||||
u'info_dict': {
|
||||
u"title": u"Postecke 120"
|
||||
'url': 'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
|
||||
'md5': '262f0adbac80317412f7e57b4808e5c4',
|
||||
'info_dict': {
|
||||
'id': '0120',
|
||||
'ext': 'flv',
|
||||
'title': 'Postecke 120',
|
||||
}
|
||||
}
|
||||
|
||||
@ -71,8 +73,7 @@ class FKTVPosteckeIE(InfoExtractor):
|
||||
video_url = 'http://dl%d.fernsehkritik.tv/postecke/postecke%d.flv' % (server, episode)
|
||||
video_title = 'Postecke %d' % episode
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'title': video_title,
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'title': video_title,
|
||||
}
|
||||
|
@ -46,7 +46,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
|
||||
f4m_format['preference'] = 1
|
||||
formats.extend(f4m_formats)
|
||||
elif video_url.endswith('.m3u8'):
|
||||
formats.extend(self._extract_m3u8_formats(video_url, video_id))
|
||||
formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
|
||||
elif video_url.startswith('rtmp'):
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
@ -58,7 +58,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'format_id': format_id,
|
||||
'preference': 2,
|
||||
'preference': -1,
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
|
||||
@ -93,7 +93,6 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
|
||||
'md5': '9cecf35f99c4079c199e9817882a9a1c',
|
||||
'info_dict': {
|
||||
'id': '84981923',
|
||||
'ext': 'flv',
|
||||
|
@ -8,7 +8,7 @@ from ..utils import ExtractorError
|
||||
|
||||
|
||||
class FunnyOrDieIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
|
||||
_VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|articles|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version',
|
||||
'md5': 'bcd81e0c4f26189ee09be362ad6e6ba9',
|
||||
@ -29,6 +29,9 @@ class FunnyOrDieIE(InfoExtractor):
|
||||
'description': 'Please use this to sell something. www.jonlajoie.com',
|
||||
'thumbnail': 're:^http:.*\.jpg$',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://www.funnyordie.com/articles/ebf5e34fc8/10-hours-of-walking-in-nyc-as-a-man',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
@ -37,7 +40,7 @@ class FunnyOrDieIE(InfoExtractor):
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
links = re.findall(r'<source src="([^"]+/v)\d+\.([^"]+)" type=\'video', webpage)
|
||||
links = re.findall(r'<source src="([^"]+/v)[^"]+\.([^"]+)" type=\'video', webpage)
|
||||
if not links:
|
||||
raise ExtractorError('No media links available for %s' % video_id)
|
||||
|
||||
|
@ -28,6 +28,7 @@ from .brightcove import BrightcoveIE
|
||||
from .ooyala import OoyalaIE
|
||||
from .rutv import RUTVIE
|
||||
from .smotri import SmotriIE
|
||||
from .condenast import CondeNastIE
|
||||
|
||||
|
||||
class GenericIE(InfoExtractor):
|
||||
@ -324,7 +325,7 @@ class GenericIE(InfoExtractor):
|
||||
'ext': 'mp4',
|
||||
'age_limit': 18,
|
||||
'uploader': 'www.handjobhub.com',
|
||||
'title': 'Busty Blonde Siri Tit Fuck While Wank at Handjob Hub',
|
||||
'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
|
||||
}
|
||||
},
|
||||
# RSS feed
|
||||
@ -379,6 +380,44 @@ class GenericIE(InfoExtractor):
|
||||
'uploader': 'education-portal.com',
|
||||
},
|
||||
},
|
||||
{
|
||||
'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
|
||||
'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
|
||||
'info_dict': {
|
||||
'id': 'uxjb0lwrcz',
|
||||
'ext': 'mp4',
|
||||
'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
|
||||
'duration': 1715.0,
|
||||
'uploader': 'thoughtworks.wistia.com',
|
||||
},
|
||||
},
|
||||
# Direct download with broken HEAD
|
||||
{
|
||||
'url': 'http://ai-radio.org:8000/radio.opus',
|
||||
'info_dict': {
|
||||
'id': 'radio',
|
||||
'ext': 'opus',
|
||||
'title': 'radio',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True, # infinite live stream
|
||||
},
|
||||
'expected_warnings': [
|
||||
r'501.*Not Implemented'
|
||||
],
|
||||
},
|
||||
# Soundcloud embed
|
||||
{
|
||||
'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
|
||||
'info_dict': {
|
||||
'id': '174391317',
|
||||
'ext': 'mp3',
|
||||
'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
|
||||
'uploader': 'Sophos Security',
|
||||
'title': 'Chet Chat 171 - Oct 29, 2014',
|
||||
'upload_date': '20141029',
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def report_following_redirect(self, new_url):
|
||||
@ -475,7 +514,8 @@ class GenericIE(InfoExtractor):
|
||||
'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
|
||||
) % (url, url), expected=True)
|
||||
else:
|
||||
assert ':' in default_search
|
||||
if ':' not in default_search:
|
||||
default_search += ':'
|
||||
return self.url_result(default_search + url)
|
||||
|
||||
url, smuggled_data = unsmuggle_url(url)
|
||||
@ -490,14 +530,14 @@ class GenericIE(InfoExtractor):
|
||||
self.to_screen('%s: Requesting header' % video_id)
|
||||
|
||||
head_req = HEADRequest(url)
|
||||
response = self._request_webpage(
|
||||
head_response = self._request_webpage(
|
||||
head_req, video_id,
|
||||
note=False, errnote='Could not send HEAD request to %s' % url,
|
||||
fatal=False)
|
||||
|
||||
if response is not False:
|
||||
if head_response is not False:
|
||||
# Check for redirect
|
||||
new_url = response.geturl()
|
||||
new_url = head_response.geturl()
|
||||
if url != new_url:
|
||||
self.report_following_redirect(new_url)
|
||||
if force_videoid:
|
||||
@ -505,34 +545,35 @@ class GenericIE(InfoExtractor):
|
||||
new_url, {'force_videoid': force_videoid})
|
||||
return self.url_result(new_url)
|
||||
|
||||
# Check for direct link to a video
|
||||
content_type = response.headers.get('Content-Type', '')
|
||||
m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
|
||||
if m:
|
||||
upload_date = response.headers.get('Last-Modified')
|
||||
if upload_date:
|
||||
upload_date = unified_strdate(upload_date)
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': os.path.splitext(url_basename(url))[0],
|
||||
'formats': [{
|
||||
'format_id': m.group('format_id'),
|
||||
'url': url,
|
||||
'vcodec': 'none' if m.group('type') == 'audio' else None
|
||||
}],
|
||||
'upload_date': upload_date,
|
||||
}
|
||||
full_response = None
|
||||
if head_response is False:
|
||||
full_response = self._request_webpage(url, video_id)
|
||||
head_response = full_response
|
||||
|
||||
# Check for direct link to a video
|
||||
content_type = head_response.headers.get('Content-Type', '')
|
||||
m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
|
||||
if m:
|
||||
upload_date = unified_strdate(
|
||||
head_response.headers.get('Last-Modified'))
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': os.path.splitext(url_basename(url))[0],
|
||||
'formats': [{
|
||||
'format_id': m.group('format_id'),
|
||||
'url': url,
|
||||
'vcodec': 'none' if m.group('type') == 'audio' else None
|
||||
}],
|
||||
'upload_date': upload_date,
|
||||
}
|
||||
|
||||
if not self._downloader.params.get('test', False) and not is_intentional:
|
||||
self._downloader.report_warning('Falling back on generic information extractor.')
|
||||
|
||||
try:
|
||||
if full_response:
|
||||
webpage = self._webpage_read_content(full_response, url, video_id)
|
||||
else:
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
except ValueError:
|
||||
# since this is the last-resort InfoExtractor, if
|
||||
# this error is thrown, it'll be thrown here
|
||||
raise ExtractorError('Failed to download URL: %s' % url)
|
||||
|
||||
self.report_extraction(video_id)
|
||||
|
||||
# Is it an RSS feed?
|
||||
@ -608,13 +649,13 @@ class GenericIE(InfoExtractor):
|
||||
if mobj:
|
||||
player_url = unescapeHTML(mobj.group('url'))
|
||||
surl = smuggle_url(player_url, {'Referer': url})
|
||||
return self.url_result(surl, 'Vimeo')
|
||||
return self.url_result(surl)
|
||||
|
||||
# Look for embedded (swf embed) Vimeo player
|
||||
mobj = re.search(
|
||||
r'<embed[^>]+?src="(https?://(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
|
||||
r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
|
||||
if mobj:
|
||||
return self.url_result(mobj.group(1), 'Vimeo')
|
||||
return self.url_result(mobj.group(1))
|
||||
|
||||
# Look for embedded YouTube player
|
||||
matches = re.findall(r'''(?x)
|
||||
@ -622,7 +663,8 @@ class GenericIE(InfoExtractor):
|
||||
<iframe[^>]+?src=|
|
||||
data-video-url=|
|
||||
<embed[^>]+?src=|
|
||||
embedSWF\(?:\s*
|
||||
embedSWF\(?:\s*|
|
||||
new\s+SWFObject\(
|
||||
)
|
||||
(["\'])
|
||||
(?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
|
||||
@ -639,19 +681,32 @@ class GenericIE(InfoExtractor):
|
||||
return _playlist_from_matches(
|
||||
matches, lambda m: unescapeHTML(m[1]))
|
||||
|
||||
# Look for embedded Dailymotion playlist player (#3822)
|
||||
m = re.search(
|
||||
r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
|
||||
if m:
|
||||
playlists = re.findall(
|
||||
r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
|
||||
if playlists:
|
||||
return _playlist_from_matches(
|
||||
playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
|
||||
|
||||
# Look for embedded Wistia player
|
||||
match = re.search(
|
||||
r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
|
||||
r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
|
||||
if match:
|
||||
embed_url = self._proto_relative_url(
|
||||
unescapeHTML(match.group('url')))
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
'url': unescapeHTML(match.group('url')),
|
||||
'url': embed_url,
|
||||
'ie_key': 'Wistia',
|
||||
'uploader': video_uploader,
|
||||
'title': video_title,
|
||||
'id': video_id,
|
||||
}
|
||||
match = re.search(r'(?:id=["\']wistia_|data-wistiaid=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
|
||||
|
||||
match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
|
||||
if match:
|
||||
return {
|
||||
'_type': 'url_transparent',
|
||||
@ -795,7 +850,7 @@ class GenericIE(InfoExtractor):
|
||||
|
||||
# Look for embeded soundcloud player
|
||||
mobj = re.search(
|
||||
r'<iframe src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
|
||||
r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
|
||||
webpage)
|
||||
if mobj is not None:
|
||||
url = unescapeHTML(mobj.group('url'))
|
||||
@ -832,55 +887,65 @@ class GenericIE(InfoExtractor):
|
||||
return self.url_result(mobj.group('url'), 'SBS')
|
||||
|
||||
mobj = re.search(
|
||||
r'<iframe[^>]+?src=(["\'])(?P<url>https?://m\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
|
||||
r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
|
||||
webpage)
|
||||
if mobj is not None:
|
||||
return self.url_result(mobj.group('url'), 'MLB')
|
||||
|
||||
mobj = re.search(
|
||||
r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
|
||||
webpage)
|
||||
if mobj is not None:
|
||||
return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
|
||||
|
||||
def check_video(vurl):
|
||||
vpath = compat_urlparse.urlparse(vurl).path
|
||||
vext = determine_ext(vpath)
|
||||
return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
|
||||
|
||||
def filter_video(urls):
|
||||
return list(filter(check_video, urls))
|
||||
|
||||
# Start with something easy: JW Player in SWFObject
|
||||
found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
|
||||
found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
|
||||
if not found:
|
||||
# Look for gorilla-vid style embedding
|
||||
found = re.findall(r'''(?sx)
|
||||
found = filter_video(re.findall(r'''(?sx)
|
||||
(?:
|
||||
jw_plugins|
|
||||
JWPlayerOptions|
|
||||
jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
|
||||
)
|
||||
.*?file\s*:\s*["\'](.*?)["\']''', webpage)
|
||||
.*?file\s*:\s*["\'](.*?)["\']''', webpage))
|
||||
if not found:
|
||||
# Broaden the search a little bit
|
||||
found = re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
|
||||
found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
|
||||
if not found:
|
||||
# Broaden the findall a little bit: JWPlayer JS loader
|
||||
found = re.findall(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
|
||||
found = filter_video(re.findall(
|
||||
r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
|
||||
if not found:
|
||||
# Flow player
|
||||
found = re.findall(r'''(?xs)
|
||||
found = filter_video(re.findall(r'''(?xs)
|
||||
flowplayer\("[^"]+",\s*
|
||||
\{[^}]+?\}\s*,
|
||||
\s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
|
||||
["']?url["']?\s*:\s*["']([^"']+)["']
|
||||
''', webpage)
|
||||
''', webpage))
|
||||
if not found:
|
||||
# Try to find twitter cards info
|
||||
found = re.findall(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
|
||||
found = filter_video(re.findall(
|
||||
r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
|
||||
if not found:
|
||||
# We look for Open Graph info:
|
||||
# We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
|
||||
m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
|
||||
# We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
|
||||
if m_video_type is not None:
|
||||
def check_video(vurl):
|
||||
vpath = compat_urlparse.urlparse(vurl).path
|
||||
vext = determine_ext(vpath)
|
||||
return '.' in vpath and vext not in ('swf', 'png', 'jpg')
|
||||
found = list(filter(
|
||||
check_video,
|
||||
re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)))
|
||||
found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
|
||||
if not found:
|
||||
# HTML5 video
|
||||
found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]+)? src="([^"]+)"', webpage)
|
||||
found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src="([^"]+)"', webpage)
|
||||
if not found:
|
||||
found = re.search(
|
||||
r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
|
||||
|
40
youtube_dl/extractor/glide.py
Normal file
40
youtube_dl/extractor/glide.py
Normal file
@ -0,0 +1,40 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class GlideIE(InfoExtractor):
|
||||
IE_DESC = 'Glide mobile video messages (glide.me)'
|
||||
_VALID_URL = r'https?://share\.glide\.me/(?P<id>[A-Za-z0-9\-=_+]+)'
|
||||
_TEST = {
|
||||
'url': 'http://share.glide.me/UZF8zlmuQbe4mr+7dCiQ0w==',
|
||||
'md5': '4466372687352851af2d131cfaa8a4c7',
|
||||
'info_dict': {
|
||||
'id': 'UZF8zlmuQbe4mr+7dCiQ0w==',
|
||||
'ext': 'mp4',
|
||||
'title': 'Damon Timm\'s Glide message',
|
||||
'thumbnail': 're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
title = self._html_search_regex(
|
||||
r'<title>(.*?)</title>', webpage, 'title')
|
||||
video_url = self.http_scheme() + self._search_regex(
|
||||
r'<source src="(.*?)" type="video/mp4">', webpage, 'video URL')
|
||||
thumbnail_url = self._search_regex(
|
||||
r'<img id="video-thumbnail" src="(.*?)"',
|
||||
webpage, 'thumbnail url', fatal=False)
|
||||
thumbnail = (
|
||||
thumbnail_url if thumbnail_url is None
|
||||
else self.http_scheme() + thumbnail_url)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'url': video_url,
|
||||
'thumbnail': thumbnail,
|
||||
}
|
398
youtube_dl/extractor/globo.py
Normal file
398
youtube_dl/extractor/globo.py
Normal file
@ -0,0 +1,398 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import random
|
||||
import math
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
compat_str,
|
||||
compat_chr,
|
||||
compat_ord,
|
||||
)
|
||||
|
||||
|
||||
class GloboIE(InfoExtractor):
|
||||
_VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
|
||||
|
||||
_API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
|
||||
_SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=2.9.9.50&resource_id=%s'
|
||||
|
||||
_VIDEOID_REGEXES = [
|
||||
r'\bdata-video-id="(\d+)"',
|
||||
r'\bdata-player-videosids="(\d+)"',
|
||||
r'<div[^>]+\bid="(\d+)"',
|
||||
]
|
||||
|
||||
_RESIGN_EXPIRATION = 86400
|
||||
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
|
||||
'md5': '03ebf41cb7ade43581608b7d9b71fab0',
|
||||
'info_dict': {
|
||||
'id': '3654973',
|
||||
'ext': 'mp4',
|
||||
'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
|
||||
'duration': 251.585,
|
||||
'uploader': 'SporTV',
|
||||
'uploader_id': 698,
|
||||
'like_count': int,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
|
||||
'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
|
||||
'info_dict': {
|
||||
'id': '3607726',
|
||||
'ext': 'mp4',
|
||||
'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
|
||||
'duration': 103.204,
|
||||
'uploader': 'Globo.com',
|
||||
'uploader_id': 265,
|
||||
'like_count': int,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
|
||||
'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
|
||||
'info_dict': {
|
||||
'id': '3652183',
|
||||
'ext': 'mp4',
|
||||
'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
|
||||
'duration': 110.711,
|
||||
'uploader': 'Rede Globo',
|
||||
'uploader_id': 196,
|
||||
'like_count': int,
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
class MD5():
|
||||
HEX_FORMAT_LOWERCASE = 0
|
||||
HEX_FORMAT_UPPERCASE = 1
|
||||
BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''
|
||||
BASE64_PAD_CHARACTER_RFC_COMPLIANCE = '='
|
||||
PADDING = '=0xFF01DD'
|
||||
hexcase = 0
|
||||
b64pad = ''
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
class JSArray(list):
|
||||
def __getitem__(self, y):
|
||||
try:
|
||||
return list.__getitem__(self, y)
|
||||
except IndexError:
|
||||
return 0
|
||||
|
||||
def __setitem__(self, i, y):
|
||||
try:
|
||||
return list.__setitem__(self, i, y)
|
||||
except IndexError:
|
||||
self.extend([0] * (i - len(self) + 1))
|
||||
self[-1] = y
|
||||
|
||||
@classmethod
|
||||
def hex_md5(cls, param1):
|
||||
return cls.rstr2hex(cls.rstr_md5(cls.str2rstr_utf8(param1)))
|
||||
|
||||
@classmethod
|
||||
def b64_md5(cls, param1, param2=None):
|
||||
return cls.rstr2b64(cls.rstr_md5(cls.str2rstr_utf8(param1, param2)))
|
||||
|
||||
@classmethod
|
||||
def any_md5(cls, param1, param2):
|
||||
return cls.rstr2any(cls.rstr_md5(cls.str2rstr_utf8(param1)), param2)
|
||||
|
||||
@classmethod
|
||||
def rstr_md5(cls, param1):
|
||||
return cls.binl2rstr(cls.binl_md5(cls.rstr2binl(param1), len(param1) * 8))
|
||||
|
||||
@classmethod
|
||||
def rstr2hex(cls, param1):
|
||||
_loc_2 = '0123456789ABCDEF' if cls.hexcase else '0123456789abcdef'
|
||||
_loc_3 = ''
|
||||
for _loc_5 in range(0, len(param1)):
|
||||
_loc_4 = compat_ord(param1[_loc_5])
|
||||
_loc_3 += _loc_2[_loc_4 >> 4 & 15] + _loc_2[_loc_4 & 15]
|
||||
return _loc_3
|
||||
|
||||
@classmethod
|
||||
def rstr2b64(cls, param1):
|
||||
_loc_2 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
|
||||
_loc_3 = ''
|
||||
_loc_4 = len(param1)
|
||||
for _loc_5 in range(0, _loc_4, 3):
|
||||
_loc_6_1 = compat_ord(param1[_loc_5]) << 16
|
||||
_loc_6_2 = compat_ord(param1[_loc_5 + 1]) << 8 if _loc_5 + 1 < _loc_4 else 0
|
||||
_loc_6_3 = compat_ord(param1[_loc_5 + 2]) if _loc_5 + 2 < _loc_4 else 0
|
||||
_loc_6 = _loc_6_1 | _loc_6_2 | _loc_6_3
|
||||
for _loc_7 in range(0, 4):
|
||||
if _loc_5 * 8 + _loc_7 * 6 > len(param1) * 8:
|
||||
_loc_3 += cls.b64pad
|
||||
else:
|
||||
_loc_3 += _loc_2[_loc_6 >> 6 * (3 - _loc_7) & 63]
|
||||
return _loc_3
|
||||
|
||||
@staticmethod
|
||||
def rstr2any(param1, param2):
|
||||
_loc_3 = len(param2)
|
||||
_loc_4 = []
|
||||
_loc_9 = [0] * ((len(param1) >> 2) + 1)
|
||||
for _loc_5 in range(0, len(_loc_9)):
|
||||
_loc_9[_loc_5] = compat_ord(param1[_loc_5 * 2]) << 8 | compat_ord(param1[_loc_5 * 2 + 1])
|
||||
|
||||
while len(_loc_9) > 0:
|
||||
_loc_8 = []
|
||||
_loc_7 = 0
|
||||
for _loc_5 in range(0, len(_loc_9)):
|
||||
_loc_7 = (_loc_7 << 16) + _loc_9[_loc_5]
|
||||
_loc_6 = math.floor(_loc_7 / _loc_3)
|
||||
_loc_7 -= _loc_6 * _loc_3
|
||||
if len(_loc_8) > 0 or _loc_6 > 0:
|
||||
_loc_8[len(_loc_8)] = _loc_6
|
||||
|
||||
_loc_4[len(_loc_4)] = _loc_7
|
||||
_loc_9 = _loc_8
|
||||
|
||||
_loc_10 = ''
|
||||
_loc_5 = len(_loc_4) - 1
|
||||
while _loc_5 >= 0:
|
||||
_loc_10 += param2[_loc_4[_loc_5]]
|
||||
_loc_5 -= 1
|
||||
|
||||
return _loc_10
|
||||
|
||||
@classmethod
|
||||
def str2rstr_utf8(cls, param1, param2=None):
|
||||
_loc_3 = ''
|
||||
_loc_4 = -1
|
||||
if not param2:
|
||||
param2 = cls.PADDING
|
||||
param1 = param1 + param2[1:9]
|
||||
while True:
|
||||
_loc_4 += 1
|
||||
if _loc_4 >= len(param1):
|
||||
break
|
||||
_loc_5 = compat_ord(param1[_loc_4])
|
||||
_loc_6 = compat_ord(param1[_loc_4 + 1]) if _loc_4 + 1 < len(param1) else 0
|
||||
if 55296 <= _loc_5 <= 56319 and 56320 <= _loc_6 <= 57343:
|
||||
_loc_5 = 65536 + ((_loc_5 & 1023) << 10) + (_loc_6 & 1023)
|
||||
_loc_4 += 1
|
||||
if _loc_5 <= 127:
|
||||
_loc_3 += compat_chr(_loc_5)
|
||||
continue
|
||||
if _loc_5 <= 2047:
|
||||
_loc_3 += compat_chr(192 | _loc_5 >> 6 & 31) + compat_chr(128 | _loc_5 & 63)
|
||||
continue
|
||||
if _loc_5 <= 65535:
|
||||
_loc_3 += compat_chr(224 | _loc_5 >> 12 & 15) + compat_chr(128 | _loc_5 >> 6 & 63) + compat_chr(
|
||||
128 | _loc_5 & 63)
|
||||
continue
|
||||
if _loc_5 <= 2097151:
|
||||
_loc_3 += compat_chr(240 | _loc_5 >> 18 & 7) + compat_chr(128 | _loc_5 >> 12 & 63) + compat_chr(
|
||||
128 | _loc_5 >> 6 & 63) + compat_chr(128 | _loc_5 & 63)
|
||||
return _loc_3
|
||||
|
||||
@staticmethod
|
||||
def rstr2binl(param1):
|
||||
_loc_2 = [0] * ((len(param1) >> 2) + 1)
|
||||
for _loc_3 in range(0, len(_loc_2)):
|
||||
_loc_2[_loc_3] = 0
|
||||
for _loc_3 in range(0, len(param1) * 8, 8):
|
||||
_loc_2[_loc_3 >> 5] |= (compat_ord(param1[_loc_3 // 8]) & 255) << _loc_3 % 32
|
||||
return _loc_2
|
||||
|
||||
@staticmethod
|
||||
def binl2rstr(param1):
|
||||
_loc_2 = ''
|
||||
for _loc_3 in range(0, len(param1) * 32, 8):
|
||||
_loc_2 += compat_chr(param1[_loc_3 >> 5] >> _loc_3 % 32 & 255)
|
||||
return _loc_2
|
||||
|
||||
@classmethod
|
||||
def binl_md5(cls, param1, param2):
|
||||
param1 = cls.JSArray(param1)
|
||||
param1[param2 >> 5] |= 128 << param2 % 32
|
||||
param1[(param2 + 64 >> 9 << 4) + 14] = param2
|
||||
_loc_3 = 1732584193
|
||||
_loc_4 = -271733879
|
||||
_loc_5 = -1732584194
|
||||
_loc_6 = 271733878
|
||||
for _loc_7 in range(0, len(param1), 16):
|
||||
_loc_8 = _loc_3
|
||||
_loc_9 = _loc_4
|
||||
_loc_10 = _loc_5
|
||||
_loc_11 = _loc_6
|
||||
_loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 0], 7, -680876936)
|
||||
_loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 1], 12, -389564586)
|
||||
_loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 2], 17, 606105819)
|
||||
_loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 3], 22, -1044525330)
|
||||
_loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 4], 7, -176418897)
|
||||
_loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 5], 12, 1200080426)
|
||||
_loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 6], 17, -1473231341)
|
||||
_loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 7], 22, -45705983)
|
||||
_loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 8], 7, 1770035416)
|
||||
_loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 9], 12, -1958414417)
|
||||
_loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 10], 17, -42063)
|
||||
_loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 11], 22, -1990404162)
|
||||
_loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 12], 7, 1804603682)
|
||||
_loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 13], 12, -40341101)
|
||||
_loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 14], 17, -1502002290)
|
||||
_loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 15], 22, 1236535329)
|
||||
_loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 1], 5, -165796510)
|
||||
_loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 6], 9, -1069501632)
|
||||
_loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 11], 14, 643717713)
|
||||
_loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 0], 20, -373897302)
|
||||
_loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 5], 5, -701558691)
|
||||
_loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 10], 9, 38016083)
|
||||
_loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 15], 14, -660478335)
|
||||
_loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 4], 20, -405537848)
|
||||
_loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 9], 5, 568446438)
|
||||
_loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 14], 9, -1019803690)
|
||||
_loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 3], 14, -187363961)
|
||||
_loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 8], 20, 1163531501)
|
||||
_loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 13], 5, -1444681467)
|
||||
_loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 2], 9, -51403784)
|
||||
_loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 7], 14, 1735328473)
|
||||
_loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 12], 20, -1926607734)
|
||||
_loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 5], 4, -378558)
|
||||
_loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 8], 11, -2022574463)
|
||||
_loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 11], 16, 1839030562)
|
||||
_loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 14], 23, -35309556)
|
||||
_loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 1], 4, -1530992060)
|
||||
_loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 4], 11, 1272893353)
|
||||
_loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 7], 16, -155497632)
|
||||
_loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 10], 23, -1094730640)
|
||||
_loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 13], 4, 681279174)
|
||||
_loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 0], 11, -358537222)
|
||||
_loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 3], 16, -722521979)
|
||||
_loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 6], 23, 76029189)
|
||||
_loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 9], 4, -640364487)
|
||||
_loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 12], 11, -421815835)
|
||||
_loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 15], 16, 530742520)
|
||||
_loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 2], 23, -995338651)
|
||||
_loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 0], 6, -198630844)
|
||||
_loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 7], 10, 1126891415)
|
||||
_loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 14], 15, -1416354905)
|
||||
_loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 5], 21, -57434055)
|
||||
_loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 12], 6, 1700485571)
|
||||
_loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 3], 10, -1894986606)
|
||||
_loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 10], 15, -1051523)
|
||||
_loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 1], 21, -2054922799)
|
||||
_loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 8], 6, 1873313359)
|
||||
_loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 15], 10, -30611744)
|
||||
_loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 6], 15, -1560198380)
|
||||
_loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 13], 21, 1309151649)
|
||||
_loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 4], 6, -145523070)
|
||||
_loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 11], 10, -1120210379)
|
||||
_loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 2], 15, 718787259)
|
||||
_loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 9], 21, -343485551)
|
||||
_loc_3 = cls.safe_add(_loc_3, _loc_8)
|
||||
_loc_4 = cls.safe_add(_loc_4, _loc_9)
|
||||
_loc_5 = cls.safe_add(_loc_5, _loc_10)
|
||||
_loc_6 = cls.safe_add(_loc_6, _loc_11)
|
||||
return [_loc_3, _loc_4, _loc_5, _loc_6]
|
||||
|
||||
@classmethod
|
||||
def md5_cmn(cls, param1, param2, param3, param4, param5, param6):
|
||||
return cls.safe_add(
|
||||
cls.bit_rol(cls.safe_add(cls.safe_add(param2, param1), cls.safe_add(param4, param6)), param5), param3)
|
||||
|
||||
@classmethod
|
||||
def md5_ff(cls, param1, param2, param3, param4, param5, param6, param7):
|
||||
return cls.md5_cmn(param2 & param3 | ~param2 & param4, param1, param2, param5, param6, param7)
|
||||
|
||||
@classmethod
|
||||
def md5_gg(cls, param1, param2, param3, param4, param5, param6, param7):
|
||||
return cls.md5_cmn(param2 & param4 | param3 & ~param4, param1, param2, param5, param6, param7)
|
||||
|
||||
@classmethod
|
||||
def md5_hh(cls, param1, param2, param3, param4, param5, param6, param7):
|
||||
return cls.md5_cmn(param2 ^ param3 ^ param4, param1, param2, param5, param6, param7)
|
||||
|
||||
@classmethod
|
||||
def md5_ii(cls, param1, param2, param3, param4, param5, param6, param7):
|
||||
return cls.md5_cmn(param3 ^ (param2 | ~param4), param1, param2, param5, param6, param7)
|
||||
|
||||
@classmethod
|
||||
def safe_add(cls, param1, param2):
|
||||
_loc_3 = (param1 & 65535) + (param2 & 65535)
|
||||
_loc_4 = (param1 >> 16) + (param2 >> 16) + (_loc_3 >> 16)
|
||||
return cls.lshift(_loc_4, 16) | _loc_3 & 65535
|
||||
|
||||
@classmethod
|
||||
def bit_rol(cls, param1, param2):
|
||||
return cls.lshift(param1, param2) | (param1 & 0xFFFFFFFF) >> (32 - param2)
|
||||
|
||||
@staticmethod
|
||||
def lshift(value, count):
|
||||
r = (0xFFFFFFFF & value) << count
|
||||
return -(~(r - 1) & 0xFFFFFFFF) if r > 0x7FFFFFFF else r
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
|
||||
|
||||
video = self._download_json(
|
||||
self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
|
||||
|
||||
title = video['title']
|
||||
duration = float_or_none(video['duration'], 1000)
|
||||
like_count = video['likes']
|
||||
uploader = video['channel']
|
||||
uploader_id = video['channel_id']
|
||||
|
||||
formats = []
|
||||
|
||||
for resource in video['resources']:
|
||||
resource_id = resource.get('_id')
|
||||
if not resource_id:
|
||||
continue
|
||||
|
||||
security = self._download_json(
|
||||
self._SECURITY_URL_TEMPLATE % (video_id, resource_id),
|
||||
video_id, 'Downloading security hash for %s' % resource_id)
|
||||
|
||||
security_hash = security.get('hash')
|
||||
if not security_hash:
|
||||
message = security.get('message')
|
||||
if message:
|
||||
raise ExtractorError(
|
||||
'%s returned error: %s' % (self.IE_NAME, message), expected=True)
|
||||
continue
|
||||
|
||||
hash_code = security_hash[:2]
|
||||
received_time = int(security_hash[2:12])
|
||||
received_random = security_hash[12:22]
|
||||
received_md5 = security_hash[22:]
|
||||
|
||||
sign_time = received_time + self._RESIGN_EXPIRATION
|
||||
padding = '%010d' % random.randint(1, 10000000000)
|
||||
|
||||
signed_md5 = self.MD5.b64_md5(received_md5 + compat_str(sign_time) + padding)
|
||||
signed_hash = hash_code + compat_str(received_time) + received_random + compat_str(sign_time) + padding + signed_md5
|
||||
|
||||
formats.append({
|
||||
'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
|
||||
'format_id': resource_id,
|
||||
'height': resource['height']
|
||||
})
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'duration': duration,
|
||||
'uploader': uploader,
|
||||
'uploader_id': uploader_id,
|
||||
'like_count': like_count,
|
||||
'formats': formats
|
||||
}
|
@ -1,13 +1,11 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import datetime
|
||||
import re
|
||||
import codecs
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
)
|
||||
from ..utils import unified_strdate
|
||||
|
||||
|
||||
class GooglePlusIE(InfoExtractor):
|
||||
@ -19,74 +17,57 @@ class GooglePlusIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': 'ZButuJc6CtH',
|
||||
'ext': 'flv',
|
||||
'title': '嘆きの天使 降臨',
|
||||
'upload_date': '20120613',
|
||||
'uploader': '井上ヨシマサ',
|
||||
'title': '嘆きの天使 降臨',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
# Extract id from URL
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
# Step 1, Retrieve post webpage to extract further information
|
||||
webpage = self._download_webpage(url, video_id, 'Downloading entry webpage')
|
||||
|
||||
self.report_extraction(video_id)
|
||||
|
||||
# Extract update date
|
||||
upload_date = self._html_search_regex(
|
||||
title = self._og_search_description(webpage).splitlines()[0]
|
||||
upload_date = unified_strdate(self._html_search_regex(
|
||||
r'''(?x)<a.+?class="o-U-s\s[^"]+"\s+style="display:\s*none"\s*>
|
||||
([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
|
||||
webpage, 'upload date', fatal=False, flags=re.VERBOSE)
|
||||
if upload_date:
|
||||
# Convert timestring to a format suitable for filename
|
||||
upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
|
||||
upload_date = upload_date.strftime('%Y%m%d')
|
||||
|
||||
# Extract uploader
|
||||
uploader = self._html_search_regex(r'rel\="author".*?>(.*?)</a>',
|
||||
webpage, 'uploader', fatal=False)
|
||||
|
||||
# Extract title
|
||||
# Get the first line for title
|
||||
video_title = self._og_search_description(webpage).splitlines()[0]
|
||||
webpage, 'upload date', fatal=False, flags=re.VERBOSE))
|
||||
uploader = self._html_search_regex(
|
||||
r'rel="author".*?>(.*?)</a>', webpage, 'uploader', fatal=False)
|
||||
|
||||
# Step 2, Simulate clicking the image box to launch video
|
||||
DOMAIN = 'https://plus.google.com/'
|
||||
video_page = self._search_regex(r'<a href="((?:%s)?photos/.*?)"' % re.escape(DOMAIN),
|
||||
video_page = self._search_regex(
|
||||
r'<a href="((?:%s)?photos/.*?)"' % re.escape(DOMAIN),
|
||||
webpage, 'video page URL')
|
||||
if not video_page.startswith(DOMAIN):
|
||||
video_page = DOMAIN + video_page
|
||||
|
||||
webpage = self._download_webpage(video_page, video_id, 'Downloading video page')
|
||||
|
||||
def unicode_escape(s):
|
||||
decoder = codecs.getdecoder('unicode_escape')
|
||||
return re.sub(
|
||||
r'\\u[0-9a-fA-F]{4,}',
|
||||
lambda m: decoder(m.group(0))[0],
|
||||
s)
|
||||
|
||||
# Extract video links all sizes
|
||||
pattern = r'\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
|
||||
mobj = re.findall(pattern, webpage)
|
||||
if len(mobj) == 0:
|
||||
raise ExtractorError('Unable to extract video links')
|
||||
|
||||
# Sort in resolution
|
||||
links = sorted(mobj)
|
||||
|
||||
# Choose the lowest of the sort, i.e. highest resolution
|
||||
video_url = links[-1]
|
||||
# Only get the url. The resolution part in the tuple has no use anymore
|
||||
video_url = video_url[-1]
|
||||
# Treat escaped \u0026 style hex
|
||||
try:
|
||||
video_url = video_url.decode("unicode_escape")
|
||||
except AttributeError: # Python 3
|
||||
video_url = bytes(video_url, 'ascii').decode('unicode-escape')
|
||||
formats = [{
|
||||
'url': unicode_escape(video_url),
|
||||
'ext': 'flv',
|
||||
'width': int(width),
|
||||
'height': int(height),
|
||||
} for width, height, video_url in re.findall(
|
||||
r'\d+,(\d+),(\d+),"(https?://redirector\.googlevideo\.com.*?)"', webpage)]
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'title': title,
|
||||
'uploader': uploader,
|
||||
'upload_date': upload_date,
|
||||
'title': video_title,
|
||||
'ext': 'flv',
|
||||
'formats': formats,
|
||||
}
|
||||
|
@ -5,6 +5,7 @@ import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
determine_ext,
|
||||
compat_urllib_parse,
|
||||
compat_urllib_request,
|
||||
@ -12,20 +13,22 @@ from ..utils import (
|
||||
|
||||
|
||||
class GorillaVidIE(InfoExtractor):
|
||||
IE_DESC = 'GorillaVid.in and daclips.in'
|
||||
IE_DESC = 'GorillaVid.in, daclips.in and movpod.in'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?P<host>(?:www\.)?
|
||||
(?:daclips\.in|gorillavid\.in))/
|
||||
(?:daclips\.in|gorillavid\.in|movpod\.in))/
|
||||
(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
|
||||
'''
|
||||
|
||||
_FILE_NOT_FOUND_REGEX = r'>(?:404 - )?File Not Found<'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://gorillavid.in/06y9juieqpmi',
|
||||
'md5': '5ae4a3580620380619678ee4875893ba',
|
||||
'info_dict': {
|
||||
'id': '06y9juieqpmi',
|
||||
'ext': 'flv',
|
||||
'title': 'Rebecca Black My Moment Official Music Video Reaction',
|
||||
'title': 'Rebecca Black My Moment Official Music Video Reaction-6GK87Rc8bzQ',
|
||||
'thumbnail': 're:http://.*\.jpg',
|
||||
},
|
||||
}, {
|
||||
@ -43,9 +46,12 @@ class GorillaVidIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '3rso4kdn6f9m',
|
||||
'ext': 'mp4',
|
||||
'title': 'Micro Pig piglets ready on 16th July 2009',
|
||||
'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc',
|
||||
'thumbnail': 're:http://.*\.jpg',
|
||||
},
|
||||
}
|
||||
}, {
|
||||
'url': 'http://movpod.in/0wguyyxi1yca',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
@ -54,6 +60,9 @@ class GorillaVidIE(InfoExtractor):
|
||||
|
||||
webpage = self._download_webpage('http://%s/%s' % (mobj.group('host'), video_id), video_id)
|
||||
|
||||
if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
|
||||
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
|
||||
|
||||
fields = dict(re.findall(r'''(?x)<input\s+
|
||||
type="hidden"\s+
|
||||
name="([^"]+)"\s+
|
||||
@ -69,14 +78,14 @@ class GorillaVidIE(InfoExtractor):
|
||||
|
||||
webpage = self._download_webpage(req, video_id, 'Downloading video page')
|
||||
|
||||
title = self._search_regex(r'style="z-index: [0-9]+;">([0-9a-zA-Z ]+)(?:-.+)?</span>', webpage, 'title')
|
||||
thumbnail = self._search_regex(r'image:\'(http[^\']+)\',', webpage, 'thumbnail')
|
||||
url = self._search_regex(r'file: \'(http[^\']+)\',', webpage, 'file url')
|
||||
title = self._search_regex(r'style="z-index: [0-9]+;">([^<]+)</span>', webpage, 'title')
|
||||
video_url = self._search_regex(r'file\s*:\s*\'(http[^\']+)\',', webpage, 'file url')
|
||||
thumbnail = self._search_regex(r'image\s*:\s*\'(http[^\']+)\',', webpage, 'thumbnail', fatal=False)
|
||||
|
||||
formats = [{
|
||||
'format_id': 'sd',
|
||||
'url': url,
|
||||
'ext': determine_ext(url),
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'quality': 1,
|
||||
}]
|
||||
|
||||
|
@ -1,37 +1,33 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import re
|
||||
import json
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import determine_ext
|
||||
|
||||
|
||||
class HarkIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://www\.hark\.com/clips/(.+?)-.+'
|
||||
_VALID_URL = r'https?://www\.hark\.com/clips/(?P<id>.+?)-.+'
|
||||
_TEST = {
|
||||
u'url': u'http://www.hark.com/clips/mmbzyhkgny-obama-beyond-the-afghan-theater-we-only-target-al-qaeda-on-may-23-2013',
|
||||
u'file': u'mmbzyhkgny.mp3',
|
||||
u'md5': u'6783a58491b47b92c7c1af5a77d4cbee',
|
||||
u'info_dict': {
|
||||
u'title': u"Obama: 'Beyond The Afghan Theater, We Only Target Al Qaeda' on May 23, 2013",
|
||||
u'description': u'President Barack Obama addressed the nation live on May 23, 2013 in a speech aimed at addressing counter-terrorism policies including the use of drone strikes, detainees at Guantanamo Bay prison facility, and American citizens who are terrorists.',
|
||||
u'duration': 11,
|
||||
'url': 'http://www.hark.com/clips/mmbzyhkgny-obama-beyond-the-afghan-theater-we-only-target-al-qaeda-on-may-23-2013',
|
||||
'md5': '6783a58491b47b92c7c1af5a77d4cbee',
|
||||
'info_dict': {
|
||||
'id': 'mmbzyhkgny',
|
||||
'ext': 'mp3',
|
||||
'title': 'Obama: \'Beyond The Afghan Theater, We Only Target Al Qaeda\' on May 23, 2013',
|
||||
'description': 'President Barack Obama addressed the nation live on May 23, 2013 in a speech aimed at addressing counter-terrorism policies including the use of drone strikes, detainees at Guantanamo Bay prison facility, and American citizens who are terrorists.',
|
||||
'duration': 11,
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group(1)
|
||||
json_url = "http://www.hark.com/clips/%s.json" %(video_id)
|
||||
info_json = self._download_webpage(json_url, video_id)
|
||||
info = json.loads(info_json)
|
||||
final_url = info['url']
|
||||
video_id = self._match_id(url)
|
||||
data = self._download_json(
|
||||
'http://www.hark.com/clips/%s.json' % video_id, video_id)
|
||||
|
||||
return {'id': video_id,
|
||||
'url' : final_url,
|
||||
'title': info['name'],
|
||||
'ext': determine_ext(final_url),
|
||||
'description': info['description'],
|
||||
'thumbnail': info['image_original'],
|
||||
'duration': info['duration'],
|
||||
}
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': data['url'],
|
||||
'title': data['name'],
|
||||
'description': data.get('description'),
|
||||
'thumbnail': data.get('image_original'),
|
||||
'duration': data.get('duration'),
|
||||
}
|
||||
|
@ -4,6 +4,7 @@ from __future__ import unicode_literals
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
get_meta_content,
|
||||
int_or_none,
|
||||
parse_iso8601,
|
||||
)
|
||||
|
||||
@ -28,20 +29,26 @@ class HeiseIE(InfoExtractor):
|
||||
'timestamp': 1411812600,
|
||||
'upload_date': '20140927',
|
||||
'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
|
||||
'thumbnail': 're:https?://.*\.jpg$',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
json_url = self._search_regex(
|
||||
r'json_url:\s*"([^"]+)"', webpage, 'json URL')
|
||||
config = self._download_json(json_url, video_id)
|
||||
|
||||
container_id = self._search_regex(
|
||||
r'<div class="videoplayerjw".*?data-container="([0-9]+)"',
|
||||
webpage, 'container ID')
|
||||
sequenz_id = self._search_regex(
|
||||
r'<div class="videoplayerjw".*?data-sequenz="([0-9]+)"',
|
||||
webpage, 'sequenz ID')
|
||||
data_url = 'http://www.heise.de/videout/feed?container=%s&sequenz=%s' % (container_id, sequenz_id)
|
||||
doc = self._download_xml(data_url, video_id)
|
||||
|
||||
info = {
|
||||
'id': video_id,
|
||||
'thumbnail': config.get('poster'),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'timestamp': parse_iso8601(get_meta_content('date', webpage)),
|
||||
'description': self._og_search_description(webpage),
|
||||
}
|
||||
@ -49,32 +56,19 @@ class HeiseIE(InfoExtractor):
|
||||
title = get_meta_content('fulltitle', webpage)
|
||||
if title:
|
||||
info['title'] = title
|
||||
elif config.get('title'):
|
||||
info['title'] = config['title']
|
||||
else:
|
||||
info['title'] = self._og_search_title(webpage)
|
||||
|
||||
formats = []
|
||||
for t, rs in config['formats'].items():
|
||||
if not rs or not hasattr(rs, 'items'):
|
||||
self._downloader.report_warning(
|
||||
'formats: {0}: no resolutions'.format(t))
|
||||
continue
|
||||
|
||||
for height_str, obj in rs.items():
|
||||
format_id = '{0}_{1}'.format(t, height_str)
|
||||
|
||||
if not obj or not obj.get('url'):
|
||||
self._downloader.report_warning(
|
||||
'formats: {0}: no url'.format(format_id))
|
||||
continue
|
||||
|
||||
formats.append({
|
||||
'url': obj['url'],
|
||||
'format_id': format_id,
|
||||
'height': self._int(height_str, 'height'),
|
||||
})
|
||||
|
||||
for source_node in doc.findall('.//{http://rss.jwpcdn.com/}source'):
|
||||
label = source_node.attrib['label']
|
||||
height = int_or_none(self._search_regex(
|
||||
r'^(.*?_)?([0-9]+)p$', label, 'height', default=None))
|
||||
formats.append({
|
||||
'url': source_node.attrib['file'],
|
||||
'format_note': label,
|
||||
'height': height,
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
info['formats'] = formats
|
||||
|
||||
|
@ -28,13 +28,13 @@ class HowStuffWorksIE(InfoExtractor):
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://adventure.howstuffworks.com/39516-deadliest-catch-jakes-farewell-pots-video.htm',
|
||||
'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
|
||||
'info_dict': {
|
||||
'id': '553470',
|
||||
'display_id': 'deadliest-catch-jakes-farewell-pots',
|
||||
'id': '453464',
|
||||
'display_id': 'survival-zone-food-and-water-in-the-savanna',
|
||||
'ext': 'mp4',
|
||||
'title': 'Deadliest Catch: Jake\'s Farewell Pots',
|
||||
'description': 'md5:9632c346d5e43ee238028c9cefd8dbbc',
|
||||
'title': 'Survival Zone: Food and Water In the Savanna',
|
||||
'description': 'md5:7e1c89f6411434970c15fa094170c371',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
},
|
||||
'params': {
|
||||
|
@ -33,8 +33,7 @@ class HuffPostIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
api_url = 'http://embed.live.huffingtonpost.com/api/segments/%s.json' % video_id
|
||||
data = self._download_json(api_url, video_id)['data']
|
||||
|
@ -1,7 +1,5 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
@ -20,13 +18,11 @@ class IconosquareIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
html_title = self._html_search_regex(
|
||||
r'<title>(.+?)</title>',
|
||||
title = self._html_search_regex(
|
||||
r'<title>(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)</title>',
|
||||
webpage, 'title')
|
||||
title = re.sub(r'(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)$', '', html_title)
|
||||
uploader_id = self._html_search_regex(
|
||||
r'@([^ ]+)', title, 'uploader name', fatal=False)
|
||||
|
||||
|
@ -1,155 +0,0 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_str,
|
||||
ExtractorError,
|
||||
formatSeconds,
|
||||
)
|
||||
|
||||
|
||||
class JustinTVIE(InfoExtractor):
|
||||
"""Information extractor for justin.tv and twitch.tv"""
|
||||
# TODO: One broadcast may be split into multiple videos. The key
|
||||
# 'broadcast_id' is the same for all parts, and 'broadcast_part'
|
||||
# starts at 1 and increases. Can we treat all parts as one video?
|
||||
|
||||
_VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/
|
||||
(?:
|
||||
(?P<channelid>[^/]+)|
|
||||
(?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
|
||||
(?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
|
||||
)
|
||||
/?(?:\#.*)?$
|
||||
"""
|
||||
_JUSTIN_PAGE_LIMIT = 100
|
||||
IE_NAME = 'justin.tv'
|
||||
IE_DESC = 'justin.tv and twitch.tv'
|
||||
_TEST = {
|
||||
'url': 'http://www.twitch.tv/thegamedevhub/b/296128360',
|
||||
'md5': 'ecaa8a790c22a40770901460af191c9a',
|
||||
'info_dict': {
|
||||
'id': '296128360',
|
||||
'ext': 'flv',
|
||||
'upload_date': '20110927',
|
||||
'uploader_id': 25114803,
|
||||
'uploader': 'thegamedevhub',
|
||||
'title': 'Beginner Series - Scripting With Python Pt.1'
|
||||
}
|
||||
}
|
||||
|
||||
# Return count of items, list of *valid* items
|
||||
def _parse_page(self, url, video_id, counter):
|
||||
info_json = self._download_webpage(
|
||||
url, video_id,
|
||||
'Downloading video info JSON on page %d' % counter,
|
||||
'Unable to download video info JSON %d' % counter)
|
||||
|
||||
response = json.loads(info_json)
|
||||
if type(response) != list:
|
||||
error_text = response.get('error', 'unknown error')
|
||||
raise ExtractorError('Justin.tv API: %s' % error_text)
|
||||
info = []
|
||||
for clip in response:
|
||||
video_url = clip['video_file_url']
|
||||
if video_url:
|
||||
video_extension = os.path.splitext(video_url)[1][1:]
|
||||
video_date = re.sub('-', '', clip['start_time'][:10])
|
||||
video_uploader_id = clip.get('user_id', clip.get('channel_id'))
|
||||
video_id = clip['id']
|
||||
video_title = clip.get('title', video_id)
|
||||
info.append({
|
||||
'id': compat_str(video_id),
|
||||
'url': video_url,
|
||||
'title': video_title,
|
||||
'uploader': clip.get('channel_name', video_uploader_id),
|
||||
'uploader_id': video_uploader_id,
|
||||
'upload_date': video_date,
|
||||
'ext': video_extension,
|
||||
})
|
||||
return (len(response), info)
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
|
||||
api_base = 'http://api.justin.tv'
|
||||
paged = False
|
||||
if mobj.group('channelid'):
|
||||
paged = True
|
||||
video_id = mobj.group('channelid')
|
||||
api = api_base + '/channel/archives/%s.json' % video_id
|
||||
elif mobj.group('chapterid'):
|
||||
chapter_id = mobj.group('chapterid')
|
||||
|
||||
webpage = self._download_webpage(url, chapter_id)
|
||||
m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage)
|
||||
if not m:
|
||||
raise ExtractorError('Cannot find archive of a chapter')
|
||||
archive_id = m.group(1)
|
||||
|
||||
api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id
|
||||
doc = self._download_xml(
|
||||
api, chapter_id,
|
||||
note='Downloading chapter information',
|
||||
errnote='Chapter information download failed')
|
||||
for a in doc.findall('.//archive'):
|
||||
if archive_id == a.find('./id').text:
|
||||
break
|
||||
else:
|
||||
raise ExtractorError('Could not find chapter in chapter information')
|
||||
|
||||
video_url = a.find('./video_file_url').text
|
||||
video_ext = video_url.rpartition('.')[2] or 'flv'
|
||||
|
||||
chapter_api_url = 'https://api.twitch.tv/kraken/videos/c' + chapter_id
|
||||
chapter_info = self._download_json(
|
||||
chapter_api_url, 'c' + chapter_id,
|
||||
note='Downloading chapter metadata',
|
||||
errnote='Download of chapter metadata failed')
|
||||
|
||||
bracket_start = int(doc.find('.//bracket_start').text)
|
||||
bracket_end = int(doc.find('.//bracket_end').text)
|
||||
|
||||
# TODO determine start (and probably fix up file)
|
||||
# youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457
|
||||
#video_url += '?start=' + TODO:start_timestamp
|
||||
# bracket_start is 13290, but we want 51670615
|
||||
self._downloader.report_warning('Chapter detected, but we can just download the whole file. '
|
||||
'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end)))
|
||||
|
||||
info = {
|
||||
'id': 'c' + chapter_id,
|
||||
'url': video_url,
|
||||
'ext': video_ext,
|
||||
'title': chapter_info['title'],
|
||||
'thumbnail': chapter_info['preview'],
|
||||
'description': chapter_info['description'],
|
||||
'uploader': chapter_info['channel']['display_name'],
|
||||
'uploader_id': chapter_info['channel']['name'],
|
||||
}
|
||||
return info
|
||||
else:
|
||||
video_id = mobj.group('videoid')
|
||||
api = api_base + '/broadcast/by_archive/%s.json' % video_id
|
||||
|
||||
entries = []
|
||||
offset = 0
|
||||
limit = self._JUSTIN_PAGE_LIMIT
|
||||
for counter in itertools.count(1):
|
||||
page_url = api + ('?offset=%d&limit=%d' % (offset, limit))
|
||||
page_count, page_info = self._parse_page(
|
||||
page_url, video_id, counter)
|
||||
entries.extend(page_info)
|
||||
if not paged or page_count != limit:
|
||||
break
|
||||
offset += limit
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'id': video_id,
|
||||
'entries': entries,
|
||||
}
|
@ -1,8 +1,6 @@
|
||||
# encoding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
@ -21,22 +19,17 @@ class KickStarterIE(InfoExtractor):
|
||||
}, {
|
||||
'note': 'Embedded video (not using the native kickstarter video service)',
|
||||
'url': 'https://www.kickstarter.com/projects/597507018/pebble-e-paper-watch-for-iphone-and-android/posts/659178',
|
||||
'playlist': [
|
||||
{
|
||||
'info_dict': {
|
||||
'id': '78704821',
|
||||
'ext': 'mp4',
|
||||
'uploader_id': 'pebble',
|
||||
'uploader': 'Pebble Technology',
|
||||
'title': 'Pebble iOS Notifications',
|
||||
}
|
||||
}
|
||||
],
|
||||
'info_dict': {
|
||||
'id': '78704821',
|
||||
'ext': 'mp4',
|
||||
'uploader_id': 'pebble',
|
||||
'uploader': 'Pebble Technology',
|
||||
'title': 'Pebble iOS Notifications',
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
m = re.match(self._VALID_URL, url)
|
||||
video_id = m.group('id')
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = self._html_search_regex(
|
||||
|
@ -34,7 +34,7 @@ class KontrTubeIE(InfoExtractor):
|
||||
video_url = self._html_search_regex(r"video_url: '(.+?)/?',", webpage, 'video URL')
|
||||
thumbnail = self._html_search_regex(r"preview_url: '(.+?)/?',", webpage, 'video thumbnail', fatal=False)
|
||||
title = self._html_search_regex(
|
||||
r'<title>(.+?) - Труба зовёт - Интересный видеохостинг</title>', webpage, 'video title')
|
||||
r'<title>(.+?)</title>', webpage, 'video title')
|
||||
description = self._html_search_meta('description', webpage, 'video description')
|
||||
|
||||
mobj = re.search(
|
||||
|
@ -1,7 +1,5 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
@ -18,11 +16,11 @@ class Ku6IE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
title = self._search_regex(r'<h1 title=.*>(.*?)</h1>', webpage, 'title')
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<h1 title=.*>(.*?)</h1>', webpage, 'title')
|
||||
dataUrl = 'http://v.ku6.com/fetchVideo4Player/%s.html' % video_id
|
||||
jsonData = self._download_json(dataUrl, video_id)
|
||||
downloadUrl = jsonData['data']['f']
|
||||
|
77
youtube_dl/extractor/laola1tv.py
Normal file
77
youtube_dl/extractor/laola1tv.py
Normal file
@ -0,0 +1,77 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import random
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class Laola1TvIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<id>[0-9]+)\.html'
|
||||
_TEST = {
|
||||
'url': 'http://www.laola1.tv/de-de/live/bwf-bitburger-open-grand-prix-gold-court-1/250019.html',
|
||||
'info_dict': {
|
||||
'id': '250019',
|
||||
'ext': 'mp4',
|
||||
'title': 'Bitburger Open Grand Prix Gold - Court 1',
|
||||
'categories': ['Badminton'],
|
||||
'uploader': 'BWF - Badminton World Federation',
|
||||
'is_live': True,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
}
|
||||
}
|
||||
|
||||
_BROKEN = True # Not really - extractor works fine, but f4m downloader does not support live streams yet.
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
lang = mobj.group('lang')
|
||||
portal = mobj.group('portal')
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
iframe_url = self._search_regex(
|
||||
r'<iframe[^>]*?class="main_tv_player"[^>]*?src="([^"]+)"',
|
||||
webpage, 'iframe URL')
|
||||
|
||||
iframe = self._download_webpage(
|
||||
iframe_url, video_id, note='Downloading iframe')
|
||||
flashvars_m = re.findall(
|
||||
r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
|
||||
flashvars = dict((m[0], m[1]) for m in flashvars_m)
|
||||
|
||||
xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
|
||||
'play=%s&partner=1&portal=%s&v5ident=&lang=%s' % (
|
||||
video_id, portal, lang))
|
||||
hd_doc = self._download_xml(xml_url, video_id)
|
||||
|
||||
title = hd_doc.find('.//video/title').text
|
||||
flash_url = hd_doc.find('.//video/url').text
|
||||
categories = hd_doc.find('.//video/meta_sports').text.split(',')
|
||||
uploader = hd_doc.find('.//video/meta_organistation').text
|
||||
|
||||
ident = random.randint(10000000, 99999999)
|
||||
token_url = '%s&ident=%s&klub=0&unikey=0×tamp=%s&auth=%s' % (
|
||||
flash_url, ident, flashvars['timestamp'], flashvars['auth'])
|
||||
|
||||
token_doc = self._download_xml(
|
||||
token_url, video_id, note='Downloading token')
|
||||
token_attrib = token_doc.find('.//token').attrib
|
||||
if token_attrib.get('auth') == 'blocked':
|
||||
raise ExtractorError('Token error: ' % token_attrib.get('comment'))
|
||||
|
||||
video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
|
||||
token_attrib['url'], token_attrib['auth'])
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'is_live': True,
|
||||
'title': title,
|
||||
'url': video_url,
|
||||
'uploader': uploader,
|
||||
'categories': categories,
|
||||
'ext': 'mp4',
|
||||
}
|
||||
|
@ -190,7 +190,8 @@ class LivestreamOriginalIE(InfoExtractor):
|
||||
'id': video_id,
|
||||
'title': item.find('title').text,
|
||||
'url': 'rtmp://extondemand.livestream.com/ondemand',
|
||||
'play_path': 'mp4:trans/dv15/mogulus-{0}.mp4'.format(path),
|
||||
'play_path': 'trans/dv15/mogulus-{0}'.format(path),
|
||||
'player_url': 'http://static.livestream.com/chromelessPlayer/v21/playerapi.swf?hash=5uetk&v=0803&classid=D27CDB6E-AE6D-11cf-96B8-444553540000&jsEnabled=false&wmode=opaque',
|
||||
'ext': 'flv',
|
||||
'thumbnail': thumbnail_url,
|
||||
}
|
||||
|
@ -22,7 +22,7 @@ class LRTIE(InfoExtractor):
|
||||
'id': '54391',
|
||||
'ext': 'mp4',
|
||||
'title': 'Septynios Kauno dienos',
|
||||
'description': 'Kauno miesto ir apskrities naujienos',
|
||||
'description': 'md5:24d84534c7dc76581e59f5689462411a',
|
||||
'duration': 1783,
|
||||
},
|
||||
'params': {
|
||||
@ -32,9 +32,7 @@ class LRTIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = remove_end(self._og_search_title(webpage), ' - LRT')
|
||||
|
@ -6,6 +6,7 @@ import json
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse,
|
||||
compat_urlparse,
|
||||
get_element_by_attribute,
|
||||
parse_duration,
|
||||
strip_jsonp,
|
||||
@ -39,13 +40,21 @@ class MiTeleIE(InfoExtractor):
|
||||
).replace('\'', '"')
|
||||
embed_data = json.loads(embed_data_json)
|
||||
|
||||
info_url = embed_data['flashvars']['host']
|
||||
domain = embed_data['mediaUrl']
|
||||
if not domain.startswith('http'):
|
||||
# only happens in telecinco.es videos
|
||||
domain = 'http://' + domain
|
||||
info_url = compat_urlparse.urljoin(
|
||||
domain,
|
||||
compat_urllib_parse.unquote(embed_data['flashvars']['host'])
|
||||
)
|
||||
info_el = self._download_xml(info_url, episode).find('./video/info')
|
||||
|
||||
video_link = info_el.find('videoUrl/link').text
|
||||
token_query = compat_urllib_parse.urlencode({'id': video_link})
|
||||
token_info = self._download_json(
|
||||
'http://token.mitele.es/?' + token_query, episode,
|
||||
embed_data['flashvars']['ov_tk'] + '?' + token_query,
|
||||
episode,
|
||||
transform_source=strip_jsonp
|
||||
)
|
||||
|
||||
|
@ -33,22 +33,22 @@ class MixcloudIE(InfoExtractor):
|
||||
},
|
||||
}
|
||||
|
||||
def check_urls(self, url_list):
|
||||
"""Returns 1st active url from list"""
|
||||
for url in url_list:
|
||||
def _get_url(self, track_id, template_url):
|
||||
server_count = 30
|
||||
for i in range(server_count):
|
||||
url = template_url % i
|
||||
try:
|
||||
# We only want to know if the request succeed
|
||||
# don't download the whole file
|
||||
self._request_webpage(HEADRequest(url), None, False)
|
||||
self._request_webpage(
|
||||
HEADRequest(url), track_id,
|
||||
'Checking URL %d/%d ...' % (i + 1, server_count + 1))
|
||||
return url
|
||||
except ExtractorError:
|
||||
url = None
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def _get_url(self, template_url):
|
||||
return self.check_urls(template_url % i for i in range(30))
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
uploader = mobj.group(1)
|
||||
@ -61,16 +61,16 @@ class MixcloudIE(InfoExtractor):
|
||||
r'\s(?:data-preview-url|m-preview)="(.+?)"', webpage, 'preview url')
|
||||
song_url = preview_url.replace('/previews/', '/c/originals/')
|
||||
template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
|
||||
final_song_url = self._get_url(template_url)
|
||||
final_song_url = self._get_url(track_id, template_url)
|
||||
if final_song_url is None:
|
||||
self.to_screen('Trying with m4a extension')
|
||||
template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
|
||||
final_song_url = self._get_url(template_url)
|
||||
final_song_url = self._get_url(track_id, template_url)
|
||||
if final_song_url is None:
|
||||
raise ExtractorError('Unable to extract track url')
|
||||
|
||||
PREFIX = (
|
||||
r'<div class="cloudcast-play-button-container"'
|
||||
r'<div class="cloudcast-play-button-container[^"]*?"'
|
||||
r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
|
||||
title = self._html_search_regex(
|
||||
PREFIX + r'm-title="([^"]+)"', webpage, 'title')
|
||||
|
@ -6,12 +6,11 @@ from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
parse_duration,
|
||||
parse_iso8601,
|
||||
find_xpath_attr,
|
||||
)
|
||||
|
||||
|
||||
class MLBIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://m\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|shared/video/embed/embed\.html\?.*?\bcontent_id=)(?P<id>n?\d+)'
|
||||
_VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
|
||||
@ -73,6 +72,14 @@ class MLBIE(InfoExtractor):
|
||||
'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
|
||||
'only_matching': True,
|
||||
},
|
||||
{
|
||||
'url': 'http://mlb.mlb.com/shared/video/embed/embed.html?content_id=36599553',
|
||||
'only_matching': True,
|
||||
},
|
||||
{
|
||||
'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
|
||||
'only_matching': True,
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
@ -88,8 +95,9 @@ class MLBIE(InfoExtractor):
|
||||
duration = parse_duration(detail.find('./duration').text)
|
||||
timestamp = parse_iso8601(detail.attrib['date'][:-5])
|
||||
|
||||
thumbnail = find_xpath_attr(
|
||||
detail, './thumbnailScenarios/thumbnailScenario', 'type', '45').text
|
||||
thumbnails = [{
|
||||
'url': thumbnail.text,
|
||||
} for thumbnail in detail.findall('./thumbnailScenarios/thumbnailScenario')]
|
||||
|
||||
formats = []
|
||||
for media_url in detail.findall('./url'):
|
||||
@ -116,5 +124,5 @@ class MLBIE(InfoExtractor):
|
||||
'duration': duration,
|
||||
'timestamp': timestamp,
|
||||
'formats': formats,
|
||||
'thumbnail': thumbnail,
|
||||
'thumbnails': thumbnails,
|
||||
}
|
||||
|
@ -5,20 +5,20 @@ import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
str_to_int,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
|
||||
class MotherlessIE(InfoExtractor):
|
||||
_VALID_URL = r'http://(?:www\.)?motherless\.com/(?P<id>[A-Z0-9]+)'
|
||||
_VALID_URL = r'http://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://motherless.com/AC3FFE1',
|
||||
'md5': '5527fef81d2e529215dad3c2d744a7d9',
|
||||
'md5': '310f62e325a9fafe64f68c0bccb6e75f',
|
||||
'info_dict': {
|
||||
'id': 'AC3FFE1',
|
||||
'ext': 'flv',
|
||||
'ext': 'mp4',
|
||||
'title': 'Fucked in the ass while playing PS3',
|
||||
'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
|
||||
'upload_date': '20100913',
|
||||
@ -40,33 +40,51 @@ class MotherlessIE(InfoExtractor):
|
||||
'thumbnail': 're:http://.*\.jpg',
|
||||
'age_limit': 18,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://motherless.com/g/cosplay/633979F',
|
||||
'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
|
||||
'info_dict': {
|
||||
'id': '633979F',
|
||||
'ext': 'mp4',
|
||||
'title': 'Turtlette',
|
||||
'categories': ['superheroine heroine superher'],
|
||||
'upload_date': '20140827',
|
||||
'uploader_id': 'shade0230',
|
||||
'thumbnail': 're:http://.*\.jpg',
|
||||
'age_limit': 18,
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self,url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = self._html_search_regex(r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
|
||||
|
||||
video_url = self._html_search_regex(r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video_url')
|
||||
title = self._html_search_regex(
|
||||
r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
|
||||
video_url = self._html_search_regex(
|
||||
r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video URL')
|
||||
age_limit = self._rta_search(webpage)
|
||||
|
||||
view_count = self._html_search_regex(r'<strong>Views</strong>\s+([^<]+)<', webpage, 'view_count')
|
||||
view_count = str_to_int(self._html_search_regex(
|
||||
r'<strong>Views</strong>\s+([^<]+)<',
|
||||
webpage, 'view count', fatal=False))
|
||||
like_count = str_to_int(self._html_search_regex(
|
||||
r'<strong>Favorited</strong>\s+([^<]+)<',
|
||||
webpage, 'like count', fatal=False))
|
||||
|
||||
upload_date = self._html_search_regex(r'<strong>Uploaded</strong>\s+([^<]+)<', webpage, 'upload_date')
|
||||
upload_date = self._html_search_regex(
|
||||
r'<strong>Uploaded</strong>\s+([^<]+)<', webpage, 'upload date')
|
||||
if 'Ago' in upload_date:
|
||||
days = int(re.search(r'([0-9]+)', upload_date).group(1))
|
||||
upload_date = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%Y%m%d')
|
||||
else:
|
||||
upload_date = unified_strdate(upload_date)
|
||||
|
||||
like_count = self._html_search_regex(r'<strong>Favorited</strong>\s+([^<]+)<', webpage, 'like_count')
|
||||
|
||||
comment_count = webpage.count('class="media-comment-contents"')
|
||||
uploader_id = self._html_search_regex(r'"thumb-member-username">\s+<a href="/m/([^"]+)"', webpage, 'uploader_id')
|
||||
uploader_id = self._html_search_regex(
|
||||
r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
|
||||
webpage, 'uploader_id')
|
||||
|
||||
categories = self._html_search_meta('keywords', webpage)
|
||||
if categories:
|
||||
@ -79,8 +97,8 @@ class MotherlessIE(InfoExtractor):
|
||||
'uploader_id': uploader_id,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'categories': categories,
|
||||
'view_count': int_or_none(view_count.replace(',', '')),
|
||||
'like_count': int_or_none(like_count.replace(',', '')),
|
||||
'view_count': view_count,
|
||||
'like_count': like_count,
|
||||
'comment_count': comment_count,
|
||||
'age_limit': age_limit,
|
||||
'url': video_url,
|
||||
|
@ -7,6 +7,7 @@ from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urllib_parse,
|
||||
ExtractorError,
|
||||
clean_html,
|
||||
)
|
||||
|
||||
|
||||
@ -31,6 +32,11 @@ class NaverIE(InfoExtractor):
|
||||
m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
|
||||
webpage)
|
||||
if m_id is None:
|
||||
m_error = re.search(
|
||||
r'(?s)<div class="nation_error">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
|
||||
webpage)
|
||||
if m_error:
|
||||
raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
|
||||
raise ExtractorError('couldn\'t extract vid and key')
|
||||
vid = m_id.group(1)
|
||||
key = m_id.group(2)
|
||||
|
@ -26,8 +26,7 @@ class NBCIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
theplatform_url = self._search_regex('class="video-player video-player-full" data-mpx-url="(.*?)"', webpage, 'theplatform url')
|
||||
if theplatform_url.startswith('//'):
|
||||
@ -57,7 +56,7 @@ class NBCNewsIE(InfoExtractor):
|
||||
'md5': 'b2421750c9f260783721d898f4c42063',
|
||||
'info_dict': {
|
||||
'id': 'I1wpAI_zmhsQ',
|
||||
'ext': 'flv',
|
||||
'ext': 'mp4',
|
||||
'title': 'How Twitter Reacted To The Snowden Interview',
|
||||
'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
|
||||
},
|
||||
@ -97,6 +96,8 @@ class NBCNewsIE(InfoExtractor):
|
||||
]
|
||||
|
||||
for base_url in base_urls:
|
||||
if not base_url:
|
||||
continue
|
||||
playlist_url = base_url + '?form=MPXNBCNewsAPI'
|
||||
all_videos = self._download_json(playlist_url, title)['videos']
|
||||
|
||||
|
@ -18,16 +18,16 @@ class NDRIE(InfoExtractor):
|
||||
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://www.ndr.de/fernsehen/media/dienordreportage325.html',
|
||||
'md5': '4a4eeafd17c3058b65f0c8f091355855',
|
||||
'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
|
||||
'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
|
||||
'note': 'Video file',
|
||||
'info_dict': {
|
||||
'id': '325',
|
||||
'id': '25866',
|
||||
'ext': 'mp4',
|
||||
'title': 'Blaue Bohnen aus Blocken',
|
||||
'description': 'md5:190d71ba2ccddc805ed01547718963bc',
|
||||
'duration': 1715,
|
||||
},
|
||||
'title': 'Kartoffeltage in der Lewitz',
|
||||
'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
|
||||
'duration': 166,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://www.ndr.de/info/audio51535.html',
|
||||
|
@ -7,7 +7,6 @@ from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
compat_urllib_parse,
|
||||
determine_ext,
|
||||
unified_strdate,
|
||||
)
|
||||
|
||||
@ -22,21 +21,23 @@ class NHLBaseInfoExtractor(InfoExtractor):
|
||||
self.report_extraction(video_id)
|
||||
|
||||
initial_video_url = info['publishPoint']
|
||||
data = compat_urllib_parse.urlencode({
|
||||
'type': 'fvod',
|
||||
'path': initial_video_url.replace('.mp4', '_sd.mp4'),
|
||||
})
|
||||
path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
|
||||
path_doc = self._download_xml(
|
||||
path_url, video_id, 'Downloading final video url')
|
||||
video_url = path_doc.find('path').text
|
||||
if info['formats'] == '1':
|
||||
data = compat_urllib_parse.urlencode({
|
||||
'type': 'fvod',
|
||||
'path': initial_video_url.replace('.mp4', '_sd.mp4'),
|
||||
})
|
||||
path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
|
||||
path_doc = self._download_xml(
|
||||
path_url, video_id, 'Downloading final video url')
|
||||
video_url = path_doc.find('path').text
|
||||
else:
|
||||
video_url = initial_video_url
|
||||
|
||||
join = compat_urlparse.urljoin
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': info['name'],
|
||||
'url': video_url,
|
||||
'ext': determine_ext(video_url),
|
||||
'description': info['description'],
|
||||
'duration': int(info['duration']),
|
||||
'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
|
||||
@ -46,10 +47,11 @@ class NHLBaseInfoExtractor(InfoExtractor):
|
||||
|
||||
class NHLIE(NHLBaseInfoExtractor):
|
||||
IE_NAME = 'nhl.com'
|
||||
_VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[0-9]+)'
|
||||
_VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[0-9a-z-]+)'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
|
||||
'md5': 'db704a4ea09e8d3988c85e36cc892d09',
|
||||
'info_dict': {
|
||||
'id': '453614',
|
||||
'ext': 'mp4',
|
||||
@ -58,6 +60,17 @@ class NHLIE(NHLBaseInfoExtractor):
|
||||
'duration': 18,
|
||||
'upload_date': '20131006',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://video.nhl.com/videocenter/console?id=2014020024-628-h',
|
||||
'md5': 'd22e82bc592f52d37d24b03531ee9696',
|
||||
'info_dict': {
|
||||
'id': '2014020024-628-h',
|
||||
'ext': 'mp4',
|
||||
'title': 'Alex Galchenyuk Goal on Ray Emery (14:40/3rd)',
|
||||
'description': 'Home broadcast - Montreal Canadiens at Philadelphia Flyers - October 11, 2014',
|
||||
'duration': 0,
|
||||
'upload_date': '20141011',
|
||||
},
|
||||
}, {
|
||||
'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
|
||||
'only_matching': True,
|
||||
|
@ -2,6 +2,7 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
@ -39,18 +40,17 @@ class NiconicoIE(InfoExtractor):
|
||||
|
||||
_VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/((?:[a-z]{2})?[0-9]+)'
|
||||
_NETRC_MACHINE = 'niconico'
|
||||
# Determine whether the downloader uses authentication to download video
|
||||
_AUTHENTICATE = False
|
||||
# Determine whether the downloader used authentication to download video
|
||||
_AUTHENTICATED = False
|
||||
|
||||
def _real_initialize(self):
|
||||
if self._downloader.params.get('username', None) is not None:
|
||||
self._AUTHENTICATE = True
|
||||
|
||||
if self._AUTHENTICATE:
|
||||
self._login()
|
||||
self._login()
|
||||
|
||||
def _login(self):
|
||||
(username, password) = self._get_login_info()
|
||||
# No authentication to be performed
|
||||
if not username:
|
||||
return True
|
||||
|
||||
# Log in
|
||||
login_form_strs = {
|
||||
@ -68,6 +68,8 @@ class NiconicoIE(InfoExtractor):
|
||||
if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
|
||||
self._downloader.report_warning('unable to log in: bad username or password')
|
||||
return False
|
||||
# Successful login
|
||||
self._AUTHENTICATED = True
|
||||
return True
|
||||
|
||||
def _real_extract(self, url):
|
||||
@ -82,7 +84,7 @@ class NiconicoIE(InfoExtractor):
|
||||
'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
|
||||
note='Downloading video info page')
|
||||
|
||||
if self._AUTHENTICATE:
|
||||
if self._AUTHENTICATED:
|
||||
# Get flv info
|
||||
flv_info_webpage = self._download_webpage(
|
||||
'http://flapi.nicovideo.jp/api/getflv?v=' + video_id,
|
||||
@ -145,3 +147,36 @@ class NiconicoIE(InfoExtractor):
|
||||
'duration': duration,
|
||||
'webpage_url': webpage_url,
|
||||
}
|
||||
|
||||
|
||||
class NiconicoPlaylistIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://www\.nicovideo\.jp/mylist/(?P<id>\d+)'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.nicovideo.jp/mylist/27411728',
|
||||
'info_dict': {
|
||||
'id': '27411728',
|
||||
'title': 'AKB48のオールナイトニッポン',
|
||||
},
|
||||
'playlist_mincount': 225,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
list_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, list_id)
|
||||
|
||||
entries_json = self._search_regex(r'Mylist\.preload\(\d+, (\[.*\])\);',
|
||||
webpage, 'entries')
|
||||
entries = json.loads(entries_json)
|
||||
entries = [{
|
||||
'_type': 'url',
|
||||
'ie_key': NiconicoIE.ie_key(),
|
||||
'url': 'http://www.nicovideo.jp/watch/%s' % entry['item_id'],
|
||||
} for entry in entries]
|
||||
|
||||
return {
|
||||
'_type': 'playlist',
|
||||
'title': self._search_regex(r'\s+name: "(.*?)"', webpage, 'title'),
|
||||
'id': list_id,
|
||||
'entries': entries,
|
||||
}
|
||||
|
@ -80,8 +80,14 @@ class PBSIE(InfoExtractor):
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
'upload_date': '20140122',
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
|
||||
'info_dict': {
|
||||
'id': 'united-states-of-secrets',
|
||||
},
|
||||
'playlist_count': 2,
|
||||
}
|
||||
|
||||
]
|
||||
|
||||
def _extract_webpage(self, url):
|
||||
@ -96,6 +102,12 @@ class PBSIE(InfoExtractor):
|
||||
r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
|
||||
webpage, 'upload date', default=None))
|
||||
|
||||
# tabbed frontline videos
|
||||
tabbed_videos = re.findall(
|
||||
r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
|
||||
if tabbed_videos:
|
||||
return tabbed_videos, presumptive_id, upload_date
|
||||
|
||||
MEDIA_ID_REGEXES = [
|
||||
r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed
|
||||
r'class="coveplayerid">([^<]+)<', # coveplayer
|
||||
@ -130,6 +142,12 @@ class PBSIE(InfoExtractor):
|
||||
def _real_extract(self, url):
|
||||
video_id, display_id, upload_date = self._extract_webpage(url)
|
||||
|
||||
if isinstance(video_id, list):
|
||||
entries = [self.url_result(
|
||||
'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id)
|
||||
for vid_id in video_id]
|
||||
return self.playlist_result(entries, display_id)
|
||||
|
||||
info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
|
||||
info = self._download_json(info_url, display_id)
|
||||
|
||||
|
31
youtube_dl/extractor/phoenix.py
Normal file
31
youtube_dl/extractor/phoenix.py
Normal file
@ -0,0 +1,31 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .zdf import extract_from_xml_url
|
||||
|
||||
|
||||
class PhoenixIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?phoenix\.de/content/(?P<id>[0-9]+)'
|
||||
_TEST = {
|
||||
'url': 'http://www.phoenix.de/content/884301',
|
||||
'md5': 'ed249f045256150c92e72dbb70eadec6',
|
||||
'info_dict': {
|
||||
'id': '884301',
|
||||
'ext': 'mp4',
|
||||
'title': 'Michael Krons mit Hans-Werner Sinn',
|
||||
'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
|
||||
'upload_date': '20141025',
|
||||
'uploader': 'Im Dialog',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
internal_id = self._search_regex(
|
||||
r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
|
||||
webpage, 'internal video ID')
|
||||
|
||||
api_url = 'http://www.phoenix.de/php/zdfplayer-v1.3/data/beitragsDetails.php?ak=web&id=%s' % internal_id
|
||||
return extract_from_xml_url(self, video_id, api_url)
|
60
youtube_dl/extractor/planetaplay.py
Normal file
60
youtube_dl/extractor/planetaplay.py
Normal file
@ -0,0 +1,60 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import ExtractorError
|
||||
|
||||
|
||||
class PlanetaPlayIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?planetaplay\.com/\?sng=(?P<id>[0-9]+)'
|
||||
_API_URL = 'http://planetaplay.com/action/playlist/?sng={0:}'
|
||||
_THUMBNAIL_URL = 'http://planetaplay.com/img/thumb/{thumb:}'
|
||||
_TEST = {
|
||||
'url': 'http://planetaplay.com/?sng=3586',
|
||||
'md5': '9d569dceb7251a4e01355d5aea60f9db',
|
||||
'info_dict': {
|
||||
'id': '3586',
|
||||
'ext': 'flv',
|
||||
'title': 'md5:e829428ee28b1deed00de90de49d1da1',
|
||||
}
|
||||
}
|
||||
|
||||
_SONG_FORMATS = {
|
||||
'lq': (0, 'http://www.planetaplay.com/videoplayback/{med_hash:}'),
|
||||
'hq': (1, 'http://www.planetaplay.com/videoplayback/hi/{med_hash:}'),
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
response = self._download_json(
|
||||
self._API_URL.format(video_id), video_id)['response']
|
||||
try:
|
||||
data = response.get('data')[0]
|
||||
except IndexError:
|
||||
raise ExtractorError(
|
||||
'%s: failed to get the playlist' % self.IE_NAME, expected=True)
|
||||
|
||||
title = '{song_artists:} - {sng_name:}'.format(**data)
|
||||
thumbnail = self._THUMBNAIL_URL.format(**data)
|
||||
|
||||
formats = []
|
||||
for format_id, (quality, url_template) in self._SONG_FORMATS.items():
|
||||
formats.append({
|
||||
'format_id': format_id,
|
||||
'url': url_template.format(**data),
|
||||
'quality': quality,
|
||||
'ext': 'flv',
|
||||
})
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'thumbnail': thumbnail,
|
||||
}
|
@ -4,19 +4,27 @@ import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import int_or_none
|
||||
from ..utils import (
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
qualities,
|
||||
determine_ext,
|
||||
)
|
||||
|
||||
|
||||
class PornHdIE(InfoExtractor):
|
||||
_VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)'
|
||||
_VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
|
||||
_TEST = {
|
||||
'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
|
||||
'md5': '956b8ca569f7f4d8ec563e2c41598441',
|
||||
'info_dict': {
|
||||
'id': '1962',
|
||||
'display_id': 'sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
|
||||
'ext': 'mp4',
|
||||
'title': 'Sierra loves doing laundry',
|
||||
'description': 'md5:8ff0523848ac2b8f9b065ba781ccf294',
|
||||
'thumbnail': 're:^https?://.*\.jpg',
|
||||
'view_count': int,
|
||||
'age_limit': 18,
|
||||
}
|
||||
}
|
||||
@ -24,8 +32,9 @@ class PornHdIE(InfoExtractor):
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
display_id = mobj.group('display_id')
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
webpage = self._download_webpage(url, display_id or video_id)
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<title>(.+) porn HD.+?</title>', webpage, 'title')
|
||||
@ -33,38 +42,21 @@ class PornHdIE(InfoExtractor):
|
||||
r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
|
||||
view_count = int_or_none(self._html_search_regex(
|
||||
r'(\d+) views\s*</span>', webpage, 'view count', fatal=False))
|
||||
thumbnail = self._search_regex(
|
||||
r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
|
||||
|
||||
videos = re.findall(
|
||||
r'var __video([\da-zA-Z]+?)(Low|High)StreamUrl = \'(http://.+?)\?noProxy=1\'', webpage)
|
||||
|
||||
mobj = re.search(r'flashVars = (?P<flashvars>{.+?});', webpage)
|
||||
if mobj:
|
||||
flashvars = json.loads(mobj.group('flashvars'))
|
||||
for key, quality in [('hashlink', 'low'), ('hd', 'high')]:
|
||||
redirect_url = flashvars.get(key)
|
||||
if redirect_url:
|
||||
videos.append(('flv', quality, redirect_url))
|
||||
thumbnail = flashvars['urlWallpaper']
|
||||
else:
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
|
||||
formats = []
|
||||
for format_, quality, redirect_url in videos:
|
||||
format_id = '%s-%s' % (format_.lower(), quality.lower())
|
||||
video_url = self._download_webpage(
|
||||
redirect_url, video_id, 'Downloading %s video link' % format_id, fatal=False)
|
||||
if not video_url:
|
||||
continue
|
||||
formats.append({
|
||||
'url': video_url,
|
||||
'ext': format_.lower(),
|
||||
'format_id': format_id,
|
||||
'quality': 1 if quality.lower() == 'high' else 0,
|
||||
})
|
||||
quality = qualities(['SD', 'HD'])
|
||||
formats = [{
|
||||
'url': source['file'],
|
||||
'format_id': '%s-%s' % (source['label'], determine_ext(source['file'])),
|
||||
'quality': quality(source['label']),
|
||||
} for source in json.loads(js_to_json(self._search_regex(
|
||||
r"(?s)'sources'\s*:\s*(\[.+?\])", webpage, 'sources')))]
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
|
@ -16,13 +16,14 @@ from ..aes import (
|
||||
|
||||
|
||||
class PornHubIE(InfoExtractor):
|
||||
_VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9a-f]+))'
|
||||
_VALID_URL = r'^https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
|
||||
_TEST = {
|
||||
'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
||||
'file': '648719015.mp4',
|
||||
'md5': '882f488fa1f0026f023f33576004a2ed',
|
||||
'info_dict': {
|
||||
"uploader": "BABES-COM",
|
||||
'id': '648719015',
|
||||
'ext': 'mp4',
|
||||
"uploader": "Babes",
|
||||
"title": "Seductive Indian beauty strips down and fingers her pink pussy",
|
||||
"age_limit": 18
|
||||
}
|
||||
@ -35,9 +36,7 @@ class PornHubIE(InfoExtractor):
|
||||
return count
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('videoid')
|
||||
url = 'http://www.' + mobj.group('url')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
req = compat_urllib_request.Request(url)
|
||||
req.add_header('Cookie', 'age_verified=1')
|
||||
@ -45,7 +44,7 @@ class PornHubIE(InfoExtractor):
|
||||
|
||||
video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
|
||||
video_uploader = self._html_search_regex(
|
||||
r'(?s)From: .+?<(?:a href="/users/|<span class="username)[^>]+>(.+?)<',
|
||||
r'(?s)From: .+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',
|
||||
webpage, 'uploader', fatal=False)
|
||||
thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
|
||||
if thumbnail:
|
||||
|
@ -14,7 +14,6 @@ from ..utils import (
|
||||
|
||||
class PromptFileIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?promptfile\.com/l/(?P<id>[0-9A-Z\-]+)'
|
||||
_FILE_NOT_FOUND_REGEX = r'<div.+id="not_found_msg".+>.+</div>[^-]'
|
||||
_TEST = {
|
||||
'url': 'http://www.promptfile.com/l/D21B4746E9-F01462F0FF',
|
||||
'md5': 'd1451b6302da7215485837aaea882c4c',
|
||||
@ -27,11 +26,10 @@ class PromptFileIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
|
||||
if re.search(r'<div.+id="not_found_msg".+>(?!We are).+</div>[^-]', webpage) is not None:
|
||||
raise ExtractorError('Video %s does not exist' % video_id,
|
||||
expected=True)
|
||||
|
||||
|
51
youtube_dl/extractor/quickvid.py
Normal file
51
youtube_dl/extractor/quickvid.py
Normal file
@ -0,0 +1,51 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_urlparse,
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
)
|
||||
|
||||
|
||||
class QuickVidIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(www\.)?quickvid\.org/watch\.php\?v=(?P<id>[a-zA-Z_0-9-]+)'
|
||||
_TEST = {
|
||||
'url': 'http://quickvid.org/watch.php?v=sUQT3RCG8dx',
|
||||
'md5': 'c0c72dd473f260c06c808a05d19acdc5',
|
||||
'info_dict': {
|
||||
'id': 'sUQT3RCG8dx',
|
||||
'ext': 'mp4',
|
||||
'title': 'Nick Offerman\'s Summer Reading Recap',
|
||||
'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
|
||||
'view_count': int,
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
title = self._html_search_regex(r'<h2>(.*?)</h2>', webpage, 'title')
|
||||
view_count = int_or_none(self._html_search_regex(
|
||||
r'(?s)<div id="views">(.*?)</div>',
|
||||
webpage, 'view count', fatal=False))
|
||||
video_code = self._search_regex(
|
||||
r'(?s)<video id="video"[^>]*>(.*?)</video>', webpage, 'video code')
|
||||
formats = [
|
||||
{
|
||||
'url': compat_urlparse.urljoin(url, src),
|
||||
'format_id': determine_ext(src, None),
|
||||
} for src in re.findall('<source\s+src="([^"]+)"', video_code)
|
||||
]
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
'view_count': view_count,
|
||||
}
|
@ -1,43 +1,43 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
compat_parse_qs,
|
||||
)
|
||||
from ..utils import compat_urllib_parse_unquote
|
||||
|
||||
|
||||
class Ro220IE(InfoExtractor):
|
||||
IE_NAME = '220.ro'
|
||||
_VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<video_id>[^/]+)'
|
||||
_VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<id>[^/]+)'
|
||||
_TEST = {
|
||||
"url": "http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/",
|
||||
'file': 'LYV6doKo7f.mp4',
|
||||
'url': 'http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/',
|
||||
'md5': '03af18b73a07b4088753930db7a34add',
|
||||
'info_dict': {
|
||||
"title": "Luati-le Banii sez 4 ep 1",
|
||||
"description": "re:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$",
|
||||
'id': 'LYV6doKo7f',
|
||||
'ext': 'mp4',
|
||||
'title': 'Luati-le Banii sez 4 ep 1',
|
||||
'description': 're:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('video_id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
flashVars_str = self._search_regex(
|
||||
r'<param name="flashVars" value="([^"]+)"',
|
||||
webpage, 'flashVars')
|
||||
flashVars = compat_parse_qs(flashVars_str)
|
||||
url = compat_urllib_parse_unquote(self._search_regex(
|
||||
r'(?s)clip\s*:\s*{.*?url\s*:\s*\'([^\']+)\'', webpage, 'url'))
|
||||
title = self._og_search_title(webpage)
|
||||
description = self._og_search_description(webpage)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
|
||||
formats = [{
|
||||
'format_id': 'sd',
|
||||
'url': url,
|
||||
'ext': 'mp4',
|
||||
}]
|
||||
|
||||
return {
|
||||
'_type': 'video',
|
||||
'id': video_id,
|
||||
'ext': 'mp4',
|
||||
'url': flashVars['videoURL'][0],
|
||||
'title': flashVars['title'][0],
|
||||
'description': clean_html(flashVars['desc'][0]),
|
||||
'thumbnail': flashVars['preview'][0],
|
||||
'formats': formats,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
}
|
||||
|
@ -81,7 +81,7 @@ class RTLnowIE(InfoExtractor):
|
||||
'id': '99205',
|
||||
'ext': 'flv',
|
||||
'title': 'Medicopter 117 - Angst!',
|
||||
'description': 'md5:895b1df01639b5f61a04fc305a5cb94d',
|
||||
'description': 're:^Im Therapiezentrum \'Sonnalm\' kommen durch eine Unachtsamkeit die für die B.handlung mit Phobikern gehaltenen Voglespinnen frei\. Eine Ausreißerin',
|
||||
'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
|
||||
'upload_date': '20080928',
|
||||
'duration': 2691,
|
||||
|
@ -1,8 +1,6 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
@ -21,19 +19,20 @@ class RUHDIE(InfoExtractor):
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_url = self._html_search_regex(
|
||||
r'<param name="src" value="([^"]+)"', webpage, 'video url')
|
||||
title = self._html_search_regex(
|
||||
r'<title>([^<]+) RUHD.ru - Видео Высокого качества №1 в России!</title>', webpage, 'title')
|
||||
r'<title>([^<]+) RUHD.ru - Видео Высокого качества №1 в России!</title>',
|
||||
webpage, 'title')
|
||||
description = self._html_search_regex(
|
||||
r'(?s)<div id="longdesc">(.+?)<span id="showlink">', webpage, 'description', fatal=False)
|
||||
r'(?s)<div id="longdesc">(.+?)<span id="showlink">',
|
||||
webpage, 'description', fatal=False)
|
||||
thumbnail = self._html_search_regex(
|
||||
r'<param name="previewImage" value="([^"]+)"', webpage, 'thumbnail', fatal=False)
|
||||
r'<param name="previewImage" value="([^"]+)"',
|
||||
webpage, 'thumbnail', fatal=False)
|
||||
if thumbnail:
|
||||
thumbnail = 'http://www.ruhd.ru' + thumbnail
|
||||
|
||||
|
117
youtube_dl/extractor/sexykarma.py
Normal file
117
youtube_dl/extractor/sexykarma.py
Normal file
@ -0,0 +1,117 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
unified_strdate,
|
||||
parse_duration,
|
||||
int_or_none,
|
||||
)
|
||||
|
||||
|
||||
class SexyKarmaIE(InfoExtractor):
|
||||
IE_DESC = 'Sexy Karma and Watch Indian Porn'
|
||||
_VALID_URL = r'https?://(?:www\.)?(?:sexykarma\.com|watchindianporn\.net)/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
|
||||
'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
|
||||
'info_dict': {
|
||||
'id': 'yHI70cOyIHt',
|
||||
'display_id': 'taking-a-quick-pee',
|
||||
'ext': 'mp4',
|
||||
'title': 'Taking a quick pee.',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
'uploader': 'wildginger7',
|
||||
'upload_date': '20141007',
|
||||
'duration': 22,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'categories': list,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
|
||||
'md5': 'dd216c68d29b49b12842b9babe762a5d',
|
||||
'info_dict': {
|
||||
'id': '8Id6EZPbuHf',
|
||||
'display_id': 'pot-pixie-tribute',
|
||||
'ext': 'mp4',
|
||||
'title': 'pot_pixie tribute',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
'uploader': 'banffite',
|
||||
'upload_date': '20141013',
|
||||
'duration': 16,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'categories': list,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
|
||||
'md5': '9afb80675550406ed9a63ac2819ef69d',
|
||||
'info_dict': {
|
||||
'id': 'dW2mtctxJfs',
|
||||
'display_id': 'desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number',
|
||||
'ext': 'mp4',
|
||||
'title': 'Desi dancer namrata stripping completely nude and dancing on a hot number',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
'uploader': 'Don',
|
||||
'upload_date': '20140213',
|
||||
'duration': 83,
|
||||
'view_count': int,
|
||||
'comment_count': int,
|
||||
'categories': list,
|
||||
}
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
display_id = mobj.group('display_id')
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
video_url = self._html_search_regex(
|
||||
r"url: escape\('([^']+)'\)", webpage, 'url')
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<h2 class="he2"><span>(.*?)</span>',
|
||||
webpage, 'title')
|
||||
thumbnail = self._html_search_regex(
|
||||
r'<span id="container"><img\s+src="([^"]+)"',
|
||||
webpage, 'thumbnail', fatal=False)
|
||||
|
||||
uploader = self._html_search_regex(
|
||||
r'class="aupa">\s*(.*?)</a>',
|
||||
webpage, 'uploader')
|
||||
upload_date = unified_strdate(self._html_search_regex(
|
||||
r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))
|
||||
|
||||
duration = parse_duration(self._search_regex(
|
||||
r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
|
||||
webpage, 'duration', fatal=False))
|
||||
|
||||
view_count = int_or_none(self._search_regex(
|
||||
r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
|
||||
webpage, 'view count', fatal=False))
|
||||
comment_count = int_or_none(self._search_regex(
|
||||
r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
|
||||
webpage, 'comment count', fatal=False))
|
||||
|
||||
categories = re.findall(
|
||||
r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
|
||||
webpage)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'url': video_url,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'uploader': uploader,
|
||||
'upload_date': upload_date,
|
||||
'duration': duration,
|
||||
'view_count': view_count,
|
||||
'comment_count': comment_count,
|
||||
'categories': categories,
|
||||
}
|
@ -40,14 +40,15 @@ class SoundcloudIE(InfoExtractor):
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
|
||||
'file': '62986583.mp3',
|
||||
'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
|
||||
'info_dict': {
|
||||
"upload_date": "20121011",
|
||||
"description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
|
||||
"uploader": "E.T. ExTerrestrial Music",
|
||||
"title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1",
|
||||
"duration": 143,
|
||||
'id': '62986583',
|
||||
'ext': 'mp3',
|
||||
'upload_date': '20121011',
|
||||
'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
|
||||
'uploader': 'E.T. ExTerrestrial Music',
|
||||
'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
|
||||
'duration': 143,
|
||||
}
|
||||
},
|
||||
# not streamable song
|
||||
@ -103,7 +104,7 @@ class SoundcloudIE(InfoExtractor):
|
||||
'id': '128590877',
|
||||
'ext': 'mp3',
|
||||
'title': 'Bus Brakes',
|
||||
'description': 'md5:0170be75dd395c96025d210d261c784e',
|
||||
'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
|
||||
'uploader': 'oddsamples',
|
||||
'upload_date': '20140109',
|
||||
'duration': 17,
|
||||
@ -140,6 +141,7 @@ class SoundcloudIE(InfoExtractor):
|
||||
'description': info['description'],
|
||||
'thumbnail': thumbnail,
|
||||
'duration': int_or_none(info.get('duration'), 1000),
|
||||
'webpage_url': info.get('permalink_url'),
|
||||
}
|
||||
formats = []
|
||||
if info.get('downloadable', False):
|
||||
|
76
youtube_dl/extractor/sportbox.py
Normal file
76
youtube_dl/extractor/sportbox.py
Normal file
@ -0,0 +1,76 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
parse_duration,
|
||||
parse_iso8601,
|
||||
)
|
||||
|
||||
|
||||
class SportBoxIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://news\.sportbox\.ru/Vidy_sporta/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
|
||||
'md5': 'ff56a598c2cf411a9a38a69709e97079',
|
||||
'info_dict': {
|
||||
'id': '80822',
|
||||
'ext': 'mp4',
|
||||
'title': 'Гонка 2 заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
|
||||
'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
'timestamp': 1411896237,
|
||||
'upload_date': '20140928',
|
||||
'duration': 4846,
|
||||
},
|
||||
'params': {
|
||||
# m3u8 download
|
||||
'skip_download': True,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
|
||||
'only_matching': True,
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
display_id = mobj.group('display_id')
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
video_id = self._search_regex(
|
||||
r'src="/vdl/player/media/(\d+)"', webpage, 'video id')
|
||||
|
||||
player = self._download_webpage(
|
||||
'http://news.sportbox.ru/vdl/player/media/%s' % video_id,
|
||||
display_id, 'Downloading player webpage')
|
||||
|
||||
hls = self._search_regex(
|
||||
r"var\s+original_hls_file\s*=\s*'([^']+)'", player, 'hls file')
|
||||
|
||||
formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
|
||||
description = self._html_search_regex(
|
||||
r'(?s)<div itemprop="description">(.+?)</div>', webpage, 'description', fatal=False)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
timestamp = parse_iso8601(self._search_regex(
|
||||
r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
|
||||
duration = parse_duration(self._html_search_regex(
|
||||
r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'timestamp': timestamp,
|
||||
'duration': duration,
|
||||
'formats': formats,
|
||||
}
|
43
youtube_dl/extractor/srmediathek.py
Normal file
43
youtube_dl/extractor/srmediathek.py
Normal file
@ -0,0 +1,43 @@
|
||||
# encoding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import js_to_json
|
||||
|
||||
|
||||
class SRMediathekIE(InfoExtractor):
|
||||
IE_DESC = 'Süddeutscher Rundfunk'
|
||||
_VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
|
||||
'info_dict': {
|
||||
'id': '28455',
|
||||
'ext': 'mp4',
|
||||
'title': 'sportarena (26.10.2014)',
|
||||
'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
murls = json.loads(js_to_json(self._search_regex(
|
||||
r'var mediaURLs\s*=\s*(.*?);\n', webpage, 'video URLs')))
|
||||
formats = [{'url': murl} for murl in murls]
|
||||
self._sort_formats(formats)
|
||||
|
||||
title = json.loads(js_to_json(self._search_regex(
|
||||
r'var mediaTitles\s*=\s*(.*?);\n', webpage, 'title')))[0]
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'description': self._og_search_description(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
}
|
@ -10,7 +10,6 @@ class SyfyIE(InfoExtractor):
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://www.syfy.com/videos/Robot%20Combat%20League/Behind%20the%20Scenes/vid:2631458',
|
||||
'md5': 'e07de1d52c7278adbb9b9b1c93a66849',
|
||||
'info_dict': {
|
||||
'id': 'NmqMrGnXvmO1',
|
||||
'ext': 'flv',
|
||||
|
@ -65,6 +65,22 @@ class TEDIE(SubtitlesInfoExtractor):
|
||||
'title': 'Who are the hackers?',
|
||||
},
|
||||
'playlist_mincount': 6,
|
||||
}, {
|
||||
# contains a youtube video
|
||||
'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
|
||||
'add_ie': ['Youtube'],
|
||||
'info_dict': {
|
||||
'id': '_ZG8HBuDjgc',
|
||||
'ext': 'mp4',
|
||||
'title': 'Douglas Adams: Parrots the Universe and Everything',
|
||||
'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
|
||||
'uploader': 'University of California Television (UCTV)',
|
||||
'uploader_id': 'UCtelevision',
|
||||
'upload_date': '20080522',
|
||||
},
|
||||
'params': {
|
||||
'skip_download': True,
|
||||
},
|
||||
}]
|
||||
|
||||
_NATIVE_FORMATS = {
|
||||
@ -114,6 +130,13 @@ class TEDIE(SubtitlesInfoExtractor):
|
||||
|
||||
talk_info = self._extract_info(webpage)['talks'][0]
|
||||
|
||||
if talk_info.get('external') is not None:
|
||||
self.to_screen('Found video from %s' % talk_info['external']['service'])
|
||||
return {
|
||||
'_type': 'url',
|
||||
'url': talk_info['external']['uri'],
|
||||
}
|
||||
|
||||
formats = [{
|
||||
'url': format_url,
|
||||
'format_id': format_id,
|
||||
|
19
youtube_dl/extractor/telecinco.py
Normal file
19
youtube_dl/extractor/telecinco.py
Normal file
@ -0,0 +1,19 @@
|
||||
#coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .mitele import MiTeleIE
|
||||
|
||||
|
||||
class TelecincoIE(MiTeleIE):
|
||||
IE_NAME = 'telecinco.es'
|
||||
_VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<episode>.*?)\.html'
|
||||
|
||||
_TEST = {
|
||||
'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
|
||||
'info_dict': {
|
||||
'id': 'MDSVID20141015_0058',
|
||||
'ext': 'mp4',
|
||||
'title': 'Con Martín Berasategui, hacer un bacalao al ...',
|
||||
'duration': 662,
|
||||
},
|
||||
}
|
70
youtube_dl/extractor/theonion.py
Normal file
70
youtube_dl/extractor/theonion.py
Normal file
@ -0,0 +1,70 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import ExtractorError
|
||||
|
||||
|
||||
class TheOnionIE(InfoExtractor):
|
||||
_VALID_URL = r'(?x)https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<article_id>[0-9]+)/?'
|
||||
_TEST = {
|
||||
'url': 'http://www.theonion.com/video/man-wearing-mm-jacket-gods-image,36918/',
|
||||
'md5': '19eaa9a39cf9b9804d982e654dc791ee',
|
||||
'info_dict': {
|
||||
'id': '2133',
|
||||
'ext': 'mp4',
|
||||
'title': 'Man Wearing M&M Jacket Apparently Made In God\'s Image',
|
||||
'description': 'md5:cc12448686b5600baae9261d3e180910',
|
||||
'thumbnail': 're:^https?://.*\.jpg\?\d+$',
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
article_id = mobj.group('article_id')
|
||||
|
||||
webpage = self._download_webpage(url, article_id)
|
||||
|
||||
video_id = self._search_regex(
|
||||
r'"videoId":\s(\d+),', webpage, 'video ID')
|
||||
title = self._og_search_title(webpage)
|
||||
description = self._og_search_description(webpage)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
|
||||
sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
|
||||
if not sources:
|
||||
raise ExtractorError(
|
||||
'No sources found for video %s' % video_id, expected=True)
|
||||
|
||||
formats = []
|
||||
for src, type_ in sources:
|
||||
if type_ == 'video/mp4':
|
||||
formats.append({
|
||||
'format_id': 'mp4_sd',
|
||||
'preference': 1,
|
||||
'url': src,
|
||||
})
|
||||
elif type_ == 'video/webm':
|
||||
formats.append({
|
||||
'format_id': 'webm_sd',
|
||||
'preference': 0,
|
||||
'url': src,
|
||||
})
|
||||
elif type_ == 'application/x-mpegURL':
|
||||
formats.extend(
|
||||
self._extract_m3u8_formats(src, video_id, preference=-1))
|
||||
else:
|
||||
self.report_warning(
|
||||
'Encountered unexpected format: %s' % type_)
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
'thumbnail': thumbnail,
|
||||
'description': description,
|
||||
}
|
@ -6,6 +6,7 @@ import json
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
compat_str,
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
xpath_with_ns,
|
||||
)
|
||||
@ -34,10 +35,21 @@ class ThePlatformIE(InfoExtractor):
|
||||
'skip_download': True,
|
||||
},
|
||||
}
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
if mobj.group('config'):
|
||||
config_url = url+ '&form=json'
|
||||
config_url = config_url.replace('swf/', 'config/')
|
||||
config_url = config_url.replace('onsite/', 'onsite/config/')
|
||||
config = self._download_json(config_url, video_id, 'Downloading config')
|
||||
smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
|
||||
else:
|
||||
smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
|
||||
'format=smil&mbr=true'.format(video_id))
|
||||
|
||||
|
||||
def _get_info(self, video_id, smil_url):
|
||||
meta = self._download_xml(smil_url, video_id)
|
||||
|
||||
try:
|
||||
error_msg = next(
|
||||
n.attrib['abstract']
|
||||
@ -89,10 +101,14 @@ class ThePlatformIE(InfoExtractor):
|
||||
for f in switch.findall(_x('smil:video')):
|
||||
attr = f.attrib
|
||||
vbr = int(attr['system-bitrate']) // 1000
|
||||
ext = determine_ext(attr['src'])
|
||||
if ext == 'once':
|
||||
ext = 'mp4'
|
||||
formats.append({
|
||||
'format_id': compat_str(vbr),
|
||||
'url': attr['src'],
|
||||
'vbr': vbr,
|
||||
'ext': ext,
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
|
||||
@ -104,17 +120,3 @@ class ThePlatformIE(InfoExtractor):
|
||||
'thumbnail': info['defaultThumbnailUrl'],
|
||||
'duration': info['duration']//1000,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
if mobj.group('config'):
|
||||
config_url = url+ '&form=json'
|
||||
config_url = config_url.replace('swf/', 'config/')
|
||||
config_url = config_url.replace('onsite/', 'onsite/config/')
|
||||
config = self._download_json(config_url, video_id, 'Downloading config')
|
||||
smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
|
||||
else:
|
||||
smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
|
||||
'format=smil&mbr=true'.format(video_id))
|
||||
return self._get_info(video_id, smil_url)
|
||||
|
100
youtube_dl/extractor/thesixtyone.py
Normal file
100
youtube_dl/extractor/thesixtyone.py
Normal file
@ -0,0 +1,100 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import unified_strdate
|
||||
|
||||
|
||||
class TheSixtyOneIE(InfoExtractor):
|
||||
_VALID_URL = r'''(?x)https?://(?:www\.)?thesixtyone\.com/
|
||||
(?:.*?/)*
|
||||
(?:
|
||||
s|
|
||||
song/comments/list|
|
||||
song
|
||||
)/(?P<id>[A-Za-z0-9]+)/?$'''
|
||||
_SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
|
||||
_SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}.thesixtyone.com/thesixtyone_production/audio/{0:}_stream'
|
||||
_THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://www.thesixtyone.com/s/SrE3zD7s1jt/',
|
||||
'md5': '821cc43b0530d3222e3e2b70bb4622ea',
|
||||
'info_dict': {
|
||||
'id': 'SrE3zD7s1jt',
|
||||
'ext': 'mp3',
|
||||
'title': 'CASIO - Unicorn War Mixtape',
|
||||
'thumbnail': 're:^https?://.*_desktop$',
|
||||
'upload_date': '20071217',
|
||||
'duration': 3208,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'http://www.thesixtyone.com/song/comments/list/SrE3zD7s1jt',
|
||||
'only_matching': True,
|
||||
},
|
||||
{
|
||||
'url': 'http://www.thesixtyone.com/s/ULoiyjuJWli#/s/SrE3zD7s1jt/',
|
||||
'only_matching': True,
|
||||
},
|
||||
{
|
||||
'url': 'http://www.thesixtyone.com/#/s/SrE3zD7s1jt/',
|
||||
'only_matching': True,
|
||||
},
|
||||
{
|
||||
'url': 'http://www.thesixtyone.com/song/SrE3zD7s1jt/',
|
||||
'only_matching': True,
|
||||
},
|
||||
]
|
||||
|
||||
_DECODE_MAP = {
|
||||
"x": "a",
|
||||
"m": "b",
|
||||
"w": "c",
|
||||
"q": "d",
|
||||
"n": "e",
|
||||
"p": "f",
|
||||
"a": "0",
|
||||
"h": "1",
|
||||
"e": "2",
|
||||
"u": "3",
|
||||
"s": "4",
|
||||
"i": "5",
|
||||
"o": "6",
|
||||
"y": "7",
|
||||
"r": "8",
|
||||
"c": "9"
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
song_id = mobj.group('id')
|
||||
|
||||
webpage = self._download_webpage(
|
||||
self._SONG_URL_TEMPLATE.format(song_id), song_id)
|
||||
|
||||
song_data = json.loads(self._search_regex(
|
||||
r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
|
||||
keys = [self._DECODE_MAP.get(s, s) for s in song_data['key']]
|
||||
url = self._SONG_FILE_URL_TEMPLATE.format(
|
||||
"".join(reversed(keys)), **song_data)
|
||||
|
||||
formats = [{
|
||||
'format_id': 'sd',
|
||||
'url': url,
|
||||
'ext': 'mp3',
|
||||
}]
|
||||
|
||||
return {
|
||||
'id': song_id,
|
||||
'title': '{artist:} - {name:}'.format(**song_data),
|
||||
'formats': formats,
|
||||
'comment_count': song_data.get('comments_count'),
|
||||
'duration': song_data.get('play_time'),
|
||||
'like_count': song_data.get('score'),
|
||||
'thumbnail': self._THUMBNAIL_URL_TEMPLATE.format(**song_data),
|
||||
'upload_date': unified_strdate(song_data.get('publish_date')),
|
||||
}
|
@ -1,13 +1,12 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import xpath_text
|
||||
|
||||
|
||||
class TruTubeIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?trutube\.tv/video/(?P<id>[0-9]+)/.*'
|
||||
_TEST = {
|
||||
_VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>[0-9]+)'
|
||||
_TESTS = [{
|
||||
'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
|
||||
'md5': 'c5b6e301b0a2040b074746cbeaa26ca1',
|
||||
'info_dict': {
|
||||
@ -16,29 +15,26 @@ class TruTubeIE(InfoExtractor):
|
||||
'title': 'Ramses II - Proven To Be A Red Headed Caucasoid',
|
||||
'thumbnail': 're:^http:.*\.jpg$',
|
||||
}
|
||||
}
|
||||
}, {
|
||||
'url': 'https://trutube.tv/nuevo/player/embed.php?v=14880',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_title = self._og_search_title(webpage).strip()
|
||||
thumbnail = self._search_regex(
|
||||
r"var splash_img = '([^']+)';", webpage, 'thumbnail', fatal=False)
|
||||
config = self._download_xml(
|
||||
'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
|
||||
video_id, transform_source=lambda s: s.strip())
|
||||
|
||||
all_formats = re.finditer(
|
||||
r"var (?P<key>[a-z]+)_video_file\s*=\s*'(?P<url>[^']+)';", webpage)
|
||||
formats = [{
|
||||
'format_id': m.group('key'),
|
||||
'quality': -i,
|
||||
'url': m.group('url'),
|
||||
} for i, m in enumerate(all_formats)]
|
||||
self._sort_formats(formats)
|
||||
# filehd is always 404
|
||||
video_url = xpath_text(config, './file', 'video URL', fatal=True)
|
||||
title = xpath_text(config, './title', 'title')
|
||||
thumbnail = xpath_text(config, './image', ' thumbnail')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': video_title,
|
||||
'formats': formats,
|
||||
'url': video_url,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
}
|
||||
|
@ -4,9 +4,6 @@ from __future__ import unicode_literals
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
)
|
||||
|
||||
|
||||
class TumblrIE(InfoExtractor):
|
||||
@ -18,7 +15,7 @@ class TumblrIE(InfoExtractor):
|
||||
'id': '54196191430',
|
||||
'ext': 'mp4',
|
||||
'title': 'tatiana maslany news, Orphan Black || DVD extra - behind the scenes ↳...',
|
||||
'description': 'md5:dfac39636969fe6bf1caa2d50405f069',
|
||||
'description': 'md5:37db8211e40b50c7c44e95da14f630b7',
|
||||
'thumbnail': 're:http://.*\.jpg',
|
||||
}
|
||||
}, {
|
||||
@ -27,7 +24,7 @@ class TumblrIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '90208453769',
|
||||
'ext': 'mp4',
|
||||
'title': '5SOS STRUM ;)',
|
||||
'title': '5SOS STRUM ;]',
|
||||
'description': 'md5:dba62ac8639482759c8eb10ce474586a',
|
||||
'thumbnail': 're:http://.*\.jpg',
|
||||
}
|
||||
@ -41,18 +38,12 @@ class TumblrIE(InfoExtractor):
|
||||
url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
re_video = r'src=\\x22(?P<video_url>http://%s\.tumblr\.com/video_file/%s/(.*?))\\x22 type=\\x22video/(?P<ext>.*?)\\x22' % (blog, video_id)
|
||||
video = re.search(re_video, webpage)
|
||||
if video is None:
|
||||
raise ExtractorError('Unable to extract video')
|
||||
video_url = video.group('video_url')
|
||||
ext = video.group('ext')
|
||||
|
||||
video_thumbnail = self._search_regex(
|
||||
r'posters.*?\[\\x22(.*?)\\x22',
|
||||
webpage, 'thumbnail', fatal=False) # We pick the first poster
|
||||
if video_thumbnail:
|
||||
video_thumbnail = video_thumbnail.replace('\\\\/', '/')
|
||||
iframe_url = self._search_regex(
|
||||
r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
|
||||
webpage, 'iframe url')
|
||||
iframe = self._download_webpage(iframe_url, video_id)
|
||||
video_url = self._search_regex(r'<source src="([^"]+)"',
|
||||
iframe, 'video url')
|
||||
|
||||
# The only place where you can get a title, it's not complete,
|
||||
# but searching in other places doesn't work for all videos
|
||||
@ -62,9 +53,9 @@ class TumblrIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'url': video_url,
|
||||
'title': video_title,
|
||||
'description': self._html_search_meta('description', webpage),
|
||||
'thumbnail': video_thumbnail,
|
||||
'ext': ext,
|
||||
'url': video_url,
|
||||
'ext': 'mp4',
|
||||
'title': video_title,
|
||||
'description': self._og_search_description(webpage),
|
||||
'thumbnail': self._og_search_thumbnail(webpage),
|
||||
}
|
||||
|
187
youtube_dl/extractor/twitch.py
Normal file
187
youtube_dl/extractor/twitch.py
Normal file
@ -0,0 +1,187 @@
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import itertools
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
parse_iso8601,
|
||||
)
|
||||
|
||||
|
||||
class TwitchIE(InfoExtractor):
|
||||
# TODO: One broadcast may be split into multiple videos. The key
|
||||
# 'broadcast_id' is the same for all parts, and 'broadcast_part'
|
||||
# starts at 1 and increases. Can we treat all parts as one video?
|
||||
_VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
|
||||
(?:
|
||||
(?P<channelid>[^/]+)|
|
||||
(?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
|
||||
(?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
|
||||
)
|
||||
/?(?:\#.*)?$
|
||||
"""
|
||||
_PAGE_LIMIT = 100
|
||||
_API_BASE = 'https://api.twitch.tv'
|
||||
_TESTS = [{
|
||||
'url': 'http://www.twitch.tv/riotgames/b/577357806',
|
||||
'info_dict': {
|
||||
'id': 'a577357806',
|
||||
'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
|
||||
},
|
||||
'playlist_mincount': 12,
|
||||
}, {
|
||||
'url': 'http://www.twitch.tv/acracingleague/c/5285812',
|
||||
'info_dict': {
|
||||
'id': 'c5285812',
|
||||
'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
|
||||
},
|
||||
'playlist_mincount': 3,
|
||||
}, {
|
||||
'url': 'http://www.twitch.tv/vanillatv',
|
||||
'info_dict': {
|
||||
'id': 'vanillatv',
|
||||
'title': 'VanillaTV',
|
||||
},
|
||||
'playlist_mincount': 412,
|
||||
}]
|
||||
|
||||
def _handle_error(self, response):
|
||||
if not isinstance(response, dict):
|
||||
return
|
||||
error = response.get('error')
|
||||
if error:
|
||||
raise ExtractorError(
|
||||
'%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
|
||||
expected=True)
|
||||
|
||||
def _download_json(self, url, video_id, note='Downloading JSON metadata'):
|
||||
response = super(TwitchIE, self)._download_json(url, video_id, note)
|
||||
self._handle_error(response)
|
||||
return response
|
||||
|
||||
def _extract_media(self, item, item_id):
|
||||
ITEMS = {
|
||||
'a': 'video',
|
||||
'c': 'chapter',
|
||||
}
|
||||
info = self._extract_info(self._download_json(
|
||||
'%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
|
||||
'Downloading %s info JSON' % ITEMS[item]))
|
||||
response = self._download_json(
|
||||
'%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
|
||||
'Downloading %s playlist JSON' % ITEMS[item])
|
||||
entries = []
|
||||
chunks = response['chunks']
|
||||
qualities = list(chunks.keys())
|
||||
for num, fragment in enumerate(zip(*chunks.values()), start=1):
|
||||
formats = []
|
||||
for fmt_num, fragment_fmt in enumerate(fragment):
|
||||
format_id = qualities[fmt_num]
|
||||
fmt = {
|
||||
'url': fragment_fmt['url'],
|
||||
'format_id': format_id,
|
||||
'quality': 1 if format_id == 'live' else 0,
|
||||
}
|
||||
m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
|
||||
if m:
|
||||
fmt['height'] = int(m.group('height'))
|
||||
formats.append(fmt)
|
||||
self._sort_formats(formats)
|
||||
entry = dict(info)
|
||||
entry['id'] = '%s_%d' % (entry['id'], num)
|
||||
entry['title'] = '%s part %d' % (entry['title'], num)
|
||||
entry['formats'] = formats
|
||||
entries.append(entry)
|
||||
return self.playlist_result(entries, info['id'], info['title'])
|
||||
|
||||
def _extract_info(self, info):
|
||||
return {
|
||||
'id': info['_id'],
|
||||
'title': info['title'],
|
||||
'description': info['description'],
|
||||
'duration': info['length'],
|
||||
'thumbnail': info['preview'],
|
||||
'uploader': info['channel']['display_name'],
|
||||
'uploader_id': info['channel']['name'],
|
||||
'timestamp': parse_iso8601(info['recorded_at']),
|
||||
'view_count': info['views'],
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
if mobj.group('chapterid'):
|
||||
return self._extract_media('c', mobj.group('chapterid'))
|
||||
|
||||
"""
|
||||
webpage = self._download_webpage(url, chapter_id)
|
||||
m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage)
|
||||
if not m:
|
||||
raise ExtractorError('Cannot find archive of a chapter')
|
||||
archive_id = m.group(1)
|
||||
|
||||
api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id
|
||||
doc = self._download_xml(
|
||||
api, chapter_id,
|
||||
note='Downloading chapter information',
|
||||
errnote='Chapter information download failed')
|
||||
for a in doc.findall('.//archive'):
|
||||
if archive_id == a.find('./id').text:
|
||||
break
|
||||
else:
|
||||
raise ExtractorError('Could not find chapter in chapter information')
|
||||
|
||||
video_url = a.find('./video_file_url').text
|
||||
video_ext = video_url.rpartition('.')[2] or 'flv'
|
||||
|
||||
chapter_api_url = 'https://api.twitch.tv/kraken/videos/c' + chapter_id
|
||||
chapter_info = self._download_json(
|
||||
chapter_api_url, 'c' + chapter_id,
|
||||
note='Downloading chapter metadata',
|
||||
errnote='Download of chapter metadata failed')
|
||||
|
||||
bracket_start = int(doc.find('.//bracket_start').text)
|
||||
bracket_end = int(doc.find('.//bracket_end').text)
|
||||
|
||||
# TODO determine start (and probably fix up file)
|
||||
# youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457
|
||||
#video_url += '?start=' + TODO:start_timestamp
|
||||
# bracket_start is 13290, but we want 51670615
|
||||
self._downloader.report_warning('Chapter detected, but we can just download the whole file. '
|
||||
'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end)))
|
||||
|
||||
info = {
|
||||
'id': 'c' + chapter_id,
|
||||
'url': video_url,
|
||||
'ext': video_ext,
|
||||
'title': chapter_info['title'],
|
||||
'thumbnail': chapter_info['preview'],
|
||||
'description': chapter_info['description'],
|
||||
'uploader': chapter_info['channel']['display_name'],
|
||||
'uploader_id': chapter_info['channel']['name'],
|
||||
}
|
||||
return info
|
||||
"""
|
||||
elif mobj.group('videoid'):
|
||||
return self._extract_media('a', mobj.group('videoid'))
|
||||
elif mobj.group('channelid'):
|
||||
channel_id = mobj.group('channelid')
|
||||
info = self._download_json(
|
||||
'%s/kraken/channels/%s' % (self._API_BASE, channel_id),
|
||||
channel_id, 'Downloading channel info JSON')
|
||||
channel_name = info.get('display_name') or info.get('name')
|
||||
entries = []
|
||||
offset = 0
|
||||
limit = self._PAGE_LIMIT
|
||||
for counter in itertools.count(1):
|
||||
response = self._download_json(
|
||||
'%s/kraken/channels/%s/videos/?offset=%d&limit=%d'
|
||||
% (self._API_BASE, channel_id, offset, limit),
|
||||
channel_id, 'Downloading channel videos JSON page %d' % counter)
|
||||
videos = response['videos']
|
||||
if not videos:
|
||||
break
|
||||
entries.extend([self.url_result(video['url'], 'Twitch') for video in videos])
|
||||
offset += limit
|
||||
return self.playlist_result(entries, channel_id, channel_name)
|
@ -72,7 +72,7 @@ class UstreamChannelIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '10874166',
|
||||
},
|
||||
'playlist_mincount': 54,
|
||||
'playlist_mincount': 17,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '84196',
|
||||
'ext': 'mp4',
|
||||
'title': 'Hevnen er søt episode 10: Abu',
|
||||
'title': 'Hevnen er søt episode 1:10 - Abu',
|
||||
'description': 'md5:e25e4badb5f544b04341e14abdc72234',
|
||||
'thumbnail': 're:^https?://.*\.jpg',
|
||||
'duration': 648.000,
|
||||
@ -67,9 +67,7 @@ class VGTVIE(InfoExtractor):
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
|
||||
video_id = self._match_id(url)
|
||||
data = self._download_json(
|
||||
'http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website' % video_id,
|
||||
video_id, 'Downloading media JSON')
|
||||
|
@ -1,55 +1,85 @@
|
||||
import json
|
||||
import re
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
)
|
||||
|
||||
|
||||
class ViddlerIE(InfoExtractor):
|
||||
_VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler\.com)/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
|
||||
_VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
|
||||
_TEST = {
|
||||
u"url": u"http://www.viddler.com/v/43903784",
|
||||
u'file': u'43903784.mp4',
|
||||
u'md5': u'fbbaedf7813e514eb7ca30410f439ac9',
|
||||
u'info_dict': {
|
||||
u"title": u"Video Made Easy",
|
||||
u"uploader": u"viddler",
|
||||
u"duration": 100.89,
|
||||
"url": "http://www.viddler.com/v/43903784",
|
||||
'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
|
||||
'info_dict': {
|
||||
'id': '43903784',
|
||||
'ext': 'mp4',
|
||||
"title": "Video Made Easy",
|
||||
'description': 'You don\'t need to be a professional to make high-quality video content. Viddler provides some quick and easy tips on how to produce great video content with limited resources. ',
|
||||
"uploader": "viddler",
|
||||
'timestamp': 1335371429,
|
||||
'upload_date': '20120425',
|
||||
"duration": 100.89,
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
'view_count': int,
|
||||
'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
video_id = self._match_id(url)
|
||||
|
||||
embed_url = mobj.group('domain') + u'/embed/' + video_id
|
||||
webpage = self._download_webpage(embed_url, video_id)
|
||||
json_url = (
|
||||
'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
|
||||
video_id)
|
||||
data = self._download_json(json_url, video_id)['video']
|
||||
|
||||
video_sources_code = self._search_regex(
|
||||
r"(?ms)sources\s*:\s*(\{.*?\})", webpage, u'video URLs')
|
||||
video_sources = json.loads(video_sources_code.replace("'", '"'))
|
||||
formats = []
|
||||
for filed in data['files']:
|
||||
if filed.get('status', 'ready') != 'ready':
|
||||
continue
|
||||
f = {
|
||||
'format_id': filed['profile_id'],
|
||||
'format_note': filed['profile_name'],
|
||||
'url': self._proto_relative_url(filed['url']),
|
||||
'width': int_or_none(filed.get('width')),
|
||||
'height': int_or_none(filed.get('height')),
|
||||
'filesize': int_or_none(filed.get('size')),
|
||||
'ext': filed.get('ext'),
|
||||
'source_preference': -1,
|
||||
}
|
||||
formats.append(f)
|
||||
|
||||
formats = [{
|
||||
'url': video_url,
|
||||
'format': format_id,
|
||||
} for video_url, format_id in video_sources.items()]
|
||||
if filed.get('cdn_url'):
|
||||
f = f.copy()
|
||||
f['url'] = self._proto_relative_url(filed['cdn_url'])
|
||||
f['format_id'] = filed['profile_id'] + '-cdn'
|
||||
f['source_preference'] = 1
|
||||
formats.append(f)
|
||||
|
||||
title = self._html_search_regex(
|
||||
r"title\s*:\s*'([^']*)'", webpage, u'title')
|
||||
uploader = self._html_search_regex(
|
||||
r"authorName\s*:\s*'([^']*)'", webpage, u'uploader', fatal=False)
|
||||
duration_s = self._html_search_regex(
|
||||
r"duration\s*:\s*([0-9.]*)", webpage, u'duration', fatal=False)
|
||||
duration = float(duration_s) if duration_s else None
|
||||
thumbnail = self._html_search_regex(
|
||||
r"thumbnail\s*:\s*'([^']*)'",
|
||||
webpage, u'thumbnail', fatal=False)
|
||||
if filed.get('html5_video_source'):
|
||||
f = f.copy()
|
||||
f['url'] = self._proto_relative_url(
|
||||
filed['html5_video_source'])
|
||||
f['format_id'] = filed['profile_id'] + '-html5'
|
||||
f['source_preference'] = 0
|
||||
formats.append(f)
|
||||
self._sort_formats(formats)
|
||||
|
||||
categories = [
|
||||
t.get('text') for t in data.get('tags', []) if 'text' in t]
|
||||
|
||||
return {
|
||||
'_type': 'video',
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'thumbnail': thumbnail,
|
||||
'uploader': uploader,
|
||||
'duration': duration,
|
||||
'title': data['title'],
|
||||
'formats': formats,
|
||||
'description': data.get('description'),
|
||||
'timestamp': int_or_none(data.get('upload_time')),
|
||||
'thumbnail': self._proto_relative_url(data.get('thumbnail_url')),
|
||||
'uploader': data.get('author'),
|
||||
'duration': float_or_none(data.get('length')),
|
||||
'view_count': int_or_none(data.get('view_count')),
|
||||
'categories': categories,
|
||||
}
|
||||
|
33
youtube_dl/extractor/vidzi.py
Normal file
33
youtube_dl/extractor/vidzi.py
Normal file
@ -0,0 +1,33 @@
|
||||
#coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
|
||||
|
||||
class VidziIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
|
||||
_TEST = {
|
||||
'url': 'http://vidzi.tv/cghql9yq6emu.html',
|
||||
'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',
|
||||
'info_dict': {
|
||||
'id': 'cghql9yq6emu',
|
||||
'ext': 'mp4',
|
||||
'title': 'youtube-dl test video 1\\\\2\'3/4<5\\\\6ä7↭',
|
||||
},
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_url = self._html_search_regex(
|
||||
r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
|
||||
title = self._html_search_regex(
|
||||
r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'url': video_url,
|
||||
}
|
||||
|
@ -8,13 +8,11 @@ import itertools
|
||||
from .common import InfoExtractor
|
||||
from .subtitles import SubtitlesInfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
compat_HTTPError,
|
||||
compat_urllib_parse,
|
||||
compat_urllib_request,
|
||||
compat_urlparse,
|
||||
ExtractorError,
|
||||
get_element_by_attribute,
|
||||
InAdvancePagedList,
|
||||
int_or_none,
|
||||
RegexNotFoundError,
|
||||
@ -56,7 +54,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||
|
||||
# _VALID_URL matches Vimeo URLs
|
||||
_VALID_URL = r'''(?x)
|
||||
(?P<proto>(?:https?:)?//)?
|
||||
https?://
|
||||
(?:(?:www|(?P<player>player))\.)?
|
||||
vimeo(?P<pro>pro)?\.com/
|
||||
(?!channels/[^/?#]+/?(?:$|[?#])|album/)
|
||||
@ -157,6 +155,18 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||
'duration': 62,
|
||||
}
|
||||
},
|
||||
{
|
||||
# from https://www.ouya.tv/game/Pier-Solar-and-the-Great-Architects/
|
||||
'url': 'https://player.vimeo.com/video/98044508',
|
||||
'note': 'The js code contains assignments to the same variable as the config',
|
||||
'info_dict': {
|
||||
'id': '98044508',
|
||||
'ext': 'mp4',
|
||||
'title': 'Pier Solar OUYA Official Trailer',
|
||||
'uploader': 'Tulio Gonçalves',
|
||||
'uploader_id': 'user28849593',
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _verify_video_password(self, url, video_id, webpage):
|
||||
@ -244,7 +254,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
|
||||
# We try to find out to which variable is assigned the config dic
|
||||
m_variable_name = re.search('(\w)\.video\.id', webpage)
|
||||
if m_variable_name is not None:
|
||||
config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1))
|
||||
config_re = r'%s=({[^}].+?});' % re.escape(m_variable_name.group(1))
|
||||
else:
|
||||
config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
|
||||
config = self._search_regex(config_re, webpage, 'info section',
|
||||
@ -502,7 +512,7 @@ class VimeoReviewIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': '91613211',
|
||||
'ext': 'mp4',
|
||||
'title': 'Death by dogma versus assembling agile - Sander Hoogendoorn',
|
||||
'title': 're:(?i)^Death by dogma versus assembling agile . Sander Hoogendoorn',
|
||||
'uploader': 'DevWeek Events',
|
||||
'duration': 2773,
|
||||
'thumbnail': 're:^https?://.*\.jpg$',
|
||||
|
@ -70,7 +70,7 @@ class VineUserIE(InfoExtractor):
|
||||
'info_dict': {
|
||||
'id': 'Visa',
|
||||
},
|
||||
'playlist_mincount': 47,
|
||||
'playlist_mincount': 46,
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
|
@ -138,9 +138,19 @@ class VKIE(InfoExtractor):
|
||||
info_url = 'http://vk.com/al_video.php?act=show&al=1&video=%s' % video_id
|
||||
info_page = self._download_webpage(info_url, video_id)
|
||||
|
||||
if re.search(r'<!>Please log in or <', info_page):
|
||||
raise ExtractorError('This video is only available for registered users, '
|
||||
'use --username and --password options to provide account credentials.', expected=True)
|
||||
ERRORS = {
|
||||
r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
|
||||
'Video %s has been removed from public access due to rightholder complaint.',
|
||||
r'<!>Please log in or <':
|
||||
'Video %s is only available for registered users, '
|
||||
'use --username and --password options to provide account credentials.',
|
||||
'<!>Unknown error':
|
||||
'Video %s does not exist.'
|
||||
}
|
||||
|
||||
for error_re, error_msg in ERRORS.items():
|
||||
if re.search(error_re, info_page):
|
||||
raise ExtractorError(error_msg % video_id, expected=True)
|
||||
|
||||
m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
|
||||
if m_yt is not None:
|
||||
|
95
youtube_dl/extractor/vrt.py
Normal file
95
youtube_dl/extractor/vrt.py
Normal file
@ -0,0 +1,95 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import float_or_none
|
||||
|
||||
|
||||
class VRTIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:deredactie|sporza|cobra)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
|
||||
_TESTS = [
|
||||
# deredactie.be
|
||||
{
|
||||
'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
|
||||
'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
|
||||
'info_dict': {
|
||||
'id': '2129880',
|
||||
'ext': 'flv',
|
||||
'title': 'Het journaal L - 25/10/14',
|
||||
'description': None,
|
||||
'timestamp': 1414271750.949,
|
||||
'upload_date': '20141025',
|
||||
'duration': 929,
|
||||
}
|
||||
},
|
||||
# sporza.be
|
||||
{
|
||||
'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
|
||||
'md5': '11f53088da9bf8e7cfc42456697953ff',
|
||||
'info_dict': {
|
||||
'id': '2124639',
|
||||
'ext': 'flv',
|
||||
'title': 'Bekijk Extra Time van 20 oktober',
|
||||
'description': 'md5:83ac5415a4f1816c6a93f8138aef2426',
|
||||
'timestamp': 1413835980.560,
|
||||
'upload_date': '20141020',
|
||||
'duration': 3238,
|
||||
}
|
||||
},
|
||||
# cobra.be
|
||||
{
|
||||
'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
|
||||
'md5': '78a2b060a5083c4f055449a72477409d',
|
||||
'info_dict': {
|
||||
'id': '2126050',
|
||||
'ext': 'flv',
|
||||
'title': 'Bret Easton Ellis in Café Corsari',
|
||||
'description': 'md5:f699986e823f32fd6036c1855a724ee9',
|
||||
'timestamp': 1413967500.494,
|
||||
'upload_date': '20141022',
|
||||
'duration': 661,
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
video_id = self._search_regex(
|
||||
r'data-video-id="([^"]+)_[^"]+"', webpage, 'video id', fatal=False)
|
||||
|
||||
formats = []
|
||||
mobj = re.search(
|
||||
r'data-video-iphone-server="(?P<server>[^"]+)"\s+data-video-iphone-path="(?P<path>[^"]+)"',
|
||||
webpage)
|
||||
if mobj:
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
'%s/%s' % (mobj.group('server'), mobj.group('path')),
|
||||
video_id, 'mp4'))
|
||||
mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
|
||||
if mobj:
|
||||
formats.extend(self._extract_f4m_formats(
|
||||
'%s/manifest.f4m' % mobj.group('src'), video_id))
|
||||
self._sort_formats(formats)
|
||||
|
||||
title = self._og_search_title(webpage)
|
||||
description = self._og_search_description(webpage, default=None)
|
||||
thumbnail = self._og_search_thumbnail(webpage)
|
||||
timestamp = float_or_none(self._search_regex(
|
||||
r'data-video-sitestat-pubdate="(\d+)"', webpage, 'timestamp', fatal=False), 1000)
|
||||
duration = float_or_none(self._search_regex(
|
||||
r'data-video-duration="(\d+)"', webpage, 'duration', fatal=False), 1000)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'timestamp': timestamp,
|
||||
'duration': duration,
|
||||
'formats': formats,
|
||||
}
|
89
youtube_dl/extractor/walla.py
Normal file
89
youtube_dl/extractor/walla.py
Normal file
@ -0,0 +1,89 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .subtitles import SubtitlesInfoExtractor
|
||||
from ..utils import (
|
||||
xpath_text,
|
||||
int_or_none,
|
||||
)
|
||||
|
||||
|
||||
class WallaIE(SubtitlesInfoExtractor):
|
||||
_VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
|
||||
_TEST = {
|
||||
'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
|
||||
'info_dict': {
|
||||
'id': '2642630',
|
||||
'display_id': 'one-direction-all-for-one',
|
||||
'ext': 'flv',
|
||||
'title': 'וואן דיירקשן: ההיסטריה',
|
||||
'description': 'md5:de9e2512a92442574cdb0913c49bc4d8',
|
||||
'thumbnail': 're:^https?://.*\.jpg',
|
||||
'duration': 3600,
|
||||
},
|
||||
'params': {
|
||||
# rtmp download
|
||||
'skip_download': True,
|
||||
}
|
||||
}
|
||||
|
||||
_SUBTITLE_LANGS = {
|
||||
'עברית': 'heb',
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
display_id = mobj.group('display_id')
|
||||
|
||||
video = self._download_xml(
|
||||
'http://video2.walla.co.il/?w=null/null/%s/@@/video/flv_pl' % video_id,
|
||||
display_id)
|
||||
|
||||
item = video.find('./items/item')
|
||||
|
||||
title = xpath_text(item, './title', 'title')
|
||||
description = xpath_text(item, './synopsis', 'description')
|
||||
thumbnail = xpath_text(item, './preview_pic', 'thumbnail')
|
||||
duration = int_or_none(xpath_text(item, './duration', 'duration'))
|
||||
|
||||
subtitles = {}
|
||||
for subtitle in item.findall('./subtitles/subtitle'):
|
||||
lang = xpath_text(subtitle, './title')
|
||||
subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = xpath_text(subtitle, './src')
|
||||
|
||||
if self._downloader.params.get('listsubtitles', False):
|
||||
self._list_available_subtitles(video_id, subtitles)
|
||||
return
|
||||
|
||||
subtitles = self.extract_subtitles(video_id, subtitles)
|
||||
|
||||
formats = []
|
||||
for quality in item.findall('./qualities/quality'):
|
||||
format_id = xpath_text(quality, './title')
|
||||
fmt = {
|
||||
'url': 'rtmp://wafla.walla.co.il/vod',
|
||||
'play_path': xpath_text(quality, './src'),
|
||||
'player_url': 'http://isc.walla.co.il/w9/swf/video_swf/vod/WallaMediaPlayerAvod.swf',
|
||||
'page_url': url,
|
||||
'ext': 'flv',
|
||||
'format_id': xpath_text(quality, './title'),
|
||||
}
|
||||
m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
|
||||
if m:
|
||||
fmt['height'] = int(m.group('height'))
|
||||
formats.append(fmt)
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'thumbnail': thumbnail,
|
||||
'duration': duration,
|
||||
'formats': formats,
|
||||
'subtitles': subtitles,
|
||||
}
|
@ -37,7 +37,7 @@ class WimpIE(InfoExtractor):
|
||||
video_id = mobj.group(1)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
video_url = self._search_regex(
|
||||
r's1\.addVariable\("file",\s*"([^"]+)"\);', webpage, 'video URL')
|
||||
r"'file'\s*:\s*'([^']+)'", webpage, 'video URL')
|
||||
if YoutubeIE.suitable(video_url):
|
||||
self.to_screen('Found YouTube video')
|
||||
return {
|
||||
|
@ -20,7 +20,7 @@ class XTubeIE(InfoExtractor):
|
||||
'id': 'kVTUy_G222_',
|
||||
'ext': 'mp4',
|
||||
'title': 'strange erotica',
|
||||
'description': 'surreal gay themed erotica...almost an ET kind of thing',
|
||||
'description': 'http://www.xtube.com an ET kind of thing',
|
||||
'uploader': 'greenshowers',
|
||||
'duration': 450,
|
||||
'age_limit': 18,
|
||||
|
@ -1,3 +1,4 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import itertools
|
||||
@ -6,6 +7,7 @@ import re
|
||||
|
||||
from .common import InfoExtractor, SearchInfoExtractor
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
compat_urllib_parse,
|
||||
compat_urlparse,
|
||||
clean_html,
|
||||
@ -15,7 +17,7 @@ from ..utils import (
|
||||
|
||||
class YahooIE(InfoExtractor):
|
||||
IE_DESC = 'Yahoo screen and movies'
|
||||
_VALID_URL = r'(?P<url>https?://(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
|
||||
_VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
|
||||
_TESTS = [
|
||||
{
|
||||
'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
|
||||
@ -25,6 +27,7 @@ class YahooIE(InfoExtractor):
|
||||
'ext': 'mp4',
|
||||
'title': 'Julian Smith & Travis Legg Watch Julian Smith',
|
||||
'description': 'Julian and Travis watch Julian Smith',
|
||||
'duration': 6863,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -34,7 +37,8 @@ class YahooIE(InfoExtractor):
|
||||
'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
|
||||
'ext': 'mp4',
|
||||
'title': 'Codefellas - The Cougar Lies with Spanish Moss',
|
||||
'description': 'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
|
||||
'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
|
||||
'duration': 151,
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -45,15 +49,95 @@ class YahooIE(InfoExtractor):
|
||||
'ext': 'mp4',
|
||||
'title': "Yahoo Saves 'Community'",
|
||||
'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
|
||||
'duration': 170,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'https://tw.screen.yahoo.com/taipei-opinion-poll/選情站報-街頭民調-台北市篇-102823042.html',
|
||||
'md5': '92a7fdd8a08783c68a174d7aa067dde8',
|
||||
'info_dict': {
|
||||
'id': '7a23b569-7bea-36cb-85b9-bd5301a0a1fb',
|
||||
'ext': 'mp4',
|
||||
'title': '選情站報 街頭民調 台北市篇',
|
||||
'description': '選情站報 街頭民調 台北市篇',
|
||||
'duration': 429,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
|
||||
'md5': '0b51660361f0e27c9789e7037ef76f4b',
|
||||
'info_dict': {
|
||||
'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
|
||||
'ext': 'mp4',
|
||||
'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
|
||||
'description': 'md5:f66c890e1490f4910a9953c941dee944',
|
||||
'duration': 97,
|
||||
}
|
||||
},
|
||||
{
|
||||
'url': 'https://ca.sports.yahoo.com/video/program-makes-hockey-more-affordable-013127711.html',
|
||||
'md5': '57e06440778b1828a6079d2f744212c4',
|
||||
'info_dict': {
|
||||
'id': 'c9fa2a36-0d4d-3937-b8f6-cc0fb1881e73',
|
||||
'ext': 'mp4',
|
||||
'title': 'Program that makes hockey more affordable not offered in Manitoba',
|
||||
'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
|
||||
'duration': 121,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://ca.finance.yahoo.com/news/20-most-valuable-brands-world-112600775.html',
|
||||
'md5': '3e401e4eed6325aa29d9b96125fd5b4f',
|
||||
'info_dict': {
|
||||
'id': 'c1b4c09c-8ed8-3b65-8b05-169c55358a83',
|
||||
'ext': 'mp4',
|
||||
'title': "Apple Is The World's Most Valuable Brand",
|
||||
'description': 'md5:73eabc1a11c6f59752593b2ceefa1262',
|
||||
'duration': 21,
|
||||
}
|
||||
}, {
|
||||
'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
|
||||
'md5': '67010fdf3a08d290e060a4dd96baa07b',
|
||||
'info_dict': {
|
||||
'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
|
||||
'ext': 'mp4',
|
||||
'title': 'China Moses Is Crazy About the Blues',
|
||||
'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
|
||||
'duration': 128,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://in.lifestyle.yahoo.com/video/connect-dots-dark-side-virgo-090247395.html',
|
||||
'md5': 'd9a083ccf1379127bf25699d67e4791b',
|
||||
'info_dict': {
|
||||
'id': '52aeeaa3-b3d1-30d8-9ef8-5d0cf05efb7c',
|
||||
'ext': 'mp4',
|
||||
'title': 'Connect the Dots: Dark Side of Virgo',
|
||||
'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
|
||||
'duration': 201,
|
||||
}
|
||||
}, {
|
||||
'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
|
||||
'only_matching': True,
|
||||
}
|
||||
]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
display_id = mobj.group('display_id')
|
||||
url = mobj.group('url')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
host = mobj.group('host')
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
|
||||
# Look for iframed media first
|
||||
iframe_m = re.search(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
|
||||
if iframe_m:
|
||||
iframepage = self._download_webpage(
|
||||
host + iframe_m.group(1), display_id, 'Downloading iframe webpage')
|
||||
items_json = self._search_regex(
|
||||
r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
|
||||
if items_json:
|
||||
items = json.loads(items_json)
|
||||
video_id = items[0]['id']
|
||||
return self._get_info(video_id, display_id, webpage)
|
||||
|
||||
items_json = self._search_regex(
|
||||
r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
|
||||
@ -64,20 +148,22 @@ class YahooIE(InfoExtractor):
|
||||
r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
|
||||
r'"first_videoid"\s*:\s*"([^"]+)"',
|
||||
]
|
||||
long_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
|
||||
video_id = long_id
|
||||
video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
|
||||
else:
|
||||
items = json.loads(items_json)
|
||||
info = items['mediaItems']['query']['results']['mediaObj'][0]
|
||||
# The 'meta' field is not always in the video webpage, we request it
|
||||
# from another page
|
||||
long_id = info['id']
|
||||
return self._get_info(long_id, video_id, webpage)
|
||||
video_id = info['id']
|
||||
return self._get_info(video_id, display_id, webpage)
|
||||
|
||||
def _get_info(self, long_id, video_id, webpage):
|
||||
def _get_info(self, video_id, display_id, webpage):
|
||||
region = self._search_regex(
|
||||
r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
|
||||
webpage, 'region', fatal=False, default='US')
|
||||
query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
|
||||
' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="US"'
|
||||
' AND protocol="http"' % long_id)
|
||||
' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="%s"'
|
||||
' AND protocol="http"' % (video_id, region))
|
||||
data = compat_urllib_parse.urlencode({
|
||||
'q': query,
|
||||
'env': 'prod',
|
||||
@ -85,9 +171,17 @@ class YahooIE(InfoExtractor):
|
||||
})
|
||||
query_result = self._download_json(
|
||||
'http://video.query.yahoo.com/v1/public/yql?' + data,
|
||||
video_id, 'Downloading video info')
|
||||
display_id, 'Downloading video info')
|
||||
|
||||
info = query_result['query']['results']['mediaObj'][0]
|
||||
meta = info['meta']
|
||||
meta = info.get('meta')
|
||||
|
||||
if not meta:
|
||||
msg = info['status'].get('msg')
|
||||
if msg:
|
||||
raise ExtractorError(
|
||||
'%s returned error: %s' % (self.IE_NAME, msg), expected=True)
|
||||
raise ExtractorError('Unable to extract media object meta')
|
||||
|
||||
formats = []
|
||||
for s in info['streams']:
|
||||
@ -114,36 +208,15 @@ class YahooIE(InfoExtractor):
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': meta['title'],
|
||||
'formats': formats,
|
||||
'description': clean_html(meta['description']),
|
||||
'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
|
||||
'duration': int_or_none(meta.get('duration')),
|
||||
}
|
||||
|
||||
|
||||
class YahooNewsIE(YahooIE):
|
||||
IE_NAME = 'yahoo:news'
|
||||
_VALID_URL = r'http://news\.yahoo\.com/video/.*?-(?P<id>\d*?)\.html'
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
|
||||
'md5': '67010fdf3a08d290e060a4dd96baa07b',
|
||||
'info_dict': {
|
||||
'id': '104538833',
|
||||
'ext': 'mp4',
|
||||
'title': 'China Moses Is Crazy About the Blues',
|
||||
'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
long_id = self._search_regex(r'contentId: \'(.+?)\',', webpage, 'long id')
|
||||
return self._get_info(long_id, video_id, webpage)
|
||||
|
||||
|
||||
class YahooSearchIE(SearchInfoExtractor):
|
||||
IE_DESC = 'Yahoo screen search'
|
||||
_MAX_RESULTS = 1000
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user