Compare commits
265 Commits
2016.06.11
...
2016.07.06
Author | SHA1 | Date | |
---|---|---|---|
0e94b4713d | |||
a6d3b89feb | |||
6c26815d63 | |||
73c4ac2c95 | |||
84f214d840 | |||
e3f88be7a9 | |||
31af3e35e0 | |||
94a5cff91d | |||
77082c7b9e | |||
252a1f75d2 | |||
5abf513cf8 | |||
c6054e3201 | |||
4080530624 | |||
c25f1a9b63 | |||
dfaa86b75e | |||
d9163ae3b6 | |||
dafafe7cf1 | |||
81953d1ae5 | |||
3a212ed62e | |||
195f084542 | |||
aa7a455b2e | |||
6a4e659c93 | |||
40f3666f6b | |||
dd801bbe18 | |||
38cce791c7 | |||
bf3ae6a543 | |||
bff98341d5 | |||
2644e911be | |||
a5f67895d3 | |||
15e4b6b758 | |||
2b28b892d8 | |||
7507fc98cb | |||
477b7a8474 | |||
034a884957 | |||
64436cb1a4 | |||
f138873900 | |||
e793338c88 | |||
369bb06206 | |||
2cb31d288e | |||
c723d1cd8d | |||
1f55234057 | |||
04006fae8d | |||
4cb13d0d6a | |||
a1f6f5c768 | |||
05c7feec77 | |||
bf83024826 | |||
a0cfd82dda | |||
1b734adb2d | |||
9b724d7277 | |||
c3a5dd3b5d | |||
e3755a624b | |||
95cf60e826 | |||
6b03e1e25d | |||
712b0b5b70 | |||
6a424391d9 | |||
dbf0157a26 | |||
7deef1ba67 | |||
fd6ca38262 | |||
bdafd88da0 | |||
7a1e71575e | |||
ac2d8f54d1 | |||
14ff6baa0e | |||
bb08101ec4 | |||
bc4b2d75ba | |||
35fc3021ba | |||
347227237b | |||
564dc3c6e8 | |||
9f4576a7eb | |||
f11315e8d4 | |||
0c2ac64bb8 | |||
a9eede3913 | |||
9e29ef13a3 | |||
eaaaaec042 | |||
3cb3b60064 | |||
044e3d91b5 | |||
c9e538a3b1 | |||
76dad392f5 | |||
9617b557aa | |||
bf4fa24414 | |||
20361b4f25 | |||
05a0068a76 | |||
66a42309fa | |||
fd94e2671a | |||
8ff6697861 | |||
eafa643715 | |||
049da7cb6c | |||
7dbeee7e22 | |||
93ad6c6bfa | |||
329179073b | |||
4d86d2008e | |||
ab47b6e881 | |||
df43389ade | |||
397b305cfe | |||
e496fa50cd | |||
06a96da15b | |||
70157c2c43 | |||
c58ed8563d | |||
4c7821227c | |||
42362fdb5e | |||
97124e572d | |||
32616c14cc | |||
8174d0fe95 | |||
8704778d95 | |||
c287f2bc60 | |||
9ea5c04c0d | |||
fd7a7498a4 | |||
e3a6747d8f | |||
f41ffc00d1 | |||
81fda15369 | |||
427cd050a3 | |||
b0c200f1ec | |||
92747e664a | |||
f1f336322d | |||
bf8dd79045 | |||
c6781156aa | |||
f484c5fa25 | |||
88d9f6c0c4 | |||
3c9c088f9c | |||
fc3996bfe1 | |||
5b6ad8630c | |||
30105f4ac0 | |||
1143535d76 | |||
7d52c052ef | |||
a2406fce3c | |||
3b34ab538c | |||
ac782306f1 | |||
0c00e889f3 | |||
ce96ed05f4 | |||
0463b77a1f | |||
2d185706ea | |||
b72b44318c | |||
46f59e89ea | |||
b4241e308e | |||
3d4b08dfc7 | |||
be49068d65 | |||
525cedb971 | |||
de3c7fe0d4 | |||
896cc72750 | |||
c1ff6e1ad0 | |||
fee70322d7 | |||
8065d6c55f | |||
494172d2e5 | |||
6e3c2047f8 | |||
011bd3221b | |||
b46eabecd3 | |||
0437307a41 | |||
22b7ac13ef | |||
96f88e91b7 | |||
3331a4644d | |||
adf1921dc1 | |||
97674f0419 | |||
73843ae8ac | |||
f2bb8c036a | |||
75ca6bcee2 | |||
089657ed1f | |||
b5eab86c24 | |||
c8e3e0974b | |||
dfc8f46e1c | |||
c143ddce5d | |||
169d836feb | |||
6ae938b295 | |||
cf40fdf5c1 | |||
23bdae0955 | |||
ca74c90bf5 | |||
7cfc1e2a10 | |||
1ac5705f62 | |||
e4f90ea0a7 | |||
cdfc187cd5 | |||
feef925f49 | |||
19e2d1cdea | |||
8369a4fe76 | |||
1f749b6658 | |||
819707920a | |||
43518503a6 | |||
5839d556e4 | |||
6c83e583b3 | |||
6aeb64b673 | |||
6cd64b6806 | |||
e154c65128 | |||
a50fd6e026 | |||
6a55bb66ee | |||
7c05097633 | |||
589568789f | |||
7577d849a6 | |||
cb23192bc4 | |||
41c1023300 | |||
90b6288cce | |||
c1823c8ad9 | |||
d7c6c656c5 | |||
b0b128049a | |||
e8f13f2637 | |||
b5aad37f6b | |||
6d0d4fc26d | |||
0278aa443f | |||
1f35745758 | |||
573c35272f | |||
09e3f91e40 | |||
1b6cf16be7 | |||
26264cb056 | |||
a72df5f36f | |||
c878e635de | |||
0f47cc2e92 | |||
5fc2757682 | |||
e3944c2621 | |||
667d96480b | |||
e6fe993c31 | |||
d0d93f76ea | |||
20a6a154fe | |||
f011876076 | |||
6929569403 | |||
eb451890da | |||
ded7511a70 | |||
d2161cade5 | |||
27e5fa8198 | |||
efbd1eb51a | |||
369ff75081 | |||
47212f7bcb | |||
4c93ee8d14 | |||
8bc4dbb1af | |||
6c3760292c | |||
4cef70db6c | |||
ff4af6ec59 | |||
d01fb21d4c | |||
a4ea28eee6 | |||
bc2a871f3e | |||
1759672eed | |||
fea55ef4a9 | |||
16b6bd01d2 | |||
14d0f4e0f3 | |||
778f969447 | |||
79cd8b3d8a | |||
b4663f12b1 | |||
b50e02c1e4 | |||
33b72ce64e | |||
cf2bf840ba | |||
bccdac6874 | |||
e69f9f5d68 | |||
77a9a9c295 | |||
84dcd1c4e4 | |||
971e3b7520 | |||
4e79011729 | |||
a936ac321c | |||
98960c911c | |||
329ca3bef6 | |||
2c3322e36e | |||
80ae228b34 | |||
6d28c408cf | |||
c83b35d4aa | |||
94e5d6aedb | |||
531a74968c | |||
c5edd147d1 | |||
856150d056 | |||
03ebea89b0 | |||
15d106787e | |||
7aab3696dd | |||
47787efa2b | |||
4a420119a6 | |||
33751818d3 | |||
698f127c1a | |||
fe458b6596 | |||
21ac1a8ac3 | |||
79027c0ea0 | |||
4cad2929cd | |||
62666af99f | |||
9ddc289f88 |
6
.github/ISSUE_TEMPLATE.md
vendored
6
.github/ISSUE_TEMPLATE.md
vendored
@ -6,8 +6,8 @@
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.06.11.1*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
|
### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2016.07.06*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
|
||||||
- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.06.11.1**
|
- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2016.07.06**
|
||||||
|
|
||||||
### Before submitting an *issue* make sure you have:
|
### Before submitting an *issue* make sure you have:
|
||||||
- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
|
- [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
|
||||||
@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
|
|||||||
[debug] User config: []
|
[debug] User config: []
|
||||||
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
|
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
|
||||||
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
|
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
|
||||||
[debug] youtube-dl version 2016.06.11.1
|
[debug] youtube-dl version 2016.07.06
|
||||||
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
|
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
|
||||||
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
|
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
|
||||||
[debug] Proxy map: {}
|
[debug] Proxy map: {}
|
||||||
|
22
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
22
.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
## Please follow the guide below
|
||||||
|
|
||||||
|
- You will be asked some questions, please read them **carefully** and answer honestly
|
||||||
|
- Put an `x` into all the boxes [ ] relevant to your *pull request* (like that [x])
|
||||||
|
- Use *Preview* tab to see how your *pull request* will actually look like
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Before submitting a *pull request* make sure you have:
|
||||||
|
- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/rg3/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/rg3/youtube-dl#youtube-dl-coding-conventions) sections
|
||||||
|
- [ ] [Searched](https://github.com/rg3/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
|
||||||
|
|
||||||
|
### What is the purpose of your *pull request*?
|
||||||
|
- [ ] Bug fix
|
||||||
|
- [ ] New extractor
|
||||||
|
- [ ] New feature
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Description of your *pull request* and other information
|
||||||
|
|
||||||
|
Explanation of your *pull request* in arbitrary form goes here. Please make sure the description explains the purpose and effect of your *pull request* and is worded well enough to be understood. Provide as much context and examples as possible.
|
4
AUTHORS
4
AUTHORS
@ -173,3 +173,7 @@ Kevin Deldycke
|
|||||||
inondle
|
inondle
|
||||||
Tomáš Čech
|
Tomáš Čech
|
||||||
Déstin Reed
|
Déstin Reed
|
||||||
|
Roman Tsiupa
|
||||||
|
Artur Krysiak
|
||||||
|
Jakub Adam Wieczorek
|
||||||
|
Aleksandar Topuzović
|
||||||
|
154
CONTRIBUTING.md
154
CONTRIBUTING.md
@ -97,9 +97,17 @@ If you want to add support for a new site, first of all **make sure** this site
|
|||||||
After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
|
After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
|
||||||
|
|
||||||
1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
|
1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
|
||||||
2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
|
2. Check out the source code with:
|
||||||
3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
|
|
||||||
|
git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git
|
||||||
|
|
||||||
|
3. Start a new git branch with
|
||||||
|
|
||||||
|
cd youtube-dl
|
||||||
|
git checkout -b yourextractor
|
||||||
|
|
||||||
4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
|
4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
@ -142,17 +150,149 @@ After you have ensured this site is distributing it's content legally, you can f
|
|||||||
```
|
```
|
||||||
5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
|
5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
|
||||||
6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
|
6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
|
||||||
7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
|
7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
|
||||||
8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
|
8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
|
||||||
9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
|
9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
|
||||||
10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
|
|
||||||
|
|
||||||
$ git add youtube_dl/extractor/extractors.py
|
$ git add youtube_dl/extractor/extractors.py
|
||||||
$ git add youtube_dl/extractor/yourextractor.py
|
$ git add youtube_dl/extractor/yourextractor.py
|
||||||
$ git commit -m '[yourextractor] Add new extractor'
|
$ git commit -m '[yourextractor] Add new extractor'
|
||||||
$ git push origin yourextractor
|
$ git push origin yourextractor
|
||||||
|
|
||||||
11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
|
10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
|
||||||
|
|
||||||
In any case, thank you very much for your contributions!
|
In any case, thank you very much for your contributions!
|
||||||
|
|
||||||
|
## youtube-dl coding conventions
|
||||||
|
|
||||||
|
This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
|
||||||
|
|
||||||
|
Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hoster out of your control and this layout tend to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize code dependency on source's layout changes and even to make the code foresee potential future changes and be ready for that. This is important because it will allow extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with fix incorporated all the previous version become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say some may never receive an update at all that is possible for non rolling release distros.
|
||||||
|
|
||||||
|
### Mandatory and optional metafields
|
||||||
|
|
||||||
|
For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L75-L257) or simply *info dict*. Only the following meta fields in *info dict* are considered mandatory for successful extraction process by youtube-dl:
|
||||||
|
|
||||||
|
- `id` (media identifier)
|
||||||
|
- `title` (media title)
|
||||||
|
- `url` (media download URL) or `formats`
|
||||||
|
|
||||||
|
In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` to be mandatory. Thus aforementioned metafields are the critical data the extraction does not make any sense without and if any of them fail to be extracted then extractor is considered completely broken.
|
||||||
|
|
||||||
|
[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Say you have some source dictionary `meta` that you've fetched as JSON with HTTP request and it has a key `summary`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
meta = self._download_json(url, video_id)
|
||||||
|
```
|
||||||
|
|
||||||
|
Assume at this point `meta`'s layout is:
|
||||||
|
|
||||||
|
```python
|
||||||
|
{
|
||||||
|
...
|
||||||
|
"summary": "some fancy summary text",
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Assume you want to extract `summary` and put into resulting info dict as `description`. Since `description` is optional metafield you should be ready that this key may be missing from the `meta` dict, so that you should extract it like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = meta.get('summary') # correct
|
||||||
|
```
|
||||||
|
|
||||||
|
and not like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = meta['summary'] # incorrect
|
||||||
|
```
|
||||||
|
|
||||||
|
The latter will break extraction process with `KeyError` if `summary` disappears from `meta` at some time later but with former approach extraction will just go ahead with `description` set to `None` that is perfectly fine (remember `None` is equivalent for absence of data).
|
||||||
|
|
||||||
|
Similarly, you should pass `fatal=False` when extracting optional data from a webpage with `_search_regex`, `_html_search_regex` or similar methods, for instance:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = self._search_regex(
|
||||||
|
r'<span[^>]+id="title"[^>]*>([^<]+)<',
|
||||||
|
webpage, 'description', fatal=False)
|
||||||
|
```
|
||||||
|
|
||||||
|
With `fatal` set to `False` if `_search_regex` fails to extract `description` it will emit a warning and continue extraction.
|
||||||
|
|
||||||
|
You can also pass `default=<some fallback value>`, for example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = self._search_regex(
|
||||||
|
r'<span[^>]+id="title"[^>]*>([^<]+)<',
|
||||||
|
webpage, 'description', default=None)
|
||||||
|
```
|
||||||
|
|
||||||
|
On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that are known to may or may not be present.
|
||||||
|
|
||||||
|
### Provide fallbacks
|
||||||
|
|
||||||
|
When extracting metadata try to provide several scenarios for that. For example if `title` is present in several places/sources try extracting from at least some of them. This would make it more future-proof in case some of the sources became unavailable.
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Say `meta` from previous example has a `title` and you are about to extract it. Since `title` is mandatory meta field you should end up with something like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = meta['title']
|
||||||
|
```
|
||||||
|
|
||||||
|
If `title` disappeares from `meta` in future due to some changes on hoster's side the extraction would fail since `title` is mandatory. That's expected.
|
||||||
|
|
||||||
|
Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback scenario:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = meta.get('title') or self._og_search_title(webpage)
|
||||||
|
```
|
||||||
|
|
||||||
|
This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
|
||||||
|
|
||||||
|
### Make regular expressions flexible
|
||||||
|
|
||||||
|
When using regular expressions try to write them fuzzy and flexible.
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Say you need to extract `title` from the following HTML code:
|
||||||
|
|
||||||
|
```html
|
||||||
|
<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">some fancy title</span>
|
||||||
|
```
|
||||||
|
|
||||||
|
The code for that task should look similar to:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<span[^>]+class="title"[^>]*>([^<]+)', webpage, 'title')
|
||||||
|
```
|
||||||
|
|
||||||
|
Or even better:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<span[^>]+class=(["\'])title\1[^>]*>(?P<title>[^<]+)',
|
||||||
|
webpage, 'title', group='title')
|
||||||
|
```
|
||||||
|
|
||||||
|
Note how you tolerate potential changes in `style` attribute's value or switch from using double quotes to single for `class` attribute:
|
||||||
|
|
||||||
|
The code definitely should not look like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">(.*?)</span>',
|
||||||
|
webpage, 'title', group='title')
|
||||||
|
```
|
||||||
|
|
||||||
|
### Use safe conversion functions
|
||||||
|
|
||||||
|
Wrap all extracted numeric data into safe functions from `utils`: `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
|
||||||
|
|
||||||
|
173
README.md
173
README.md
@ -44,7 +44,7 @@ Or with [MacPorts](https://www.macports.org/):
|
|||||||
Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
|
Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://rg3.github.io/youtube-dl/download.html).
|
||||||
|
|
||||||
# DESCRIPTION
|
# DESCRIPTION
|
||||||
**youtube-dl** is a small command-line program to download videos from
|
**youtube-dl** is a command-line program to download videos from
|
||||||
YouTube.com and a few more sites. It requires the Python interpreter, version
|
YouTube.com and a few more sites. It requires the Python interpreter, version
|
||||||
2.6, 2.7, or 3.2+, and it is not platform specific. It should work on
|
2.6, 2.7, or 3.2+, and it is not platform specific. It should work on
|
||||||
your Unix box, on Windows or on Mac OS X. It is released to the public domain,
|
your Unix box, on Windows or on Mac OS X. It is released to the public domain,
|
||||||
@ -103,9 +103,9 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
(experimental)
|
(experimental)
|
||||||
-6, --force-ipv6 Make all connections via IPv6
|
-6, --force-ipv6 Make all connections via IPv6
|
||||||
(experimental)
|
(experimental)
|
||||||
--cn-verification-proxy URL Use this proxy to verify the IP address for
|
--geo-verification-proxy URL Use this proxy to verify the IP address for
|
||||||
some Chinese sites. The default proxy
|
some geo-restricted sites. The default
|
||||||
specified by --proxy (or none, if the
|
proxy specified by --proxy (or none, if the
|
||||||
options is not present) is used for the
|
options is not present) is used for the
|
||||||
actual downloading. (experimental)
|
actual downloading. (experimental)
|
||||||
|
|
||||||
@ -424,7 +424,7 @@ which means you can modify it, redistribute it or use it however you like.
|
|||||||
|
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
|
|
||||||
You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`.
|
You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and OS X, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
|
||||||
|
|
||||||
For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
|
For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
|
||||||
```
|
```
|
||||||
@ -511,6 +511,9 @@ The basic usage is not to set any template arguments when downloading a single f
|
|||||||
- `autonumber`: Five-digit number that will be increased with each download, starting at zero
|
- `autonumber`: Five-digit number that will be increased with each download, starting at zero
|
||||||
- `playlist`: Name or id of the playlist that contains the video
|
- `playlist`: Name or id of the playlist that contains the video
|
||||||
- `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
|
- `playlist_index`: Index of the video in the playlist padded with leading zeros according to the total length of the playlist
|
||||||
|
- `playlist_id`: Playlist identifier
|
||||||
|
- `playlist_title`: Playlist title
|
||||||
|
|
||||||
|
|
||||||
Available for the video that belongs to some logical chapter or section:
|
Available for the video that belongs to some logical chapter or section:
|
||||||
- `chapter`: Name or title of the chapter the video belongs to
|
- `chapter`: Name or title of the chapter the video belongs to
|
||||||
@ -550,6 +553,10 @@ The current default template is `%(title)s-%(id)s.%(ext)s`.
|
|||||||
|
|
||||||
In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
|
In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
|
||||||
|
|
||||||
|
#### Output template and Windows batch files
|
||||||
|
|
||||||
|
If you are using output template inside a Windows batch file then you must escape plain percent characters (`%`) by doubling, so that `-o "%(title)s-%(id)s.%(ext)s"` should become `-o "%%(title)s-%%(id)s.%%(ext)s"`. However you should not touch `%`'s that are not plain characters, e.g. environment variables for expansion should stay intact: `-o "C:\%HOMEPATH%\Desktop\%%(title)s.%%(ext)s"`.
|
||||||
|
|
||||||
#### Output template examples
|
#### Output template examples
|
||||||
|
|
||||||
Note on Windows you may need to use double quotes instead of single.
|
Note on Windows you may need to use double quotes instead of single.
|
||||||
@ -883,9 +890,17 @@ If you want to add support for a new site, first of all **make sure** this site
|
|||||||
After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
|
After you have ensured this site is distributing it's content legally, you can follow this quick list (assuming your service is called `yourextractor`):
|
||||||
|
|
||||||
1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
|
1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
|
||||||
2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
|
2. Check out the source code with:
|
||||||
3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
|
|
||||||
|
git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git
|
||||||
|
|
||||||
|
3. Start a new git branch with
|
||||||
|
|
||||||
|
cd youtube-dl
|
||||||
|
git checkout -b yourextractor
|
||||||
|
|
||||||
4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
|
4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
@ -928,20 +943,152 @@ After you have ensured this site is distributing it's content legally, you can f
|
|||||||
```
|
```
|
||||||
5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
|
5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
|
||||||
6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
|
6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
|
||||||
7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L68-L226). Add tests and code for as many as you want.
|
7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L74-L252). Add tests and code for as many as you want.
|
||||||
8. Keep in mind that the only mandatory fields in info dict for successful extraction process are `id`, `title` and either `url` or `formats`, i.e. these are the critical data the extraction does not make any sense without. This means that [any field](https://github.com/rg3/youtube-dl/blob/58525c94d547be1c8167d16c298bdd75506db328/youtube_dl/extractor/common.py#L138-L226) apart from aforementioned mandatory ones should be treated **as optional** and extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields. For example, if you have some intermediate dict `meta` that is a source of metadata and it has a key `summary` that you want to extract and put into resulting info dict as `description`, you should be ready that this key may be missing from the `meta` dict, i.e. you should extract it as `meta.get('summary')` and not `meta['summary']`. Similarly, you should pass `fatal=False` when extracting data from a webpage with `_search_regex/_html_search_regex`.
|
8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://pypi.python.org/pypi/flake8). Also make sure your code works under all [Python](http://www.python.org/) versions claimed supported by youtube-dl, namely 2.6, 2.7, and 3.2+.
|
||||||
9. Check the code with [flake8](https://pypi.python.org/pypi/flake8).
|
9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
|
||||||
10. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
|
|
||||||
|
|
||||||
$ git add youtube_dl/extractor/extractors.py
|
$ git add youtube_dl/extractor/extractors.py
|
||||||
$ git add youtube_dl/extractor/yourextractor.py
|
$ git add youtube_dl/extractor/yourextractor.py
|
||||||
$ git commit -m '[yourextractor] Add new extractor'
|
$ git commit -m '[yourextractor] Add new extractor'
|
||||||
$ git push origin yourextractor
|
$ git push origin yourextractor
|
||||||
|
|
||||||
11. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
|
10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
|
||||||
|
|
||||||
In any case, thank you very much for your contributions!
|
In any case, thank you very much for your contributions!
|
||||||
|
|
||||||
|
## youtube-dl coding conventions
|
||||||
|
|
||||||
|
This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
|
||||||
|
|
||||||
|
Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hoster out of your control and this layout tend to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize code dependency on source's layout changes and even to make the code foresee potential future changes and be ready for that. This is important because it will allow extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with fix incorporated all the previous version become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say some may never receive an update at all that is possible for non rolling release distros.
|
||||||
|
|
||||||
|
### Mandatory and optional metafields
|
||||||
|
|
||||||
|
For extraction to work youtube-dl relies on metadata your extractor extracts and provides to youtube-dl expressed by [information dictionary](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L75-L257) or simply *info dict*. Only the following meta fields in *info dict* are considered mandatory for successful extraction process by youtube-dl:
|
||||||
|
|
||||||
|
- `id` (media identifier)
|
||||||
|
- `title` (media title)
|
||||||
|
- `url` (media download URL) or `formats`
|
||||||
|
|
||||||
|
In fact only the last option is technically mandatory (i.e. if you can't figure out the download location of the media the extraction does not make any sense). But by convention youtube-dl also treats `id` and `title` to be mandatory. Thus aforementioned metafields are the critical data the extraction does not make any sense without and if any of them fail to be extracted then extractor is considered completely broken.
|
||||||
|
|
||||||
|
[Any field](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L149-L257) apart from the aforementioned ones are considered **optional**. That means that extraction should be **tolerate** to situations when sources for these fields can potentially be unavailable (even if they are always available at the moment) and **future-proof** in order not to break the extraction of general purpose mandatory fields.
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Say you have some source dictionary `meta` that you've fetched as JSON with HTTP request and it has a key `summary`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
meta = self._download_json(url, video_id)
|
||||||
|
```
|
||||||
|
|
||||||
|
Assume at this point `meta`'s layout is:
|
||||||
|
|
||||||
|
```python
|
||||||
|
{
|
||||||
|
...
|
||||||
|
"summary": "some fancy summary text",
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Assume you want to extract `summary` and put into resulting info dict as `description`. Since `description` is optional metafield you should be ready that this key may be missing from the `meta` dict, so that you should extract it like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = meta.get('summary') # correct
|
||||||
|
```
|
||||||
|
|
||||||
|
and not like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = meta['summary'] # incorrect
|
||||||
|
```
|
||||||
|
|
||||||
|
The latter will break extraction process with `KeyError` if `summary` disappears from `meta` at some time later but with former approach extraction will just go ahead with `description` set to `None` that is perfectly fine (remember `None` is equivalent for absence of data).
|
||||||
|
|
||||||
|
Similarly, you should pass `fatal=False` when extracting optional data from a webpage with `_search_regex`, `_html_search_regex` or similar methods, for instance:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = self._search_regex(
|
||||||
|
r'<span[^>]+id="title"[^>]*>([^<]+)<',
|
||||||
|
webpage, 'description', fatal=False)
|
||||||
|
```
|
||||||
|
|
||||||
|
With `fatal` set to `False` if `_search_regex` fails to extract `description` it will emit a warning and continue extraction.
|
||||||
|
|
||||||
|
You can also pass `default=<some fallback value>`, for example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
description = self._search_regex(
|
||||||
|
r'<span[^>]+id="title"[^>]*>([^<]+)<',
|
||||||
|
webpage, 'description', default=None)
|
||||||
|
```
|
||||||
|
|
||||||
|
On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that are known to may or may not be present.
|
||||||
|
|
||||||
|
### Provide fallbacks
|
||||||
|
|
||||||
|
When extracting metadata try to provide several scenarios for that. For example if `title` is present in several places/sources try extracting from at least some of them. This would make it more future-proof in case some of the sources became unavailable.
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Say `meta` from previous example has a `title` and you are about to extract it. Since `title` is mandatory meta field you should end up with something like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = meta['title']
|
||||||
|
```
|
||||||
|
|
||||||
|
If `title` disappeares from `meta` in future due to some changes on hoster's side the extraction would fail since `title` is mandatory. That's expected.
|
||||||
|
|
||||||
|
Assume that you have some another source you can extract `title` from, for example `og:title` HTML meta of a `webpage`. In this case you can provide a fallback scenario:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = meta.get('title') or self._og_search_title(webpage)
|
||||||
|
```
|
||||||
|
|
||||||
|
This code will try to extract from `meta` first and if it fails it will try extracting `og:title` from a `webpage`.
|
||||||
|
|
||||||
|
### Make regular expressions flexible
|
||||||
|
|
||||||
|
When using regular expressions try to write them fuzzy and flexible.
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
|
||||||
|
Say you need to extract `title` from the following HTML code:
|
||||||
|
|
||||||
|
```html
|
||||||
|
<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">some fancy title</span>
|
||||||
|
```
|
||||||
|
|
||||||
|
The code for that task should look similar to:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<span[^>]+class="title"[^>]*>([^<]+)', webpage, 'title')
|
||||||
|
```
|
||||||
|
|
||||||
|
Or even better:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<span[^>]+class=(["\'])title\1[^>]*>(?P<title>[^<]+)',
|
||||||
|
webpage, 'title', group='title')
|
||||||
|
```
|
||||||
|
|
||||||
|
Note how you tolerate potential changes in `style` attribute's value or switch from using double quotes to single for `class` attribute:
|
||||||
|
|
||||||
|
The code definitely should not look like:
|
||||||
|
|
||||||
|
```python
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<span style="position: absolute; left: 910px; width: 90px; float: right; z-index: 9999;" class="title">(.*?)</span>',
|
||||||
|
webpage, 'title', group='title')
|
||||||
|
```
|
||||||
|
|
||||||
|
### Use safe conversion functions
|
||||||
|
|
||||||
|
Wrap all extracted numeric data into safe functions from `utils`: `int_or_none`, `float_or_none`. Use them for string to number conversions as well.
|
||||||
|
|
||||||
# EMBEDDING YOUTUBE-DL
|
# EMBEDDING YOUTUBE-DL
|
||||||
|
|
||||||
youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
|
youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
|
||||||
@ -957,7 +1104,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
|
|||||||
ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
|
ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
|
||||||
```
|
```
|
||||||
|
|
||||||
Most likely, you'll want to use various options. For a list of what can be done, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L121-L269). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
|
Most likely, you'll want to use various options. For a list of options available, have a look at [`youtube_dl/YoutubeDL.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L128-L278). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
|
||||||
|
|
||||||
Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
|
Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
|
||||||
|
|
||||||
|
@ -14,15 +14,17 @@ if os.path.exists(lazy_extractors_filename):
|
|||||||
os.remove(lazy_extractors_filename)
|
os.remove(lazy_extractors_filename)
|
||||||
|
|
||||||
from youtube_dl.extractor import _ALL_CLASSES
|
from youtube_dl.extractor import _ALL_CLASSES
|
||||||
from youtube_dl.extractor.common import InfoExtractor
|
from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
|
||||||
|
|
||||||
with open('devscripts/lazy_load_template.py', 'rt') as f:
|
with open('devscripts/lazy_load_template.py', 'rt') as f:
|
||||||
module_template = f.read()
|
module_template = f.read()
|
||||||
|
|
||||||
module_contents = [module_template + '\n' + getsource(InfoExtractor.suitable)]
|
module_contents = [
|
||||||
|
module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
|
||||||
|
'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n']
|
||||||
|
|
||||||
ie_template = '''
|
ie_template = '''
|
||||||
class {name}(LazyLoadExtractor):
|
class {name}({bases}):
|
||||||
_VALID_URL = {valid_url!r}
|
_VALID_URL = {valid_url!r}
|
||||||
_module = '{module}'
|
_module = '{module}'
|
||||||
'''
|
'''
|
||||||
@ -34,10 +36,20 @@ make_valid_template = '''
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def get_base_name(base):
|
||||||
|
if base is InfoExtractor:
|
||||||
|
return 'LazyLoadExtractor'
|
||||||
|
elif base is SearchInfoExtractor:
|
||||||
|
return 'LazyLoadSearchExtractor'
|
||||||
|
else:
|
||||||
|
return base.__name__
|
||||||
|
|
||||||
|
|
||||||
def build_lazy_ie(ie, name):
|
def build_lazy_ie(ie, name):
|
||||||
valid_url = getattr(ie, '_VALID_URL', None)
|
valid_url = getattr(ie, '_VALID_URL', None)
|
||||||
s = ie_template.format(
|
s = ie_template.format(
|
||||||
name=name,
|
name=name,
|
||||||
|
bases=', '.join(map(get_base_name, ie.__bases__)),
|
||||||
valid_url=valid_url,
|
valid_url=valid_url,
|
||||||
module=ie.__module__)
|
module=ie.__module__)
|
||||||
if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
|
if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
|
||||||
@ -47,12 +59,35 @@ def build_lazy_ie(ie, name):
|
|||||||
s += make_valid_template.format(valid_url=ie._make_valid_url())
|
s += make_valid_template.format(valid_url=ie._make_valid_url())
|
||||||
return s
|
return s
|
||||||
|
|
||||||
|
# find the correct sorting and add the required base classes so that sublcasses
|
||||||
|
# can be correctly created
|
||||||
|
classes = _ALL_CLASSES[:-1]
|
||||||
|
ordered_cls = []
|
||||||
|
while classes:
|
||||||
|
for c in classes[:]:
|
||||||
|
bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor))
|
||||||
|
stop = False
|
||||||
|
for b in bases:
|
||||||
|
if b not in classes and b not in ordered_cls:
|
||||||
|
if b.__name__ == 'GenericIE':
|
||||||
|
exit()
|
||||||
|
classes.insert(0, b)
|
||||||
|
stop = True
|
||||||
|
if stop:
|
||||||
|
break
|
||||||
|
if all(b in ordered_cls for b in bases):
|
||||||
|
ordered_cls.append(c)
|
||||||
|
classes.remove(c)
|
||||||
|
break
|
||||||
|
ordered_cls.append(_ALL_CLASSES[-1])
|
||||||
|
|
||||||
names = []
|
names = []
|
||||||
for ie in list(sorted(_ALL_CLASSES[:-1], key=lambda cls: cls.ie_key())) + _ALL_CLASSES[-1:]:
|
for ie in ordered_cls:
|
||||||
name = ie.ie_key() + 'IE'
|
name = ie.__name__
|
||||||
src = build_lazy_ie(ie, name)
|
src = build_lazy_ie(ie, name)
|
||||||
module_contents.append(src)
|
module_contents.append(src)
|
||||||
names.append(name)
|
if ie in _ALL_CLASSES:
|
||||||
|
names.append(name)
|
||||||
|
|
||||||
module_contents.append(
|
module_contents.append(
|
||||||
'_ALL_CLASSES = [{0}]'.format(', '.join(names)))
|
'_ALL_CLASSES = [{0}]'.format(', '.join(names)))
|
||||||
|
@ -15,6 +15,7 @@
|
|||||||
set -e
|
set -e
|
||||||
|
|
||||||
skip_tests=true
|
skip_tests=true
|
||||||
|
gpg_sign_commits=""
|
||||||
buildserver='localhost:8142'
|
buildserver='localhost:8142'
|
||||||
|
|
||||||
while true
|
while true
|
||||||
@ -24,6 +25,10 @@ case "$1" in
|
|||||||
skip_tests=false
|
skip_tests=false
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
|
--gpg-sign-commits|-S)
|
||||||
|
gpg_sign_commits="-S"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
--buildserver)
|
--buildserver)
|
||||||
buildserver="$2"
|
buildserver="$2"
|
||||||
shift 2
|
shift 2
|
||||||
@ -69,7 +74,7 @@ sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
|
|||||||
/bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
|
/bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..."
|
||||||
make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites
|
make README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md supportedsites
|
||||||
git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
|
git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE.md docs/supportedsites.md youtube_dl/version.py
|
||||||
git commit -m "release $version"
|
git commit $gpg_sign_commits -m "release $version"
|
||||||
|
|
||||||
/bin/echo -e "\n### Now tagging, signing and pushing..."
|
/bin/echo -e "\n### Now tagging, signing and pushing..."
|
||||||
git tag -s -m "Release $version" "$version"
|
git tag -s -m "Release $version" "$version"
|
||||||
@ -116,7 +121,7 @@ git clone --branch gh-pages --single-branch . build/gh-pages
|
|||||||
"$ROOT/devscripts/gh-pages/update-copyright.py"
|
"$ROOT/devscripts/gh-pages/update-copyright.py"
|
||||||
"$ROOT/devscripts/gh-pages/update-sites.py"
|
"$ROOT/devscripts/gh-pages/update-sites.py"
|
||||||
git add *.html *.html.in update
|
git add *.html *.html.in update
|
||||||
git commit -m "release $version"
|
git commit $gpg_sign_commits -m "release $version"
|
||||||
git push "$ROOT" gh-pages
|
git push "$ROOT" gh-pages
|
||||||
git push "$ORIGIN_URL" gh-pages
|
git push "$ORIGIN_URL" gh-pages
|
||||||
)
|
)
|
||||||
|
41
devscripts/show-downloads-statistics.py
Normal file
41
devscripts/show-downloads-statistics.py
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from youtube_dl.compat import (
|
||||||
|
compat_print,
|
||||||
|
compat_urllib_request,
|
||||||
|
)
|
||||||
|
from youtube_dl.utils import format_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def format_size(bytes):
|
||||||
|
return '%s (%d bytes)' % (format_bytes(bytes), bytes)
|
||||||
|
|
||||||
|
|
||||||
|
total_bytes = 0
|
||||||
|
|
||||||
|
releases = json.loads(compat_urllib_request.urlopen(
|
||||||
|
'https://api.github.com/repos/rg3/youtube-dl/releases').read().decode('utf-8'))
|
||||||
|
|
||||||
|
for release in releases:
|
||||||
|
compat_print(release['name'])
|
||||||
|
for asset in release['assets']:
|
||||||
|
asset_name = asset['name']
|
||||||
|
total_bytes += asset['download_count'] * asset['size']
|
||||||
|
if all(not re.match(p, asset_name) for p in (
|
||||||
|
r'^youtube-dl$',
|
||||||
|
r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$',
|
||||||
|
r'^youtube-dl\.exe$')):
|
||||||
|
continue
|
||||||
|
compat_print(
|
||||||
|
' %s size: %s downloads: %d'
|
||||||
|
% (asset_name, format_size(asset['size']), asset['download_count']))
|
||||||
|
|
||||||
|
compat_print('total downloads traffic: %s' % format_size(total_bytes))
|
@ -45,7 +45,6 @@
|
|||||||
- **archive.org**: archive.org videos
|
- **archive.org**: archive.org videos
|
||||||
- **ARD**
|
- **ARD**
|
||||||
- **ARD:mediathek**
|
- **ARD:mediathek**
|
||||||
- **ARD:mediathek**: Saarländischer Rundfunk
|
|
||||||
- **arte.tv**
|
- **arte.tv**
|
||||||
- **arte.tv:+7**
|
- **arte.tv:+7**
|
||||||
- **arte.tv:cinema**
|
- **arte.tv:cinema**
|
||||||
@ -74,6 +73,8 @@
|
|||||||
- **bbc**: BBC
|
- **bbc**: BBC
|
||||||
- **bbc.co.uk**: BBC iPlayer
|
- **bbc.co.uk**: BBC iPlayer
|
||||||
- **bbc.co.uk:article**: BBC articles
|
- **bbc.co.uk:article**: BBC articles
|
||||||
|
- **bbc.co.uk:iplayer:playlist**
|
||||||
|
- **bbc.co.uk:playlist**
|
||||||
- **BeatportPro**
|
- **BeatportPro**
|
||||||
- **Beeg**
|
- **Beeg**
|
||||||
- **BehindKink**
|
- **BehindKink**
|
||||||
@ -104,6 +105,8 @@
|
|||||||
- **canalc2.tv**
|
- **canalc2.tv**
|
||||||
- **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
|
- **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
|
||||||
- **Canvas**
|
- **Canvas**
|
||||||
|
- **CarambaTV**
|
||||||
|
- **CarambaTVPage**
|
||||||
- **CBC**
|
- **CBC**
|
||||||
- **CBCPlayer**
|
- **CBCPlayer**
|
||||||
- **CBS**
|
- **CBS**
|
||||||
@ -124,6 +127,7 @@
|
|||||||
- **cliphunter**
|
- **cliphunter**
|
||||||
- **ClipRs**
|
- **ClipRs**
|
||||||
- **Clipsyndicate**
|
- **Clipsyndicate**
|
||||||
|
- **CloserToTruth**
|
||||||
- **cloudtime**: CloudTime
|
- **cloudtime**: CloudTime
|
||||||
- **Cloudy**
|
- **Cloudy**
|
||||||
- **Clubic**
|
- **Clubic**
|
||||||
@ -148,6 +152,8 @@
|
|||||||
- **CSNNE**
|
- **CSNNE**
|
||||||
- **CSpan**: C-SPAN
|
- **CSpan**: C-SPAN
|
||||||
- **CtsNews**: 華視新聞
|
- **CtsNews**: 華視新聞
|
||||||
|
- **CTV**
|
||||||
|
- **CTVNews**
|
||||||
- **culturebox.francetvinfo.fr**
|
- **culturebox.francetvinfo.fr**
|
||||||
- **CultureUnplugged**
|
- **CultureUnplugged**
|
||||||
- **CWTV**
|
- **CWTV**
|
||||||
@ -236,6 +242,7 @@
|
|||||||
- **FreeVideo**
|
- **FreeVideo**
|
||||||
- **Funimation**
|
- **Funimation**
|
||||||
- **FunnyOrDie**
|
- **FunnyOrDie**
|
||||||
|
- **Fusion**
|
||||||
- **GameInformer**
|
- **GameInformer**
|
||||||
- **Gamekings**
|
- **Gamekings**
|
||||||
- **GameOne**
|
- **GameOne**
|
||||||
@ -243,7 +250,6 @@
|
|||||||
- **Gamersyde**
|
- **Gamersyde**
|
||||||
- **GameSpot**
|
- **GameSpot**
|
||||||
- **GameStar**
|
- **GameStar**
|
||||||
- **Gametrailers**
|
|
||||||
- **Gazeta**
|
- **Gazeta**
|
||||||
- **GDCVault**
|
- **GDCVault**
|
||||||
- **generic**: Generic downloader that works on some sites
|
- **generic**: Generic downloader that works on some sites
|
||||||
@ -269,6 +275,7 @@
|
|||||||
- **Helsinki**: helsinki.fi
|
- **Helsinki**: helsinki.fi
|
||||||
- **HentaiStigma**
|
- **HentaiStigma**
|
||||||
- **HistoricFilms**
|
- **HistoricFilms**
|
||||||
|
- **history:topic**: History.com Topic
|
||||||
- **hitbox**
|
- **hitbox**
|
||||||
- **hitbox:live**
|
- **hitbox:live**
|
||||||
- **HornBunny**
|
- **HornBunny**
|
||||||
@ -276,6 +283,8 @@
|
|||||||
- **HotStar**
|
- **HotStar**
|
||||||
- **Howcast**
|
- **Howcast**
|
||||||
- **HowStuffWorks**
|
- **HowStuffWorks**
|
||||||
|
- **HRTi**
|
||||||
|
- **HRTiPlaylist**
|
||||||
- **HuffPost**: Huffington Post
|
- **HuffPost**: Huffington Post
|
||||||
- **Hypem**
|
- **Hypem**
|
||||||
- **Iconosquare**
|
- **Iconosquare**
|
||||||
@ -322,7 +331,7 @@
|
|||||||
- **kuwo:mv**: 酷我音乐 - MV
|
- **kuwo:mv**: 酷我音乐 - MV
|
||||||
- **kuwo:singer**: 酷我音乐 - 歌手
|
- **kuwo:singer**: 酷我音乐 - 歌手
|
||||||
- **kuwo:song**: 酷我音乐
|
- **kuwo:song**: 酷我音乐
|
||||||
- **la7.tv**
|
- **la7.it**
|
||||||
- **Laola1Tv**
|
- **Laola1Tv**
|
||||||
- **Le**: 乐视网
|
- **Le**: 乐视网
|
||||||
- **Learnr**
|
- **Learnr**
|
||||||
@ -355,6 +364,7 @@
|
|||||||
- **MatchTV**
|
- **MatchTV**
|
||||||
- **MDR**: MDR.DE and KiKA
|
- **MDR**: MDR.DE and KiKA
|
||||||
- **media.ccc.de**
|
- **media.ccc.de**
|
||||||
|
- **META**
|
||||||
- **metacafe**
|
- **metacafe**
|
||||||
- **Metacritic**
|
- **Metacritic**
|
||||||
- **Mgoon**
|
- **Mgoon**
|
||||||
@ -381,7 +391,7 @@
|
|||||||
- **MovieFap**
|
- **MovieFap**
|
||||||
- **Moviezine**
|
- **Moviezine**
|
||||||
- **MPORA**
|
- **MPORA**
|
||||||
- **MSNBC**
|
- **MSN**
|
||||||
- **MTV**
|
- **MTV**
|
||||||
- **mtv.de**
|
- **mtv.de**
|
||||||
- **mtviggy.com**
|
- **mtviggy.com**
|
||||||
@ -432,8 +442,10 @@
|
|||||||
- **nhl.com:videocenter**
|
- **nhl.com:videocenter**
|
||||||
- **nhl.com:videocenter:category**: NHL videocenter category
|
- **nhl.com:videocenter:category**: NHL videocenter category
|
||||||
- **nick.com**
|
- **nick.com**
|
||||||
|
- **nick.de**
|
||||||
- **niconico**: ニコニコ動画
|
- **niconico**: ニコニコ動画
|
||||||
- **NiconicoPlaylist**
|
- **NiconicoPlaylist**
|
||||||
|
- **NineCNineMedia**
|
||||||
- **njoy**: N-JOY
|
- **njoy**: N-JOY
|
||||||
- **njoy:embed**
|
- **njoy:embed**
|
||||||
- **Noco**
|
- **Noco**
|
||||||
@ -497,8 +509,9 @@
|
|||||||
- **plus.google**: Google Plus
|
- **plus.google**: Google Plus
|
||||||
- **pluzz.francetv.fr**
|
- **pluzz.francetv.fr**
|
||||||
- **podomatic**
|
- **podomatic**
|
||||||
|
- **PolskieRadio**
|
||||||
- **PornHd**
|
- **PornHd**
|
||||||
- **PornHub**
|
- **PornHub**: PornHub and Thumbzilla
|
||||||
- **PornHubPlaylist**
|
- **PornHubPlaylist**
|
||||||
- **PornHubUserVideos**
|
- **PornHubUserVideos**
|
||||||
- **Pornotube**
|
- **Pornotube**
|
||||||
@ -516,6 +529,7 @@
|
|||||||
- **qqmusic:singer**: QQ音乐 - 歌手
|
- **qqmusic:singer**: QQ音乐 - 歌手
|
||||||
- **qqmusic:toplist**: QQ音乐 - 排行榜
|
- **qqmusic:toplist**: QQ音乐 - 排行榜
|
||||||
- **R7**
|
- **R7**
|
||||||
|
- **R7Article**
|
||||||
- **radio.de**
|
- **radio.de**
|
||||||
- **radiobremen**
|
- **radiobremen**
|
||||||
- **radiocanada**
|
- **radiocanada**
|
||||||
@ -535,6 +549,7 @@
|
|||||||
- **revision3:embed**
|
- **revision3:embed**
|
||||||
- **RICE**
|
- **RICE**
|
||||||
- **RingTV**
|
- **RingTV**
|
||||||
|
- **RockstarGames**
|
||||||
- **RottenTomatoes**
|
- **RottenTomatoes**
|
||||||
- **Roxwel**
|
- **Roxwel**
|
||||||
- **RTBF**
|
- **RTBF**
|
||||||
@ -580,8 +595,10 @@
|
|||||||
- **Shared**: shared.sx and vivo.sx
|
- **Shared**: shared.sx and vivo.sx
|
||||||
- **ShareSix**
|
- **ShareSix**
|
||||||
- **Sina**
|
- **Sina**
|
||||||
|
- **SixPlay**
|
||||||
|
- **skynewsarabia:article**
|
||||||
- **skynewsarabia:video**
|
- **skynewsarabia:video**
|
||||||
- **skynewsarabia:video**
|
- **SkySports**
|
||||||
- **Slideshare**
|
- **Slideshare**
|
||||||
- **Slutload**
|
- **Slutload**
|
||||||
- **smotri**: Smotri.com
|
- **smotri**: Smotri.com
|
||||||
@ -613,6 +630,7 @@
|
|||||||
- **SportBoxEmbed**
|
- **SportBoxEmbed**
|
||||||
- **SportDeutschland**
|
- **SportDeutschland**
|
||||||
- **Sportschau**
|
- **Sportschau**
|
||||||
|
- **sr:mediathek**: Saarländischer Rundfunk
|
||||||
- **SRGSSR**
|
- **SRGSSR**
|
||||||
- **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
|
- **SRGSSRPlay**: srf.ch, rts.ch, rsi.ch, rtr.ch and swissinfo.ch play sites
|
||||||
- **SSA**
|
- **SSA**
|
||||||
@ -647,6 +665,7 @@
|
|||||||
- **Telegraaf**
|
- **Telegraaf**
|
||||||
- **TeleMB**
|
- **TeleMB**
|
||||||
- **TeleTask**
|
- **TeleTask**
|
||||||
|
- **Telewebion**
|
||||||
- **TF1**
|
- **TF1**
|
||||||
- **TheIntercept**
|
- **TheIntercept**
|
||||||
- **ThePlatform**
|
- **ThePlatform**
|
||||||
@ -698,6 +717,7 @@
|
|||||||
- **TVPlay**: TV3Play and related services
|
- **TVPlay**: TV3Play and related services
|
||||||
- **Tweakers**
|
- **Tweakers**
|
||||||
- **twitch:chapter**
|
- **twitch:chapter**
|
||||||
|
- **twitch:clips**
|
||||||
- **twitch:past_broadcasts**
|
- **twitch:past_broadcasts**
|
||||||
- **twitch:profile**
|
- **twitch:profile**
|
||||||
- **twitch:stream**
|
- **twitch:stream**
|
||||||
@ -711,6 +731,7 @@
|
|||||||
- **UDNEmbed**: 聯合影音
|
- **UDNEmbed**: 聯合影音
|
||||||
- **Unistra**
|
- **Unistra**
|
||||||
- **Urort**: NRK P3 Urørt
|
- **Urort**: NRK P3 Urørt
|
||||||
|
- **URPlay**
|
||||||
- **USAToday**
|
- **USAToday**
|
||||||
- **ustream**
|
- **ustream**
|
||||||
- **ustream:channel**
|
- **ustream:channel**
|
||||||
@ -728,6 +749,7 @@
|
|||||||
- **vh1.com**
|
- **vh1.com**
|
||||||
- **Vice**
|
- **Vice**
|
||||||
- **ViceShow**
|
- **ViceShow**
|
||||||
|
- **Vidbit**
|
||||||
- **Viddler**
|
- **Viddler**
|
||||||
- **video.google:search**: Google Video search
|
- **video.google:search**: Google Video search
|
||||||
- **video.mit.edu**
|
- **video.mit.edu**
|
||||||
@ -792,10 +814,11 @@
|
|||||||
- **WNL**
|
- **WNL**
|
||||||
- **WorldStarHipHop**
|
- **WorldStarHipHop**
|
||||||
- **wrzuta.pl**
|
- **wrzuta.pl**
|
||||||
|
- **wrzuta.pl:playlist**
|
||||||
- **WSJ**: Wall Street Journal
|
- **WSJ**: Wall Street Journal
|
||||||
- **XBef**
|
- **XBef**
|
||||||
- **XboxClips**
|
- **XboxClips**
|
||||||
- **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To
|
- **XFileShare**: XFileShare based sites: DaClips, FileHoot, GorillaVid, MovPod, PowerWatch, Rapidvideo.ws, TheVideoBee, Vidto, Streamin.To, XVIDSTAGE
|
||||||
- **XHamster**
|
- **XHamster**
|
||||||
- **XHamsterEmbed**
|
- **XHamsterEmbed**
|
||||||
- **xiami:album**: 虾米音乐 - 专辑
|
- **xiami:album**: 虾米音乐 - 专辑
|
||||||
@ -834,6 +857,7 @@
|
|||||||
- **youtube:search**: YouTube.com searches
|
- **youtube:search**: YouTube.com searches
|
||||||
- **youtube:search:date**: YouTube.com searches, newest videos first
|
- **youtube:search:date**: YouTube.com searches, newest videos first
|
||||||
- **youtube:search_url**: YouTube.com search URLs
|
- **youtube:search_url**: YouTube.com search URLs
|
||||||
|
- **youtube:shared**
|
||||||
- **youtube:show**: YouTube.com (multi-season) shows
|
- **youtube:show**: YouTube.com (multi-season) shows
|
||||||
- **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
|
- **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
|
||||||
- **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
|
- **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
|
||||||
|
62
setup.py
62
setup.py
@ -21,25 +21,37 @@ try:
|
|||||||
import py2exe
|
import py2exe
|
||||||
except ImportError:
|
except ImportError:
|
||||||
if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
|
if len(sys.argv) >= 2 and sys.argv[1] == 'py2exe':
|
||||||
print("Cannot import py2exe", file=sys.stderr)
|
print('Cannot import py2exe', file=sys.stderr)
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
py2exe_options = {
|
py2exe_options = {
|
||||||
"bundle_files": 1,
|
'bundle_files': 1,
|
||||||
"compressed": 1,
|
'compressed': 1,
|
||||||
"optimize": 2,
|
'optimize': 2,
|
||||||
"dist_dir": '.',
|
'dist_dir': '.',
|
||||||
"dll_excludes": ['w9xpopen.exe', 'crypt32.dll'],
|
'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Get the version from youtube_dl/version.py without importing the package
|
||||||
|
exec(compile(open('youtube_dl/version.py').read(),
|
||||||
|
'youtube_dl/version.py', 'exec'))
|
||||||
|
|
||||||
|
DESCRIPTION = 'YouTube video downloader'
|
||||||
|
LONG_DESCRIPTION = 'Command-line program to download videos from YouTube.com and other video sites'
|
||||||
|
|
||||||
py2exe_console = [{
|
py2exe_console = [{
|
||||||
"script": "./youtube_dl/__main__.py",
|
'script': './youtube_dl/__main__.py',
|
||||||
"dest_base": "youtube-dl",
|
'dest_base': 'youtube-dl',
|
||||||
|
'version': __version__,
|
||||||
|
'description': DESCRIPTION,
|
||||||
|
'comments': LONG_DESCRIPTION,
|
||||||
|
'product_name': 'youtube-dl',
|
||||||
|
'product_version': __version__,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
py2exe_params = {
|
py2exe_params = {
|
||||||
'console': py2exe_console,
|
'console': py2exe_console,
|
||||||
'options': {"py2exe": py2exe_options},
|
'options': {'py2exe': py2exe_options},
|
||||||
'zipfile': None
|
'zipfile': None
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -72,7 +84,7 @@ else:
|
|||||||
params['scripts'] = ['bin/youtube-dl']
|
params['scripts'] = ['bin/youtube-dl']
|
||||||
|
|
||||||
class build_lazy_extractors(Command):
|
class build_lazy_extractors(Command):
|
||||||
description = "Build the extractor lazy loading module"
|
description = 'Build the extractor lazy loading module'
|
||||||
user_options = []
|
user_options = []
|
||||||
|
|
||||||
def initialize_options(self):
|
def initialize_options(self):
|
||||||
@ -87,16 +99,11 @@ class build_lazy_extractors(Command):
|
|||||||
dry_run=self.dry_run,
|
dry_run=self.dry_run,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Get the version from youtube_dl/version.py without importing the package
|
|
||||||
exec(compile(open('youtube_dl/version.py').read(),
|
|
||||||
'youtube_dl/version.py', 'exec'))
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name='youtube_dl',
|
name='youtube_dl',
|
||||||
version=__version__,
|
version=__version__,
|
||||||
description='YouTube video downloader',
|
description=DESCRIPTION,
|
||||||
long_description='Small command-line program to download videos from'
|
long_description=LONG_DESCRIPTION,
|
||||||
' YouTube.com and other video sites.',
|
|
||||||
url='https://github.com/rg3/youtube-dl',
|
url='https://github.com/rg3/youtube-dl',
|
||||||
author='Ricardo Garcia',
|
author='Ricardo Garcia',
|
||||||
author_email='ytdl@yt-dl.org',
|
author_email='ytdl@yt-dl.org',
|
||||||
@ -112,16 +119,17 @@ setup(
|
|||||||
# test_requires = ['nosetest'],
|
# test_requires = ['nosetest'],
|
||||||
|
|
||||||
classifiers=[
|
classifiers=[
|
||||||
"Topic :: Multimedia :: Video",
|
'Topic :: Multimedia :: Video',
|
||||||
"Development Status :: 5 - Production/Stable",
|
'Development Status :: 5 - Production/Stable',
|
||||||
"Environment :: Console",
|
'Environment :: Console',
|
||||||
"License :: Public Domain",
|
'License :: Public Domain',
|
||||||
"Programming Language :: Python :: 2.6",
|
'Programming Language :: Python :: 2.6',
|
||||||
"Programming Language :: Python :: 2.7",
|
'Programming Language :: Python :: 2.7',
|
||||||
"Programming Language :: Python :: 3",
|
'Programming Language :: Python :: 3',
|
||||||
"Programming Language :: Python :: 3.2",
|
'Programming Language :: Python :: 3.2',
|
||||||
"Programming Language :: Python :: 3.3",
|
'Programming Language :: Python :: 3.3',
|
||||||
"Programming Language :: Python :: 3.4",
|
'Programming Language :: Python :: 3.4',
|
||||||
|
'Programming Language :: Python :: 3.5',
|
||||||
],
|
],
|
||||||
|
|
||||||
cmdclass={'build_lazy_extractors': build_lazy_extractors},
|
cmdclass={'build_lazy_extractors': build_lazy_extractors},
|
||||||
|
@ -11,7 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|||||||
from test.helper import FakeYDL
|
from test.helper import FakeYDL
|
||||||
from youtube_dl.extractor.common import InfoExtractor
|
from youtube_dl.extractor.common import InfoExtractor
|
||||||
from youtube_dl.extractor import YoutubeIE, get_info_extractor
|
from youtube_dl.extractor import YoutubeIE, get_info_extractor
|
||||||
from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError
|
from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
|
||||||
|
|
||||||
|
|
||||||
class TestIE(InfoExtractor):
|
class TestIE(InfoExtractor):
|
||||||
@ -66,6 +66,11 @@ class TestInfoExtractor(unittest.TestCase):
|
|||||||
self.assertEqual(ie._html_search_meta('d', html), '4')
|
self.assertEqual(ie._html_search_meta('d', html), '4')
|
||||||
self.assertEqual(ie._html_search_meta('e', html), '5')
|
self.assertEqual(ie._html_search_meta('e', html), '5')
|
||||||
self.assertEqual(ie._html_search_meta('f', html), '6')
|
self.assertEqual(ie._html_search_meta('f', html), '6')
|
||||||
|
self.assertEqual(ie._html_search_meta(('a', 'b', 'c'), html), '1')
|
||||||
|
self.assertEqual(ie._html_search_meta(('c', 'b', 'a'), html), '3')
|
||||||
|
self.assertEqual(ie._html_search_meta(('z', 'x', 'c'), html), '3')
|
||||||
|
self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
|
||||||
|
self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
|
||||||
|
|
||||||
def test_download_json(self):
|
def test_download_json(self):
|
||||||
uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
|
uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
|
||||||
|
@ -6,6 +6,7 @@ from __future__ import unicode_literals
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
|
import collections
|
||||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
|
||||||
@ -130,6 +131,15 @@ class TestAllURLsMatching(unittest.TestCase):
|
|||||||
'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
|
'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
|
||||||
['Yahoo'])
|
['Yahoo'])
|
||||||
|
|
||||||
|
def test_no_duplicated_ie_names(self):
|
||||||
|
name_accu = collections.defaultdict(list)
|
||||||
|
for ie in self.ies:
|
||||||
|
name_accu[ie.IE_NAME.lower()].append(type(ie).__name__)
|
||||||
|
for (ie_name, ie_list) in name_accu.items():
|
||||||
|
self.assertEqual(
|
||||||
|
len(ie_list), 1,
|
||||||
|
'Multiple extractors with the same IE_NAME "%s" (%s)' % (ie_name, ', '.join(ie_list)))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
@ -87,6 +87,7 @@ class TestCompat(unittest.TestCase):
|
|||||||
|
|
||||||
def test_compat_shlex_split(self):
|
def test_compat_shlex_split(self):
|
||||||
self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
|
self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
|
||||||
|
self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag'])
|
||||||
|
|
||||||
def test_compat_etree_fromstring(self):
|
def test_compat_etree_fromstring(self):
|
||||||
xml = '''
|
xml = '''
|
||||||
|
@ -138,27 +138,27 @@ class TestProxy(unittest.TestCase):
|
|||||||
self.proxy_thread.daemon = True
|
self.proxy_thread.daemon = True
|
||||||
self.proxy_thread.start()
|
self.proxy_thread.start()
|
||||||
|
|
||||||
self.cn_proxy = compat_http_server.HTTPServer(
|
self.geo_proxy = compat_http_server.HTTPServer(
|
||||||
('localhost', 0), _build_proxy_handler('cn'))
|
('localhost', 0), _build_proxy_handler('geo'))
|
||||||
self.cn_port = http_server_port(self.cn_proxy)
|
self.geo_port = http_server_port(self.geo_proxy)
|
||||||
self.cn_proxy_thread = threading.Thread(target=self.cn_proxy.serve_forever)
|
self.geo_proxy_thread = threading.Thread(target=self.geo_proxy.serve_forever)
|
||||||
self.cn_proxy_thread.daemon = True
|
self.geo_proxy_thread.daemon = True
|
||||||
self.cn_proxy_thread.start()
|
self.geo_proxy_thread.start()
|
||||||
|
|
||||||
def test_proxy(self):
|
def test_proxy(self):
|
||||||
cn_proxy = 'localhost:{0}'.format(self.cn_port)
|
geo_proxy = 'localhost:{0}'.format(self.geo_port)
|
||||||
ydl = YoutubeDL({
|
ydl = YoutubeDL({
|
||||||
'proxy': 'localhost:{0}'.format(self.port),
|
'proxy': 'localhost:{0}'.format(self.port),
|
||||||
'cn_verification_proxy': cn_proxy,
|
'geo_verification_proxy': geo_proxy,
|
||||||
})
|
})
|
||||||
url = 'http://foo.com/bar'
|
url = 'http://foo.com/bar'
|
||||||
response = ydl.urlopen(url).read().decode('utf-8')
|
response = ydl.urlopen(url).read().decode('utf-8')
|
||||||
self.assertEqual(response, 'normal: {0}'.format(url))
|
self.assertEqual(response, 'normal: {0}'.format(url))
|
||||||
|
|
||||||
req = compat_urllib_request.Request(url)
|
req = compat_urllib_request.Request(url)
|
||||||
req.add_header('Ytdl-request-proxy', cn_proxy)
|
req.add_header('Ytdl-request-proxy', geo_proxy)
|
||||||
response = ydl.urlopen(req).read().decode('utf-8')
|
response = ydl.urlopen(req).read().decode('utf-8')
|
||||||
self.assertEqual(response, 'cn: {0}'.format(url))
|
self.assertEqual(response, 'geo: {0}'.format(url))
|
||||||
|
|
||||||
def test_proxy_with_idn(self):
|
def test_proxy_with_idn(self):
|
||||||
ydl = YoutubeDL({
|
ydl = YoutubeDL({
|
||||||
|
@ -60,11 +60,13 @@ from youtube_dl.utils import (
|
|||||||
timeconvert,
|
timeconvert,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
|
unified_timestamp,
|
||||||
unsmuggle_url,
|
unsmuggle_url,
|
||||||
uppercase_escape,
|
uppercase_escape,
|
||||||
lowercase_escape,
|
lowercase_escape,
|
||||||
url_basename,
|
url_basename,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
|
urshift,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
version_tuple,
|
version_tuple,
|
||||||
xpath_with_ns,
|
xpath_with_ns,
|
||||||
@ -283,8 +285,28 @@ class TestUtil(unittest.TestCase):
|
|||||||
'20150202')
|
'20150202')
|
||||||
self.assertEqual(unified_strdate('Feb 14th 2016 5:45PM'), '20160214')
|
self.assertEqual(unified_strdate('Feb 14th 2016 5:45PM'), '20160214')
|
||||||
self.assertEqual(unified_strdate('25-09-2014'), '20140925')
|
self.assertEqual(unified_strdate('25-09-2014'), '20140925')
|
||||||
|
self.assertEqual(unified_strdate('27.02.2016 17:30'), '20160227')
|
||||||
self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
|
self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
|
||||||
|
|
||||||
|
def test_unified_timestamps(self):
|
||||||
|
self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)
|
||||||
|
self.assertEqual(unified_timestamp('8/7/2009'), 1247011200)
|
||||||
|
self.assertEqual(unified_timestamp('Dec 14, 2012'), 1355443200)
|
||||||
|
self.assertEqual(unified_timestamp('2012/10/11 01:56:38 +0000'), 1349920598)
|
||||||
|
self.assertEqual(unified_timestamp('1968 12 10'), -33436800)
|
||||||
|
self.assertEqual(unified_timestamp('1968-12-10'), -33436800)
|
||||||
|
self.assertEqual(unified_timestamp('28/01/2014 21:00:00 +0100'), 1390939200)
|
||||||
|
self.assertEqual(
|
||||||
|
unified_timestamp('11/26/2014 11:30:00 AM PST', day_first=False),
|
||||||
|
1417001400)
|
||||||
|
self.assertEqual(
|
||||||
|
unified_timestamp('2/2/2015 6:47:40 PM', day_first=False),
|
||||||
|
1422902860)
|
||||||
|
self.assertEqual(unified_timestamp('Feb 14th 2016 5:45PM'), 1455471900)
|
||||||
|
self.assertEqual(unified_timestamp('25-09-2014'), 1411603200)
|
||||||
|
self.assertEqual(unified_timestamp('27.02.2016 17:30'), 1456594200)
|
||||||
|
self.assertEqual(unified_timestamp('UNKNOWN DATE FORMAT'), None)
|
||||||
|
|
||||||
def test_determine_ext(self):
|
def test_determine_ext(self):
|
||||||
self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
|
self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
|
||||||
self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
|
self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
|
||||||
@ -383,6 +405,12 @@ class TestUtil(unittest.TestCase):
|
|||||||
self.assertEqual(res_url, url)
|
self.assertEqual(res_url, url)
|
||||||
self.assertEqual(res_data, None)
|
self.assertEqual(res_data, None)
|
||||||
|
|
||||||
|
smug_url = smuggle_url(url, {'a': 'b'})
|
||||||
|
smug_smug_url = smuggle_url(smug_url, {'c': 'd'})
|
||||||
|
res_url, res_data = unsmuggle_url(smug_smug_url)
|
||||||
|
self.assertEqual(res_url, url)
|
||||||
|
self.assertEqual(res_data, {'a': 'b', 'c': 'd'})
|
||||||
|
|
||||||
def test_shell_quote(self):
|
def test_shell_quote(self):
|
||||||
args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
|
args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')]
|
||||||
self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
|
self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
|
||||||
@ -640,6 +668,9 @@ class TestUtil(unittest.TestCase):
|
|||||||
"1":{"src":"skipped", "type": "application/vnd.apple.mpegURL"}
|
"1":{"src":"skipped", "type": "application/vnd.apple.mpegURL"}
|
||||||
}''')
|
}''')
|
||||||
|
|
||||||
|
inp = '''{"foo":101}'''
|
||||||
|
self.assertEqual(js_to_json(inp), '''{"foo":101}''')
|
||||||
|
|
||||||
def test_js_to_json_edgecases(self):
|
def test_js_to_json_edgecases(self):
|
||||||
on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
|
on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
|
||||||
self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
|
self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
|
||||||
@ -956,5 +987,9 @@ The first line
|
|||||||
self.assertRaises(ValueError, encode_base_n, 0, 70)
|
self.assertRaises(ValueError, encode_base_n, 0, 70)
|
||||||
self.assertRaises(ValueError, encode_base_n, 0, 60, custom_table)
|
self.assertRaises(ValueError, encode_base_n, 0, 60, custom_table)
|
||||||
|
|
||||||
|
def test_urshift(self):
|
||||||
|
self.assertEqual(urshift(3, 1), 1)
|
||||||
|
self.assertEqual(urshift(-3, 1), 2147483646)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
@ -196,8 +196,8 @@ class YoutubeDL(object):
|
|||||||
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
|
prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
|
||||||
At the moment, this is only supported by YouTube.
|
At the moment, this is only supported by YouTube.
|
||||||
proxy: URL of the proxy server to use
|
proxy: URL of the proxy server to use
|
||||||
cn_verification_proxy: URL of the proxy to use for IP address verification
|
geo_verification_proxy: URL of the proxy to use for IP address verification
|
||||||
on Chinese sites. (Experimental)
|
on geo-restricted sites. (Experimental)
|
||||||
socket_timeout: Time to wait for unresponsive hosts, in seconds
|
socket_timeout: Time to wait for unresponsive hosts, in seconds
|
||||||
bidi_workaround: Work around buggy terminals without bidirectional text
|
bidi_workaround: Work around buggy terminals without bidirectional text
|
||||||
support, using fridibi
|
support, using fridibi
|
||||||
@ -304,6 +304,11 @@ class YoutubeDL(object):
|
|||||||
self.params.update(params)
|
self.params.update(params)
|
||||||
self.cache = Cache(self)
|
self.cache = Cache(self)
|
||||||
|
|
||||||
|
if self.params.get('cn_verification_proxy') is not None:
|
||||||
|
self.report_warning('--cn-verification-proxy is deprecated. Use --geo-verification-proxy instead.')
|
||||||
|
if self.params.get('geo_verification_proxy') is None:
|
||||||
|
self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
|
||||||
|
|
||||||
if params.get('bidi_workaround', False):
|
if params.get('bidi_workaround', False):
|
||||||
try:
|
try:
|
||||||
import pty
|
import pty
|
||||||
|
@ -382,6 +382,8 @@ def _real_main(argv=None):
|
|||||||
'external_downloader_args': external_downloader_args,
|
'external_downloader_args': external_downloader_args,
|
||||||
'postprocessor_args': postprocessor_args,
|
'postprocessor_args': postprocessor_args,
|
||||||
'cn_verification_proxy': opts.cn_verification_proxy,
|
'cn_verification_proxy': opts.cn_verification_proxy,
|
||||||
|
'geo_verification_proxy': opts.geo_verification_proxy,
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
with YoutubeDL(ydl_opts) as ydl:
|
with YoutubeDL(ydl_opts) as ydl:
|
||||||
|
@ -85,7 +85,7 @@ class ExternalFD(FileDownloader):
|
|||||||
cmd, stderr=subprocess.PIPE)
|
cmd, stderr=subprocess.PIPE)
|
||||||
_, stderr = p.communicate()
|
_, stderr = p.communicate()
|
||||||
if p.returncode != 0:
|
if p.returncode != 0:
|
||||||
self.to_stderr(stderr)
|
self.to_stderr(stderr.decode('utf-8', 'replace'))
|
||||||
return p.returncode
|
return p.returncode
|
||||||
|
|
||||||
|
|
||||||
|
@ -196,6 +196,11 @@ def build_fragments_list(boot_info):
|
|||||||
first_frag_number = fragment_run_entry_table[0]['first']
|
first_frag_number = fragment_run_entry_table[0]['first']
|
||||||
fragments_counter = itertools.count(first_frag_number)
|
fragments_counter = itertools.count(first_frag_number)
|
||||||
for segment, fragments_count in segment_run_table['segment_run']:
|
for segment, fragments_count in segment_run_table['segment_run']:
|
||||||
|
# In some live HDS streams (for example Rai), `fragments_count` is
|
||||||
|
# abnormal and causing out-of-memory errors. It's OK to change the
|
||||||
|
# number of fragments for live streams as they are updated periodically
|
||||||
|
if fragments_count == 4294967295 and boot_info['live']:
|
||||||
|
fragments_count = 2
|
||||||
for _ in range(fragments_count):
|
for _ in range(fragments_count):
|
||||||
res.append((segment, next(fragments_counter)))
|
res.append((segment, next(fragments_counter)))
|
||||||
|
|
||||||
@ -329,7 +334,11 @@ class F4mFD(FragmentFD):
|
|||||||
|
|
||||||
base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
|
base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
|
||||||
bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
|
bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
|
||||||
boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, base_url)
|
# From Adobe F4M 3.0 spec:
|
||||||
|
# The <baseURL> element SHALL be the base URL for all relative
|
||||||
|
# (HTTP-based) URLs in the manifest. If <baseURL> is not present, said
|
||||||
|
# URLs should be relative to the location of the containing document.
|
||||||
|
boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, man_url)
|
||||||
live = boot_info['live']
|
live = boot_info['live']
|
||||||
metadata_node = media.find(_add_ns('metadata'))
|
metadata_node = media.find(_add_ns('metadata'))
|
||||||
if metadata_node is not None:
|
if metadata_node is not None:
|
||||||
|
@ -2,14 +2,24 @@ from __future__ import unicode_literals
|
|||||||
|
|
||||||
import os.path
|
import os.path
|
||||||
import re
|
import re
|
||||||
|
import binascii
|
||||||
|
try:
|
||||||
|
from Crypto.Cipher import AES
|
||||||
|
can_decrypt_frag = True
|
||||||
|
except ImportError:
|
||||||
|
can_decrypt_frag = False
|
||||||
|
|
||||||
from .fragment import FragmentFD
|
from .fragment import FragmentFD
|
||||||
from .external import FFmpegFD
|
from .external import FFmpegFD
|
||||||
|
|
||||||
from ..compat import compat_urlparse
|
from ..compat import (
|
||||||
|
compat_urlparse,
|
||||||
|
compat_struct_pack,
|
||||||
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
encodeFilename,
|
encodeFilename,
|
||||||
sanitize_open,
|
sanitize_open,
|
||||||
|
parse_m3u8_attributes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -21,7 +31,7 @@ class HlsFD(FragmentFD):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def can_download(manifest):
|
def can_download(manifest):
|
||||||
UNSUPPORTED_FEATURES = (
|
UNSUPPORTED_FEATURES = (
|
||||||
r'#EXT-X-KEY:METHOD=(?!NONE)', # encrypted streams [1]
|
r'#EXT-X-KEY:METHOD=(?!NONE|AES-128)', # encrypted streams [1]
|
||||||
r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
|
r'#EXT-X-BYTERANGE', # playlists composed of byte ranges of media files [2]
|
||||||
|
|
||||||
# Live streams heuristic does not always work (e.g. geo restricted to Germany
|
# Live streams heuristic does not always work (e.g. geo restricted to Germany
|
||||||
@ -39,7 +49,9 @@ class HlsFD(FragmentFD):
|
|||||||
# 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
|
# 3. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.2
|
||||||
# 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
|
# 4. https://tools.ietf.org/html/draft-pantos-http-live-streaming-17#section-4.3.3.5
|
||||||
)
|
)
|
||||||
return all(not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
|
check_results = [not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES]
|
||||||
|
check_results.append(can_decrypt_frag or '#EXT-X-KEY:METHOD=AES-128' not in manifest)
|
||||||
|
return all(check_results)
|
||||||
|
|
||||||
def real_download(self, filename, info_dict):
|
def real_download(self, filename, info_dict):
|
||||||
man_url = info_dict['url']
|
man_url = info_dict['url']
|
||||||
@ -57,36 +69,60 @@ class HlsFD(FragmentFD):
|
|||||||
fd.add_progress_hook(ph)
|
fd.add_progress_hook(ph)
|
||||||
return fd.real_download(filename, info_dict)
|
return fd.real_download(filename, info_dict)
|
||||||
|
|
||||||
fragment_urls = []
|
total_frags = 0
|
||||||
for line in s.splitlines():
|
for line in s.splitlines():
|
||||||
line = line.strip()
|
line = line.strip()
|
||||||
if line and not line.startswith('#'):
|
if line and not line.startswith('#'):
|
||||||
segment_url = (
|
total_frags += 1
|
||||||
line
|
|
||||||
if re.match(r'^https?://', line)
|
|
||||||
else compat_urlparse.urljoin(man_url, line))
|
|
||||||
fragment_urls.append(segment_url)
|
|
||||||
# We only download the first fragment during the test
|
|
||||||
if self.params.get('test', False):
|
|
||||||
break
|
|
||||||
|
|
||||||
ctx = {
|
ctx = {
|
||||||
'filename': filename,
|
'filename': filename,
|
||||||
'total_frags': len(fragment_urls),
|
'total_frags': total_frags,
|
||||||
}
|
}
|
||||||
|
|
||||||
self._prepare_and_start_frag_download(ctx)
|
self._prepare_and_start_frag_download(ctx)
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
media_sequence = 0
|
||||||
|
decrypt_info = {'METHOD': 'NONE'}
|
||||||
frags_filenames = []
|
frags_filenames = []
|
||||||
for i, frag_url in enumerate(fragment_urls):
|
for line in s.splitlines():
|
||||||
frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
|
line = line.strip()
|
||||||
success = ctx['dl'].download(frag_filename, {'url': frag_url})
|
if line:
|
||||||
if not success:
|
if not line.startswith('#'):
|
||||||
return False
|
frag_url = (
|
||||||
down, frag_sanitized = sanitize_open(frag_filename, 'rb')
|
line
|
||||||
ctx['dest_stream'].write(down.read())
|
if re.match(r'^https?://', line)
|
||||||
down.close()
|
else compat_urlparse.urljoin(man_url, line))
|
||||||
frags_filenames.append(frag_sanitized)
|
frag_filename = '%s-Frag%d' % (ctx['tmpfilename'], i)
|
||||||
|
success = ctx['dl'].download(frag_filename, {'url': frag_url})
|
||||||
|
if not success:
|
||||||
|
return False
|
||||||
|
down, frag_sanitized = sanitize_open(frag_filename, 'rb')
|
||||||
|
frag_content = down.read()
|
||||||
|
down.close()
|
||||||
|
if decrypt_info['METHOD'] == 'AES-128':
|
||||||
|
iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
|
||||||
|
frag_content = AES.new(
|
||||||
|
decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
|
||||||
|
ctx['dest_stream'].write(frag_content)
|
||||||
|
frags_filenames.append(frag_sanitized)
|
||||||
|
# We only download the first fragment during the test
|
||||||
|
if self.params.get('test', False):
|
||||||
|
break
|
||||||
|
i += 1
|
||||||
|
media_sequence += 1
|
||||||
|
elif line.startswith('#EXT-X-KEY'):
|
||||||
|
decrypt_info = parse_m3u8_attributes(line[11:])
|
||||||
|
if decrypt_info['METHOD'] == 'AES-128':
|
||||||
|
if 'IV' in decrypt_info:
|
||||||
|
decrypt_info['IV'] = binascii.unhexlify(decrypt_info['IV'][2:])
|
||||||
|
if not re.match(r'^https?://', decrypt_info['URI']):
|
||||||
|
decrypt_info['URI'] = compat_urlparse.urljoin(
|
||||||
|
man_url, decrypt_info['URI'])
|
||||||
|
decrypt_info['KEY'] = self.ydl.urlopen(decrypt_info['URI']).read()
|
||||||
|
elif line.startswith('#EXT-X-MEDIA-SEQUENCE'):
|
||||||
|
media_sequence = int(line[22:])
|
||||||
|
|
||||||
self._finish_frag_download(ctx)
|
self._finish_frag_download(ctx)
|
||||||
|
|
||||||
|
@ -156,7 +156,10 @@ class AdobeTVVideoIE(InfoExtractor):
|
|||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
video_data = self._download_json(url + '?format=json', video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
video_data = self._parse_json(self._search_regex(
|
||||||
|
r'var\s+bridge\s*=\s*([^;]+);', webpage, 'bridged data'), video_id)
|
||||||
|
|
||||||
formats = [{
|
formats = [{
|
||||||
'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')),
|
'format_id': '%s-%s' % (determine_ext(source['src']), source.get('height')),
|
||||||
|
@ -2,23 +2,137 @@ from __future__ import unicode_literals
|
|||||||
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .theplatform import ThePlatformIE
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
|
extract_attributes,
|
||||||
|
get_element_by_attribute,
|
||||||
|
)
|
||||||
|
from ..compat import (
|
||||||
|
compat_urlparse,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class AENetworksIE(InfoExtractor):
|
class AENetworksBaseIE(ThePlatformIE):
|
||||||
|
_THEPLATFORM_KEY = 'crazyjava'
|
||||||
|
_THEPLATFORM_SECRET = 's3cr3t'
|
||||||
|
|
||||||
|
|
||||||
|
class AENetworksIE(AENetworksBaseIE):
|
||||||
IE_NAME = 'aenetworks'
|
IE_NAME = 'aenetworks'
|
||||||
IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
|
IE_DESC = 'A+E Networks: A&E, Lifetime, History.com, FYI Network'
|
||||||
_VALID_URL = r'https?://(?:www\.)?(?:(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?P<type>[^/]+)/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
|
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|aetv|mylifetime)\.com|fyi\.tv)/(?:shows/(?P<show_path>[^/]+(?:/[^/]+){0,2})|movies/(?P<movie_display_id>[^/]+)/full-movie)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
|
||||||
|
'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '22253814',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Winter Is Coming',
|
||||||
|
'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
|
||||||
|
'timestamp': 1338306241,
|
||||||
|
'upload_date': '20120529',
|
||||||
|
'uploader': 'AENE-NEW',
|
||||||
|
},
|
||||||
|
'add_ie': ['ThePlatform'],
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.history.com/shows/ancient-aliens/season-1',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '71889446852',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 5,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.mylifetime.com/shows/atlanta-plastic',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'SERIES4317',
|
||||||
|
'title': 'Atlanta Plastic',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 2,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
|
||||||
|
'only_matching': True
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
|
||||||
|
'only_matching': True
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.mylifetime.com/shows/project-runway-junior/season-1/episode-6',
|
||||||
|
'only_matching': True
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.mylifetime.com/movies/center-stage-on-pointe/full-movie',
|
||||||
|
'only_matching': True
|
||||||
|
}]
|
||||||
|
_DOMAIN_TO_REQUESTOR_ID = {
|
||||||
|
'history.com': 'HISTORY',
|
||||||
|
'aetv.com': 'AETV',
|
||||||
|
'mylifetime.com': 'LIFETIME',
|
||||||
|
'fyi.tv': 'FYI',
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
domain, show_path, movie_display_id = re.match(self._VALID_URL, url).groups()
|
||||||
|
display_id = show_path or movie_display_id
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
if show_path:
|
||||||
|
url_parts = show_path.split('/')
|
||||||
|
url_parts_len = len(url_parts)
|
||||||
|
if url_parts_len == 1:
|
||||||
|
entries = []
|
||||||
|
for season_url_path in re.findall(r'(?s)<li[^>]+data-href="(/shows/%s/season-\d+)"' % url_parts[0], webpage):
|
||||||
|
entries.append(self.url_result(
|
||||||
|
compat_urlparse.urljoin(url, season_url_path), 'AENetworks'))
|
||||||
|
return self.playlist_result(
|
||||||
|
entries, self._html_search_meta('aetn:SeriesId', webpage),
|
||||||
|
self._html_search_meta('aetn:SeriesTitle', webpage))
|
||||||
|
elif url_parts_len == 2:
|
||||||
|
entries = []
|
||||||
|
for episode_item in re.findall(r'(?s)<div[^>]+class="[^"]*episode-item[^"]*"[^>]*>', webpage):
|
||||||
|
episode_attributes = extract_attributes(episode_item)
|
||||||
|
episode_url = compat_urlparse.urljoin(
|
||||||
|
url, episode_attributes['data-canonical'])
|
||||||
|
entries.append(self.url_result(
|
||||||
|
episode_url, 'AENetworks',
|
||||||
|
episode_attributes['data-videoid']))
|
||||||
|
return self.playlist_result(
|
||||||
|
entries, self._html_search_meta('aetn:SeasonId', webpage))
|
||||||
|
|
||||||
|
query = {
|
||||||
|
'mbr': 'true',
|
||||||
|
'assetTypes': 'medium_video_s3'
|
||||||
|
}
|
||||||
|
video_id = self._html_search_meta('aetn:VideoID', webpage)
|
||||||
|
media_url = self._search_regex(
|
||||||
|
r"media_url\s*=\s*'([^']+)'", webpage, 'video url')
|
||||||
|
theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
|
||||||
|
r'https?://link.theplatform.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
|
||||||
|
info = self._parse_theplatform_metadata(theplatform_metadata)
|
||||||
|
if theplatform_metadata.get('AETN$isBehindWall'):
|
||||||
|
requestor_id = self._DOMAIN_TO_REQUESTOR_ID[domain]
|
||||||
|
resource = '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>%s</title><item><title>%s</title><guid>%s</guid><media:rating scheme="urn:v-chip">%s</media:rating></item></channel></rss>' % (requestor_id, theplatform_metadata['title'], theplatform_metadata['AETN$PPL_pplProgramId'], theplatform_metadata['ratings'][0]['rating'])
|
||||||
|
query['auth'] = self._extract_mvpd_auth(
|
||||||
|
url, video_id, requestor_id, resource)
|
||||||
|
info.update(self._search_json_ld(webpage, video_id, fatal=False))
|
||||||
|
media_url = update_url_query(media_url, query)
|
||||||
|
media_url = self._sign_url(media_url, self._THEPLATFORM_KEY, self._THEPLATFORM_SECRET)
|
||||||
|
formats, subtitles = self._extract_theplatform_smil(media_url, video_id)
|
||||||
|
self._sort_formats(formats)
|
||||||
|
info.update({
|
||||||
|
'id': video_id,
|
||||||
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
})
|
||||||
|
return info
|
||||||
|
|
||||||
|
|
||||||
|
class HistoryTopicIE(AENetworksBaseIE):
|
||||||
|
IE_NAME = 'history:topic'
|
||||||
|
IE_DESC = 'History.com Topic'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?history\.com/topics/(?:[^/]+/)?(?P<topic_id>[^/]+)(?:/[^/]+(?:/(?P<video_display_id>[^/?#]+))?)?'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
|
'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'g12m5Gyt3fdR',
|
'id': '40700995724',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': "Bet You Didn't Know: Valentine's Day",
|
'title': "Bet You Didn't Know: Valentine's Day",
|
||||||
'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
|
'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
|
||||||
@ -31,57 +145,61 @@ class AENetworksIE(InfoExtractor):
|
|||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
'add_ie': ['ThePlatform'],
|
'add_ie': ['ThePlatform'],
|
||||||
'expected_warnings': ['JSON-LD'],
|
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.history.com/shows/mountain-men/season-1/episode-1',
|
'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/videos',
|
||||||
'md5': '8ff93eb073449f151d6b90c0ae1ef0c7',
|
'info_dict':
|
||||||
'info_dict': {
|
{
|
||||||
'id': 'eg47EERs_JsZ',
|
'id': 'world-war-i-history',
|
||||||
'ext': 'mp4',
|
'title': 'World War I History',
|
||||||
'title': 'Winter Is Coming',
|
|
||||||
'description': 'md5:641f424b7a19d8e24f26dea22cf59d74',
|
|
||||||
'timestamp': 1338306241,
|
|
||||||
'upload_date': '20120529',
|
|
||||||
'uploader': 'AENE-NEW',
|
|
||||||
},
|
},
|
||||||
'add_ie': ['ThePlatform'],
|
'playlist_mincount': 24,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.aetv.com/shows/duck-dynasty/video/inlawful-entry',
|
'url': 'http://www.history.com/topics/world-war-i-history/videos',
|
||||||
'only_matching': True
|
'only_matching': True,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.fyi.tv/shows/tiny-house-nation/videos/207-sq-ft-minnesota-prairie-cottage',
|
'url': 'http://www.history.com/topics/world-war-i/world-war-i-history',
|
||||||
'only_matching': True
|
'only_matching': True,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.mylifetime.com/shows/project-runway-junior/video/season-1/episode-6/superstar-clients',
|
'url': 'http://www.history.com/topics/world-war-i/world-war-i-history/speeches',
|
||||||
'only_matching': True
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def theplatform_url_result(self, theplatform_url, video_id, query):
|
||||||
page_type, video_id = re.match(self._VALID_URL, url).groups()
|
return {
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
|
||||||
|
|
||||||
video_url_re = [
|
|
||||||
r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
|
|
||||||
r"media_url\s*=\s*'([^']+)'"
|
|
||||||
]
|
|
||||||
video_url = unescapeHTML(self._search_regex(video_url_re, webpage, 'video url'))
|
|
||||||
query = {'mbr': 'true'}
|
|
||||||
if page_type == 'shows':
|
|
||||||
query['assetTypes'] = 'medium_video_s3'
|
|
||||||
if 'switch=hds' in video_url:
|
|
||||||
query['switch'] = 'hls'
|
|
||||||
|
|
||||||
info = self._search_json_ld(webpage, video_id, fatal=False)
|
|
||||||
info.update({
|
|
||||||
'_type': 'url_transparent',
|
'_type': 'url_transparent',
|
||||||
|
'id': video_id,
|
||||||
'url': smuggle_url(
|
'url': smuggle_url(
|
||||||
update_url_query(video_url, query),
|
update_url_query(theplatform_url, query),
|
||||||
{
|
{
|
||||||
'sig': {
|
'sig': {
|
||||||
'key': 'crazyjava',
|
'key': self._THEPLATFORM_KEY,
|
||||||
'secret': 's3cr3t'},
|
'secret': self._THEPLATFORM_SECRET,
|
||||||
|
},
|
||||||
'force_smil_url': True
|
'force_smil_url': True
|
||||||
}),
|
}),
|
||||||
})
|
'ie_key': 'ThePlatform',
|
||||||
return info
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
topic_id, video_display_id = re.match(self._VALID_URL, url).groups()
|
||||||
|
if video_display_id:
|
||||||
|
webpage = self._download_webpage(url, video_display_id)
|
||||||
|
release_url, video_id = re.search(r"_videoPlayer.play\('([^']+)'\s*,\s*'[^']+'\s*,\s*'(\d+)'\)", webpage).groups()
|
||||||
|
release_url = unescapeHTML(release_url)
|
||||||
|
|
||||||
|
return self.theplatform_url_result(
|
||||||
|
release_url, video_id, {
|
||||||
|
'mbr': 'true',
|
||||||
|
'switch': 'hls'
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
webpage = self._download_webpage(url, topic_id)
|
||||||
|
entries = []
|
||||||
|
for episode_item in re.findall(r'<a.+?data-release-url="[^"]+"[^>]*>', webpage):
|
||||||
|
video_attributes = extract_attributes(episode_item)
|
||||||
|
entries.append(self.theplatform_url_result(
|
||||||
|
video_attributes['data-release-url'], video_attributes['data-id'], {
|
||||||
|
'mbr': 'true',
|
||||||
|
'switch': 'hls'
|
||||||
|
}))
|
||||||
|
return self.playlist_result(entries, topic_id, get_element_by_attribute('class', 'show-title', webpage))
|
||||||
|
@ -24,10 +24,10 @@ class AftonbladetIE(InfoExtractor):
|
|||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
# find internal video meta data
|
# find internal video meta data
|
||||||
meta_url = 'http://aftonbladet-play.drlib.aptoma.no/video/%s.json'
|
meta_url = 'http://aftonbladet-play-metadata.cdn.drvideo.aptoma.no/video/%s.json'
|
||||||
player_config = self._parse_json(self._html_search_regex(
|
player_config = self._parse_json(self._html_search_regex(
|
||||||
r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
|
r'data-player-config="([^"]+)"', webpage, 'player config'), video_id)
|
||||||
internal_meta_id = player_config['videoId']
|
internal_meta_id = player_config['aptomaVideoId']
|
||||||
internal_meta_url = meta_url % internal_meta_id
|
internal_meta_url = meta_url % internal_meta_id
|
||||||
internal_meta_json = self._download_json(
|
internal_meta_json = self._download_json(
|
||||||
internal_meta_url, video_id, 'Downloading video meta data')
|
internal_meta_url, video_id, 'Downloading video meta data')
|
||||||
|
@ -7,6 +7,8 @@ from .common import InfoExtractor
|
|||||||
from ..compat import compat_urlparse
|
from ..compat import compat_urlparse
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
parse_duration,
|
||||||
|
unified_strdate,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -16,7 +18,8 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
|
'url': 'http://trailers.apple.com/trailers/wb/manofsteel/',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'manofsteel',
|
'id': '5111',
|
||||||
|
'title': 'Man of Steel',
|
||||||
},
|
},
|
||||||
'playlist': [
|
'playlist': [
|
||||||
{
|
{
|
||||||
@ -70,6 +73,15 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
'id': 'blackthorn',
|
'id': 'blackthorn',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 2,
|
'playlist_mincount': 2,
|
||||||
|
'expected_warnings': ['Unable to download JSON metadata'],
|
||||||
|
}, {
|
||||||
|
# json data only available from http://trailers.apple.com/trailers/feeds/data/15881.json
|
||||||
|
'url': 'http://trailers.apple.com/trailers/fox/kungfupanda3/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '15881',
|
||||||
|
'title': 'Kung Fu Panda 3',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 4,
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://trailers.apple.com/ca/metropole/autrui/',
|
'url': 'http://trailers.apple.com/ca/metropole/autrui/',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
@ -85,6 +97,45 @@ class AppleTrailersIE(InfoExtractor):
|
|||||||
movie = mobj.group('movie')
|
movie = mobj.group('movie')
|
||||||
uploader_id = mobj.group('company')
|
uploader_id = mobj.group('company')
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, movie)
|
||||||
|
film_id = self._search_regex(r"FilmId\s*=\s*'(\d+)'", webpage, 'film id')
|
||||||
|
film_data = self._download_json(
|
||||||
|
'http://trailers.apple.com/trailers/feeds/data/%s.json' % film_id,
|
||||||
|
film_id, fatal=False)
|
||||||
|
|
||||||
|
if film_data:
|
||||||
|
entries = []
|
||||||
|
for clip in film_data.get('clips', []):
|
||||||
|
clip_title = clip['title']
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
for version, version_data in clip.get('versions', {}).items():
|
||||||
|
for size, size_data in version_data.get('sizes', {}).items():
|
||||||
|
src = size_data.get('src')
|
||||||
|
if not src:
|
||||||
|
continue
|
||||||
|
formats.append({
|
||||||
|
'format_id': '%s-%s' % (version, size),
|
||||||
|
'url': re.sub(r'_(\d+p.mov)', r'_h\1', src),
|
||||||
|
'width': int_or_none(size_data.get('width')),
|
||||||
|
'height': int_or_none(size_data.get('height')),
|
||||||
|
'language': version[:2],
|
||||||
|
})
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
entries.append({
|
||||||
|
'id': movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', clip_title).lower(),
|
||||||
|
'formats': formats,
|
||||||
|
'title': clip_title,
|
||||||
|
'thumbnail': clip.get('screen') or clip.get('thumb'),
|
||||||
|
'duration': parse_duration(clip.get('runtime') or clip.get('faded')),
|
||||||
|
'upload_date': unified_strdate(clip.get('posted')),
|
||||||
|
'uploader_id': uploader_id,
|
||||||
|
})
|
||||||
|
|
||||||
|
page_data = film_data.get('page', {})
|
||||||
|
return self.playlist_result(entries, film_id, page_data.get('movie_title'))
|
||||||
|
|
||||||
playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
|
playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
|
||||||
|
|
||||||
def fix_html(s):
|
def fix_html(s):
|
||||||
|
@ -8,7 +8,6 @@ from .generic import GenericIE
|
|||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
get_element_by_attribute,
|
|
||||||
qualities,
|
qualities,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
@ -274,41 +273,3 @@ class ARDIE(InfoExtractor):
|
|||||||
'upload_date': upload_date,
|
'upload_date': upload_date,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class SportschauIE(ARDMediathekIE):
|
|
||||||
IE_NAME = 'Sportschau'
|
|
||||||
_VALID_URL = r'(?P<baseurl>https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video(?P<id>[^/#?]+))\.html'
|
|
||||||
_TESTS = [{
|
|
||||||
'url': 'http://www.sportschau.de/tourdefrance/videoseppeltkokainhatnichtsmitklassischemdopingzutun100.html',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'seppeltkokainhatnichtsmitklassischemdopingzutun100',
|
|
||||||
'ext': 'mp4',
|
|
||||||
'title': 'Seppelt: "Kokain hat nichts mit klassischem Doping zu tun"',
|
|
||||||
'thumbnail': 're:^https?://.*\.jpg$',
|
|
||||||
'description': 'Der ARD-Doping Experte Hajo Seppelt gibt seine Einschätzung zum ersten Dopingfall der diesjährigen Tour de France um den Italiener Luca Paolini ab.',
|
|
||||||
},
|
|
||||||
'params': {
|
|
||||||
# m3u8 download
|
|
||||||
'skip_download': True,
|
|
||||||
},
|
|
||||||
}]
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
|
||||||
mobj = re.match(self._VALID_URL, url)
|
|
||||||
video_id = mobj.group('id')
|
|
||||||
base_url = mobj.group('baseurl')
|
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
|
||||||
title = get_element_by_attribute('class', 'headline', webpage)
|
|
||||||
description = self._html_search_meta('description', webpage, 'description')
|
|
||||||
|
|
||||||
info = self._extract_media_info(
|
|
||||||
base_url + '-mc_defaultQuality-h.json', webpage, video_id)
|
|
||||||
|
|
||||||
info.update({
|
|
||||||
'title': title,
|
|
||||||
'description': description,
|
|
||||||
})
|
|
||||||
|
|
||||||
return info
|
|
||||||
|
@ -180,11 +180,14 @@ class ArteTVBaseIE(InfoExtractor):
|
|||||||
|
|
||||||
class ArteTVPlus7IE(ArteTVBaseIE):
|
class ArteTVPlus7IE(ArteTVBaseIE):
|
||||||
IE_NAME = 'arte.tv:+7'
|
IE_NAME = 'arte.tv:+7'
|
||||||
_VALID_URL = r'https?://(?:www\.)?arte\.tv/guide/(?P<lang>fr|de|en|es)/(?:(?:sendungen|emissions|embed)/)?(?P<id>[^/]+)/(?P<name>[^/?#&]+)'
|
_VALID_URL = r'https?://(?:(?:www|sites)\.)?arte\.tv/[^/]+/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
|
'url': 'http://www.arte.tv/guide/de/sendungen/XEN/xenius/?vid=055918-015_PLUS7-D',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://sites.arte.tv/karambolage/de/video/karambolage-22',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -240,10 +243,10 @@ class ArteTVPlus7IE(ArteTVBaseIE):
|
|||||||
return self._extract_from_json_url(json_url, video_id, lang, title=title)
|
return self._extract_from_json_url(json_url, video_id, lang, title=title)
|
||||||
# Different kind of embed URL (e.g.
|
# Different kind of embed URL (e.g.
|
||||||
# http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
|
# http://www.arte.tv/magazine/trepalium/fr/episode-0406-replay-trepalium)
|
||||||
embed_url = self._search_regex(
|
entries = [
|
||||||
r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1',
|
self.url_result(url)
|
||||||
webpage, 'embed url', group='url')
|
for _, url in re.findall(r'<iframe[^>]+src=(["\'])(?P<url>.+?)\1', webpage)]
|
||||||
return self.url_result(embed_url)
|
return self.playlist_result(entries)
|
||||||
|
|
||||||
|
|
||||||
# It also uses the arte_vp_url url from the webpage to extract the information
|
# It also uses the arte_vp_url url from the webpage to extract the information
|
||||||
@ -252,22 +255,17 @@ class ArteTVCreativeIE(ArteTVPlus7IE):
|
|||||||
_VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
_VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de|en|es)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design',
|
'url': 'http://creative.arte.tv/fr/episode/osmosis-episode-1',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '72176',
|
'id': '057405-001-A',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Folge 2 - Corporate Design',
|
'title': 'OSMOSIS - N\'AYEZ PLUS PEUR D\'AIMER (1)',
|
||||||
'upload_date': '20131004',
|
'upload_date': '20150716',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion',
|
'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion',
|
||||||
'info_dict': {
|
'playlist_count': 11,
|
||||||
'id': '160676',
|
'add_ie': ['Youtube'],
|
||||||
'ext': 'mp4',
|
|
||||||
'title': 'Monty Python live (mostly)',
|
|
||||||
'description': 'Événement ! Quarante-cinq ans après leurs premiers succès, les légendaires Monty Python remontent sur scène.\n',
|
|
||||||
'upload_date': '20140805',
|
|
||||||
}
|
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://creative.arte.tv/de/episode/agentur-amateur-4-der-erste-kunde',
|
'url': 'http://creative.arte.tv/de/episode/agentur-amateur-4-der-erste-kunde',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
@ -349,14 +347,13 @@ class ArteTVCinemaIE(ArteTVPlus7IE):
|
|||||||
_VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
|
_VALID_URL = r'https?://cinema\.arte\.tv/(?P<lang>fr|de|en|es)/(?P<id>.+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://cinema.arte.tv/de/node/38291',
|
'url': 'http://cinema.arte.tv/fr/article/les-ailes-du-desir-de-julia-reck',
|
||||||
'md5': '6b275511a5107c60bacbeeda368c3aa1',
|
'md5': 'a5b9dd5575a11d93daf0e3f404f45438',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '055876-000_PWA12025-D',
|
'id': '062494-000-A',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Tod auf dem Nil',
|
'title': 'Film lauréat du concours web - "Les ailes du désir" de Julia Reck',
|
||||||
'upload_date': '20160122',
|
'upload_date': '20150807',
|
||||||
'description': 'md5:7f749bbb77d800ef2be11d54529b96bc',
|
|
||||||
},
|
},
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@ -422,6 +419,7 @@ class ArteTVPlaylistIE(ArteTVBaseIE):
|
|||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'PL-013263',
|
'id': 'PL-013263',
|
||||||
'title': 'Areva & Uramin',
|
'title': 'Areva & Uramin',
|
||||||
|
'description': 'md5:a1dc0312ce357c262259139cfd48c9bf',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 6,
|
'playlist_mincount': 6,
|
||||||
}, {
|
}, {
|
||||||
|
@ -46,6 +46,7 @@ class AzubuIE(InfoExtractor):
|
|||||||
'uploader_id': 272749,
|
'uploader_id': 272749,
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
},
|
},
|
||||||
|
'skip': 'Channel offline',
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
@ -56,22 +57,26 @@ class AzubuIE(InfoExtractor):
|
|||||||
'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
|
'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
|
||||||
|
|
||||||
title = data['title'].strip()
|
title = data['title'].strip()
|
||||||
description = data['description']
|
description = data.get('description')
|
||||||
thumbnail = data['thumbnail']
|
thumbnail = data.get('thumbnail')
|
||||||
view_count = data['view_count']
|
view_count = data.get('view_count')
|
||||||
uploader = data['user']['username']
|
user = data.get('user', {})
|
||||||
uploader_id = data['user']['id']
|
uploader = user.get('username')
|
||||||
|
uploader_id = user.get('id')
|
||||||
|
|
||||||
stream_params = json.loads(data['stream_params'])
|
stream_params = json.loads(data['stream_params'])
|
||||||
|
|
||||||
timestamp = float_or_none(stream_params['creationDate'], 1000)
|
timestamp = float_or_none(stream_params.get('creationDate'), 1000)
|
||||||
duration = float_or_none(stream_params['length'], 1000)
|
duration = float_or_none(stream_params.get('length'), 1000)
|
||||||
|
|
||||||
renditions = stream_params.get('renditions') or []
|
renditions = stream_params.get('renditions') or []
|
||||||
video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
|
video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
|
||||||
if video:
|
if video:
|
||||||
renditions.append(video)
|
renditions.append(video)
|
||||||
|
|
||||||
|
if not renditions and not user.get('channel', {}).get('is_live', True):
|
||||||
|
raise ExtractorError('%s said: channel is offline.' % self.IE_NAME, expected=True)
|
||||||
|
|
||||||
formats = [{
|
formats = [{
|
||||||
'url': fmt['url'],
|
'url': fmt['url'],
|
||||||
'width': fmt['frameWidth'],
|
'width': fmt['frameWidth'],
|
||||||
|
@ -31,7 +31,7 @@ class BBCCoUkIE(InfoExtractor):
|
|||||||
music/clips[/#]|
|
music/clips[/#]|
|
||||||
radio/player/
|
radio/player/
|
||||||
)
|
)
|
||||||
(?P<id>%s)
|
(?P<id>%s)(?!/(?:episodes|broadcasts|clips))
|
||||||
''' % _ID_REGEX
|
''' % _ID_REGEX
|
||||||
|
|
||||||
_MEDIASELECTOR_URLS = [
|
_MEDIASELECTOR_URLS = [
|
||||||
@ -192,6 +192,7 @@ class BBCCoUkIE(InfoExtractor):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'Now it\'s really geo-restricted',
|
||||||
}, {
|
}, {
|
||||||
# compact player (https://github.com/rg3/youtube-dl/issues/8147)
|
# compact player (https://github.com/rg3/youtube-dl/issues/8147)
|
||||||
'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
|
'url': 'http://www.bbc.co.uk/programmes/p028bfkf/player',
|
||||||
@ -698,7 +699,9 @@ class BBCIE(BBCCoUkIE):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def suitable(cls, url):
|
def suitable(cls, url):
|
||||||
return False if BBCCoUkIE.suitable(url) or BBCCoUkArticleIE.suitable(url) else super(BBCIE, cls).suitable(url)
|
EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerPlaylistIE, BBCCoUkPlaylistIE)
|
||||||
|
return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
|
||||||
|
else super(BBCIE, cls).suitable(url))
|
||||||
|
|
||||||
def _extract_from_media_meta(self, media_meta, video_id):
|
def _extract_from_media_meta(self, media_meta, video_id):
|
||||||
# Direct links to media in media metadata (e.g.
|
# Direct links to media in media metadata (e.g.
|
||||||
@ -975,3 +978,72 @@ class BBCCoUkArticleIE(InfoExtractor):
|
|||||||
r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
|
r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
|
||||||
|
|
||||||
return self.playlist_result(entries, playlist_id, title, description)
|
return self.playlist_result(entries, playlist_id, title, description)
|
||||||
|
|
||||||
|
|
||||||
|
class BBCCoUkPlaylistBaseIE(InfoExtractor):
|
||||||
|
def _real_extract(self, url):
|
||||||
|
playlist_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, playlist_id)
|
||||||
|
|
||||||
|
entries = [
|
||||||
|
self.url_result(self._URL_TEMPLATE % video_id, BBCCoUkIE.ie_key())
|
||||||
|
for video_id in re.findall(
|
||||||
|
self._VIDEO_ID_TEMPLATE % BBCCoUkIE._ID_REGEX, webpage)]
|
||||||
|
|
||||||
|
title, description = self._extract_title_and_description(webpage)
|
||||||
|
|
||||||
|
return self.playlist_result(entries, playlist_id, title, description)
|
||||||
|
|
||||||
|
|
||||||
|
class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
|
||||||
|
IE_NAME = 'bbc.co.uk:iplayer:playlist'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/episodes/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
|
||||||
|
_URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
|
||||||
|
_VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
|
||||||
|
_TEST = {
|
||||||
|
'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'b05rcz9v',
|
||||||
|
'title': 'The Disappearance',
|
||||||
|
'description': 'French thriller serial about a missing teenager.',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 6,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _extract_title_and_description(self, webpage):
|
||||||
|
title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
|
||||||
|
description = self._search_regex(
|
||||||
|
r'<p[^>]+class=(["\'])subtitle\1[^>]*>(?P<value>[^<]+)</p>',
|
||||||
|
webpage, 'description', fatal=False, group='value')
|
||||||
|
return title, description
|
||||||
|
|
||||||
|
|
||||||
|
class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):
|
||||||
|
IE_NAME = 'bbc.co.uk:playlist'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/programmes/(?P<id>%s)/(?:episodes|broadcasts|clips)' % BBCCoUkIE._ID_REGEX
|
||||||
|
_URL_TEMPLATE = 'http://www.bbc.co.uk/programmes/%s'
|
||||||
|
_VIDEO_ID_TEMPLATE = r'data-pid=["\'](%s)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'b05rcz9v',
|
||||||
|
'title': 'The Disappearance - Clips - BBC Four',
|
||||||
|
'description': 'French thriller serial about a missing teenager.',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 7,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/broadcasts/2016/06',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.bbc.co.uk/programmes/b05rcz9v/clips',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.bbc.co.uk/programmes/b055jkys/episodes/player',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _extract_title_and_description(self, webpage):
|
||||||
|
title = self._og_search_title(webpage, fatal=False)
|
||||||
|
description = self._og_search_description(webpage)
|
||||||
|
return title, description
|
||||||
|
@ -1,31 +1,27 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .mtv import MTVServicesInfoExtractor
|
||||||
from ..compat import compat_urllib_parse_unquote
|
from ..utils import unified_strdate
|
||||||
from ..utils import (
|
from ..compat import compat_urllib_parse_urlencode
|
||||||
xpath_text,
|
|
||||||
xpath_with_ns,
|
|
||||||
int_or_none,
|
|
||||||
parse_iso8601,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class BetIE(InfoExtractor):
|
class BetIE(MTVServicesInfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
|
_VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
{
|
{
|
||||||
'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
|
'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'news/national/2014/a-conversation-with-president-obama',
|
'id': '07e96bd3-8850-3051-b856-271b457f0ab8',
|
||||||
'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
|
'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
|
||||||
'ext': 'flv',
|
'ext': 'flv',
|
||||||
'title': 'A Conversation With President Obama',
|
'title': 'A Conversation With President Obama',
|
||||||
'description': 'md5:699d0652a350cf3e491cd15cc745b5da',
|
'description': 'President Obama urges persistence in confronting racism and bias.',
|
||||||
'duration': 1534,
|
'duration': 1534,
|
||||||
'timestamp': 1418075340,
|
|
||||||
'upload_date': '20141208',
|
'upload_date': '20141208',
|
||||||
'uploader': 'admin',
|
|
||||||
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
||||||
|
'subtitles': {
|
||||||
|
'en': 'mincount:2',
|
||||||
|
}
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
# rtmp download
|
# rtmp download
|
||||||
@ -35,16 +31,17 @@ class BetIE(InfoExtractor):
|
|||||||
{
|
{
|
||||||
'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
|
'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'news/national/2014/justice-for-ferguson-a-community-reacts',
|
'id': '9f516bf1-7543-39c4-8076-dd441b459ba9',
|
||||||
'display_id': 'justice-for-ferguson-a-community-reacts',
|
'display_id': 'justice-for-ferguson-a-community-reacts',
|
||||||
'ext': 'flv',
|
'ext': 'flv',
|
||||||
'title': 'Justice for Ferguson: A Community Reacts',
|
'title': 'Justice for Ferguson: A Community Reacts',
|
||||||
'description': 'A BET News special.',
|
'description': 'A BET News special.',
|
||||||
'duration': 1696,
|
'duration': 1696,
|
||||||
'timestamp': 1416942360,
|
|
||||||
'upload_date': '20141125',
|
'upload_date': '20141125',
|
||||||
'uploader': 'admin',
|
|
||||||
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
||||||
|
'subtitles': {
|
||||||
|
'en': 'mincount:2',
|
||||||
|
}
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
# rtmp download
|
# rtmp download
|
||||||
@ -53,57 +50,32 @@ class BetIE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
_FEED_URL = "http://feeds.mtvnservices.com/od/feed/bet-mrss-player"
|
||||||
|
|
||||||
|
def _get_feed_query(self, uri):
|
||||||
|
return compat_urllib_parse_urlencode({
|
||||||
|
'uuid': uri,
|
||||||
|
})
|
||||||
|
|
||||||
|
def _extract_mgid(self, webpage):
|
||||||
|
return self._search_regex(r'data-uri="([^"]+)', webpage, 'mgid')
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
|
|
||||||
webpage = self._download_webpage(url, display_id)
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
mgid = self._extract_mgid(webpage)
|
||||||
|
videos_info = self._get_videos_info(mgid)
|
||||||
|
|
||||||
media_url = compat_urllib_parse_unquote(self._search_regex(
|
info_dict = videos_info['entries'][0]
|
||||||
[r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
|
|
||||||
webpage, 'media URL'))
|
|
||||||
|
|
||||||
video_id = self._search_regex(
|
upload_date = unified_strdate(self._html_search_meta('date', webpage))
|
||||||
r'/video/(.*)/_jcr_content/', media_url, 'video id')
|
description = self._html_search_meta('description', webpage)
|
||||||
|
|
||||||
mrss = self._download_xml(media_url, display_id)
|
info_dict.update({
|
||||||
|
|
||||||
item = mrss.find('./channel/item')
|
|
||||||
|
|
||||||
NS_MAP = {
|
|
||||||
'dc': 'http://purl.org/dc/elements/1.1/',
|
|
||||||
'media': 'http://search.yahoo.com/mrss/',
|
|
||||||
'ka': 'http://kickapps.com/karss',
|
|
||||||
}
|
|
||||||
|
|
||||||
title = xpath_text(item, './title', 'title')
|
|
||||||
description = xpath_text(
|
|
||||||
item, './description', 'description', fatal=False)
|
|
||||||
|
|
||||||
timestamp = parse_iso8601(xpath_text(
|
|
||||||
item, xpath_with_ns('./dc:date', NS_MAP),
|
|
||||||
'upload date', fatal=False))
|
|
||||||
uploader = xpath_text(
|
|
||||||
item, xpath_with_ns('./dc:creator', NS_MAP),
|
|
||||||
'uploader', fatal=False)
|
|
||||||
|
|
||||||
media_content = item.find(
|
|
||||||
xpath_with_ns('./media:content', NS_MAP))
|
|
||||||
duration = int_or_none(media_content.get('duration'))
|
|
||||||
smil_url = media_content.get('url')
|
|
||||||
|
|
||||||
thumbnail = media_content.find(
|
|
||||||
xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
|
|
||||||
|
|
||||||
formats = self._extract_smil_formats(smil_url, display_id)
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
return {
|
|
||||||
'id': video_id,
|
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
'title': title,
|
|
||||||
'description': description,
|
'description': description,
|
||||||
'thumbnail': thumbnail,
|
'upload_date': upload_date,
|
||||||
'timestamp': timestamp,
|
})
|
||||||
'uploader': uploader,
|
|
||||||
'duration': duration,
|
return info_dict
|
||||||
'formats': formats,
|
|
||||||
}
|
|
||||||
|
@ -29,7 +29,8 @@ class BRIE(InfoExtractor):
|
|||||||
'duration': 180,
|
'duration': 180,
|
||||||
'uploader': 'Reinhard Weber',
|
'uploader': 'Reinhard Weber',
|
||||||
'upload_date': '20150422',
|
'upload_date': '20150422',
|
||||||
}
|
},
|
||||||
|
'skip': '404 not found',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
|
'url': 'http://www.br.de/nachrichten/oberbayern/inhalt/muenchner-polizeipraesident-schreiber-gestorben-100.html',
|
||||||
@ -40,7 +41,8 @@ class BRIE(InfoExtractor):
|
|||||||
'title': 'Manfred Schreiber ist tot',
|
'title': 'Manfred Schreiber ist tot',
|
||||||
'description': 'md5:b454d867f2a9fc524ebe88c3f5092d97',
|
'description': 'md5:b454d867f2a9fc524ebe88c3f5092d97',
|
||||||
'duration': 26,
|
'duration': 26,
|
||||||
}
|
},
|
||||||
|
'skip': '404 not found',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'https://www.br-klassik.de/audio/peeping-tom-premierenkritik-dance-festival-muenchen-100.html',
|
'url': 'https://www.br-klassik.de/audio/peeping-tom-premierenkritik-dance-festival-muenchen-100.html',
|
||||||
@ -51,7 +53,8 @@ class BRIE(InfoExtractor):
|
|||||||
'title': 'Kurzweilig und sehr bewegend',
|
'title': 'Kurzweilig und sehr bewegend',
|
||||||
'description': 'md5:0351996e3283d64adeb38ede91fac54e',
|
'description': 'md5:0351996e3283d64adeb38ede91fac54e',
|
||||||
'duration': 296,
|
'duration': 296,
|
||||||
}
|
},
|
||||||
|
'skip': '404 not found',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',
|
'url': 'http://www.br.de/radio/bayern1/service/team/videos/team-video-erdelt100.html',
|
||||||
|
@ -585,6 +585,13 @@ class BrightcoveNewIE(InfoExtractor):
|
|||||||
'format_id': build_format_id('rtmp'),
|
'format_id': build_format_id('rtmp'),
|
||||||
})
|
})
|
||||||
formats.append(f)
|
formats.append(f)
|
||||||
|
|
||||||
|
errors = json_data.get('errors')
|
||||||
|
if not formats and errors:
|
||||||
|
error = errors[0]
|
||||||
|
raise ExtractorError(
|
||||||
|
error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
|
||||||
|
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
|
@ -5,6 +5,7 @@ import json
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
from .facebook import FacebookIE
|
||||||
|
|
||||||
|
|
||||||
class BuzzFeedIE(InfoExtractor):
|
class BuzzFeedIE(InfoExtractor):
|
||||||
@ -20,11 +21,11 @@ class BuzzFeedIE(InfoExtractor):
|
|||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'aVCR29aE_OQ',
|
'id': 'aVCR29aE_OQ',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
'title': 'Angry Ram destroys a punching bag..',
|
||||||
|
'description': 'md5:c59533190ef23fd4458a5e8c8c872345',
|
||||||
'upload_date': '20141024',
|
'upload_date': '20141024',
|
||||||
'uploader_id': 'Buddhanz1',
|
'uploader_id': 'Buddhanz1',
|
||||||
'description': 'He likes to stay in shape with his heavy bag, he wont stop until its on the ground\n\nFollow Angry Ram on Facebook for regular updates -\nhttps://www.facebook.com/pages/Angry-Ram/1436897249899558?ref=hl',
|
'uploader': 'Angry Ram',
|
||||||
'uploader': 'Buddhanz',
|
|
||||||
'title': 'Angry Ram destroys a punching bag',
|
|
||||||
}
|
}
|
||||||
}]
|
}]
|
||||||
}, {
|
}, {
|
||||||
@ -41,13 +42,30 @@ class BuzzFeedIE(InfoExtractor):
|
|||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'mVmBL8B-In0',
|
'id': 'mVmBL8B-In0',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
|
'title': 're:Munchkin the Teddy Bear gets her exercise',
|
||||||
|
'description': 'md5:28faab95cda6e361bcff06ec12fc21d8',
|
||||||
'upload_date': '20141124',
|
'upload_date': '20141124',
|
||||||
'uploader_id': 'CindysMunchkin',
|
'uploader_id': 'CindysMunchkin',
|
||||||
'description': 're:© 2014 Munchkin the',
|
|
||||||
'uploader': 're:^Munchkin the',
|
'uploader': 're:^Munchkin the',
|
||||||
'title': 're:Munchkin the Teddy Bear gets her exercise',
|
|
||||||
},
|
},
|
||||||
}]
|
}]
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.buzzfeed.com/craigsilverman/the-most-adorable-crash-landing-ever#.eq7pX0BAmK',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'the-most-adorable-crash-landing-ever',
|
||||||
|
'title': 'Watch This Baby Goose Make The Most Adorable Crash Landing',
|
||||||
|
'description': 'This gosling knows how to stick a landing.',
|
||||||
|
},
|
||||||
|
'playlist': [{
|
||||||
|
'md5': '763ca415512f91ca62e4621086900a23',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '971793786185728',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'We set up crash pads so that the goslings on our roof would have a safe landi...',
|
||||||
|
'uploader': 'Calgary Outdoor Centre-University of Calgary',
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
'add_ie': ['Facebook'],
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
@ -66,6 +84,10 @@ class BuzzFeedIE(InfoExtractor):
|
|||||||
continue
|
continue
|
||||||
entries.append(self.url_result(video['url']))
|
entries.append(self.url_result(video['url']))
|
||||||
|
|
||||||
|
facebook_url = FacebookIE._extract_url(webpage)
|
||||||
|
if facebook_url:
|
||||||
|
entries.append(self.url_result(facebook_url))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'_type': 'playlist',
|
'_type': 'playlist',
|
||||||
'id': playlist_id,
|
'id': playlist_id,
|
||||||
|
88
youtube_dl/extractor/carambatv.py
Normal file
88
youtube_dl/extractor/carambatv.py
Normal file
@ -0,0 +1,88 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..compat import compat_str
|
||||||
|
from ..utils import (
|
||||||
|
float_or_none,
|
||||||
|
int_or_none,
|
||||||
|
try_get,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CarambaTVIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'(?:carambatv:|https?://video1\.carambatv\.ru/v/)(?P<id>\d+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://video1.carambatv.ru/v/191910501',
|
||||||
|
'md5': '2f4a81b7cfd5ab866ee2d7270cb34a2a',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '191910501',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg',
|
||||||
|
'duration': 2678.31,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'carambatv:191910501',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
video = self._download_json(
|
||||||
|
'http://video1.carambatv.ru/v/%s/videoinfo.js' % video_id,
|
||||||
|
video_id)
|
||||||
|
|
||||||
|
title = video['title']
|
||||||
|
|
||||||
|
base_url = video.get('video') or 'http://video1.carambatv.ru/v/%s/' % video_id
|
||||||
|
|
||||||
|
formats = [{
|
||||||
|
'url': base_url + f['fn'],
|
||||||
|
'height': int_or_none(f.get('height')),
|
||||||
|
'format_id': '%sp' % f['height'] if f.get('height') else None,
|
||||||
|
} for f in video['qualities'] if f.get('fn')]
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
thumbnail = video.get('splash')
|
||||||
|
duration = float_or_none(try_get(
|
||||||
|
video, lambda x: x['annotations'][0]['end_time'], compat_str))
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': title,
|
||||||
|
'thumbnail': thumbnail,
|
||||||
|
'duration': duration,
|
||||||
|
'formats': formats,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CarambaTVPageIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://carambatv\.ru/(?:[^/]+/)+(?P<id>[^/?#&]+)'
|
||||||
|
_TEST = {
|
||||||
|
'url': 'http://carambatv.ru/movie/bad-comedian/razborka-v-manile/',
|
||||||
|
'md5': '',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '191910501',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
|
'duration': 2678.31,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
video_url = self._og_search_property('video:iframe', webpage, default=None)
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
video_id = self._search_regex(
|
||||||
|
r'(?:video_id|crmb_vuid)\s*[:=]\s*["\']?(\d+)',
|
||||||
|
webpage, 'video id')
|
||||||
|
video_url = 'carambatv:%s' % video_id
|
||||||
|
|
||||||
|
return self.url_result(video_url, CarambaTVIE.ie_key())
|
@ -1,17 +1,13 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
from .theplatform import ThePlatformFeedIE
|
||||||
|
|
||||||
from .theplatform import ThePlatformIE
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
xpath_text,
|
|
||||||
xpath_element,
|
|
||||||
int_or_none,
|
int_or_none,
|
||||||
find_xpath_attr,
|
find_xpath_attr,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class CBSBaseIE(ThePlatformIE):
|
class CBSBaseIE(ThePlatformFeedIE):
|
||||||
def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
|
def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
|
||||||
closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
|
closed_caption_e = find_xpath_attr(smil, self._xpath_ns('.//param', namespace), 'name', 'ClosedCaptionURL')
|
||||||
return {
|
return {
|
||||||
@ -21,9 +17,22 @@ class CBSBaseIE(ThePlatformIE):
|
|||||||
}]
|
}]
|
||||||
} if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
|
} if closed_caption_e is not None and closed_caption_e.attrib.get('value') else []
|
||||||
|
|
||||||
|
def _extract_video_info(self, filter_query, video_id):
|
||||||
|
return self._extract_feed_info(
|
||||||
|
'dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id, lambda entry: {
|
||||||
|
'series': entry.get('cbs$SeriesTitle'),
|
||||||
|
'season_number': int_or_none(entry.get('cbs$SeasonNumber')),
|
||||||
|
'episode': entry.get('cbs$EpisodeTitle'),
|
||||||
|
'episode_number': int_or_none(entry.get('cbs$EpisodeNumber')),
|
||||||
|
}, {
|
||||||
|
'StreamPack': {
|
||||||
|
'manifest': 'm3u',
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
class CBSIE(CBSBaseIE):
|
class CBSIE(CBSBaseIE):
|
||||||
_VALID_URL = r'(?:cbs:(?P<content_id>\w+)|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/(?:video|artist)|colbertlateshow\.com/(?:video|podcasts))/[^/]+/(?P<display_id>[^/]+))'
|
_VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
|
'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
|
||||||
@ -38,25 +47,7 @@ class CBSIE(CBSBaseIE):
|
|||||||
'upload_date': '20131127',
|
'upload_date': '20131127',
|
||||||
'uploader': 'CBSI-NEW',
|
'uploader': 'CBSI-NEW',
|
||||||
},
|
},
|
||||||
'params': {
|
'expected_warnings': ['Failed to download m3u8 information'],
|
||||||
# rtmp download
|
|
||||||
'skip_download': True,
|
|
||||||
},
|
|
||||||
'_skip': 'Blocked outside the US',
|
|
||||||
}, {
|
|
||||||
'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'WWF_5KqY3PK1',
|
|
||||||
'display_id': 'st-vincent',
|
|
||||||
'ext': 'flv',
|
|
||||||
'title': 'Live on Letterman - St. Vincent',
|
|
||||||
'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.',
|
|
||||||
'duration': 3221,
|
|
||||||
},
|
|
||||||
'params': {
|
|
||||||
# rtmp download
|
|
||||||
'skip_download': True,
|
|
||||||
},
|
|
||||||
'_skip': 'Blocked outside the US',
|
'_skip': 'Blocked outside the US',
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
|
'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/',
|
||||||
@ -68,44 +59,5 @@ class CBSIE(CBSBaseIE):
|
|||||||
TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true'
|
TP_RELEASE_URL_TEMPLATE = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
content_id, display_id = re.match(self._VALID_URL, url).groups()
|
content_id = self._match_id(url)
|
||||||
if not content_id:
|
return self._extract_video_info('byGuid=%s' % content_id, content_id)
|
||||||
webpage = self._download_webpage(url, display_id)
|
|
||||||
content_id = self._search_regex(
|
|
||||||
[r"video\.settings\.content_id\s*=\s*'([^']+)';", r"cbsplayer\.contentId\s*=\s*'([^']+)';"],
|
|
||||||
webpage, 'content id')
|
|
||||||
items_data = self._download_xml(
|
|
||||||
'http://can.cbs.com/thunder/player/videoPlayerService.php',
|
|
||||||
content_id, query={'partner': 'cbs', 'contentId': content_id})
|
|
||||||
video_data = xpath_element(items_data, './/item')
|
|
||||||
title = xpath_text(video_data, 'videoTitle', 'title', True)
|
|
||||||
|
|
||||||
subtitles = {}
|
|
||||||
formats = []
|
|
||||||
for item in items_data.findall('.//item'):
|
|
||||||
pid = xpath_text(item, 'pid')
|
|
||||||
if not pid:
|
|
||||||
continue
|
|
||||||
tp_release_url = self.TP_RELEASE_URL_TEMPLATE % pid
|
|
||||||
if '.m3u8' in xpath_text(item, 'contentUrl', default=''):
|
|
||||||
tp_release_url += '&manifest=m3u'
|
|
||||||
tp_formats, tp_subtitles = self._extract_theplatform_smil(
|
|
||||||
tp_release_url, content_id, 'Downloading %s SMIL data' % pid)
|
|
||||||
formats.extend(tp_formats)
|
|
||||||
subtitles = self._merge_subtitles(subtitles, tp_subtitles)
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
info = self.get_metadata('dJ5BDC/media/guid/2198311517/%s' % content_id, content_id)
|
|
||||||
info.update({
|
|
||||||
'id': content_id,
|
|
||||||
'display_id': display_id,
|
|
||||||
'title': title,
|
|
||||||
'series': xpath_text(video_data, 'seriesTitle'),
|
|
||||||
'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')),
|
|
||||||
'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')),
|
|
||||||
'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000),
|
|
||||||
'thumbnail': xpath_text(video_data, 'previewImageURL'),
|
|
||||||
'formats': formats,
|
|
||||||
'subtitles': subtitles,
|
|
||||||
})
|
|
||||||
return info
|
|
||||||
|
@ -80,9 +80,6 @@ class CBSInteractiveIE(ThePlatformIE):
|
|||||||
|
|
||||||
media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
|
media_guid_path = 'media/guid/%d/%s' % (self.MPX_ACCOUNTS[site], vdata['mpxRefId'])
|
||||||
formats, subtitles = [], {}
|
formats, subtitles = [], {}
|
||||||
if site == 'cnet':
|
|
||||||
formats, subtitles = self._extract_theplatform_smil(
|
|
||||||
self.TP_RELEASE_URL_TEMPLATE % media_guid_path, video_id)
|
|
||||||
for (fkey, vid) in vdata['files'].items():
|
for (fkey, vid) in vdata['files'].items():
|
||||||
if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
|
if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']:
|
||||||
continue
|
continue
|
||||||
@ -94,7 +91,7 @@ class CBSInteractiveIE(ThePlatformIE):
|
|||||||
subtitles = self._merge_subtitles(subtitles, tp_subtitles)
|
subtitles = self._merge_subtitles(subtitles, tp_subtitles)
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
info = self.get_metadata('kYEXFC/%s' % media_guid_path, video_id)
|
info = self._extract_theplatform_metadata('kYEXFC/%s' % media_guid_path, video_id)
|
||||||
info.update({
|
info.update({
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
|
@ -30,9 +30,12 @@ class CBSNewsIE(CBSBaseIE):
|
|||||||
{
|
{
|
||||||
'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
|
'url': 'http://www.cbsnews.com/videos/fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack/',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'fort-hood-shooting-army-downplays-mental-illness-as-cause-of-attack',
|
'id': 'SNJBOYzXiWBOvaLsdzwH8fmtP1SCd91Y',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
|
'title': 'Fort Hood shooting: Army downplays mental illness as cause of attack',
|
||||||
|
'description': 'md5:4a6983e480542d8b333a947bfc64ddc7',
|
||||||
|
'upload_date': '19700101',
|
||||||
|
'uploader': 'CBSI-NEW',
|
||||||
'thumbnail': 're:^https?://.*\.jpg$',
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
'duration': 205,
|
'duration': 205,
|
||||||
'subtitles': {
|
'subtitles': {
|
||||||
@ -58,30 +61,8 @@ class CBSNewsIE(CBSBaseIE):
|
|||||||
webpage, 'video JSON info'), video_id)
|
webpage, 'video JSON info'), video_id)
|
||||||
|
|
||||||
item = video_info['item'] if 'item' in video_info else video_info
|
item = video_info['item'] if 'item' in video_info else video_info
|
||||||
title = item.get('articleTitle') or item.get('hed')
|
guid = item['mpxRefId']
|
||||||
duration = item.get('duration')
|
return self._extract_video_info('byGuid=%s' % guid, guid)
|
||||||
thumbnail = item.get('mediaImage') or item.get('thumbnail')
|
|
||||||
|
|
||||||
subtitles = {}
|
|
||||||
formats = []
|
|
||||||
for format_id in ['RtmpMobileLow', 'RtmpMobileHigh', 'Hls', 'RtmpDesktop']:
|
|
||||||
pid = item.get('media' + format_id)
|
|
||||||
if not pid:
|
|
||||||
continue
|
|
||||||
release_url = 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true' % pid
|
|
||||||
tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % pid)
|
|
||||||
formats.extend(tp_formats)
|
|
||||||
subtitles = self._merge_subtitles(subtitles, tp_subtitles)
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
return {
|
|
||||||
'id': video_id,
|
|
||||||
'title': title,
|
|
||||||
'thumbnail': thumbnail,
|
|
||||||
'duration': duration,
|
|
||||||
'formats': formats,
|
|
||||||
'subtitles': subtitles,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class CBSNewsLiveVideoIE(InfoExtractor):
|
class CBSNewsLiveVideoIE(InfoExtractor):
|
||||||
|
@ -1,30 +1,28 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
from .cbs import CBSBaseIE
|
||||||
|
|
||||||
from .common import InfoExtractor
|
|
||||||
|
|
||||||
|
|
||||||
class CBSSportsIE(InfoExtractor):
|
class CBSSportsIE(CBSBaseIE):
|
||||||
_VALID_URL = r'https?://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
|
_VALID_URL = r'https?://www\.cbssports\.com/video/player/[^/]+/(?P<id>\d+)'
|
||||||
|
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
|
'url': 'http://www.cbssports.com/video/player/videos/708337219968/0/ben-simmons-the-next-lebron?-not-so-fast',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '_d5_GbO8p1sT',
|
'id': '708337219968',
|
||||||
'ext': 'flv',
|
'ext': 'mp4',
|
||||||
'title': 'US Open flashbacks: 1990s',
|
'title': 'Ben Simmons the next LeBron? Not so fast',
|
||||||
'description': 'Bill Macatee relives the best moments in US Open history from the 1990s.',
|
'description': 'md5:854294f627921baba1f4b9a990d87197',
|
||||||
|
'timestamp': 1466293740,
|
||||||
|
'upload_date': '20160618',
|
||||||
|
'uploader': 'CBSI-NEW',
|
||||||
},
|
},
|
||||||
}
|
'params': {
|
||||||
|
# m3u8 download
|
||||||
|
'skip_download': True,
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
video_id = self._match_id(url)
|
||||||
section = mobj.group('section')
|
return self._extract_video_info('byId=%s' % video_id, video_id)
|
||||||
video_id = mobj.group('id')
|
|
||||||
all_videos = self._download_json(
|
|
||||||
'http://www.cbssports.com/data/video/player/getVideos/%s?as=json' % section,
|
|
||||||
video_id)
|
|
||||||
# The json file contains the info of all the videos in the section
|
|
||||||
video_info = next(v for v in all_videos if v['pcid'] == video_id)
|
|
||||||
return self.url_result('theplatform:%s' % video_info['pid'], 'ThePlatform')
|
|
||||||
|
@ -58,7 +58,8 @@ class CDAIE(InfoExtractor):
|
|||||||
def extract_format(page, version):
|
def extract_format(page, version):
|
||||||
unpacked = decode_packed_codes(page)
|
unpacked = decode_packed_codes(page)
|
||||||
format_url = self._search_regex(
|
format_url = self._search_regex(
|
||||||
r"url:\\'(.+?)\\'", unpacked, '%s url' % version, fatal=False)
|
r"(?:file|url)\s*:\s*(\\?[\"'])(?P<url>http.+?)\1", unpacked,
|
||||||
|
'%s url' % version, fatal=False, group='url')
|
||||||
if not format_url:
|
if not format_url:
|
||||||
return
|
return
|
||||||
f = {
|
f = {
|
||||||
@ -75,7 +76,8 @@ class CDAIE(InfoExtractor):
|
|||||||
info_dict['formats'].append(f)
|
info_dict['formats'].append(f)
|
||||||
if not info_dict['duration']:
|
if not info_dict['duration']:
|
||||||
info_dict['duration'] = parse_duration(self._search_regex(
|
info_dict['duration'] = parse_duration(self._search_regex(
|
||||||
r"duration:\\'(.+?)\\'", unpacked, 'duration', fatal=False))
|
r"duration\s*:\s*(\\?[\"'])(?P<duration>.+?)\1",
|
||||||
|
unpacked, 'duration', fatal=False, group='duration'))
|
||||||
|
|
||||||
extract_format(webpage, 'default')
|
extract_format(webpage, 'default')
|
||||||
|
|
||||||
|
92
youtube_dl/extractor/closertotruth.py
Normal file
92
youtube_dl/extractor/closertotruth.py
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class CloserToTruthIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?closertotruth\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://closertotruth.com/series/solutions-the-mind-body-problem#video-3688',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '0_zof1ktre',
|
||||||
|
'display_id': 'solutions-the-mind-body-problem',
|
||||||
|
'ext': 'mov',
|
||||||
|
'title': 'Solutions to the Mind-Body Problem?',
|
||||||
|
'upload_date': '20140221',
|
||||||
|
'timestamp': 1392956007,
|
||||||
|
'uploader_id': 'CTTXML'
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'http://closertotruth.com/episodes/how-do-brains-work',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '0_iuxai6g6',
|
||||||
|
'display_id': 'how-do-brains-work',
|
||||||
|
'ext': 'mov',
|
||||||
|
'title': 'How do Brains Work?',
|
||||||
|
'upload_date': '20140221',
|
||||||
|
'timestamp': 1392956024,
|
||||||
|
'uploader_id': 'CTTXML'
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'http://closertotruth.com/interviews/1725',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1725',
|
||||||
|
'title': 'AyaFr-002',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 2,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
display_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
|
partner_id = self._search_regex(
|
||||||
|
r'<script[^>]+src=["\'].*?\b(?:partner_id|p)/(\d+)',
|
||||||
|
webpage, 'kaltura partner_id')
|
||||||
|
|
||||||
|
title = self._search_regex(
|
||||||
|
r'<title>(.+?)\s*\|\s*.+?</title>', webpage, 'video title')
|
||||||
|
|
||||||
|
select = self._search_regex(
|
||||||
|
r'(?s)<select[^>]+id="select-version"[^>]*>(.+?)</select>',
|
||||||
|
webpage, 'select version', default=None)
|
||||||
|
if select:
|
||||||
|
entry_ids = set()
|
||||||
|
entries = []
|
||||||
|
for mobj in re.finditer(
|
||||||
|
r'<option[^>]+value=(["\'])(?P<id>[0-9a-z_]+)(?:#.+?)?\1[^>]*>(?P<title>[^<]+)',
|
||||||
|
webpage):
|
||||||
|
entry_id = mobj.group('id')
|
||||||
|
if entry_id in entry_ids:
|
||||||
|
continue
|
||||||
|
entry_ids.add(entry_id)
|
||||||
|
entries.append({
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'url': 'kaltura:%s:%s' % (partner_id, entry_id),
|
||||||
|
'ie_key': 'Kaltura',
|
||||||
|
'title': mobj.group('title'),
|
||||||
|
})
|
||||||
|
if entries:
|
||||||
|
return self.playlist_result(entries, display_id, title)
|
||||||
|
|
||||||
|
entry_id = self._search_regex(
|
||||||
|
r'<a[^>]+id=(["\'])embed-kaltura\1[^>]+data-kaltura=(["\'])(?P<id>[0-9a-z_]+)\2',
|
||||||
|
webpage, 'kaltura entry_id', group='id')
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'display_id': display_id,
|
||||||
|
'url': 'kaltura:%s:%s' % (partner_id, entry_id),
|
||||||
|
'ie_key': 'Kaltura',
|
||||||
|
'title': title
|
||||||
|
}
|
@ -53,6 +53,7 @@ from ..utils import (
|
|||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
update_Request,
|
update_Request,
|
||||||
update_url_query,
|
update_url_query,
|
||||||
|
parse_m3u8_attributes,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -748,10 +749,12 @@ class InfoExtractor(object):
|
|||||||
return self._og_search_property('url', html, **kargs)
|
return self._og_search_property('url', html, **kargs)
|
||||||
|
|
||||||
def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
|
def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
|
||||||
|
if not isinstance(name, (list, tuple)):
|
||||||
|
name = [name]
|
||||||
if display_name is None:
|
if display_name is None:
|
||||||
display_name = name
|
display_name = name[0]
|
||||||
return self._html_search_regex(
|
return self._html_search_regex(
|
||||||
self._meta_regex(name),
|
[self._meta_regex(n) for n in name],
|
||||||
html, display_name, fatal=fatal, group='content', **kwargs)
|
html, display_name, fatal=fatal, group='content', **kwargs)
|
||||||
|
|
||||||
def _dc_search_uploader(self, html):
|
def _dc_search_uploader(self, html):
|
||||||
@ -875,7 +878,11 @@ class InfoExtractor(object):
|
|||||||
f['ext'] = determine_ext(f['url'])
|
f['ext'] = determine_ext(f['url'])
|
||||||
|
|
||||||
if isinstance(field_preference, (list, tuple)):
|
if isinstance(field_preference, (list, tuple)):
|
||||||
return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
|
return tuple(
|
||||||
|
f.get(field)
|
||||||
|
if f.get(field) is not None
|
||||||
|
else ('' if field == 'format_id' else -1)
|
||||||
|
for field in field_preference)
|
||||||
|
|
||||||
preference = f.get('preference')
|
preference = f.get('preference')
|
||||||
if preference is None:
|
if preference is None:
|
||||||
@ -1150,23 +1157,11 @@ class InfoExtractor(object):
|
|||||||
}]
|
}]
|
||||||
last_info = None
|
last_info = None
|
||||||
last_media = None
|
last_media = None
|
||||||
kv_rex = re.compile(
|
|
||||||
r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
|
|
||||||
for line in m3u8_doc.splitlines():
|
for line in m3u8_doc.splitlines():
|
||||||
if line.startswith('#EXT-X-STREAM-INF:'):
|
if line.startswith('#EXT-X-STREAM-INF:'):
|
||||||
last_info = {}
|
last_info = parse_m3u8_attributes(line)
|
||||||
for m in kv_rex.finditer(line):
|
|
||||||
v = m.group('val')
|
|
||||||
if v.startswith('"'):
|
|
||||||
v = v[1:-1]
|
|
||||||
last_info[m.group('key')] = v
|
|
||||||
elif line.startswith('#EXT-X-MEDIA:'):
|
elif line.startswith('#EXT-X-MEDIA:'):
|
||||||
last_media = {}
|
last_media = parse_m3u8_attributes(line)
|
||||||
for m in kv_rex.finditer(line):
|
|
||||||
v = m.group('val')
|
|
||||||
if v.startswith('"'):
|
|
||||||
v = v[1:-1]
|
|
||||||
last_media[m.group('key')] = v
|
|
||||||
elif line.startswith('#') or not line.strip():
|
elif line.startswith('#') or not line.strip():
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
@ -1734,6 +1729,13 @@ class InfoExtractor(object):
|
|||||||
def _mark_watched(self, *args, **kwargs):
|
def _mark_watched(self, *args, **kwargs):
|
||||||
raise NotImplementedError('This method must be implemented by subclasses')
|
raise NotImplementedError('This method must be implemented by subclasses')
|
||||||
|
|
||||||
|
def geo_verification_headers(self):
|
||||||
|
headers = {}
|
||||||
|
geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
|
||||||
|
if geo_verification_proxy:
|
||||||
|
headers['Ytdl-request-proxy'] = geo_verification_proxy
|
||||||
|
return headers
|
||||||
|
|
||||||
|
|
||||||
class SearchInfoExtractor(InfoExtractor):
|
class SearchInfoExtractor(InfoExtractor):
|
||||||
"""
|
"""
|
||||||
|
30
youtube_dl/extractor/ctv.py
Normal file
30
youtube_dl/extractor/ctv.py
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class CTVIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?ctv\.ca/video/player\?vid=(?P<id>[0-9.]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.ctv.ca/video/player?vid=706966',
|
||||||
|
'md5': 'ff2ebbeae0aa2dcc32a830c3fd69b7b0',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '706966',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Larry Day and Richard Jutras on the TIFF red carpet of \'Stonewall\'',
|
||||||
|
'description': 'etalk catches up with Larry Day and Richard Jutras on the TIFF red carpet of "Stonewall”.',
|
||||||
|
'upload_date': '20150919',
|
||||||
|
'timestamp': 1442624700,
|
||||||
|
},
|
||||||
|
'expected_warnings': ['HTTP Error 404'],
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'id': video_id,
|
||||||
|
'url': '9c9media:ctv_web:%s' % video_id,
|
||||||
|
'ie_key': 'NineCNineMedia',
|
||||||
|
}
|
65
youtube_dl/extractor/ctvnews.py
Normal file
65
youtube_dl/extractor/ctvnews.py
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import orderedSet
|
||||||
|
|
||||||
|
|
||||||
|
class CTVNewsIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?ctvnews\.ca/(?:video\?(?:clip|playlist|bin)Id=|.*?)(?P<id>[0-9.]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.ctvnews.ca/video?clipId=901995',
|
||||||
|
'md5': '10deb320dc0ccb8d01d34d12fc2ea672',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '901995',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Extended: \'That person cannot be me\' Johnson says',
|
||||||
|
'description': 'md5:958dd3b4f5bbbf0ed4d045c790d89285',
|
||||||
|
'timestamp': 1467286284,
|
||||||
|
'upload_date': '20160630',
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.ctvnews.ca/video?playlistId=1.2966224',
|
||||||
|
'info_dict':
|
||||||
|
{
|
||||||
|
'id': '1.2966224',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 19,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.ctvnews.ca/video?binId=1.2876780',
|
||||||
|
'info_dict':
|
||||||
|
{
|
||||||
|
'id': '1.2876780',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 100,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.ctvnews.ca/1.810401',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.ctvnews.ca/canadiens-send-p-k-subban-to-nashville-in-blockbuster-trade-1.2967231',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
page_id = self._match_id(url)
|
||||||
|
|
||||||
|
def ninecninemedia_url_result(clip_id):
|
||||||
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'id': clip_id,
|
||||||
|
'url': '9c9media:ctvnews_web:%s' % clip_id,
|
||||||
|
'ie_key': 'NineCNineMedia',
|
||||||
|
}
|
||||||
|
|
||||||
|
if page_id.isdigit():
|
||||||
|
return ninecninemedia_url_result(page_id)
|
||||||
|
else:
|
||||||
|
webpage = self._download_webpage('http://www.ctvnews.ca/%s' % page_id, page_id, query={
|
||||||
|
'ot': 'example.AjaxPageLayout.ot',
|
||||||
|
'maxItemsPerPage': 1000000,
|
||||||
|
})
|
||||||
|
entries = [ninecninemedia_url_result(clip_id) for clip_id in orderedSet(
|
||||||
|
re.findall(r'clip\.id\s*=\s*(\d+);', webpage))]
|
||||||
|
return self.playlist_result(entries, page_id)
|
@ -20,7 +20,7 @@ from ..utils import (
|
|||||||
|
|
||||||
|
|
||||||
class DCNIE(InfoExtractor):
|
class DCNIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
|
_VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?show/(?P<show_id>\d+)/[^/]+(?:/(?P<video_id>\d+)/(?P<season_id>\d+))?'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
|
show_id, video_id, season_id = re.match(self._VALID_URL, url).groups()
|
||||||
@ -55,30 +55,32 @@ class DCNBaseIE(InfoExtractor):
|
|||||||
'is_live': is_live,
|
'is_live': is_live,
|
||||||
}
|
}
|
||||||
|
|
||||||
def _extract_video_formats(self, webpage, video_id, entry_protocol):
|
def _extract_video_formats(self, webpage, video_id, m3u8_entry_protocol):
|
||||||
formats = []
|
formats = []
|
||||||
m3u8_url = self._html_search_regex(
|
format_url_base = 'http' + self._html_search_regex(
|
||||||
r'file\s*:\s*"([^"]+)', webpage, 'm3u8 url', fatal=False)
|
[
|
||||||
if m3u8_url:
|
r'file\s*:\s*"https?(://[^"]+)/playlist.m3u8',
|
||||||
formats.extend(self._extract_m3u8_formats(
|
r'<a[^>]+href="rtsp(://[^"]+)"'
|
||||||
m3u8_url, video_id, 'mp4', entry_protocol, m3u8_id='hls', fatal=None))
|
], webpage, 'format url')
|
||||||
|
# TODO: Current DASH formats are broken - $Time$ pattern in
|
||||||
rtsp_url = self._search_regex(
|
# <SegmentTemplate> not implemented yet
|
||||||
r'<a[^>]+href="(rtsp://[^"]+)"', webpage, 'rtsp url', fatal=False)
|
# formats.extend(self._extract_mpd_formats(
|
||||||
if rtsp_url:
|
# format_url_base + '/manifest.mpd',
|
||||||
formats.append({
|
# video_id, mpd_id='dash', fatal=False))
|
||||||
'url': rtsp_url,
|
formats.extend(self._extract_m3u8_formats(
|
||||||
'format_id': 'rtsp',
|
format_url_base + '/playlist.m3u8', video_id, 'mp4',
|
||||||
})
|
m3u8_entry_protocol, m3u8_id='hls', fatal=False))
|
||||||
|
formats.extend(self._extract_f4m_formats(
|
||||||
|
format_url_base + '/manifest.f4m',
|
||||||
|
video_id, f4m_id='hds', fatal=False))
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
return formats
|
return formats
|
||||||
|
|
||||||
|
|
||||||
class DCNVideoIE(DCNBaseIE):
|
class DCNVideoIE(DCNBaseIE):
|
||||||
IE_NAME = 'dcn:video'
|
IE_NAME = 'dcn:video'
|
||||||
_VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?(?:video/[^/]+|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
|
_VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?(?:video(?:/[^/]+)?|media|catchup/[^/]+/[^/]+)/(?P<id>\d+)'
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
|
'url': 'http://www.dcndigital.ae/#/video/%D8%B1%D8%AD%D9%84%D8%A9-%D8%A7%D9%84%D8%B9%D9%85%D8%B1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/17375',
|
||||||
'info_dict':
|
'info_dict':
|
||||||
{
|
{
|
||||||
@ -94,7 +96,10 @@ class DCNVideoIE(DCNBaseIE):
|
|||||||
# m3u8 download
|
# m3u8 download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
}
|
}, {
|
||||||
|
'url': 'http://awaan.ae/video/26723981/%D8%AF%D8%A7%D8%B1-%D8%A7%D9%84%D8%B3%D9%84%D8%A7%D9%85:-%D8%AE%D9%8A%D8%B1-%D8%AF%D9%88%D8%B1-%D8%A7%D9%84%D8%A3%D9%86%D8%B5%D8%A7%D8%B1',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
@ -120,7 +125,7 @@ class DCNVideoIE(DCNBaseIE):
|
|||||||
|
|
||||||
class DCNLiveIE(DCNBaseIE):
|
class DCNLiveIE(DCNBaseIE):
|
||||||
IE_NAME = 'dcn:live'
|
IE_NAME = 'dcn:live'
|
||||||
_VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?live/(?P<id>\d+)'
|
_VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?live/(?P<id>\d+)'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
channel_id = self._match_id(url)
|
channel_id = self._match_id(url)
|
||||||
@ -147,7 +152,7 @@ class DCNLiveIE(DCNBaseIE):
|
|||||||
|
|
||||||
class DCNSeasonIE(InfoExtractor):
|
class DCNSeasonIE(InfoExtractor):
|
||||||
IE_NAME = 'dcn:season'
|
IE_NAME = 'dcn:season'
|
||||||
_VALID_URL = r'https?://(?:www\.)?dcndigital\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
|
_VALID_URL = r'https?://(?:www\.)?(?:awaan|dcndigital)\.ae/(?:#/)?program/(?:(?P<show_id>\d+)|season/(?P<season_id>\d+))'
|
||||||
_TEST = {
|
_TEST = {
|
||||||
'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
|
'url': 'http://dcndigital.ae/#/program/205024/%D9%85%D8%AD%D8%A7%D8%B6%D8%B1%D8%A7%D8%AA-%D8%A7%D9%84%D8%B4%D9%8A%D8%AE-%D8%A7%D9%84%D8%B4%D8%B9%D8%B1%D8%A7%D9%88%D9%8A',
|
||||||
'info_dict':
|
'info_dict':
|
||||||
|
@ -50,6 +50,14 @@ class EaglePlatformIE(InfoExtractor):
|
|||||||
'skip': 'Georestricted',
|
'skip': 'Georestricted',
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_url(webpage):
|
||||||
|
mobj = re.search(
|
||||||
|
r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//.+?\.media\.eagleplatform\.com/index/player\?.+?)\1',
|
||||||
|
webpage)
|
||||||
|
if mobj is not None:
|
||||||
|
return mobj.group('url')
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _handle_error(response):
|
def _handle_error(response):
|
||||||
status = int_or_none(response.get('status', 200))
|
status = int_or_none(response.get('status', 200))
|
||||||
|
@ -20,7 +20,10 @@ from .adobetv import (
|
|||||||
AdobeTVVideoIE,
|
AdobeTVVideoIE,
|
||||||
)
|
)
|
||||||
from .adultswim import AdultSwimIE
|
from .adultswim import AdultSwimIE
|
||||||
from .aenetworks import AENetworksIE
|
from .aenetworks import (
|
||||||
|
AENetworksIE,
|
||||||
|
HistoryTopicIE,
|
||||||
|
)
|
||||||
from .afreecatv import AfreecaTVIE
|
from .afreecatv import AfreecaTVIE
|
||||||
from .aftonbladet import AftonbladetIE
|
from .aftonbladet import AftonbladetIE
|
||||||
from .airmozilla import AirMozillaIE
|
from .airmozilla import AirMozillaIE
|
||||||
@ -44,7 +47,6 @@ from .archiveorg import ArchiveOrgIE
|
|||||||
from .ard import (
|
from .ard import (
|
||||||
ARDIE,
|
ARDIE,
|
||||||
ARDMediathekIE,
|
ARDMediathekIE,
|
||||||
SportschauIE,
|
|
||||||
)
|
)
|
||||||
from .arte import (
|
from .arte import (
|
||||||
ArteTvIE,
|
ArteTvIE,
|
||||||
@ -71,6 +73,8 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
|
|||||||
from .bbc import (
|
from .bbc import (
|
||||||
BBCCoUkIE,
|
BBCCoUkIE,
|
||||||
BBCCoUkArticleIE,
|
BBCCoUkArticleIE,
|
||||||
|
BBCCoUkIPlayerPlaylistIE,
|
||||||
|
BBCCoUkPlaylistIE,
|
||||||
BBCIE,
|
BBCIE,
|
||||||
)
|
)
|
||||||
from .beeg import BeegIE
|
from .beeg import BeegIE
|
||||||
@ -108,6 +112,10 @@ from .camwithher import CamWithHerIE
|
|||||||
from .canalplus import CanalplusIE
|
from .canalplus import CanalplusIE
|
||||||
from .canalc2 import Canalc2IE
|
from .canalc2 import Canalc2IE
|
||||||
from .canvas import CanvasIE
|
from .canvas import CanvasIE
|
||||||
|
from .carambatv import (
|
||||||
|
CarambaTVIE,
|
||||||
|
CarambaTVPageIE,
|
||||||
|
)
|
||||||
from .cbc import (
|
from .cbc import (
|
||||||
CBCIE,
|
CBCIE,
|
||||||
CBCPlayerIE,
|
CBCPlayerIE,
|
||||||
@ -135,6 +143,7 @@ from .cliprs import ClipRsIE
|
|||||||
from .clipfish import ClipfishIE
|
from .clipfish import ClipfishIE
|
||||||
from .cliphunter import CliphunterIE
|
from .cliphunter import CliphunterIE
|
||||||
from .clipsyndicate import ClipsyndicateIE
|
from .clipsyndicate import ClipsyndicateIE
|
||||||
|
from .closertotruth import CloserToTruthIE
|
||||||
from .cloudy import CloudyIE
|
from .cloudy import CloudyIE
|
||||||
from .clubic import ClubicIE
|
from .clubic import ClubicIE
|
||||||
from .clyp import ClypIE
|
from .clyp import ClypIE
|
||||||
@ -162,6 +171,8 @@ from .crunchyroll import (
|
|||||||
)
|
)
|
||||||
from .cspan import CSpanIE
|
from .cspan import CSpanIE
|
||||||
from .ctsnews import CtsNewsIE
|
from .ctsnews import CtsNewsIE
|
||||||
|
from .ctv import CTVIE
|
||||||
|
from .ctvnews import CTVNewsIE
|
||||||
from .cultureunplugged import CultureUnpluggedIE
|
from .cultureunplugged import CultureUnpluggedIE
|
||||||
from .cwtv import CWTVIE
|
from .cwtv import CWTVIE
|
||||||
from .dailymail import DailyMailIE
|
from .dailymail import DailyMailIE
|
||||||
@ -270,6 +281,7 @@ from .freespeech import FreespeechIE
|
|||||||
from .freevideo import FreeVideoIE
|
from .freevideo import FreeVideoIE
|
||||||
from .funimation import FunimationIE
|
from .funimation import FunimationIE
|
||||||
from .funnyordie import FunnyOrDieIE
|
from .funnyordie import FunnyOrDieIE
|
||||||
|
from .fusion import FusionIE
|
||||||
from .gameinformer import GameInformerIE
|
from .gameinformer import GameInformerIE
|
||||||
from .gamekings import GamekingsIE
|
from .gamekings import GamekingsIE
|
||||||
from .gameone import (
|
from .gameone import (
|
||||||
@ -279,7 +291,6 @@ from .gameone import (
|
|||||||
from .gamersyde import GamersydeIE
|
from .gamersyde import GamersydeIE
|
||||||
from .gamespot import GameSpotIE
|
from .gamespot import GameSpotIE
|
||||||
from .gamestar import GameStarIE
|
from .gamestar import GameStarIE
|
||||||
from .gametrailers import GametrailersIE
|
|
||||||
from .gazeta import GazetaIE
|
from .gazeta import GazetaIE
|
||||||
from .gdcvault import GDCVaultIE
|
from .gdcvault import GDCVaultIE
|
||||||
from .generic import GenericIE
|
from .generic import GenericIE
|
||||||
@ -315,6 +326,10 @@ from .hotnewhiphop import HotNewHipHopIE
|
|||||||
from .hotstar import HotStarIE
|
from .hotstar import HotStarIE
|
||||||
from .howcast import HowcastIE
|
from .howcast import HowcastIE
|
||||||
from .howstuffworks import HowStuffWorksIE
|
from .howstuffworks import HowStuffWorksIE
|
||||||
|
from .hrti import (
|
||||||
|
HRTiIE,
|
||||||
|
HRTiPlaylistIE,
|
||||||
|
)
|
||||||
from .huffpost import HuffPostIE
|
from .huffpost import HuffPostIE
|
||||||
from .hypem import HypemIE
|
from .hypem import HypemIE
|
||||||
from .iconosquare import IconosquareIE
|
from .iconosquare import IconosquareIE
|
||||||
@ -417,6 +432,7 @@ from .makerschannel import MakersChannelIE
|
|||||||
from .makertv import MakerTVIE
|
from .makertv import MakerTVIE
|
||||||
from .matchtv import MatchTVIE
|
from .matchtv import MatchTVIE
|
||||||
from .mdr import MDRIE
|
from .mdr import MDRIE
|
||||||
|
from .meta import METAIE
|
||||||
from .metacafe import MetacafeIE
|
from .metacafe import MetacafeIE
|
||||||
from .metacritic import MetacriticIE
|
from .metacritic import MetacriticIE
|
||||||
from .mgoon import MgoonIE
|
from .mgoon import MgoonIE
|
||||||
@ -449,6 +465,7 @@ from .motherless import MotherlessIE
|
|||||||
from .motorsport import MotorsportIE
|
from .motorsport import MotorsportIE
|
||||||
from .movieclips import MovieClipsIE
|
from .movieclips import MovieClipsIE
|
||||||
from .moviezine import MoviezineIE
|
from .moviezine import MoviezineIE
|
||||||
|
from .msn import MSNIE
|
||||||
from .mtv import (
|
from .mtv import (
|
||||||
MTVIE,
|
MTVIE,
|
||||||
MTVServicesEmbeddedIE,
|
MTVServicesEmbeddedIE,
|
||||||
@ -475,7 +492,6 @@ from .nbc import (
|
|||||||
NBCNewsIE,
|
NBCNewsIE,
|
||||||
NBCSportsIE,
|
NBCSportsIE,
|
||||||
NBCSportsVPlayerIE,
|
NBCSportsVPlayerIE,
|
||||||
MSNBCIE,
|
|
||||||
)
|
)
|
||||||
from .ndr import (
|
from .ndr import (
|
||||||
NDRIE,
|
NDRIE,
|
||||||
@ -512,8 +528,12 @@ from .nhl import (
|
|||||||
NHLVideocenterCategoryIE,
|
NHLVideocenterCategoryIE,
|
||||||
NHLIE,
|
NHLIE,
|
||||||
)
|
)
|
||||||
from .nick import NickIE
|
from .nick import (
|
||||||
|
NickIE,
|
||||||
|
NickDeIE,
|
||||||
|
)
|
||||||
from .niconico import NiconicoIE, NiconicoPlaylistIE
|
from .niconico import NiconicoIE, NiconicoPlaylistIE
|
||||||
|
from .ninecninemedia import NineCNineMediaIE
|
||||||
from .ninegag import NineGagIE
|
from .ninegag import NineGagIE
|
||||||
from .noco import NocoIE
|
from .noco import NocoIE
|
||||||
from .normalboots import NormalbootsIE
|
from .normalboots import NormalbootsIE
|
||||||
@ -599,6 +619,7 @@ from .pluralsight import (
|
|||||||
PluralsightCourseIE,
|
PluralsightCourseIE,
|
||||||
)
|
)
|
||||||
from .podomatic import PodomaticIE
|
from .podomatic import PodomaticIE
|
||||||
|
from .polskieradio import PolskieRadioIE
|
||||||
from .porn91 import Porn91IE
|
from .porn91 import Porn91IE
|
||||||
from .pornhd import PornHdIE
|
from .pornhd import PornHdIE
|
||||||
from .pornhub import (
|
from .pornhub import (
|
||||||
@ -622,7 +643,10 @@ from .qqmusic import (
|
|||||||
QQMusicToplistIE,
|
QQMusicToplistIE,
|
||||||
QQMusicPlaylistIE,
|
QQMusicPlaylistIE,
|
||||||
)
|
)
|
||||||
from .r7 import R7IE
|
from .r7 import (
|
||||||
|
R7IE,
|
||||||
|
R7ArticleIE,
|
||||||
|
)
|
||||||
from .radiocanada import (
|
from .radiocanada import (
|
||||||
RadioCanadaIE,
|
RadioCanadaIE,
|
||||||
RadioCanadaAudioVideoIE,
|
RadioCanadaAudioVideoIE,
|
||||||
@ -649,6 +673,7 @@ from .revision3 import (
|
|||||||
from .rice import RICEIE
|
from .rice import RICEIE
|
||||||
from .ringtv import RingTVIE
|
from .ringtv import RingTVIE
|
||||||
from .ro220 import Ro220IE
|
from .ro220 import Ro220IE
|
||||||
|
from .rockstargames import RockstarGamesIE
|
||||||
from .rottentomatoes import RottenTomatoesIE
|
from .rottentomatoes import RottenTomatoesIE
|
||||||
from .roxwel import RoxwelIE
|
from .roxwel import RoxwelIE
|
||||||
from .rtbf import RTBFIE
|
from .rtbf import RTBFIE
|
||||||
@ -693,10 +718,12 @@ from .shahid import ShahidIE
|
|||||||
from .shared import SharedIE
|
from .shared import SharedIE
|
||||||
from .sharesix import ShareSixIE
|
from .sharesix import ShareSixIE
|
||||||
from .sina import SinaIE
|
from .sina import SinaIE
|
||||||
|
from .sixplay import SixPlayIE
|
||||||
from .skynewsarabia import (
|
from .skynewsarabia import (
|
||||||
SkyNewsArabiaIE,
|
SkyNewsArabiaIE,
|
||||||
SkyNewsArabiaArticleIE,
|
SkyNewsArabiaArticleIE,
|
||||||
)
|
)
|
||||||
|
from .skysports import SkySportsIE
|
||||||
from .slideshare import SlideshareIE
|
from .slideshare import SlideshareIE
|
||||||
from .slutload import SlutloadIE
|
from .slutload import SlutloadIE
|
||||||
from .smotri import (
|
from .smotri import (
|
||||||
@ -737,6 +764,7 @@ from .sportbox import (
|
|||||||
SportBoxEmbedIE,
|
SportBoxEmbedIE,
|
||||||
)
|
)
|
||||||
from .sportdeutschland import SportDeutschlandIE
|
from .sportdeutschland import SportDeutschlandIE
|
||||||
|
from .sportschau import SportschauIE
|
||||||
from .srgssr import (
|
from .srgssr import (
|
||||||
SRGSSRIE,
|
SRGSSRIE,
|
||||||
SRGSSRPlayIE,
|
SRGSSRPlayIE,
|
||||||
@ -777,6 +805,7 @@ from .telecinco import TelecincoIE
|
|||||||
from .telegraaf import TelegraafIE
|
from .telegraaf import TelegraafIE
|
||||||
from .telemb import TeleMBIE
|
from .telemb import TeleMBIE
|
||||||
from .teletask import TeleTaskIE
|
from .teletask import TeleTaskIE
|
||||||
|
from .telewebion import TelewebionIE
|
||||||
from .testurl import TestURLIE
|
from .testurl import TestURLIE
|
||||||
from .tf1 import TF1IE
|
from .tf1 import TF1IE
|
||||||
from .theintercept import TheInterceptIE
|
from .theintercept import TheInterceptIE
|
||||||
@ -861,6 +890,7 @@ from .twitch import (
|
|||||||
TwitchProfileIE,
|
TwitchProfileIE,
|
||||||
TwitchPastBroadcastsIE,
|
TwitchPastBroadcastsIE,
|
||||||
TwitchStreamIE,
|
TwitchStreamIE,
|
||||||
|
TwitchClipsIE,
|
||||||
)
|
)
|
||||||
from .twitter import (
|
from .twitter import (
|
||||||
TwitterCardIE,
|
TwitterCardIE,
|
||||||
@ -875,6 +905,7 @@ from .udn import UDNEmbedIE
|
|||||||
from .digiteka import DigitekaIE
|
from .digiteka import DigitekaIE
|
||||||
from .unistra import UnistraIE
|
from .unistra import UnistraIE
|
||||||
from .urort import UrortIE
|
from .urort import UrortIE
|
||||||
|
from .urplay import URPlayIE
|
||||||
from .usatoday import USATodayIE
|
from .usatoday import USATodayIE
|
||||||
from .ustream import UstreamIE, UstreamChannelIE
|
from .ustream import UstreamIE, UstreamChannelIE
|
||||||
from .ustudio import (
|
from .ustudio import (
|
||||||
@ -901,6 +932,7 @@ from .vice import (
|
|||||||
ViceIE,
|
ViceIE,
|
||||||
ViceShowIE,
|
ViceShowIE,
|
||||||
)
|
)
|
||||||
|
from .vidbit import VidbitIE
|
||||||
from .viddler import ViddlerIE
|
from .viddler import ViddlerIE
|
||||||
from .videodetective import VideoDetectiveIE
|
from .videodetective import VideoDetectiveIE
|
||||||
from .videofyme import VideofyMeIE
|
from .videofyme import VideofyMeIE
|
||||||
@ -977,7 +1009,10 @@ from .weiqitv import WeiqiTVIE
|
|||||||
from .wimp import WimpIE
|
from .wimp import WimpIE
|
||||||
from .wistia import WistiaIE
|
from .wistia import WistiaIE
|
||||||
from .worldstarhiphop import WorldStarHipHopIE
|
from .worldstarhiphop import WorldStarHipHopIE
|
||||||
from .wrzuta import WrzutaIE
|
from .wrzuta import (
|
||||||
|
WrzutaIE,
|
||||||
|
WrzutaPlaylistIE,
|
||||||
|
)
|
||||||
from .wsj import WSJIE
|
from .wsj import WSJIE
|
||||||
from .xbef import XBefIE
|
from .xbef import XBefIE
|
||||||
from .xboxclips import XboxClipsIE
|
from .xboxclips import XboxClipsIE
|
||||||
@ -1031,6 +1066,7 @@ from .youtube import (
|
|||||||
YoutubeSearchDateIE,
|
YoutubeSearchDateIE,
|
||||||
YoutubeSearchIE,
|
YoutubeSearchIE,
|
||||||
YoutubeSearchURLIE,
|
YoutubeSearchURLIE,
|
||||||
|
YoutubeSharedVideoIE,
|
||||||
YoutubeShowIE,
|
YoutubeShowIE,
|
||||||
YoutubeSubscriptionsIE,
|
YoutubeSubscriptionsIE,
|
||||||
YoutubeTruncatedIDIE,
|
YoutubeTruncatedIDIE,
|
||||||
|
@ -129,6 +129,21 @@ class FacebookIE(InfoExtractor):
|
|||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _extract_url(webpage):
|
||||||
|
mobj = re.search(
|
||||||
|
r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
|
||||||
|
if mobj is not None:
|
||||||
|
return mobj.group('url')
|
||||||
|
|
||||||
|
# Facebook API embed
|
||||||
|
# see https://developers.facebook.com/docs/plugins/embedded-video-player
|
||||||
|
mobj = re.search(r'''(?x)<div[^>]+
|
||||||
|
class=(?P<q1>[\'"])[^\'"]*\bfb-(?:video|post)\b[^\'"]*(?P=q1)[^>]+
|
||||||
|
data-href=(?P<q2>[\'"])(?P<url>(?:https?:)?//(?:www\.)?facebook.com/.+?)(?P=q2)''', webpage)
|
||||||
|
if mobj is not None:
|
||||||
|
return mobj.group('url')
|
||||||
|
|
||||||
def _login(self):
|
def _login(self):
|
||||||
(useremail, password) = self._get_login_info()
|
(useremail, password) = self._get_login_info()
|
||||||
if useremail is None:
|
if useremail is None:
|
||||||
@ -239,6 +254,8 @@ class FacebookIE(InfoExtractor):
|
|||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for format_id, f in video_data.items():
|
for format_id, f in video_data.items():
|
||||||
|
if f and isinstance(f, dict):
|
||||||
|
f = [f]
|
||||||
if not f or not isinstance(f, list):
|
if not f or not isinstance(f, list):
|
||||||
continue
|
continue
|
||||||
for quality in ('sd', 'hd'):
|
for quality in ('sd', 'hd'):
|
||||||
|
@ -1,7 +1,10 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import smuggle_url
|
from ..utils import (
|
||||||
|
smuggle_url,
|
||||||
|
update_url_query,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FoxSportsIE(InfoExtractor):
|
class FoxSportsIE(InfoExtractor):
|
||||||
@ -9,11 +12,15 @@ class FoxSportsIE(InfoExtractor):
|
|||||||
|
|
||||||
_TEST = {
|
_TEST = {
|
||||||
'url': 'http://www.foxsports.com/video?vid=432609859715',
|
'url': 'http://www.foxsports.com/video?vid=432609859715',
|
||||||
|
'md5': 'b49050e955bebe32c301972e4012ac17',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'gA0bHB3Ladz3',
|
'id': 'i0qKWsk3qJaM',
|
||||||
'ext': 'flv',
|
'ext': 'mp4',
|
||||||
'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
|
'title': 'Courtney Lee on going up 2-0 in series vs. Blazers',
|
||||||
'description': 'Courtney Lee talks about Memphis being focused.',
|
'description': 'Courtney Lee talks about Memphis being focused.',
|
||||||
|
'upload_date': '20150423',
|
||||||
|
'timestamp': 1429761109,
|
||||||
|
'uploader': 'NEWA-FNG-FOXSPORTS',
|
||||||
},
|
},
|
||||||
'add_ie': ['ThePlatform'],
|
'add_ie': ['ThePlatform'],
|
||||||
}
|
}
|
||||||
@ -28,5 +35,8 @@ class FoxSportsIE(InfoExtractor):
|
|||||||
r"data-player-config='([^']+)'", webpage, 'data player config'),
|
r"data-player-config='([^']+)'", webpage, 'data player config'),
|
||||||
video_id)
|
video_id)
|
||||||
|
|
||||||
return self.url_result(smuggle_url(
|
return self.url_result(smuggle_url(update_url_query(
|
||||||
config['releaseURL'] + '&manifest=f4m', {'force_smil_url': True}))
|
config['releaseURL'], {
|
||||||
|
'mbr': 'true',
|
||||||
|
'switch': 'http',
|
||||||
|
}), {'force_smil_url': True}))
|
||||||
|
35
youtube_dl/extractor/fusion.py
Normal file
35
youtube_dl/extractor/fusion.py
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from .ooyala import OoyalaIE
|
||||||
|
|
||||||
|
|
||||||
|
class FusionIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?fusion\.net/video/(?P<id>\d+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://fusion.net/video/201781/u-s-and-panamanian-forces-work-together-to-stop-a-vessel-smuggling-drugs/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'ZpcWNoMTE6x6uVIIWYpHh0qQDjxBuq5P',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'U.S. and Panamanian forces work together to stop a vessel smuggling drugs',
|
||||||
|
'description': 'md5:0cc84a9943c064c0f46b128b41b1b0d7',
|
||||||
|
'duration': 140.0,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
'add_ie': ['Ooyala'],
|
||||||
|
}, {
|
||||||
|
'url': 'http://fusion.net/video/201781',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
display_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
|
ooyala_code = self._search_regex(
|
||||||
|
r'data-video-id=(["\'])(?P<code>.+?)\1',
|
||||||
|
webpage, 'ooyala code', group='code')
|
||||||
|
|
||||||
|
return OoyalaIE._build_url_result(ooyala_code)
|
@ -1,19 +1,19 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import json
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .once import OnceIE
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse_unquote,
|
compat_urllib_parse_unquote,
|
||||||
compat_urlparse,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
|
url_basename,
|
||||||
|
dict_get,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class GameSpotIE(InfoExtractor):
|
class GameSpotIE(OnceIE):
|
||||||
_VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
|
_VALID_URL = r'https?://(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
|
'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
|
||||||
@ -39,29 +39,73 @@ class GameSpotIE(InfoExtractor):
|
|||||||
webpage = self._download_webpage(url, page_id)
|
webpage = self._download_webpage(url, page_id)
|
||||||
data_video_json = self._search_regex(
|
data_video_json = self._search_regex(
|
||||||
r'data-video=["\'](.*?)["\']', webpage, 'data video')
|
r'data-video=["\'](.*?)["\']', webpage, 'data video')
|
||||||
data_video = json.loads(unescapeHTML(data_video_json))
|
data_video = self._parse_json(unescapeHTML(data_video_json), page_id)
|
||||||
streams = data_video['videoStreams']
|
streams = data_video['videoStreams']
|
||||||
|
|
||||||
|
manifest_url = None
|
||||||
formats = []
|
formats = []
|
||||||
f4m_url = streams.get('f4m_stream')
|
f4m_url = streams.get('f4m_stream')
|
||||||
if f4m_url is not None:
|
if f4m_url:
|
||||||
# Transform the manifest url to a link to the mp4 files
|
manifest_url = f4m_url
|
||||||
# they are used in mobile devices.
|
formats.extend(self._extract_f4m_formats(
|
||||||
f4m_path = compat_urlparse.urlparse(f4m_url).path
|
f4m_url + '?hdcore=3.7.0', page_id, f4m_id='hds', fatal=False))
|
||||||
QUALITIES_RE = r'((,\d+)+,?)'
|
m3u8_url = streams.get('m3u8_stream')
|
||||||
qualities = self._search_regex(QUALITIES_RE, f4m_path, 'qualities').strip(',').split(',')
|
if m3u8_url:
|
||||||
http_path = f4m_path[1:].split('/', 1)[1]
|
manifest_url = m3u8_url
|
||||||
http_template = re.sub(QUALITIES_RE, r'%s', http_path)
|
m3u8_formats = self._extract_m3u8_formats(
|
||||||
http_template = http_template.replace('.csmil/manifest.f4m', '')
|
m3u8_url, page_id, 'mp4', 'm3u8_native',
|
||||||
http_template = compat_urlparse.urljoin(
|
m3u8_id='hls', fatal=False)
|
||||||
'http://video.gamespotcdn.com/', http_template)
|
formats.extend(m3u8_formats)
|
||||||
for q in qualities:
|
progressive_url = dict_get(
|
||||||
formats.append({
|
streams, ('progressive_hd', 'progressive_high', 'progressive_low'))
|
||||||
'url': http_template % q,
|
if progressive_url and manifest_url:
|
||||||
'ext': 'mp4',
|
qualities_basename = self._search_regex(
|
||||||
'format_id': q,
|
'/([^/]+)\.csmil/',
|
||||||
})
|
manifest_url, 'qualities basename', default=None)
|
||||||
else:
|
if qualities_basename:
|
||||||
|
QUALITIES_RE = r'((,\d+)+,?)'
|
||||||
|
qualities = self._search_regex(
|
||||||
|
QUALITIES_RE, qualities_basename,
|
||||||
|
'qualities', default=None)
|
||||||
|
if qualities:
|
||||||
|
qualities = list(map(lambda q: int(q), qualities.strip(',').split(',')))
|
||||||
|
qualities.sort()
|
||||||
|
http_template = re.sub(QUALITIES_RE, r'%d', qualities_basename)
|
||||||
|
http_url_basename = url_basename(progressive_url)
|
||||||
|
if m3u8_formats:
|
||||||
|
self._sort_formats(m3u8_formats)
|
||||||
|
m3u8_formats = list(filter(
|
||||||
|
lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
|
||||||
|
m3u8_formats))
|
||||||
|
if len(qualities) == len(m3u8_formats):
|
||||||
|
for q, m3u8_format in zip(qualities, m3u8_formats):
|
||||||
|
f = m3u8_format.copy()
|
||||||
|
f.update({
|
||||||
|
'url': progressive_url.replace(
|
||||||
|
http_url_basename, http_template % q),
|
||||||
|
'format_id': f['format_id'].replace('hls', 'http'),
|
||||||
|
'protocol': 'http',
|
||||||
|
})
|
||||||
|
formats.append(f)
|
||||||
|
else:
|
||||||
|
for q in qualities:
|
||||||
|
formats.append({
|
||||||
|
'url': progressive_url.replace(
|
||||||
|
http_url_basename, http_template % q),
|
||||||
|
'ext': 'mp4',
|
||||||
|
'format_id': 'http-%d' % q,
|
||||||
|
'tbr': q,
|
||||||
|
})
|
||||||
|
|
||||||
|
onceux_json = self._search_regex(
|
||||||
|
r'data-onceux-options=["\'](.*?)["\']', webpage, 'data video', default=None)
|
||||||
|
if onceux_json:
|
||||||
|
onceux_url = self._parse_json(unescapeHTML(onceux_json), page_id).get('metadataUri')
|
||||||
|
if onceux_url:
|
||||||
|
formats.extend(self._extract_once_formats(re.sub(
|
||||||
|
r'https?://[^/]+', 'http://once.unicornmedia.com', onceux_url).replace('ads/vmap/', '')))
|
||||||
|
|
||||||
|
if not formats:
|
||||||
for quality in ['sd', 'hd']:
|
for quality in ['sd', 'hd']:
|
||||||
# It's actually a link to a flv file
|
# It's actually a link to a flv file
|
||||||
flv_url = streams.get('f4m_{0}'.format(quality))
|
flv_url = streams.get('f4m_{0}'.format(quality))
|
||||||
@ -71,6 +115,7 @@ class GameSpotIE(InfoExtractor):
|
|||||||
'ext': 'flv',
|
'ext': 'flv',
|
||||||
'format_id': quality,
|
'format_id': quality,
|
||||||
})
|
})
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': data_video['guid'],
|
'id': data_video['guid'],
|
||||||
|
@ -1,62 +0,0 @@
|
|||||||
from __future__ import unicode_literals
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
|
||||||
from ..utils import (
|
|
||||||
int_or_none,
|
|
||||||
parse_age_limit,
|
|
||||||
url_basename,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class GametrailersIE(InfoExtractor):
|
|
||||||
_VALID_URL = r'https?://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
|
|
||||||
|
|
||||||
_TEST = {
|
|
||||||
'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
|
|
||||||
'md5': 'f28c4efa0bdfaf9b760f6507955b6a6a',
|
|
||||||
'info_dict': {
|
|
||||||
'id': '2983958',
|
|
||||||
'ext': 'mp4',
|
|
||||||
'display_id': '116437-Just-Cause-3-Review',
|
|
||||||
'title': 'Just Cause 3 - Review',
|
|
||||||
'description': 'It\'s a lot of fun to shoot at things and then watch them explode in Just Cause 3, but should there be more to the experience than that?',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
|
||||||
display_id = self._match_id(url)
|
|
||||||
webpage = self._download_webpage(url, display_id)
|
|
||||||
title = self._html_search_regex(
|
|
||||||
r'<title>(.+?)\|', webpage, 'title').strip()
|
|
||||||
embed_url = self._proto_relative_url(
|
|
||||||
self._search_regex(
|
|
||||||
r'src=\'(//embed.gametrailers.com/embed/[^\']+)\'', webpage,
|
|
||||||
'embed url'),
|
|
||||||
scheme='http:')
|
|
||||||
video_id = url_basename(embed_url)
|
|
||||||
embed_page = self._download_webpage(embed_url, video_id)
|
|
||||||
embed_vars_json = self._search_regex(
|
|
||||||
r'(?s)var embedVars = (\{.*?\})\s*</script>', embed_page,
|
|
||||||
'embed vars')
|
|
||||||
info = self._parse_json(embed_vars_json, video_id)
|
|
||||||
|
|
||||||
formats = []
|
|
||||||
for media in info['media']:
|
|
||||||
if media['mediaPurpose'] == 'play':
|
|
||||||
formats.append({
|
|
||||||
'url': media['uri'],
|
|
||||||
'height': media['height'],
|
|
||||||
'width:': media['width'],
|
|
||||||
})
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
return {
|
|
||||||
'id': video_id,
|
|
||||||
'display_id': display_id,
|
|
||||||
'title': title,
|
|
||||||
'formats': formats,
|
|
||||||
'thumbnail': info.get('thumbUri'),
|
|
||||||
'description': self._og_search_description(webpage),
|
|
||||||
'duration': int_or_none(info.get('videoLengthInSeconds')),
|
|
||||||
'age_limit': parse_age_limit(info.get('audienceRating')),
|
|
||||||
}
|
|
@ -64,6 +64,9 @@ from .liveleak import LiveLeakIE
|
|||||||
from .threeqsdn import ThreeQSDNIE
|
from .threeqsdn import ThreeQSDNIE
|
||||||
from .theplatform import ThePlatformIE
|
from .theplatform import ThePlatformIE
|
||||||
from .vessel import VesselIE
|
from .vessel import VesselIE
|
||||||
|
from .kaltura import KalturaIE
|
||||||
|
from .eagleplatform import EaglePlatformIE
|
||||||
|
from .facebook import FacebookIE
|
||||||
|
|
||||||
|
|
||||||
class GenericIE(InfoExtractor):
|
class GenericIE(InfoExtractor):
|
||||||
@ -920,6 +923,24 @@ class GenericIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
'add_ie': ['Kaltura'],
|
'add_ie': ['Kaltura'],
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
# Kaltura embedded via quoted entry_id
|
||||||
|
'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '0_utuok90b',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '06_matthew_brender_raj_dutt',
|
||||||
|
'timestamp': 1466638791,
|
||||||
|
'upload_date': '20160622',
|
||||||
|
},
|
||||||
|
'add_ie': ['Kaltura'],
|
||||||
|
'expected_warnings': [
|
||||||
|
'Could not send HEAD request'
|
||||||
|
],
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
}
|
||||||
|
},
|
||||||
# Eagle.Platform embed (generic URL)
|
# Eagle.Platform embed (generic URL)
|
||||||
{
|
{
|
||||||
'url': 'http://lenta.ru/news/2015/03/06/navalny/',
|
'url': 'http://lenta.ru/news/2015/03/06/navalny/',
|
||||||
@ -1073,20 +1094,6 @@ class GenericIE(InfoExtractor):
|
|||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
# Contains a SMIL manifest
|
|
||||||
{
|
|
||||||
'url': 'http://www.telewebion.com/fa/1263668/%D9%82%D8%B1%D8%B9%D9%87%E2%80%8C%DA%A9%D8%B4%DB%8C-%D9%84%DB%8C%DA%AF-%D9%82%D9%87%D8%B1%D9%85%D8%A7%D9%86%D8%A7%D9%86-%D8%A7%D8%B1%D9%88%D9%BE%D8%A7/%2B-%D9%81%D9%88%D8%AA%D8%A8%D8%A7%D9%84.html',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'file',
|
|
||||||
'ext': 'flv',
|
|
||||||
'title': '+ Football: Lottery Champions League Europe',
|
|
||||||
'uploader': 'www.telewebion.com',
|
|
||||||
},
|
|
||||||
'params': {
|
|
||||||
# rtmpe downloads
|
|
||||||
'skip_download': True,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
# Brightcove URL in single quotes
|
# Brightcove URL in single quotes
|
||||||
{
|
{
|
||||||
'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
|
'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
|
||||||
@ -1105,12 +1112,17 @@ class GenericIE(InfoExtractor):
|
|||||||
# Dailymotion Cloud video
|
# Dailymotion Cloud video
|
||||||
{
|
{
|
||||||
'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
|
'url': 'http://replay.publicsenat.fr/vod/le-debat/florent-kolandjian,dominique-cena,axel-decourtye,laurence-abeille,bruno-parmentier/175910',
|
||||||
'md5': '49444254273501a64675a7e68c502681',
|
'md5': 'dcaf23ad0c67a256f4278bce6e0bae38',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '5585de919473990de4bee11b',
|
'id': 'x2uy8t3',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Le débat',
|
'title': 'Sauvons les abeilles ! - Le débat',
|
||||||
|
'description': 'md5:d9082128b1c5277987825d684939ca26',
|
||||||
'thumbnail': 're:^https?://.*\.jpe?g$',
|
'thumbnail': 're:^https?://.*\.jpe?g$',
|
||||||
|
'timestamp': 1434970506,
|
||||||
|
'upload_date': '20150622',
|
||||||
|
'uploader': 'Public Sénat',
|
||||||
|
'uploader_id': 'xa9gza',
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
# OnionStudios embed
|
# OnionStudios embed
|
||||||
@ -1234,6 +1246,70 @@ class GenericIE(InfoExtractor):
|
|||||||
'uploader': 'www.hudl.com',
|
'uploader': 'www.hudl.com',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
# twitter:player embed
|
||||||
|
{
|
||||||
|
'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
|
||||||
|
'md5': 'a3e0df96369831de324f0778e126653c',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '4909620399001',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'What Do Black Holes Sound Like?',
|
||||||
|
'description': 'what do black holes sound like',
|
||||||
|
'upload_date': '20160524',
|
||||||
|
'uploader_id': '29913724001',
|
||||||
|
'timestamp': 1464107587,
|
||||||
|
'uploader': 'TheAtlantic',
|
||||||
|
},
|
||||||
|
'add_ie': ['BrightcoveLegacy'],
|
||||||
|
},
|
||||||
|
# Facebook <iframe> embed
|
||||||
|
{
|
||||||
|
'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
|
||||||
|
'md5': 'fbcde74f534176ecb015849146dd3aee',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '599637780109885',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Facebook video #599637780109885',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
# Facebook API embed
|
||||||
|
{
|
||||||
|
'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
|
||||||
|
'md5': 'a47372ee61b39a7b90287094d447d94e',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '10153467542406923',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Facebook video #10153467542406923',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
# Wordpress "YouTube Video Importer" plugin
|
||||||
|
{
|
||||||
|
'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
|
||||||
|
'md5': 'd16797741b560b485194eddda8121b48',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'HNTXWDXV9Is',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Blue Devils Drumline Stanford lot 2016',
|
||||||
|
'upload_date': '20160627',
|
||||||
|
'uploader_id': 'GENOCIDE8GENERAL10',
|
||||||
|
'uploader': 'cylus cyrus',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# video stored on custom kaltura server
|
||||||
|
'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
|
||||||
|
'md5': '537617d06e64dfed891fa1593c4b30cc',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '0_1iotm5bh',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Elecciones británicas: 5 lecciones para Rajoy',
|
||||||
|
'description': 'md5:435a89d68b9760b92ce67ed227055f16',
|
||||||
|
'uploader_id': 'videos.expansion@el-mundo.net',
|
||||||
|
'upload_date': '20150429',
|
||||||
|
'timestamp': 1430303472,
|
||||||
|
},
|
||||||
|
'add_ie': ['Kaltura'],
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
def report_following_redirect(self, new_url):
|
def report_following_redirect(self, new_url):
|
||||||
@ -1590,6 +1666,13 @@ class GenericIE(InfoExtractor):
|
|||||||
if matches:
|
if matches:
|
||||||
return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
|
return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
|
||||||
|
|
||||||
|
# Look for Wordpress "YouTube Video Importer" plugin
|
||||||
|
matches = re.findall(r'''(?x)<div[^>]+
|
||||||
|
class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
|
||||||
|
data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
|
||||||
|
if matches:
|
||||||
|
return _playlist_from_matches(matches, lambda m: m[-1])
|
||||||
|
|
||||||
# Look for embedded Dailymotion player
|
# Look for embedded Dailymotion player
|
||||||
matches = re.findall(
|
matches = re.findall(
|
||||||
r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
|
r'<(?:(?:embed|iframe)[^>]+?src=|input[^>]+id=[\'"]dmcloudUrlEmissionSelect[\'"][^>]+value=)(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/(?:embed|swf)/video/.+?)\1', webpage)
|
||||||
@ -1732,10 +1815,9 @@ class GenericIE(InfoExtractor):
|
|||||||
return self.url_result(mobj.group('url'))
|
return self.url_result(mobj.group('url'))
|
||||||
|
|
||||||
# Look for embedded Facebook player
|
# Look for embedded Facebook player
|
||||||
mobj = re.search(
|
facebook_url = FacebookIE._extract_url(webpage)
|
||||||
r'<iframe[^>]+?src=(["\'])(?P<url>https://www\.facebook\.com/video/embed.+?)\1', webpage)
|
if facebook_url is not None:
|
||||||
if mobj is not None:
|
return self.url_result(facebook_url, 'Facebook')
|
||||||
return self.url_result(mobj.group('url'), 'Facebook')
|
|
||||||
|
|
||||||
# Look for embedded VK player
|
# Look for embedded VK player
|
||||||
mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
|
mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
|
||||||
@ -1917,18 +1999,14 @@ class GenericIE(InfoExtractor):
|
|||||||
return self.url_result(mobj.group('url'), 'Zapiks')
|
return self.url_result(mobj.group('url'), 'Zapiks')
|
||||||
|
|
||||||
# Look for Kaltura embeds
|
# Look for Kaltura embeds
|
||||||
mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?(?P<q1>['\"])wid(?P=q1)\s*:\s*(?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?(?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*(?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),", webpage) or
|
kaltura_url = KalturaIE._extract_url(webpage)
|
||||||
re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
|
if kaltura_url:
|
||||||
if mobj is not None:
|
return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key())
|
||||||
return self.url_result(smuggle_url(
|
|
||||||
'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
|
|
||||||
{'source_url': url}), 'Kaltura')
|
|
||||||
|
|
||||||
# Look for Eagle.Platform embeds
|
# Look for Eagle.Platform embeds
|
||||||
mobj = re.search(
|
eagleplatform_url = EaglePlatformIE._extract_url(webpage)
|
||||||
r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
|
if eagleplatform_url:
|
||||||
if mobj is not None:
|
return self.url_result(eagleplatform_url, EaglePlatformIE.ie_key())
|
||||||
return self.url_result(mobj.group('url'), 'EaglePlatform')
|
|
||||||
|
|
||||||
# Look for ClipYou (uses Eagle.Platform) embeds
|
# Look for ClipYou (uses Eagle.Platform) embeds
|
||||||
mobj = re.search(
|
mobj = re.search(
|
||||||
@ -2074,6 +2152,11 @@ class GenericIE(InfoExtractor):
|
|||||||
'uploader': video_uploader,
|
'uploader': video_uploader,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser
|
||||||
|
embed_url = self._html_search_meta('twitter:player', webpage, default=None)
|
||||||
|
if embed_url:
|
||||||
|
return self.url_result(embed_url)
|
||||||
|
|
||||||
def check_video(vurl):
|
def check_video(vurl):
|
||||||
if YoutubeIE.suitable(vurl):
|
if YoutubeIE.suitable(vurl):
|
||||||
return True
|
return True
|
||||||
|
202
youtube_dl/extractor/hrti.py
Normal file
202
youtube_dl/extractor/hrti.py
Normal file
@ -0,0 +1,202 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..compat import compat_HTTPError
|
||||||
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
|
ExtractorError,
|
||||||
|
int_or_none,
|
||||||
|
parse_age_limit,
|
||||||
|
sanitized_Request,
|
||||||
|
try_get,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class HRTiBaseIE(InfoExtractor):
|
||||||
|
"""
|
||||||
|
Base Information Extractor for Croatian Radiotelevision
|
||||||
|
video on demand site https://hrti.hrt.hr
|
||||||
|
Reverse engineered from the JavaScript app in app.min.js
|
||||||
|
"""
|
||||||
|
_NETRC_MACHINE = 'hrti'
|
||||||
|
|
||||||
|
_APP_LANGUAGE = 'hr'
|
||||||
|
_APP_VERSION = '1.1'
|
||||||
|
_APP_PUBLICATION_ID = 'all_in_one'
|
||||||
|
_API_URL = 'http://clientapi.hrt.hr/client_api.php/config/identify/format/json'
|
||||||
|
|
||||||
|
def _initialize_api(self):
|
||||||
|
init_data = {
|
||||||
|
'application_publication_id': self._APP_PUBLICATION_ID
|
||||||
|
}
|
||||||
|
|
||||||
|
uuid = self._download_json(
|
||||||
|
self._API_URL, None, note='Downloading uuid',
|
||||||
|
errnote='Unable to download uuid',
|
||||||
|
data=json.dumps(init_data).encode('utf-8'))['uuid']
|
||||||
|
|
||||||
|
app_data = {
|
||||||
|
'uuid': uuid,
|
||||||
|
'application_publication_id': self._APP_PUBLICATION_ID,
|
||||||
|
'application_version': self._APP_VERSION
|
||||||
|
}
|
||||||
|
|
||||||
|
req = sanitized_Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
|
||||||
|
req.get_method = lambda: 'PUT'
|
||||||
|
|
||||||
|
resources = self._download_json(
|
||||||
|
req, None, note='Downloading session information',
|
||||||
|
errnote='Unable to download session information')
|
||||||
|
|
||||||
|
self._session_id = resources['session_id']
|
||||||
|
|
||||||
|
modules = resources['modules']
|
||||||
|
|
||||||
|
self._search_url = modules['vod_catalog']['resources']['search']['uri'].format(
|
||||||
|
language=self._APP_LANGUAGE,
|
||||||
|
application_id=self._APP_PUBLICATION_ID)
|
||||||
|
|
||||||
|
self._login_url = (modules['user']['resources']['login']['uri'] +
|
||||||
|
'/format/json').format(session_id=self._session_id)
|
||||||
|
|
||||||
|
self._logout_url = modules['user']['resources']['logout']['uri']
|
||||||
|
|
||||||
|
def _login(self):
|
||||||
|
(username, password) = self._get_login_info()
|
||||||
|
# TODO: figure out authentication with cookies
|
||||||
|
if username is None or password is None:
|
||||||
|
self.raise_login_required()
|
||||||
|
|
||||||
|
auth_data = {
|
||||||
|
'username': username,
|
||||||
|
'password': password,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
auth_info = self._download_json(
|
||||||
|
self._login_url, None, note='Logging in', errnote='Unable to log in',
|
||||||
|
data=json.dumps(auth_data).encode('utf-8'))
|
||||||
|
except ExtractorError as e:
|
||||||
|
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406:
|
||||||
|
auth_info = self._parse_json(e.cause.read().encode('utf-8'), None)
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
error_message = auth_info.get('error', {}).get('message')
|
||||||
|
if error_message:
|
||||||
|
raise ExtractorError(
|
||||||
|
'%s said: %s' % (self.IE_NAME, error_message),
|
||||||
|
expected=True)
|
||||||
|
|
||||||
|
self._token = auth_info['secure_streaming_token']
|
||||||
|
|
||||||
|
def _real_initialize(self):
|
||||||
|
self._initialize_api()
|
||||||
|
self._login()
|
||||||
|
|
||||||
|
|
||||||
|
class HRTiIE(HRTiBaseIE):
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
(?:
|
||||||
|
hrti:(?P<short_id>[0-9]+)|
|
||||||
|
https?://
|
||||||
|
hrti\.hrt\.hr/\#/video/show/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?
|
||||||
|
)
|
||||||
|
'''
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://hrti.hrt.hr/#/video/show/2181385/republika-dokumentarna-serija-16-hd',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '2181385',
|
||||||
|
'display_id': 'republika-dokumentarna-serija-16-hd',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'REPUBLIKA, dokumentarna serija (1/6) (HD)',
|
||||||
|
'description': 'md5:48af85f620e8e0e1df4096270568544f',
|
||||||
|
'duration': 2922,
|
||||||
|
'view_count': int,
|
||||||
|
'average_rating': int,
|
||||||
|
'episode_number': int,
|
||||||
|
'season_number': int,
|
||||||
|
'age_limit': 12,
|
||||||
|
},
|
||||||
|
'skip': 'Requires account credentials',
|
||||||
|
}, {
|
||||||
|
'url': 'https://hrti.hrt.hr/#/video/show/2181385/',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'hrti:2181385',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id = mobj.group('short_id') or mobj.group('id')
|
||||||
|
display_id = mobj.group('display_id') or video_id
|
||||||
|
|
||||||
|
video = self._download_json(
|
||||||
|
'%s/video_id/%s/format/json' % (self._search_url, video_id),
|
||||||
|
display_id, 'Downloading video metadata JSON')['video'][0]
|
||||||
|
|
||||||
|
title_info = video['title']
|
||||||
|
title = title_info['title_long']
|
||||||
|
|
||||||
|
movie = video['video_assets']['movie'][0]
|
||||||
|
m3u8_url = movie['url'].format(TOKEN=self._token)
|
||||||
|
formats = self._extract_m3u8_formats(
|
||||||
|
m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native',
|
||||||
|
m3u8_id='hls')
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
description = clean_html(title_info.get('summary_long'))
|
||||||
|
age_limit = parse_age_limit(video.get('parental_control', {}).get('rating'))
|
||||||
|
view_count = int_or_none(video.get('views'))
|
||||||
|
average_rating = int_or_none(video.get('user_rating'))
|
||||||
|
duration = int_or_none(movie.get('duration'))
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'display_id': display_id,
|
||||||
|
'title': title,
|
||||||
|
'description': description,
|
||||||
|
'duration': duration,
|
||||||
|
'view_count': view_count,
|
||||||
|
'average_rating': average_rating,
|
||||||
|
'age_limit': age_limit,
|
||||||
|
'formats': formats,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class HRTiPlaylistIE(HRTiBaseIE):
|
||||||
|
_VALID_URL = r'https?://hrti.hrt.hr/#/video/list/category/(?P<id>[0-9]+)/(?P<display_id>[^/]+)?'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://hrti.hrt.hr/#/video/list/category/212/ekumena',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '212',
|
||||||
|
'title': 'ekumena',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 8,
|
||||||
|
'skip': 'Requires account credentials',
|
||||||
|
}, {
|
||||||
|
'url': 'https://hrti.hrt.hr/#/video/list/category/212/',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
category_id = mobj.group('id')
|
||||||
|
display_id = mobj.group('display_id') or category_id
|
||||||
|
|
||||||
|
response = self._download_json(
|
||||||
|
'%s/category_id/%s/format/json' % (self._search_url, category_id),
|
||||||
|
display_id, 'Downloading video metadata JSON')
|
||||||
|
|
||||||
|
video_ids = try_get(
|
||||||
|
response, lambda x: x['video_listings'][0]['alternatives'][0]['list'],
|
||||||
|
list) or [video['id'] for video in response.get('videos', []) if video.get('id')]
|
||||||
|
|
||||||
|
entries = [self.url_result('hrti:%s' % video_id) for video_id in video_ids]
|
||||||
|
|
||||||
|
return self.playlist_result(entries, category_id, display_id)
|
@ -12,7 +12,7 @@ from ..utils import (
|
|||||||
class ImdbIE(InfoExtractor):
|
class ImdbIE(InfoExtractor):
|
||||||
IE_NAME = 'imdb'
|
IE_NAME = 'imdb'
|
||||||
IE_DESC = 'Internet Movie Database trailers'
|
IE_DESC = 'Internet Movie Database trailers'
|
||||||
_VALID_URL = r'https?://(?:www|m)\.imdb\.com/video/[^/]+/vi(?P<id>\d+)'
|
_VALID_URL = r'https?://(?:www|m)\.imdb\.com/(?:video/[^/]+/|title/tt\d+.*?#lb-)vi(?P<id>\d+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.imdb.com/video/imdb/vi2524815897',
|
'url': 'http://www.imdb.com/video/imdb/vi2524815897',
|
||||||
@ -25,6 +25,12 @@ class ImdbIE(InfoExtractor):
|
|||||||
}, {
|
}, {
|
||||||
'url': 'http://www.imdb.com/video/_/vi2524815897',
|
'url': 'http://www.imdb.com/video/_/vi2524815897',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.imdb.com/title/tt1667889/?ref_=ext_shr_eml_vi#lb-vi2524815897',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.imdb.com/title/tt1667889/#lb-vi2524815897',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
|
@ -60,7 +60,8 @@ class IndavideoEmbedIE(InfoExtractor):
|
|||||||
|
|
||||||
formats = [{
|
formats = [{
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'height': self._search_regex(r'\.(\d{3,4})\.mp4$', video_url, 'height', default=None),
|
'height': int_or_none(self._search_regex(
|
||||||
|
r'\.(\d{3,4})\.mp4(?:\?|$)', video_url, 'height', default=None)),
|
||||||
} for video_url in video_urls]
|
} for video_url in video_urls]
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
@ -8,6 +8,7 @@ from ..utils import (
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
limit_length,
|
limit_length,
|
||||||
lowercase_escape,
|
lowercase_escape,
|
||||||
|
try_get,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -19,10 +20,16 @@ class InstagramIE(InfoExtractor):
|
|||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'aye83DjauH',
|
'id': 'aye83DjauH',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'uploader_id': 'naomipq',
|
|
||||||
'title': 'Video by naomipq',
|
'title': 'Video by naomipq',
|
||||||
'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
|
'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
|
||||||
}
|
'thumbnail': 're:^https?://.*\.jpg',
|
||||||
|
'timestamp': 1371748545,
|
||||||
|
'upload_date': '20130620',
|
||||||
|
'uploader_id': 'naomipq',
|
||||||
|
'uploader': 'Naomi Leonor Phan-Quang',
|
||||||
|
'like_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
},
|
||||||
}, {
|
}, {
|
||||||
# missing description
|
# missing description
|
||||||
'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
|
'url': 'https://www.instagram.com/p/BA-pQFBG8HZ/?taken-by=britneyspears',
|
||||||
@ -31,6 +38,13 @@ class InstagramIE(InfoExtractor):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'uploader_id': 'britneyspears',
|
'uploader_id': 'britneyspears',
|
||||||
'title': 'Video by britneyspears',
|
'title': 'Video by britneyspears',
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg',
|
||||||
|
'timestamp': 1453760977,
|
||||||
|
'upload_date': '20160125',
|
||||||
|
'uploader_id': 'britneyspears',
|
||||||
|
'uploader': 'Britney Spears',
|
||||||
|
'like_count': int,
|
||||||
|
'comment_count': int,
|
||||||
},
|
},
|
||||||
'params': {
|
'params': {
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
@ -67,21 +81,57 @@ class InstagramIE(InfoExtractor):
|
|||||||
url = mobj.group('url')
|
url = mobj.group('url')
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(url, video_id)
|
||||||
uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
|
|
||||||
webpage, 'uploader id', fatal=False)
|
(video_url, description, thumbnail, timestamp, uploader,
|
||||||
desc = self._search_regex(
|
uploader_id, like_count, comment_count) = [None] * 8
|
||||||
r'"caption":"(.+?)"', webpage, 'description', default=None)
|
|
||||||
if desc is not None:
|
shared_data = self._parse_json(
|
||||||
desc = lowercase_escape(desc)
|
self._search_regex(
|
||||||
|
r'window\._sharedData\s*=\s*({.+?});',
|
||||||
|
webpage, 'shared data', default='{}'),
|
||||||
|
video_id, fatal=False)
|
||||||
|
if shared_data:
|
||||||
|
media = try_get(
|
||||||
|
shared_data, lambda x: x['entry_data']['PostPage'][0]['media'], dict)
|
||||||
|
if media:
|
||||||
|
video_url = media.get('video_url')
|
||||||
|
description = media.get('caption')
|
||||||
|
thumbnail = media.get('display_src')
|
||||||
|
timestamp = int_or_none(media.get('date'))
|
||||||
|
uploader = media.get('owner', {}).get('full_name')
|
||||||
|
uploader_id = media.get('owner', {}).get('username')
|
||||||
|
like_count = int_or_none(media.get('likes', {}).get('count'))
|
||||||
|
comment_count = int_or_none(media.get('comments', {}).get('count'))
|
||||||
|
|
||||||
|
if not video_url:
|
||||||
|
video_url = self._og_search_video_url(webpage, secure=False)
|
||||||
|
|
||||||
|
if not uploader_id:
|
||||||
|
uploader_id = self._search_regex(
|
||||||
|
r'"owner"\s*:\s*{\s*"username"\s*:\s*"(.+?)"',
|
||||||
|
webpage, 'uploader id', fatal=False)
|
||||||
|
|
||||||
|
if not description:
|
||||||
|
description = self._search_regex(
|
||||||
|
r'"caption"\s*:\s*"(.+?)"', webpage, 'description', default=None)
|
||||||
|
if description is not None:
|
||||||
|
description = lowercase_escape(description)
|
||||||
|
|
||||||
|
if not thumbnail:
|
||||||
|
thumbnail = self._og_search_thumbnail(webpage)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'url': self._og_search_video_url(webpage, secure=False),
|
'url': video_url,
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Video by %s' % uploader_id,
|
'title': 'Video by %s' % uploader_id,
|
||||||
'thumbnail': self._og_search_thumbnail(webpage),
|
'description': description,
|
||||||
|
'thumbnail': thumbnail,
|
||||||
|
'timestamp': timestamp,
|
||||||
'uploader_id': uploader_id,
|
'uploader_id': uploader_id,
|
||||||
'description': desc,
|
'uploader': uploader,
|
||||||
|
'like_count': like_count,
|
||||||
|
'comment_count': comment_count,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -3,28 +3,22 @@ from __future__ import unicode_literals
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import itertools
|
import itertools
|
||||||
import math
|
|
||||||
import os
|
|
||||||
import random
|
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
import uuid
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_parse_qs,
|
|
||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse_urlencode,
|
compat_urllib_parse_urlencode,
|
||||||
compat_urllib_parse_urlparse,
|
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
clean_html,
|
||||||
decode_packed_codes,
|
decode_packed_codes,
|
||||||
|
get_element_by_id,
|
||||||
|
get_element_by_attribute,
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
ohdave_rsa_encrypt,
|
ohdave_rsa_encrypt,
|
||||||
remove_start,
|
remove_start,
|
||||||
sanitized_Request,
|
|
||||||
urlencode_postdata,
|
|
||||||
url_basename,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -171,70 +165,21 @@ class IqiyiIE(InfoExtractor):
|
|||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
|
'url': 'http://www.iqiyi.com/v_19rrojlavg.html',
|
||||||
'md5': '2cb594dc2781e6c941a110d8f358118b',
|
# MD5 checksum differs on my machine and Travis CI
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
|
'id': '9c1fb1b99d192b21c559e5a1a2cb3c73',
|
||||||
|
'ext': 'mp4',
|
||||||
'title': '美国德州空中惊现奇异云团 酷似UFO',
|
'title': '美国德州空中惊现奇异云团 酷似UFO',
|
||||||
'ext': 'f4v',
|
|
||||||
}
|
}
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
|
'url': 'http://www.iqiyi.com/v_19rrhnnclk.html',
|
||||||
|
'md5': '667171934041350c5de3f5015f7f1152',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb',
|
'id': 'e3f585b550a280af23c98b6cb2be19fb',
|
||||||
'title': '名侦探柯南第752集',
|
'ext': 'mp4',
|
||||||
},
|
'title': '名侦探柯南 国语版:第752集 迫近灰原秘密的黑影 下篇',
|
||||||
'playlist': [{
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part1',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part2',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part3',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part4',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part5',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part6',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part7',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'e3f585b550a280af23c98b6cb2be19fb_part8',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '名侦探柯南第752集',
|
|
||||||
},
|
|
||||||
}],
|
|
||||||
'params': {
|
|
||||||
'skip_download': True,
|
|
||||||
},
|
},
|
||||||
|
'skip': 'Geo-restricted to China',
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
|
'url': 'http://www.iqiyi.com/w_19rt6o8t9p.html',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
@ -250,22 +195,10 @@ class IqiyiIE(InfoExtractor):
|
|||||||
'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
|
'url': 'http://www.iqiyi.com/v_19rrny4w8w.html',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'f3cf468b39dddb30d676f89a91200dc1',
|
'id': 'f3cf468b39dddb30d676f89a91200dc1',
|
||||||
|
'ext': 'mp4',
|
||||||
'title': '泰坦尼克号',
|
'title': '泰坦尼克号',
|
||||||
},
|
},
|
||||||
'playlist': [{
|
'skip': 'Geo-restricted to China',
|
||||||
'info_dict': {
|
|
||||||
'id': 'f3cf468b39dddb30d676f89a91200dc1_part1',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '泰坦尼克号',
|
|
||||||
},
|
|
||||||
}, {
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'f3cf468b39dddb30d676f89a91200dc1_part2',
|
|
||||||
'ext': 'f4v',
|
|
||||||
'title': '泰坦尼克号',
|
|
||||||
},
|
|
||||||
}],
|
|
||||||
'expected_warnings': ['Needs a VIP account for full video'],
|
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
|
'url': 'http://www.iqiyi.com/a_19rrhb8ce1.html',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
@ -278,20 +211,15 @@ class IqiyiIE(InfoExtractor):
|
|||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
_FORMATS_MAP = [
|
_FORMATS_MAP = {
|
||||||
('1', 'h6'),
|
'96': 1, # 216p, 240p
|
||||||
('2', 'h5'),
|
'1': 2, # 336p, 360p
|
||||||
('3', 'h4'),
|
'2': 3, # 480p, 504p
|
||||||
('4', 'h3'),
|
'21': 4, # 504p
|
||||||
('5', 'h2'),
|
'4': 5, # 720p
|
||||||
('10', 'h1'),
|
'17': 5, # 720p
|
||||||
]
|
'5': 6, # 1072p, 1080p
|
||||||
|
'18': 7, # 1080p
|
||||||
AUTH_API_ERRORS = {
|
|
||||||
# No preview available (不允许试看鉴权失败)
|
|
||||||
'Q00505': 'This video requires a VIP account',
|
|
||||||
# End of preview time (试看结束鉴权失败)
|
|
||||||
'Q00506': 'Needs a VIP account for full video',
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def _real_initialize(self):
|
def _real_initialize(self):
|
||||||
@ -352,177 +280,23 @@ class IqiyiIE(InfoExtractor):
|
|||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _authenticate_vip_video(self, api_video_url, video_id, tvid, _uuid, do_report_warning):
|
def get_raw_data(self, tvid, video_id):
|
||||||
auth_params = {
|
tm = int(time.time() * 1000)
|
||||||
# version and platform hard-coded in com/qiyi/player/core/model/remote/AuthenticationRemote.as
|
|
||||||
'version': '2.0',
|
key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
|
||||||
'platform': 'b6c13e26323c537d',
|
sc = md5_text(compat_str(tm) + key + tvid)
|
||||||
'aid': tvid,
|
params = {
|
||||||
'tvid': tvid,
|
'tvid': tvid,
|
||||||
'uid': '',
|
|
||||||
'deviceId': _uuid,
|
|
||||||
'playType': 'main', # XXX: always main?
|
|
||||||
'filename': os.path.splitext(url_basename(api_video_url))[0],
|
|
||||||
}
|
|
||||||
|
|
||||||
qd_items = compat_parse_qs(compat_urllib_parse_urlparse(api_video_url).query)
|
|
||||||
for key, val in qd_items.items():
|
|
||||||
auth_params[key] = val[0]
|
|
||||||
|
|
||||||
auth_req = sanitized_Request(
|
|
||||||
'http://api.vip.iqiyi.com/services/ckn.action',
|
|
||||||
urlencode_postdata(auth_params))
|
|
||||||
# iQiyi server throws HTTP 405 error without the following header
|
|
||||||
auth_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
|
|
||||||
auth_result = self._download_json(
|
|
||||||
auth_req, video_id,
|
|
||||||
note='Downloading video authentication JSON',
|
|
||||||
errnote='Unable to download video authentication JSON')
|
|
||||||
|
|
||||||
code = auth_result.get('code')
|
|
||||||
msg = self.AUTH_API_ERRORS.get(code) or auth_result.get('msg') or code
|
|
||||||
if code == 'Q00506':
|
|
||||||
if do_report_warning:
|
|
||||||
self.report_warning(msg)
|
|
||||||
return False
|
|
||||||
if 'data' not in auth_result:
|
|
||||||
if msg is not None:
|
|
||||||
raise ExtractorError('%s said: %s' % (self.IE_NAME, msg), expected=True)
|
|
||||||
raise ExtractorError('Unexpected error from Iqiyi auth API')
|
|
||||||
|
|
||||||
return auth_result['data']
|
|
||||||
|
|
||||||
def construct_video_urls(self, data, video_id, _uuid, tvid):
|
|
||||||
def do_xor(x, y):
|
|
||||||
a = y % 3
|
|
||||||
if a == 1:
|
|
||||||
return x ^ 121
|
|
||||||
if a == 2:
|
|
||||||
return x ^ 72
|
|
||||||
return x ^ 103
|
|
||||||
|
|
||||||
def get_encode_code(l):
|
|
||||||
a = 0
|
|
||||||
b = l.split('-')
|
|
||||||
c = len(b)
|
|
||||||
s = ''
|
|
||||||
for i in range(c - 1, -1, -1):
|
|
||||||
a = do_xor(int(b[c - i - 1], 16), i)
|
|
||||||
s += chr(a)
|
|
||||||
return s[::-1]
|
|
||||||
|
|
||||||
def get_path_key(x, format_id, segment_index):
|
|
||||||
mg = ')(*&^flash@#$%a'
|
|
||||||
tm = self._download_json(
|
|
||||||
'http://data.video.qiyi.com/t?tn=' + str(random.random()), video_id,
|
|
||||||
note='Download path key of segment %d for format %s' % (segment_index + 1, format_id)
|
|
||||||
)['t']
|
|
||||||
t = str(int(math.floor(int(tm) / (600.0))))
|
|
||||||
return md5_text(t + mg + x)
|
|
||||||
|
|
||||||
video_urls_dict = {}
|
|
||||||
need_vip_warning_report = True
|
|
||||||
for format_item in data['vp']['tkl'][0]['vs']:
|
|
||||||
if 0 < int(format_item['bid']) <= 10:
|
|
||||||
format_id = self.get_format(format_item['bid'])
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
|
|
||||||
video_urls = []
|
|
||||||
|
|
||||||
video_urls_info = format_item['fs']
|
|
||||||
if not format_item['fs'][0]['l'].startswith('/'):
|
|
||||||
t = get_encode_code(format_item['fs'][0]['l'])
|
|
||||||
if t.endswith('mp4'):
|
|
||||||
video_urls_info = format_item['flvs']
|
|
||||||
|
|
||||||
for segment_index, segment in enumerate(video_urls_info):
|
|
||||||
vl = segment['l']
|
|
||||||
if not vl.startswith('/'):
|
|
||||||
vl = get_encode_code(vl)
|
|
||||||
is_vip_video = '/vip/' in vl
|
|
||||||
filesize = segment['b']
|
|
||||||
base_url = data['vp']['du'].split('/')
|
|
||||||
if not is_vip_video:
|
|
||||||
key = get_path_key(
|
|
||||||
vl.split('/')[-1].split('.')[0], format_id, segment_index)
|
|
||||||
base_url.insert(-1, key)
|
|
||||||
base_url = '/'.join(base_url)
|
|
||||||
param = {
|
|
||||||
'su': _uuid,
|
|
||||||
'qyid': uuid.uuid4().hex,
|
|
||||||
'client': '',
|
|
||||||
'z': '',
|
|
||||||
'bt': '',
|
|
||||||
'ct': '',
|
|
||||||
'tn': str(int(time.time()))
|
|
||||||
}
|
|
||||||
api_video_url = base_url + vl
|
|
||||||
if is_vip_video:
|
|
||||||
api_video_url = api_video_url.replace('.f4v', '.hml')
|
|
||||||
auth_result = self._authenticate_vip_video(
|
|
||||||
api_video_url, video_id, tvid, _uuid, need_vip_warning_report)
|
|
||||||
if auth_result is False:
|
|
||||||
need_vip_warning_report = False
|
|
||||||
break
|
|
||||||
param.update({
|
|
||||||
't': auth_result['t'],
|
|
||||||
# cid is hard-coded in com/qiyi/player/core/player/RuntimeData.as
|
|
||||||
'cid': 'afbe8fd3d73448c9',
|
|
||||||
'vid': video_id,
|
|
||||||
'QY00001': auth_result['u'],
|
|
||||||
})
|
|
||||||
api_video_url += '?' if '?' not in api_video_url else '&'
|
|
||||||
api_video_url += compat_urllib_parse_urlencode(param)
|
|
||||||
js = self._download_json(
|
|
||||||
api_video_url, video_id,
|
|
||||||
note='Download video info of segment %d for format %s' % (segment_index + 1, format_id))
|
|
||||||
video_url = js['l']
|
|
||||||
video_urls.append(
|
|
||||||
(video_url, filesize))
|
|
||||||
|
|
||||||
video_urls_dict[format_id] = video_urls
|
|
||||||
return video_urls_dict
|
|
||||||
|
|
||||||
def get_format(self, bid):
|
|
||||||
matched_format_ids = [_format_id for _bid, _format_id in self._FORMATS_MAP if _bid == str(bid)]
|
|
||||||
return matched_format_ids[0] if len(matched_format_ids) else None
|
|
||||||
|
|
||||||
def get_bid(self, format_id):
|
|
||||||
matched_bids = [_bid for _bid, _format_id in self._FORMATS_MAP if _format_id == format_id]
|
|
||||||
return matched_bids[0] if len(matched_bids) else None
|
|
||||||
|
|
||||||
def get_raw_data(self, tvid, video_id, enc_key, _uuid):
|
|
||||||
tm = str(int(time.time()))
|
|
||||||
tail = tm + tvid
|
|
||||||
param = {
|
|
||||||
'key': 'fvip',
|
|
||||||
'src': md5_text('youtube-dl'),
|
|
||||||
'tvId': tvid,
|
|
||||||
'vid': video_id,
|
'vid': video_id,
|
||||||
'vinfo': 1,
|
'src': '76f90cbd92f94a2e925d83e8ccd22cb7',
|
||||||
'tm': tm,
|
'sc': sc,
|
||||||
'enc': md5_text(enc_key + tail),
|
't': tm,
|
||||||
'qyid': _uuid,
|
|
||||||
'tn': random.random(),
|
|
||||||
# In iQiyi's flash player, um is set to 1 if there's a logged user
|
|
||||||
# Some 1080P formats are only available with a logged user.
|
|
||||||
# Here force um=1 to trick the iQiyi server
|
|
||||||
'um': 1,
|
|
||||||
'authkey': md5_text(md5_text('') + tail),
|
|
||||||
'k_tag': 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
api_url = 'http://cache.video.qiyi.com/vms' + '?' + \
|
return self._download_json(
|
||||||
compat_urllib_parse_urlencode(param)
|
'http://cache.m.iqiyi.com/jp/tmts/%s/%s/' % (tvid, video_id),
|
||||||
raw_data = self._download_json(api_url, video_id)
|
video_id, transform_source=lambda s: remove_start(s, 'var tvInfoJs='),
|
||||||
return raw_data
|
query=params, headers=self.geo_verification_headers())
|
||||||
|
|
||||||
def get_enc_key(self, video_id):
|
|
||||||
# TODO: automatic key extraction
|
|
||||||
# last update at 2016-01-22 for Zombie::bite
|
|
||||||
enc_key = '4a1caba4b4465345366f28da7c117d20'
|
|
||||||
return enc_key
|
|
||||||
|
|
||||||
def _extract_playlist(self, webpage):
|
def _extract_playlist(self, webpage):
|
||||||
PAGE_SIZE = 50
|
PAGE_SIZE = 50
|
||||||
@ -571,58 +345,41 @@ class IqiyiIE(InfoExtractor):
|
|||||||
r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
|
r'data-player-tvid\s*=\s*[\'"](\d+)', webpage, 'tvid')
|
||||||
video_id = self._search_regex(
|
video_id = self._search_regex(
|
||||||
r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
|
r'data-player-videoid\s*=\s*[\'"]([a-f\d]+)', webpage, 'video_id')
|
||||||
_uuid = uuid.uuid4().hex
|
|
||||||
|
|
||||||
enc_key = self.get_enc_key(video_id)
|
formats = []
|
||||||
|
for _ in range(5):
|
||||||
|
raw_data = self.get_raw_data(tvid, video_id)
|
||||||
|
|
||||||
raw_data = self.get_raw_data(tvid, video_id, enc_key, _uuid)
|
if raw_data['code'] != 'A00000':
|
||||||
|
if raw_data['code'] == 'A00111':
|
||||||
|
self.raise_geo_restricted()
|
||||||
|
raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
|
||||||
|
|
||||||
if raw_data['code'] != 'A000000':
|
data = raw_data['data']
|
||||||
raise ExtractorError('Unable to load data. Error code: ' + raw_data['code'])
|
|
||||||
|
|
||||||
data = raw_data['data']
|
for stream in data['vidl']:
|
||||||
|
if 'm3utx' not in stream:
|
||||||
|
continue
|
||||||
|
vd = compat_str(stream['vd'])
|
||||||
|
formats.append({
|
||||||
|
'url': stream['m3utx'],
|
||||||
|
'format_id': vd,
|
||||||
|
'ext': 'mp4',
|
||||||
|
'preference': self._FORMATS_MAP.get(vd, -1),
|
||||||
|
'protocol': 'm3u8_native',
|
||||||
|
})
|
||||||
|
|
||||||
title = data['vi']['vn']
|
if formats:
|
||||||
|
break
|
||||||
|
|
||||||
# generate video_urls_dict
|
self._sleep(5, video_id)
|
||||||
video_urls_dict = self.construct_video_urls(
|
|
||||||
data, video_id, _uuid, tvid)
|
|
||||||
|
|
||||||
# construct info
|
self._sort_formats(formats)
|
||||||
entries = []
|
title = (get_element_by_id('widget-videotitle', webpage) or
|
||||||
for format_id in video_urls_dict:
|
clean_html(get_element_by_attribute('class', 'mod-play-tit', webpage)))
|
||||||
video_urls = video_urls_dict[format_id]
|
|
||||||
for i, video_url_info in enumerate(video_urls):
|
|
||||||
if len(entries) < i + 1:
|
|
||||||
entries.append({'formats': []})
|
|
||||||
entries[i]['formats'].append(
|
|
||||||
{
|
|
||||||
'url': video_url_info[0],
|
|
||||||
'filesize': video_url_info[-1],
|
|
||||||
'format_id': format_id,
|
|
||||||
'preference': int(self.get_bid(format_id))
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
for i in range(len(entries)):
|
return {
|
||||||
self._sort_formats(entries[i]['formats'])
|
'id': video_id,
|
||||||
entries[i].update(
|
'title': title,
|
||||||
{
|
'formats': formats,
|
||||||
'id': '%s_part%d' % (video_id, i + 1),
|
}
|
||||||
'title': title,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
if len(entries) > 1:
|
|
||||||
info = {
|
|
||||||
'_type': 'multi_video',
|
|
||||||
'id': video_id,
|
|
||||||
'title': title,
|
|
||||||
'entries': entries,
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
info = entries[0]
|
|
||||||
info['id'] = video_id
|
|
||||||
info['title'] = title
|
|
||||||
|
|
||||||
return info
|
|
||||||
|
@ -12,9 +12,35 @@ from ..utils import (
|
|||||||
|
|
||||||
|
|
||||||
class JWPlatformBaseIE(InfoExtractor):
|
class JWPlatformBaseIE(InfoExtractor):
|
||||||
|
@staticmethod
|
||||||
|
def _find_jwplayer_data(webpage):
|
||||||
|
# TODO: Merge this with JWPlayer-related codes in generic.py
|
||||||
|
|
||||||
|
mobj = re.search(
|
||||||
|
'jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)\.setup\((?P<options>[^)]+)\)',
|
||||||
|
webpage)
|
||||||
|
if mobj:
|
||||||
|
return mobj.group('options')
|
||||||
|
|
||||||
|
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
|
||||||
|
jwplayer_data = self._parse_json(
|
||||||
|
self._find_jwplayer_data(webpage), video_id)
|
||||||
|
return self._parse_jwplayer_data(
|
||||||
|
jwplayer_data, video_id, *args, **kwargs)
|
||||||
|
|
||||||
def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None):
|
def _parse_jwplayer_data(self, jwplayer_data, video_id, require_title=True, m3u8_id=None, rtmp_params=None):
|
||||||
|
# JWPlayer backward compatibility: flattened playlists
|
||||||
|
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
|
||||||
|
if 'playlist' not in jwplayer_data:
|
||||||
|
jwplayer_data = {'playlist': [jwplayer_data]}
|
||||||
|
|
||||||
video_data = jwplayer_data['playlist'][0]
|
video_data = jwplayer_data['playlist'][0]
|
||||||
|
|
||||||
|
# JWPlayer backward compatibility: flattened sources
|
||||||
|
# https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
|
||||||
|
if 'sources' not in video_data:
|
||||||
|
video_data['sources'] = [video_data]
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for source in video_data['sources']:
|
for source in video_data['sources']:
|
||||||
source_url = self._proto_relative_url(source['file'])
|
source_url = self._proto_relative_url(source['file'])
|
||||||
|
@ -6,7 +6,6 @@ import base64
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse_urlencode,
|
|
||||||
compat_urlparse,
|
compat_urlparse,
|
||||||
compat_parse_qs,
|
compat_parse_qs,
|
||||||
)
|
)
|
||||||
@ -15,6 +14,7 @@ from ..utils import (
|
|||||||
ExtractorError,
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
unsmuggle_url,
|
unsmuggle_url,
|
||||||
|
smuggle_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -34,7 +34,8 @@ class KalturaIE(InfoExtractor):
|
|||||||
)(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
|
)(?:/(?P<path>[^?]+))?(?:\?(?P<query>.*))?
|
||||||
)
|
)
|
||||||
'''
|
'''
|
||||||
_API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
|
_SERVICE_URL = 'http://cdnapi.kaltura.com'
|
||||||
|
_SERVICE_BASE = '/api_v3/index.php'
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
{
|
{
|
||||||
'url': 'kaltura:269692:1_1jc2y3e4',
|
'url': 'kaltura:269692:1_1jc2y3e4',
|
||||||
@ -64,16 +65,50 @@ class KalturaIE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
|
@staticmethod
|
||||||
|
def _extract_url(webpage):
|
||||||
|
mobj = (
|
||||||
|
re.search(
|
||||||
|
r"""(?xs)
|
||||||
|
kWidget\.(?:thumb)?[Ee]mbed\(
|
||||||
|
\{.*?
|
||||||
|
(?P<q1>['\"])wid(?P=q1)\s*:\s*
|
||||||
|
(?P<q2>['\"])_?(?P<partner_id>[^'\"]+)(?P=q2),.*?
|
||||||
|
(?P<q3>['\"])entry_?[Ii]d(?P=q3)\s*:\s*
|
||||||
|
(?P<q4>['\"])(?P<id>[^'\"]+)(?P=q4),
|
||||||
|
""", webpage) or
|
||||||
|
re.search(
|
||||||
|
r'''(?xs)
|
||||||
|
(?P<q1>["\'])
|
||||||
|
(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?
|
||||||
|
(?P=q1).*?
|
||||||
|
(?:
|
||||||
|
entry_?[Ii]d|
|
||||||
|
(?P<q2>["\'])entry_?[Ii]d(?P=q2)
|
||||||
|
)\s*:\s*
|
||||||
|
(?P<q3>["\'])(?P<id>.+?)(?P=q3)
|
||||||
|
''', webpage))
|
||||||
|
if mobj:
|
||||||
|
embed_info = mobj.groupdict()
|
||||||
|
url = 'kaltura:%(partner_id)s:%(id)s' % embed_info
|
||||||
|
escaped_pid = re.escape(embed_info['partner_id'])
|
||||||
|
service_url = re.search(
|
||||||
|
r'<script[^>]+src=["\']((?:https?:)?//.+?)/p/%s/sp/%s00/embedIframeJs' % (escaped_pid, escaped_pid),
|
||||||
|
webpage)
|
||||||
|
if service_url:
|
||||||
|
url = smuggle_url(url, {'service_url': service_url.group(1)})
|
||||||
|
return url
|
||||||
|
|
||||||
|
def _kaltura_api_call(self, video_id, actions, service_url=None, *args, **kwargs):
|
||||||
params = actions[0]
|
params = actions[0]
|
||||||
if len(actions) > 1:
|
if len(actions) > 1:
|
||||||
for i, a in enumerate(actions[1:], start=1):
|
for i, a in enumerate(actions[1:], start=1):
|
||||||
for k, v in a.items():
|
for k, v in a.items():
|
||||||
params['%d:%s' % (i, k)] = v
|
params['%d:%s' % (i, k)] = v
|
||||||
|
|
||||||
query = compat_urllib_parse_urlencode(params)
|
data = self._download_json(
|
||||||
url = self._API_BASE + query
|
(service_url or self._SERVICE_URL) + self._SERVICE_BASE,
|
||||||
data = self._download_json(url, video_id, *args, **kwargs)
|
video_id, query=params, *args, **kwargs)
|
||||||
|
|
||||||
status = data if len(actions) == 1 else data[0]
|
status = data if len(actions) == 1 else data[0]
|
||||||
if status.get('objectType') == 'KalturaAPIException':
|
if status.get('objectType') == 'KalturaAPIException':
|
||||||
@ -82,7 +117,7 @@ class KalturaIE(InfoExtractor):
|
|||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def _get_kaltura_signature(self, video_id, partner_id):
|
def _get_kaltura_signature(self, video_id, partner_id, service_url=None):
|
||||||
actions = [{
|
actions = [{
|
||||||
'apiVersion': '3.1',
|
'apiVersion': '3.1',
|
||||||
'expiry': 86400,
|
'expiry': 86400,
|
||||||
@ -92,10 +127,10 @@ class KalturaIE(InfoExtractor):
|
|||||||
'widgetId': '_%s' % partner_id,
|
'widgetId': '_%s' % partner_id,
|
||||||
}]
|
}]
|
||||||
return self._kaltura_api_call(
|
return self._kaltura_api_call(
|
||||||
video_id, actions, note='Downloading Kaltura signature')['ks']
|
video_id, actions, service_url, note='Downloading Kaltura signature')['ks']
|
||||||
|
|
||||||
def _get_video_info(self, video_id, partner_id):
|
def _get_video_info(self, video_id, partner_id, service_url=None):
|
||||||
signature = self._get_kaltura_signature(video_id, partner_id)
|
signature = self._get_kaltura_signature(video_id, partner_id, service_url)
|
||||||
actions = [
|
actions = [
|
||||||
{
|
{
|
||||||
'action': 'null',
|
'action': 'null',
|
||||||
@ -118,7 +153,7 @@ class KalturaIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
]
|
]
|
||||||
return self._kaltura_api_call(
|
return self._kaltura_api_call(
|
||||||
video_id, actions, note='Downloading video info JSON')
|
video_id, actions, service_url, note='Downloading video info JSON')
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
url, smuggled_data = unsmuggle_url(url, {})
|
url, smuggled_data = unsmuggle_url(url, {})
|
||||||
@ -127,7 +162,7 @@ class KalturaIE(InfoExtractor):
|
|||||||
partner_id, entry_id = mobj.group('partner_id', 'id')
|
partner_id, entry_id = mobj.group('partner_id', 'id')
|
||||||
ks = None
|
ks = None
|
||||||
if partner_id and entry_id:
|
if partner_id and entry_id:
|
||||||
info, flavor_assets = self._get_video_info(entry_id, partner_id)
|
info, flavor_assets = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'))
|
||||||
else:
|
else:
|
||||||
path, query = mobj.group('path', 'query')
|
path, query = mobj.group('path', 'query')
|
||||||
if not path and not query:
|
if not path and not query:
|
||||||
@ -175,12 +210,17 @@ class KalturaIE(InfoExtractor):
|
|||||||
unsigned_url += '?referrer=%s' % referrer
|
unsigned_url += '?referrer=%s' % referrer
|
||||||
return unsigned_url
|
return unsigned_url
|
||||||
|
|
||||||
|
data_url = info['dataUrl']
|
||||||
|
if '/flvclipper/' in data_url:
|
||||||
|
data_url = re.sub(r'/flvclipper/.*', '/serveFlavor', data_url)
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for f in flavor_assets:
|
for f in flavor_assets:
|
||||||
# Continue if asset is not ready
|
# Continue if asset is not ready
|
||||||
if f['status'] != 2:
|
if f['status'] != 2:
|
||||||
continue
|
continue
|
||||||
video_url = sign_url('%s/flavorId/%s' % (info['dataUrl'], f['id']))
|
video_url = sign_url(
|
||||||
|
'%s/flavorId/%s' % (data_url, f['id']))
|
||||||
formats.append({
|
formats.append({
|
||||||
'format_id': '%(fileExt)s-%(bitrate)s' % f,
|
'format_id': '%(fileExt)s-%(bitrate)s' % f,
|
||||||
'ext': f.get('fileExt'),
|
'ext': f.get('fileExt'),
|
||||||
@ -193,9 +233,12 @@ class KalturaIE(InfoExtractor):
|
|||||||
'width': int_or_none(f.get('width')),
|
'width': int_or_none(f.get('width')),
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
})
|
})
|
||||||
m3u8_url = sign_url(info['dataUrl'].replace('format/url', 'format/applehttp'))
|
if '/playManifest/' in data_url:
|
||||||
formats.extend(self._extract_m3u8_formats(
|
m3u8_url = sign_url(data_url.replace(
|
||||||
m3u8_url, entry_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
|
'format/url', 'format/applehttp'))
|
||||||
|
formats.extend(self._extract_m3u8_formats(
|
||||||
|
m3u8_url, entry_id, 'mp4', 'm3u8_native',
|
||||||
|
m3u8_id='hls', fatal=False))
|
||||||
|
|
||||||
self._check_formats(formats, entry_id)
|
self._check_formats(formats, entry_id)
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
@ -26,11 +26,6 @@ class KuwoBaseIE(InfoExtractor):
|
|||||||
def _get_formats(self, song_id, tolerate_ip_deny=False):
|
def _get_formats(self, song_id, tolerate_ip_deny=False):
|
||||||
formats = []
|
formats = []
|
||||||
for file_format in self._FORMATS:
|
for file_format in self._FORMATS:
|
||||||
headers = {}
|
|
||||||
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
|
||||||
if cn_verification_proxy:
|
|
||||||
headers['Ytdl-request-proxy'] = cn_verification_proxy
|
|
||||||
|
|
||||||
query = {
|
query = {
|
||||||
'format': file_format['ext'],
|
'format': file_format['ext'],
|
||||||
'br': file_format.get('br', ''),
|
'br': file_format.get('br', ''),
|
||||||
@ -42,7 +37,7 @@ class KuwoBaseIE(InfoExtractor):
|
|||||||
song_url = self._download_webpage(
|
song_url = self._download_webpage(
|
||||||
'http://antiserver.kuwo.cn/anti.s',
|
'http://antiserver.kuwo.cn/anti.s',
|
||||||
song_id, note='Download %s url info' % file_format['format'],
|
song_id, note='Download %s url info' % file_format['format'],
|
||||||
query=query, headers=headers,
|
query=query, headers=self.geo_verification_headers(),
|
||||||
)
|
)
|
||||||
|
|
||||||
if song_url == 'IPDeny' and not tolerate_ip_deny:
|
if song_url == 'IPDeny' and not tolerate_ip_deny:
|
||||||
@ -148,8 +143,8 @@ class KuwoAlbumIE(InfoExtractor):
|
|||||||
'url': 'http://www.kuwo.cn/album/502294/',
|
'url': 'http://www.kuwo.cn/album/502294/',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '502294',
|
'id': '502294',
|
||||||
'title': 'M',
|
'title': 'Made\xa0Series\xa0《M》',
|
||||||
'description': 'md5:6a7235a84cc6400ec3b38a7bdaf1d60c',
|
'description': 'md5:d463f0d8a0ff3c3ea3d6ed7452a9483f',
|
||||||
},
|
},
|
||||||
'playlist_count': 2,
|
'playlist_count': 2,
|
||||||
}
|
}
|
||||||
@ -209,7 +204,7 @@ class KuwoSingerIE(InfoExtractor):
|
|||||||
'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
|
'url': 'http://www.kuwo.cn/mingxing/bruno+mars/',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'bruno+mars',
|
'id': 'bruno+mars',
|
||||||
'title': 'Bruno Mars',
|
'title': 'Bruno\xa0Mars',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 329,
|
'playlist_mincount': 329,
|
||||||
}, {
|
}, {
|
||||||
@ -306,7 +301,7 @@ class KuwoMvIE(KuwoBaseIE):
|
|||||||
'id': '6480076',
|
'id': '6480076',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'My HouseMV',
|
'title': 'My HouseMV',
|
||||||
'creator': '2PM',
|
'creator': 'PM02:00',
|
||||||
},
|
},
|
||||||
# In this video, music URLs (anti.s) are blocked outside China and
|
# In this video, music URLs (anti.s) are blocked outside China and
|
||||||
# USA, while the MV URL (mvurl) is available globally, so force the MV
|
# USA, while the MV URL (mvurl) is available globally, so force the MV
|
||||||
|
@ -1,60 +1,65 @@
|
|||||||
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_duration,
|
js_to_json,
|
||||||
|
smuggle_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class LA7IE(InfoExtractor):
|
class LA7IE(InfoExtractor):
|
||||||
IE_NAME = 'la7.tv'
|
IE_NAME = 'la7.it'
|
||||||
_VALID_URL = r'''(?x)
|
_VALID_URL = r'''(?x)(https?://)?(?:
|
||||||
https?://(?:www\.)?la7\.tv/
|
(?:www\.)?la7\.it/([^/]+)/(?:rivedila7|video)/|
|
||||||
(?:
|
tg\.la7\.it/repliche-tgla7\?id=
|
||||||
richplayer/\?assetid=|
|
)(?P<id>.+)'''
|
||||||
\?contentId=
|
|
||||||
)
|
|
||||||
(?P<id>[0-9]+)'''
|
|
||||||
|
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
|
# 'src' is a plain URL
|
||||||
'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
|
'url': 'http://www.la7.it/crozza/video/inccool8-02-10-2015-163722',
|
||||||
|
'md5': '8b613ffc0c4bf9b9e377169fc19c214c',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '50355319',
|
'id': 'inccool8-02-10-2015-163722',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'IL DIVO',
|
'title': 'Inc.Cool8',
|
||||||
'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti e Flavio Bucci',
|
'description': 'Benvenuti nell\'incredibile mondo della INC. COOL. 8. dove “INC.” sta per “Incorporated” “COOL” sta per “fashion” ed Eight sta per il gesto atletico',
|
||||||
'duration': 6254,
|
'thumbnail': 're:^https?://.*',
|
||||||
|
'uploader_id': 'kdla7pillole@iltrovatore.it',
|
||||||
|
'timestamp': 1443814869,
|
||||||
|
'upload_date': '20151002',
|
||||||
},
|
},
|
||||||
'skip': 'Blocked in the US',
|
}, {
|
||||||
}
|
# 'src' is a dictionary
|
||||||
|
'url': 'http://tg.la7.it/repliche-tgla7?id=189080',
|
||||||
|
'md5': '6b0d8888d286e39870208dfeceaf456b',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '189080',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'TG LA7',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.la7.it/omnibus/rivedila7/omnibus-news-02-07-2016-189077',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
|
|
||||||
doc = self._download_xml(xml_url, video_id)
|
|
||||||
|
|
||||||
video_title = doc.find('title').text
|
webpage = self._download_webpage(url, video_id)
|
||||||
description = doc.find('description').text
|
|
||||||
duration = parse_duration(doc.find('duration').text)
|
|
||||||
thumbnail = doc.find('img').text
|
|
||||||
view_count = int(doc.find('views').text)
|
|
||||||
|
|
||||||
prefix = doc.find('.//fqdn').text.strip().replace('auto:', 'http:')
|
player_data = self._parse_json(
|
||||||
|
self._search_regex(r'videoLa7\(({[^;]+})\);', webpage, 'player data'),
|
||||||
formats = [{
|
video_id, transform_source=js_to_json)
|
||||||
'format': vnode.find('quality').text,
|
|
||||||
'tbr': int(vnode.find('quality').text),
|
|
||||||
'url': vnode.find('fms').text.strip().replace('mp4:', prefix),
|
|
||||||
} for vnode in doc.findall('.//videos/video')]
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'url': smuggle_url('kaltura:103:%s' % player_data['vid'], {
|
||||||
|
'service_url': 'http://kdam.iltrovatore.it',
|
||||||
|
}),
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': video_title,
|
'title': player_data['title'],
|
||||||
'description': description,
|
'description': self._og_search_description(webpage, default=None),
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': player_data.get('poster'),
|
||||||
'duration': duration,
|
'ie_key': 'Kaltura',
|
||||||
'formats': formats,
|
|
||||||
'view_count': view_count,
|
|
||||||
}
|
}
|
||||||
|
@ -20,15 +20,15 @@ from ..utils import (
|
|||||||
int_or_none,
|
int_or_none,
|
||||||
orderedSet,
|
orderedSet,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
sanitized_Request,
|
|
||||||
str_or_none,
|
str_or_none,
|
||||||
url_basename,
|
url_basename,
|
||||||
|
urshift,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class LeIE(InfoExtractor):
|
class LeIE(InfoExtractor):
|
||||||
IE_DESC = '乐视网'
|
IE_DESC = '乐视网'
|
||||||
_VALID_URL = r'https?://www\.le\.com/ptv/vplay/(?P<id>\d+)\.html'
|
_VALID_URL = r'https?://(?:www\.le\.com/ptv/vplay|sports\.le\.com/video)/(?P<id>\d+)\.html'
|
||||||
|
|
||||||
_URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
|
_URL_TEMPLATE = 'http://www.le.com/ptv/vplay/%s.html'
|
||||||
|
|
||||||
@ -69,17 +69,16 @@ class LeIE(InfoExtractor):
|
|||||||
'hls_prefer_native': True,
|
'hls_prefer_native': True,
|
||||||
},
|
},
|
||||||
'skip': 'Only available in China',
|
'skip': 'Only available in China',
|
||||||
|
}, {
|
||||||
|
'url': 'http://sports.le.com/video/25737697.html',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def urshift(val, n):
|
|
||||||
return val >> n if val >= 0 else (val + 0x100000000) >> n
|
|
||||||
|
|
||||||
# ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
|
# ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
|
||||||
def ror(self, param1, param2):
|
def ror(self, param1, param2):
|
||||||
_loc3_ = 0
|
_loc3_ = 0
|
||||||
while _loc3_ < param2:
|
while _loc3_ < param2:
|
||||||
param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
|
param1 = urshift(param1, 1) + ((param1 & 1) << 31)
|
||||||
_loc3_ += 1
|
_loc3_ += 1
|
||||||
return param1
|
return param1
|
||||||
|
|
||||||
@ -121,16 +120,11 @@ class LeIE(InfoExtractor):
|
|||||||
'tkey': self.calc_time_key(int(time.time())),
|
'tkey': self.calc_time_key(int(time.time())),
|
||||||
'domain': 'www.le.com'
|
'domain': 'www.le.com'
|
||||||
}
|
}
|
||||||
play_json_req = sanitized_Request(
|
|
||||||
'http://api.le.com/mms/out/video/playJson?' + compat_urllib_parse_urlencode(params)
|
|
||||||
)
|
|
||||||
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
|
||||||
if cn_verification_proxy:
|
|
||||||
play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
|
|
||||||
|
|
||||||
play_json = self._download_json(
|
play_json = self._download_json(
|
||||||
play_json_req,
|
'http://api.le.com/mms/out/video/playJson',
|
||||||
media_id, 'Downloading playJson data')
|
media_id, 'Downloading playJson data', query=params,
|
||||||
|
headers=self.geo_verification_headers())
|
||||||
|
|
||||||
# Check for errors
|
# Check for errors
|
||||||
playstatus = play_json['playstatus']
|
playstatus = play_json['playstatus']
|
||||||
@ -196,7 +190,7 @@ class LeIE(InfoExtractor):
|
|||||||
|
|
||||||
|
|
||||||
class LePlaylistIE(InfoExtractor):
|
class LePlaylistIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://[a-z]+\.le\.com/[a-z]+/(?P<id>[a-z0-9_]+)'
|
_VALID_URL = r'https?://[a-z]+\.le\.com/(?!video)[a-z]+/(?P<id>[a-z0-9_]+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.le.com/tv/46177.html',
|
'url': 'http://www.le.com/tv/46177.html',
|
||||||
|
@ -98,13 +98,19 @@ class LimelightBaseIE(InfoExtractor):
|
|||||||
} for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
|
} for thumbnail in properties.get('thumbnails', []) if thumbnail.get('url')]
|
||||||
|
|
||||||
subtitles = {}
|
subtitles = {}
|
||||||
for caption in properties.get('captions', {}):
|
for caption in properties.get('captions', []):
|
||||||
lang = caption.get('language_code')
|
lang = caption.get('language_code')
|
||||||
subtitles_url = caption.get('url')
|
subtitles_url = caption.get('url')
|
||||||
if lang and subtitles_url:
|
if lang and subtitles_url:
|
||||||
subtitles[lang] = [{
|
subtitles.setdefault(lang, []).append({
|
||||||
'url': subtitles_url,
|
'url': subtitles_url,
|
||||||
}]
|
})
|
||||||
|
closed_captions_url = properties.get('closed_captions_url')
|
||||||
|
if closed_captions_url:
|
||||||
|
subtitles.setdefault('en', []).append({
|
||||||
|
'url': closed_captions_url,
|
||||||
|
'ext': 'ttml',
|
||||||
|
})
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
@ -123,7 +129,18 @@ class LimelightBaseIE(InfoExtractor):
|
|||||||
|
|
||||||
class LimelightMediaIE(LimelightBaseIE):
|
class LimelightMediaIE(LimelightBaseIE):
|
||||||
IE_NAME = 'limelight'
|
IE_NAME = 'limelight'
|
||||||
_VALID_URL = r'(?:limelight:media:|https?://link\.videoplatform\.limelight\.com/media/\??\bmediaId=)(?P<id>[a-z0-9]{32})'
|
_VALID_URL = r'''(?x)
|
||||||
|
(?:
|
||||||
|
limelight:media:|
|
||||||
|
https?://
|
||||||
|
(?:
|
||||||
|
link\.videoplatform\.limelight\.com/media/|
|
||||||
|
assets\.delvenetworks\.com/player/loader\.swf
|
||||||
|
)
|
||||||
|
\?.*?\bmediaId=
|
||||||
|
)
|
||||||
|
(?P<id>[a-z0-9]{32})
|
||||||
|
'''
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
|
'url': 'http://link.videoplatform.limelight.com/media/?mediaId=3ffd040b522b4485b6d84effc750cd86',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
@ -158,6 +175,9 @@ class LimelightMediaIE(LimelightBaseIE):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'https://assets.delvenetworks.com/player/loader.swf?mediaId=8018a574f08d416e95ceaccae4ba0452',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
_PLAYLIST_SERVICE_PATH = 'media'
|
_PLAYLIST_SERVICE_PATH = 'media'
|
||||||
_API_PATH = 'media'
|
_API_PATH = 'media'
|
||||||
@ -176,15 +196,29 @@ class LimelightMediaIE(LimelightBaseIE):
|
|||||||
|
|
||||||
class LimelightChannelIE(LimelightBaseIE):
|
class LimelightChannelIE(LimelightBaseIE):
|
||||||
IE_NAME = 'limelight:channel'
|
IE_NAME = 'limelight:channel'
|
||||||
_VALID_URL = r'(?:limelight:channel:|https?://link\.videoplatform\.limelight\.com/media/\??\bchannelId=)(?P<id>[a-z0-9]{32})'
|
_VALID_URL = r'''(?x)
|
||||||
_TEST = {
|
(?:
|
||||||
|
limelight:channel:|
|
||||||
|
https?://
|
||||||
|
(?:
|
||||||
|
link\.videoplatform\.limelight\.com/media/|
|
||||||
|
assets\.delvenetworks\.com/player/loader\.swf
|
||||||
|
)
|
||||||
|
\?.*?\bchannelId=
|
||||||
|
)
|
||||||
|
(?P<id>[a-z0-9]{32})
|
||||||
|
'''
|
||||||
|
_TESTS = [{
|
||||||
'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
|
'url': 'http://link.videoplatform.limelight.com/media/?channelId=ab6a524c379342f9b23642917020c082',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'ab6a524c379342f9b23642917020c082',
|
'id': 'ab6a524c379342f9b23642917020c082',
|
||||||
'title': 'Javascript Sample Code',
|
'title': 'Javascript Sample Code',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 3,
|
'playlist_mincount': 3,
|
||||||
}
|
}, {
|
||||||
|
'url': 'http://assets.delvenetworks.com/player/loader.swf?channelId=ab6a524c379342f9b23642917020c082',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
_PLAYLIST_SERVICE_PATH = 'channel'
|
_PLAYLIST_SERVICE_PATH = 'channel'
|
||||||
_API_PATH = 'channels'
|
_API_PATH = 'channels'
|
||||||
|
|
||||||
@ -207,15 +241,29 @@ class LimelightChannelIE(LimelightBaseIE):
|
|||||||
|
|
||||||
class LimelightChannelListIE(LimelightBaseIE):
|
class LimelightChannelListIE(LimelightBaseIE):
|
||||||
IE_NAME = 'limelight:channel_list'
|
IE_NAME = 'limelight:channel_list'
|
||||||
_VALID_URL = r'(?:limelight:channel_list:|https?://link\.videoplatform\.limelight\.com/media/\?.*?\bchannelListId=)(?P<id>[a-z0-9]{32})'
|
_VALID_URL = r'''(?x)
|
||||||
_TEST = {
|
(?:
|
||||||
|
limelight:channel_list:|
|
||||||
|
https?://
|
||||||
|
(?:
|
||||||
|
link\.videoplatform\.limelight\.com/media/|
|
||||||
|
assets\.delvenetworks\.com/player/loader\.swf
|
||||||
|
)
|
||||||
|
\?.*?\bchannelListId=
|
||||||
|
)
|
||||||
|
(?P<id>[a-z0-9]{32})
|
||||||
|
'''
|
||||||
|
_TESTS = [{
|
||||||
'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
|
'url': 'http://link.videoplatform.limelight.com/media/?channelListId=301b117890c4465c8179ede21fd92e2b',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '301b117890c4465c8179ede21fd92e2b',
|
'id': '301b117890c4465c8179ede21fd92e2b',
|
||||||
'title': 'Website - Hero Player',
|
'title': 'Website - Hero Player',
|
||||||
},
|
},
|
||||||
'playlist_mincount': 2,
|
'playlist_mincount': 2,
|
||||||
}
|
}, {
|
||||||
|
'url': 'https://assets.delvenetworks.com/player/loader.swf?channelListId=301b117890c4465c8179ede21fd92e2b',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
_PLAYLIST_SERVICE_PATH = 'channel_list'
|
_PLAYLIST_SERVICE_PATH = 'channel_list'
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
|
@ -95,7 +95,6 @@ class LyndaIE(LyndaBaseIE):
|
|||||||
IE_NAME = 'lynda'
|
IE_NAME = 'lynda'
|
||||||
IE_DESC = 'lynda.com videos'
|
IE_DESC = 'lynda.com videos'
|
||||||
_VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
|
_VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
|
||||||
_NETRC_MACHINE = 'lynda'
|
|
||||||
|
|
||||||
_TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
|
_TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
|
||||||
|
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
# encoding: utf-8
|
# encoding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
@ -23,34 +21,5 @@ class M6IE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
video_id = self._match_id(url)
|
||||||
video_id = mobj.group('id')
|
return self.url_result('6play:%s' % video_id, 'SixPlay', video_id)
|
||||||
|
|
||||||
rss = self._download_xml('http://ws.m6.fr/v1/video/info/m6/bonus/%s' % video_id, video_id,
|
|
||||||
'Downloading video RSS')
|
|
||||||
|
|
||||||
title = rss.find('./channel/item/title').text
|
|
||||||
description = rss.find('./channel/item/description').text
|
|
||||||
thumbnail = rss.find('./channel/item/visuel_clip_big').text
|
|
||||||
duration = int(rss.find('./channel/item/duration').text)
|
|
||||||
view_count = int(rss.find('./channel/item/nombre_vues').text)
|
|
||||||
|
|
||||||
formats = []
|
|
||||||
for format_id in ['lq', 'sd', 'hq', 'hd']:
|
|
||||||
video_url = rss.find('./channel/item/url_video_%s' % format_id)
|
|
||||||
if video_url is None:
|
|
||||||
continue
|
|
||||||
formats.append({
|
|
||||||
'url': video_url.text,
|
|
||||||
'format_id': format_id,
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
'id': video_id,
|
|
||||||
'title': title,
|
|
||||||
'description': description,
|
|
||||||
'thumbnail': thumbnail,
|
|
||||||
'duration': duration,
|
|
||||||
'view_count': view_count,
|
|
||||||
'formats': formats,
|
|
||||||
}
|
|
||||||
|
@ -4,16 +4,12 @@ from __future__ import unicode_literals
|
|||||||
import random
|
import random
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_parse_urlencode
|
from ..utils import xpath_text
|
||||||
from ..utils import (
|
|
||||||
sanitized_Request,
|
|
||||||
xpath_text,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class MatchTVIE(InfoExtractor):
|
class MatchTVIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://matchtv\.ru/?#live-player'
|
_VALID_URL = r'https?://matchtv\.ru(?:/on-air|/?#live-player)'
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://matchtv.ru/#live-player',
|
'url': 'http://matchtv.ru/#live-player',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'matchtv-live',
|
'id': 'matchtv-live',
|
||||||
@ -24,12 +20,16 @@ class MatchTVIE(InfoExtractor):
|
|||||||
'params': {
|
'params': {
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
}
|
}, {
|
||||||
|
'url': 'http://matchtv.ru/on-air/',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = 'matchtv-live'
|
video_id = 'matchtv-live'
|
||||||
request = sanitized_Request(
|
video_url = self._download_json(
|
||||||
'http://player.matchtv.ntvplus.tv/player/smil?%s' % compat_urllib_parse_urlencode({
|
'http://player.matchtv.ntvplus.tv/player/smil', video_id,
|
||||||
|
query={
|
||||||
'ts': '',
|
'ts': '',
|
||||||
'quality': 'SD',
|
'quality': 'SD',
|
||||||
'contentId': '561d2c0df7159b37178b4567',
|
'contentId': '561d2c0df7159b37178b4567',
|
||||||
@ -40,11 +40,10 @@ class MatchTVIE(InfoExtractor):
|
|||||||
'contentType': 'channel',
|
'contentType': 'channel',
|
||||||
'timeShift': '0',
|
'timeShift': '0',
|
||||||
'platform': 'portal',
|
'platform': 'portal',
|
||||||
}),
|
},
|
||||||
headers={
|
headers={
|
||||||
'Referer': 'http://player.matchtv.ntvplus.tv/embed-player/NTVEmbedPlayer.swf',
|
'Referer': 'http://player.matchtv.ntvplus.tv/embed-player/NTVEmbedPlayer.swf',
|
||||||
})
|
})['data']['videoUrl']
|
||||||
video_url = self._download_json(request, video_id)['data']['videoUrl']
|
|
||||||
f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
|
f4m_url = xpath_text(self._download_xml(video_url, video_id), './to')
|
||||||
formats = self._extract_f4m_formats(f4m_url, video_id)
|
formats = self._extract_f4m_formats(f4m_url, video_id)
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
73
youtube_dl/extractor/meta.py
Normal file
73
youtube_dl/extractor/meta.py
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from .pladform import PladformIE
|
||||||
|
from ..utils import (
|
||||||
|
unescapeHTML,
|
||||||
|
int_or_none,
|
||||||
|
ExtractorError,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class METAIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://video\.meta\.ua/(?:iframe/)?(?P<id>[0-9]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://video.meta.ua/5502115.video',
|
||||||
|
'md5': '71b6f3ee274bef16f1ab410f7f56b476',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '5502115',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Sony Xperia Z camera test [HQ]',
|
||||||
|
'description': 'Xperia Z shoots video in FullHD HDR.',
|
||||||
|
'uploader_id': 'nomobile',
|
||||||
|
'uploader': 'CHЁZA.TV',
|
||||||
|
'upload_date': '20130211',
|
||||||
|
},
|
||||||
|
'add_ie': ['Youtube'],
|
||||||
|
}, {
|
||||||
|
'url': 'http://video.meta.ua/iframe/5502115',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# pladform embed
|
||||||
|
'url': 'http://video.meta.ua/7121015.video',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
st_html5 = self._search_regex(
|
||||||
|
r"st_html5\s*=\s*'#([^']+)'", webpage, 'uppod html5 st', default=None)
|
||||||
|
|
||||||
|
if st_html5:
|
||||||
|
# uppod st decryption algorithm is reverse engineered from function un(s) at uppod.js
|
||||||
|
json_str = ''
|
||||||
|
for i in range(0, len(st_html5), 3):
|
||||||
|
json_str += '�%s;' % st_html5[i:i + 3]
|
||||||
|
uppod_data = self._parse_json(unescapeHTML(json_str), video_id)
|
||||||
|
error = uppod_data.get('customnotfound')
|
||||||
|
if error:
|
||||||
|
raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
|
||||||
|
|
||||||
|
video_url = uppod_data['file']
|
||||||
|
info = {
|
||||||
|
'id': video_id,
|
||||||
|
'url': video_url,
|
||||||
|
'title': uppod_data.get('comment') or self._og_search_title(webpage),
|
||||||
|
'description': self._og_search_description(webpage, default=None),
|
||||||
|
'thumbnail': uppod_data.get('poster') or self._og_search_thumbnail(webpage),
|
||||||
|
'duration': int_or_none(self._og_search_property(
|
||||||
|
'video:duration', webpage, default=None)),
|
||||||
|
}
|
||||||
|
if 'youtube.com/' in video_url:
|
||||||
|
info.update({
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'ie_key': 'Youtube',
|
||||||
|
})
|
||||||
|
return info
|
||||||
|
|
||||||
|
pladform_url = PladformIE._extract_url(webpage)
|
||||||
|
if pladform_url:
|
||||||
|
return self.url_result(pladform_url)
|
@ -1,5 +1,8 @@
|
|||||||
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse_urlencode,
|
compat_urllib_parse_urlencode,
|
||||||
@ -8,6 +11,7 @@ from ..compat import (
|
|||||||
from ..utils import (
|
from ..utils import (
|
||||||
get_element_by_attribute,
|
get_element_by_attribute,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
remove_start,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -15,7 +19,7 @@ class MiTeleIE(InfoExtractor):
|
|||||||
IE_DESC = 'mitele.es'
|
IE_DESC = 'mitele.es'
|
||||||
_VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
|
_VALID_URL = r'https?://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
|
||||||
|
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
|
'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
|
||||||
# MD5 is unstable
|
# MD5 is unstable
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
@ -24,10 +28,31 @@ class MiTeleIE(InfoExtractor):
|
|||||||
'ext': 'flv',
|
'ext': 'flv',
|
||||||
'title': 'Tor, la web invisible',
|
'title': 'Tor, la web invisible',
|
||||||
'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
|
'description': 'md5:3b6fce7eaa41b2d97358726378d9369f',
|
||||||
|
'series': 'Diario de',
|
||||||
|
'season': 'La redacción',
|
||||||
|
'episode': 'Programa 144',
|
||||||
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
||||||
'duration': 2913,
|
'duration': 2913,
|
||||||
},
|
},
|
||||||
}
|
}, {
|
||||||
|
# no explicit title
|
||||||
|
'url': 'http://www.mitele.es/programas-tv/cuarto-milenio/temporada-6/programa-226/',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'eLZSwoEd1S3pVyUm8lc6F',
|
||||||
|
'display_id': 'programa-226',
|
||||||
|
'ext': 'flv',
|
||||||
|
'title': 'Cuarto Milenio - Temporada 6 - Programa 226',
|
||||||
|
'description': 'md5:50daf9fadefa4e62d9fc866d0c015701',
|
||||||
|
'series': 'Cuarto Milenio',
|
||||||
|
'season': 'Temporada 6',
|
||||||
|
'episode': 'Programa 226',
|
||||||
|
'thumbnail': 're:(?i)^https?://.*\.jpg$',
|
||||||
|
'duration': 7312,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
display_id = self._match_id(url)
|
display_id = self._match_id(url)
|
||||||
@ -70,7 +95,22 @@ class MiTeleIE(InfoExtractor):
|
|||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
title = self._search_regex(
|
title = self._search_regex(
|
||||||
r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>', webpage, 'title')
|
r'class="Destacado-text"[^>]*>\s*<strong>([^<]+)</strong>',
|
||||||
|
webpage, 'title', default=None)
|
||||||
|
|
||||||
|
mobj = re.search(r'''(?sx)
|
||||||
|
class="Destacado-text"[^>]*>.*?<h1>\s*
|
||||||
|
<span>(?P<series>[^<]+)</span>\s*
|
||||||
|
<span>(?P<season>[^<]+)</span>\s*
|
||||||
|
<span>(?P<episode>[^<]+)</span>''', webpage)
|
||||||
|
series, season, episode = mobj.groups() if mobj else [None] * 3
|
||||||
|
|
||||||
|
if not title:
|
||||||
|
if mobj:
|
||||||
|
title = '%s - %s - %s' % (series, season, episode)
|
||||||
|
else:
|
||||||
|
title = remove_start(self._search_regex(
|
||||||
|
r'<title>([^<]+)</title>', webpage, 'title'), 'Ver online ')
|
||||||
|
|
||||||
video_id = self._search_regex(
|
video_id = self._search_regex(
|
||||||
r'data-media-id\s*=\s*"([^"]+)"', webpage,
|
r'data-media-id\s*=\s*"([^"]+)"', webpage,
|
||||||
@ -83,6 +123,9 @@ class MiTeleIE(InfoExtractor):
|
|||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': get_element_by_attribute('class', 'text', webpage),
|
'description': get_element_by_attribute('class', 'text', webpage),
|
||||||
|
'series': series,
|
||||||
|
'season': season,
|
||||||
|
'episode': episode,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
|
@ -102,11 +102,11 @@ class MixcloudIE(InfoExtractor):
|
|||||||
description = self._og_search_description(webpage)
|
description = self._og_search_description(webpage)
|
||||||
like_count = parse_count(self._search_regex(
|
like_count = parse_count(self._search_regex(
|
||||||
r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
|
r'\bbutton-favorite[^>]+>.*?<span[^>]+class=["\']toggle-number[^>]+>\s*([^<]+)',
|
||||||
webpage, 'like count', fatal=False))
|
webpage, 'like count', default=None))
|
||||||
view_count = str_to_int(self._search_regex(
|
view_count = str_to_int(self._search_regex(
|
||||||
[r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
|
[r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
|
||||||
r'/listeners/?">([0-9,.]+)</a>'],
|
r'/listeners/?">([0-9,.]+)</a>'],
|
||||||
webpage, 'play count', fatal=False))
|
webpage, 'play count', default=None))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': track_id,
|
'id': track_id,
|
||||||
|
122
youtube_dl/extractor/msn.py
Normal file
122
youtube_dl/extractor/msn.py
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..compat import compat_str
|
||||||
|
from ..utils import (
|
||||||
|
determine_ext,
|
||||||
|
ExtractorError,
|
||||||
|
int_or_none,
|
||||||
|
unescapeHTML,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MSNIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?msn\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/[a-z]{2}-(?P<id>[\da-zA-Z]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/criminal-minds-shemar-moore-shares-a-touching-goodbye-message/vp-BBqQYNE',
|
||||||
|
'md5': '8442f66c116cbab1ff7098f986983458',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'BBqQYNE',
|
||||||
|
'display_id': 'criminal-minds-shemar-moore-shares-a-touching-goodbye-message',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Criminal Minds - Shemar Moore Shares A Touching Goodbye Message',
|
||||||
|
'description': 'md5:e8e89b897b222eb33a6b5067a8f1bc25',
|
||||||
|
'duration': 104,
|
||||||
|
'uploader': 'CBS Entertainment',
|
||||||
|
'uploader_id': 'IT0X5aoJ6bJgYerJXSDCgFmYPB1__54v',
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.msn.com/en-ae/news/offbeat/meet-the-nine-year-old-self-made-millionaire/ar-BBt6ZKf',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.msn.com/en-ae/video/watch/obama-a-lot-of-people-will-be-disappointed/vi-AAhxUMH',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# geo restricted
|
||||||
|
'url': 'http://www.msn.com/en-ae/foodanddrink/joinourtable/the-first-fart-makes-you-laugh-the-last-fart-makes-you-cry/vp-AAhzIBU',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.msn.com/en-ae/entertainment/bollywood/watch-how-salman-khan-reacted-when-asked-if-he-would-apologize-for-his-‘raped-woman’-comment/vi-AAhvzW6',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
mobj = re.match(self._VALID_URL, url)
|
||||||
|
video_id, display_id = mobj.group('id', 'display_id')
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
|
video = self._parse_json(
|
||||||
|
self._search_regex(
|
||||||
|
r'data-metadata\s*=\s*(["\'])(?P<data>.+?)\1',
|
||||||
|
webpage, 'video data', default='{}', group='data'),
|
||||||
|
display_id, transform_source=unescapeHTML)
|
||||||
|
|
||||||
|
if not video:
|
||||||
|
error = unescapeHTML(self._search_regex(
|
||||||
|
r'data-error=(["\'])(?P<error>.+?)\1',
|
||||||
|
webpage, 'error', group='error'))
|
||||||
|
raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
|
||||||
|
|
||||||
|
title = video['title']
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
for file_ in video.get('videoFiles', []):
|
||||||
|
format_url = file_.get('url')
|
||||||
|
if not format_url:
|
||||||
|
continue
|
||||||
|
ext = determine_ext(format_url)
|
||||||
|
# .ism is not yet supported (see
|
||||||
|
# https://github.com/rg3/youtube-dl/issues/8118)
|
||||||
|
if ext == 'ism':
|
||||||
|
continue
|
||||||
|
if 'm3u8' in format_url:
|
||||||
|
# m3u8_native should not be used here until
|
||||||
|
# https://github.com/rg3/youtube-dl/issues/9913 is fixed
|
||||||
|
m3u8_formats = self._extract_m3u8_formats(
|
||||||
|
format_url, display_id, 'mp4',
|
||||||
|
m3u8_id='hls', fatal=False)
|
||||||
|
# Despite metadata in m3u8 all video+audio formats are
|
||||||
|
# actually video-only (no audio)
|
||||||
|
for f in m3u8_formats:
|
||||||
|
if f.get('acodec') != 'none' and f.get('vcodec') != 'none':
|
||||||
|
f['acodec'] = 'none'
|
||||||
|
formats.extend(m3u8_formats)
|
||||||
|
else:
|
||||||
|
formats.append({
|
||||||
|
'url': format_url,
|
||||||
|
'ext': 'mp4',
|
||||||
|
'format_id': 'http',
|
||||||
|
'width': int_or_none(file_.get('width')),
|
||||||
|
'height': int_or_none(file_.get('height')),
|
||||||
|
})
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
subtitles = {}
|
||||||
|
for file_ in video.get('files', []):
|
||||||
|
format_url = file_.get('url')
|
||||||
|
format_code = file_.get('formatCode')
|
||||||
|
if not format_url or not format_code:
|
||||||
|
continue
|
||||||
|
if compat_str(format_code) == '3100':
|
||||||
|
subtitles.setdefault(file_.get('culture', 'en'), []).append({
|
||||||
|
'ext': determine_ext(format_url, 'ttml'),
|
||||||
|
'url': format_url,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'display_id': display_id,
|
||||||
|
'title': title,
|
||||||
|
'description': video.get('description'),
|
||||||
|
'thumbnail': video.get('headlineImage', {}).get('url'),
|
||||||
|
'duration': int_or_none(video.get('durationSecs')),
|
||||||
|
'uploader': video.get('sourceFriendly'),
|
||||||
|
'uploader_id': video.get('providerId'),
|
||||||
|
'creator': video.get('creator'),
|
||||||
|
'subtitles': subtitles,
|
||||||
|
'formats': formats,
|
||||||
|
}
|
@ -6,6 +6,7 @@ from .common import InfoExtractor
|
|||||||
from ..compat import (
|
from ..compat import (
|
||||||
compat_urllib_parse_urlencode,
|
compat_urllib_parse_urlencode,
|
||||||
compat_str,
|
compat_str,
|
||||||
|
compat_xpath,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
@ -84,9 +85,10 @@ class MTVServicesInfoExtractor(InfoExtractor):
|
|||||||
rtmp_video_url = rendition.find('./src').text
|
rtmp_video_url = rendition.find('./src').text
|
||||||
if rtmp_video_url.endswith('siteunavail.png'):
|
if rtmp_video_url.endswith('siteunavail.png'):
|
||||||
continue
|
continue
|
||||||
|
new_url = self._transform_rtmp_url(rtmp_video_url)
|
||||||
formats.append({
|
formats.append({
|
||||||
'ext': ext,
|
'ext': 'flv' if new_url.startswith('rtmp') else ext,
|
||||||
'url': self._transform_rtmp_url(rtmp_video_url),
|
'url': new_url,
|
||||||
'format_id': rendition.get('bitrate'),
|
'format_id': rendition.get('bitrate'),
|
||||||
'width': int(rendition.get('width')),
|
'width': int(rendition.get('width')),
|
||||||
'height': int(rendition.get('height')),
|
'height': int(rendition.get('height')),
|
||||||
@ -139,9 +141,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
|
|||||||
itemdoc, './/{http://search.yahoo.com/mrss/}category',
|
itemdoc, './/{http://search.yahoo.com/mrss/}category',
|
||||||
'scheme', 'urn:mtvn:video_title')
|
'scheme', 'urn:mtvn:video_title')
|
||||||
if title_el is None:
|
if title_el is None:
|
||||||
title_el = itemdoc.find('.//{http://search.yahoo.com/mrss/}title')
|
title_el = itemdoc.find(compat_xpath('.//{http://search.yahoo.com/mrss/}title'))
|
||||||
if title_el is None:
|
if title_el is None:
|
||||||
title_el = itemdoc.find('.//title') or itemdoc.find('./title')
|
title_el = itemdoc.find(compat_xpath('.//title'))
|
||||||
if title_el.text is None:
|
if title_el.text is None:
|
||||||
title_el = None
|
title_el = None
|
||||||
|
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
|
from .theplatform import ThePlatformIE
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
url_basename,
|
url_basename,
|
||||||
@ -61,7 +62,7 @@ class NationalGeographicIE(InfoExtractor):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class NationalGeographicChannelIE(InfoExtractor):
|
class NationalGeographicChannelIE(ThePlatformIE):
|
||||||
IE_NAME = 'natgeo:channel'
|
IE_NAME = 'natgeo:channel'
|
||||||
_VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/videos/(?P<id>[^/?]+)'
|
_VALID_URL = r'https?://channel\.nationalgeographic\.com/(?:wild/)?[^/]+/videos/(?P<id>[^/?]+)'
|
||||||
|
|
||||||
@ -102,12 +103,22 @@ class NationalGeographicChannelIE(InfoExtractor):
|
|||||||
release_url = self._search_regex(
|
release_url = self._search_regex(
|
||||||
r'video_auth_playlist_url\s*=\s*"([^"]+)"',
|
r'video_auth_playlist_url\s*=\s*"([^"]+)"',
|
||||||
webpage, 'release url')
|
webpage, 'release url')
|
||||||
|
query = {
|
||||||
|
'mbr': 'true',
|
||||||
|
'switch': 'http',
|
||||||
|
}
|
||||||
|
is_auth = self._search_regex(r'video_is_auth\s*=\s*"([^"]+)"', webpage, 'is auth', fatal=False)
|
||||||
|
if is_auth == 'auth':
|
||||||
|
auth_resource_id = self._search_regex(
|
||||||
|
r"video_auth_resourceId\s*=\s*'([^']+)'",
|
||||||
|
webpage, 'auth resource id')
|
||||||
|
query['auth'] = self._extract_mvpd_auth(url, display_id, 'natgeo', auth_resource_id) or ''
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'_type': 'url_transparent',
|
'_type': 'url_transparent',
|
||||||
'ie_key': 'ThePlatform',
|
'ie_key': 'ThePlatform',
|
||||||
'url': smuggle_url(
|
'url': smuggle_url(
|
||||||
update_url_query(release_url, {'mbr': 'true', 'switch': 'http'}),
|
update_url_query(release_url, query),
|
||||||
{'force_smil_url': True}),
|
{'force_smil_url': True}),
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
}
|
}
|
||||||
|
@ -9,10 +9,6 @@ from ..utils import (
|
|||||||
lowercase_escape,
|
lowercase_escape,
|
||||||
smuggle_url,
|
smuggle_url,
|
||||||
unescapeHTML,
|
unescapeHTML,
|
||||||
update_url_query,
|
|
||||||
int_or_none,
|
|
||||||
HEADRequest,
|
|
||||||
parse_iso8601,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -192,9 +188,9 @@ class CSNNEIE(InfoExtractor):
|
|||||||
|
|
||||||
|
|
||||||
class NBCNewsIE(ThePlatformIE):
|
class NBCNewsIE(ThePlatformIE):
|
||||||
_VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today)\.com/
|
_VALID_URL = r'''(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/
|
||||||
(?:video/.+?/(?P<id>\d+)|
|
(?:video/.+?/(?P<id>\d+)|
|
||||||
([^/]+/)*(?P<display_id>[^/?]+))
|
([^/]+/)*(?:.*-)?(?P<mpx_id>[^/?]+))
|
||||||
'''
|
'''
|
||||||
|
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
@ -216,13 +212,16 @@ class NBCNewsIE(ThePlatformIE):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'How Twitter Reacted To The Snowden Interview',
|
'title': 'How Twitter Reacted To The Snowden Interview',
|
||||||
'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
|
'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
|
||||||
|
'uploader': 'NBCU-NEWS',
|
||||||
|
'timestamp': 1401363060,
|
||||||
|
'upload_date': '20140529',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
|
'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
|
||||||
'md5': 'fdbf39ab73a72df5896b6234ff98518a',
|
'md5': 'fdbf39ab73a72df5896b6234ff98518a',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'Wjf9EDR3A_60',
|
'id': '529953347624',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'FULL EPISODE: Family Business',
|
'title': 'FULL EPISODE: Family Business',
|
||||||
'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
|
'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
|
||||||
@ -237,6 +236,9 @@ class NBCNewsIE(ThePlatformIE):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
|
'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
|
||||||
'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
|
'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
|
||||||
|
'timestamp': 1423104900,
|
||||||
|
'uploader': 'NBCU-NEWS',
|
||||||
|
'upload_date': '20150205',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -245,10 +247,12 @@ class NBCNewsIE(ThePlatformIE):
|
|||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '529953347624',
|
'id': '529953347624',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Volkswagen U.S. Chief: We \'Totally Screwed Up\'',
|
'title': 'Volkswagen U.S. Chief:\xa0 We Have Totally Screwed Up',
|
||||||
'description': 'md5:d22d1281a24f22ea0880741bb4dd6301',
|
'description': 'md5:c8be487b2d80ff0594c005add88d8351',
|
||||||
|
'upload_date': '20150922',
|
||||||
|
'timestamp': 1442917800,
|
||||||
|
'uploader': 'NBCU-NEWS',
|
||||||
},
|
},
|
||||||
'expected_warnings': ['http-6000 is not available']
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
|
'url': 'http://www.today.com/video/see-the-aurora-borealis-from-space-in-stunning-new-nasa-video-669831235788',
|
||||||
@ -260,6 +264,22 @@ class NBCNewsIE(ThePlatformIE):
|
|||||||
'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
|
'description': 'md5:74752b7358afb99939c5f8bb2d1d04b1',
|
||||||
'upload_date': '20160420',
|
'upload_date': '20160420',
|
||||||
'timestamp': 1461152093,
|
'timestamp': 1461152093,
|
||||||
|
'uploader': 'NBCU-NEWS',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
|
||||||
|
'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '314487875924',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'The chaotic GOP immigration vote',
|
||||||
|
'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
|
'timestamp': 1406937606,
|
||||||
|
'upload_date': '20140802',
|
||||||
|
'uploader': 'NBCU-NEWS',
|
||||||
|
'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -290,105 +310,28 @@ class NBCNewsIE(ThePlatformIE):
|
|||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
# "feature" and "nightly-news" pages use theplatform.com
|
# "feature" and "nightly-news" pages use theplatform.com
|
||||||
display_id = mobj.group('display_id')
|
video_id = mobj.group('mpx_id')
|
||||||
webpage = self._download_webpage(url, display_id)
|
if not video_id.isdigit():
|
||||||
info = None
|
webpage = self._download_webpage(url, video_id)
|
||||||
bootstrap_json = self._search_regex(
|
info = None
|
||||||
[r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
|
bootstrap_json = self._search_regex(
|
||||||
r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"'],
|
[r'(?m)(?:var\s+(?:bootstrapJson|playlistData)|NEWS\.videoObj)\s*=\s*({.+});?\s*$',
|
||||||
webpage, 'bootstrap json', default=None)
|
r'videoObj\s*:\s*({.+})', r'data-video="([^"]+)"'],
|
||||||
bootstrap = self._parse_json(
|
webpage, 'bootstrap json', default=None)
|
||||||
bootstrap_json, display_id, transform_source=unescapeHTML)
|
bootstrap = self._parse_json(
|
||||||
if 'results' in bootstrap:
|
bootstrap_json, video_id, transform_source=unescapeHTML)
|
||||||
info = bootstrap['results'][0]['video']
|
if 'results' in bootstrap:
|
||||||
elif 'video' in bootstrap:
|
info = bootstrap['results'][0]['video']
|
||||||
info = bootstrap['video']
|
elif 'video' in bootstrap:
|
||||||
else:
|
info = bootstrap['video']
|
||||||
info = bootstrap
|
|
||||||
video_id = info['mpxId']
|
|
||||||
title = info['title']
|
|
||||||
|
|
||||||
subtitles = {}
|
|
||||||
caption_links = info.get('captionLinks')
|
|
||||||
if caption_links:
|
|
||||||
for (sub_key, sub_ext) in (('smpte-tt', 'ttml'), ('web-vtt', 'vtt'), ('srt', 'srt')):
|
|
||||||
sub_url = caption_links.get(sub_key)
|
|
||||||
if sub_url:
|
|
||||||
subtitles.setdefault('en', []).append({
|
|
||||||
'url': sub_url,
|
|
||||||
'ext': sub_ext,
|
|
||||||
})
|
|
||||||
|
|
||||||
formats = []
|
|
||||||
for video_asset in info['videoAssets']:
|
|
||||||
video_url = video_asset.get('publicUrl')
|
|
||||||
if not video_url:
|
|
||||||
continue
|
|
||||||
container = video_asset.get('format')
|
|
||||||
asset_type = video_asset.get('assetType') or ''
|
|
||||||
if container == 'ISM' or asset_type == 'FireTV-Once':
|
|
||||||
continue
|
|
||||||
elif asset_type == 'OnceURL':
|
|
||||||
tp_formats, tp_subtitles = self._extract_theplatform_smil(
|
|
||||||
video_url, video_id)
|
|
||||||
formats.extend(tp_formats)
|
|
||||||
subtitles = self._merge_subtitles(subtitles, tp_subtitles)
|
|
||||||
else:
|
else:
|
||||||
tbr = int_or_none(video_asset.get('bitRate') or video_asset.get('bitrate'), 1000)
|
info = bootstrap
|
||||||
format_id = 'http%s' % ('-%d' % tbr if tbr else '')
|
video_id = info['mpxId']
|
||||||
video_url = update_url_query(
|
|
||||||
video_url, {'format': 'redirect'})
|
|
||||||
# resolve the url so that we can check availability and detect the correct extension
|
|
||||||
head = self._request_webpage(
|
|
||||||
HEADRequest(video_url), video_id,
|
|
||||||
'Checking %s url' % format_id,
|
|
||||||
'%s is not available' % format_id,
|
|
||||||
fatal=False)
|
|
||||||
if head:
|
|
||||||
video_url = head.geturl()
|
|
||||||
formats.append({
|
|
||||||
'format_id': format_id,
|
|
||||||
'url': video_url,
|
|
||||||
'width': int_or_none(video_asset.get('width')),
|
|
||||||
'height': int_or_none(video_asset.get('height')),
|
|
||||||
'tbr': tbr,
|
|
||||||
'container': video_asset.get('format'),
|
|
||||||
})
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
# http://feed.theplatform.com/f/2E2eJC/nbcnews also works
|
||||||
'description': info.get('description'),
|
'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byId=%s' % video_id,
|
||||||
'thumbnail': info.get('thumbnail'),
|
'ie_key': 'ThePlatformFeed',
|
||||||
'duration': int_or_none(info.get('duration')),
|
|
||||||
'timestamp': parse_iso8601(info.get('pubDate') or info.get('pub_date')),
|
|
||||||
'formats': formats,
|
|
||||||
'subtitles': subtitles,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class MSNBCIE(InfoExtractor):
|
|
||||||
# https URLs redirect to corresponding http ones
|
|
||||||
_VALID_URL = r'https?://www\.msnbc\.com/[^/]+/watch/(?P<id>[^/]+)'
|
|
||||||
_TEST = {
|
|
||||||
'url': 'http://www.msnbc.com/all-in-with-chris-hayes/watch/the-chaotic-gop-immigration-vote-314487875924',
|
|
||||||
'md5': '6d236bf4f3dddc226633ce6e2c3f814d',
|
|
||||||
'info_dict': {
|
|
||||||
'id': 'n_hayes_Aimm_140801_272214',
|
|
||||||
'ext': 'mp4',
|
|
||||||
'title': 'The chaotic GOP immigration vote',
|
|
||||||
'description': 'The Republican House votes on a border bill that has no chance of getting through the Senate or signed by the President and is drawing criticism from all sides.',
|
|
||||||
'thumbnail': 're:^https?://.*\.jpg$',
|
|
||||||
'timestamp': 1406937606,
|
|
||||||
'upload_date': '20140802',
|
|
||||||
'uploader': 'NBCU-NEWS',
|
|
||||||
'categories': ['MSNBC/Topics/Franchise/Best of last night', 'MSNBC/Topics/General/Congress'],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
def _real_extract(self, url):
|
|
||||||
video_id = self._match_id(url)
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
|
||||||
embed_url = self._html_search_meta('embedURL', webpage)
|
|
||||||
return self.url_result(embed_url)
|
|
||||||
|
@ -3,6 +3,7 @@ from __future__ import unicode_literals
|
|||||||
|
|
||||||
from .mtv import MTVServicesInfoExtractor
|
from .mtv import MTVServicesInfoExtractor
|
||||||
from ..compat import compat_urllib_parse_urlencode
|
from ..compat import compat_urllib_parse_urlencode
|
||||||
|
from ..utils import update_url_query
|
||||||
|
|
||||||
|
|
||||||
class NickIE(MTVServicesInfoExtractor):
|
class NickIE(MTVServicesInfoExtractor):
|
||||||
@ -61,3 +62,26 @@ class NickIE(MTVServicesInfoExtractor):
|
|||||||
|
|
||||||
def _extract_mgid(self, webpage):
|
def _extract_mgid(self, webpage):
|
||||||
return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')
|
return self._search_regex(r'data-contenturi="([^"]+)', webpage, 'mgid')
|
||||||
|
|
||||||
|
|
||||||
|
class NickDeIE(MTVServicesInfoExtractor):
|
||||||
|
IE_NAME = 'nick.de'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?nick\.de/(?:playlist|shows)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.nick.de/playlist/3773-top-videos/videos/episode/17306-zu-wasser-und-zu-land-rauchende-erdnusse',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.nick.de/shows/342-icarly',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
mrss_url = update_url_query(self._search_regex(
|
||||||
|
r'data-mrss=(["\'])(?P<url>http.+?)\1', webpage, 'mrss url', group='url'),
|
||||||
|
{'siteKey': 'nick.de'})
|
||||||
|
|
||||||
|
return self._get_videos_info_from_url(mrss_url, video_id)
|
||||||
|
55
youtube_dl/extractor/ninecninemedia.py
Normal file
55
youtube_dl/extractor/ninecninemedia.py
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
parse_iso8601,
|
||||||
|
parse_duration,
|
||||||
|
ExtractorError
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class NineCNineMediaIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'9c9media:(?P<destination_code>[^:]+):(?P<id>\d+)'
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
destination_code, video_id = re.match(self._VALID_URL, url).groups()
|
||||||
|
api_base_url = 'http://capi.9c9media.com/destinations/%s/platforms/desktop/contents/%s/' % (destination_code, video_id)
|
||||||
|
content = self._download_json(api_base_url, video_id, query={
|
||||||
|
'$include': '[contentpackages]',
|
||||||
|
})
|
||||||
|
title = content['Name']
|
||||||
|
if len(content['ContentPackages']) > 1:
|
||||||
|
raise ExtractorError('multiple content packages')
|
||||||
|
content_package = content['ContentPackages'][0]
|
||||||
|
stacks_base_url = api_base_url + 'contentpackages/%s/stacks/' % content_package['Id']
|
||||||
|
stacks = self._download_json(stacks_base_url, video_id)['Items']
|
||||||
|
if len(stacks) > 1:
|
||||||
|
raise ExtractorError('multiple stacks')
|
||||||
|
stack = stacks[0]
|
||||||
|
stack_base_url = '%s%s/manifest.' % (stacks_base_url, stack['Id'])
|
||||||
|
formats = []
|
||||||
|
formats.extend(self._extract_m3u8_formats(
|
||||||
|
stack_base_url + 'm3u8', video_id, 'mp4',
|
||||||
|
'm3u8_native', m3u8_id='hls', fatal=False))
|
||||||
|
formats.extend(self._extract_f4m_formats(
|
||||||
|
stack_base_url + 'f4m', video_id,
|
||||||
|
f4m_id='hds', fatal=False))
|
||||||
|
mp4_url = self._download_webpage(stack_base_url + 'pd', video_id, fatal=False)
|
||||||
|
if mp4_url:
|
||||||
|
formats.append({
|
||||||
|
'url': mp4_url,
|
||||||
|
'format_id': 'mp4',
|
||||||
|
})
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': title,
|
||||||
|
'description': content.get('Desc') or content.get('ShortDesc'),
|
||||||
|
'timestamp': parse_iso8601(content.get('BroadcastDateTime')),
|
||||||
|
'duration': parse_duration(content.get('BroadcastTime')),
|
||||||
|
'formats': formats,
|
||||||
|
}
|
@ -163,7 +163,7 @@ class NRKTVIE(NRKBaseIE):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': '20 spørsmål 23.05.2014',
|
'title': '20 spørsmål 23.05.2014',
|
||||||
'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
|
'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
|
||||||
'duration': 1741.52,
|
'duration': 1741,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'https://tv.nrk.no/program/mdfp15000514',
|
'url': 'https://tv.nrk.no/program/mdfp15000514',
|
||||||
@ -173,7 +173,7 @@ class NRKTVIE(NRKBaseIE):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
|
'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
|
||||||
'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
|
'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
|
||||||
'duration': 4605.08,
|
'duration': 4605,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
# single playlist video
|
# single playlist video
|
||||||
@ -260,30 +260,34 @@ class NRKPlaylistIE(InfoExtractor):
|
|||||||
|
|
||||||
class NRKSkoleIE(InfoExtractor):
|
class NRKSkoleIE(InfoExtractor):
|
||||||
IE_DESC = 'NRK Skole'
|
IE_DESC = 'NRK Skole'
|
||||||
_VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=(?P<id>[^/?#&]+)'
|
_VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/?\?.*\bmediaId=(?P<id>\d+)'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
|
'url': 'https://www.nrk.no/skole/?page=search&q=&mediaId=14099',
|
||||||
'md5': '04cd85877cc1913bce73c5d28a47e00f',
|
'md5': '6bc936b01f9dd8ed45bc58b252b2d9b6',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '6021',
|
'id': '6021',
|
||||||
'ext': 'flv',
|
'ext': 'mp4',
|
||||||
'title': 'Genetikk og eneggede tvillinger',
|
'title': 'Genetikk og eneggede tvillinger',
|
||||||
'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
|
'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
|
||||||
'duration': 399,
|
'duration': 399,
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
|
'url': 'https://www.nrk.no/skole/?page=objectives&subject=naturfag&objective=K15114&mediaId=19355',
|
||||||
'only_matching': True,
|
|
||||||
}, {
|
|
||||||
'url': 'http://www.nrk.no/skole/klippdetalj?topic=urn:x-mediadb:21379',
|
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = compat_urllib_parse_unquote(self._match_id(url))
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
webpage = self._download_webpage(
|
||||||
|
'https://mimir.nrk.no/plugin/1.0/static?mediaId=%s' % video_id,
|
||||||
|
video_id)
|
||||||
|
|
||||||
|
nrk_id = self._parse_json(
|
||||||
|
self._search_regex(
|
||||||
|
r'<script[^>]+type=["\']application/json["\'][^>]*>({.+?})</script>',
|
||||||
|
webpage, 'application json'),
|
||||||
|
video_id)['activeMedia']['psId']
|
||||||
|
|
||||||
nrk_id = self._search_regex(r'data-nrk-id=["\'](\d+)', webpage, 'nrk id')
|
|
||||||
return self.url_result('nrk:%s' % nrk_id)
|
return self.url_result('nrk:%s' % nrk_id)
|
||||||
|
@ -7,6 +7,7 @@ from .common import InfoExtractor
|
|||||||
from ..utils import (
|
from ..utils import (
|
||||||
determine_ext,
|
determine_ext,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
|
float_or_none,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -15,15 +16,14 @@ class OnionStudiosIE(InfoExtractor):
|
|||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
|
'url': 'http://www.onionstudios.com/videos/hannibal-charges-forward-stops-for-a-cocktail-2937',
|
||||||
'md5': 'd4851405d31adfadf71cd7a487b765bb',
|
'md5': 'e49f947c105b8a78a675a0ee1bddedfe',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '2937',
|
'id': '2937',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Hannibal charges forward, stops for a cocktail',
|
'title': 'Hannibal charges forward, stops for a cocktail',
|
||||||
'description': 'md5:e786add7f280b7f0fe237b64cc73df76',
|
|
||||||
'thumbnail': 're:^https?://.*\.jpg$',
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
'uploader': 'The A.V. Club',
|
'uploader': 'The A.V. Club',
|
||||||
'uploader_id': 'TheAVClub',
|
'uploader_id': 'the-av-club',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
|
'url': 'http://www.onionstudios.com/embed?id=2855&autoplay=true',
|
||||||
@ -40,50 +40,39 @@ class OnionStudiosIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
webpage = self._download_webpage(
|
video_data = self._download_json(
|
||||||
'http://www.onionstudios.com/embed?id=%s' % video_id, video_id)
|
'http://www.onionstudios.com/video/%s.json' % video_id, video_id)
|
||||||
|
|
||||||
|
title = video_data['title']
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for src in re.findall(r'<source[^>]+src="([^"]+)"', webpage):
|
for source in video_data.get('sources', []):
|
||||||
ext = determine_ext(src)
|
source_url = source.get('url')
|
||||||
if ext == 'm3u8':
|
if not source_url:
|
||||||
|
continue
|
||||||
|
content_type = source.get('content_type')
|
||||||
|
ext = determine_ext(source_url)
|
||||||
|
if content_type == 'application/x-mpegURL' or ext == 'm3u8':
|
||||||
formats.extend(self._extract_m3u8_formats(
|
formats.extend(self._extract_m3u8_formats(
|
||||||
src, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
|
source_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False))
|
||||||
else:
|
else:
|
||||||
height = int_or_none(self._search_regex(
|
tbr = int_or_none(source.get('bitrate'))
|
||||||
r'/(\d+)\.%s' % ext, src, 'height', default=None))
|
|
||||||
formats.append({
|
formats.append({
|
||||||
'format_id': ext + ('-%sp' % height if height else ''),
|
'format_id': ext + ('-%d' % tbr if tbr else ''),
|
||||||
'url': src,
|
'url': source_url,
|
||||||
'height': height,
|
'width': int_or_none(source.get('width')),
|
||||||
|
'tbr': tbr,
|
||||||
'ext': ext,
|
'ext': ext,
|
||||||
'preference': 1,
|
|
||||||
})
|
})
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
title = self._search_regex(
|
|
||||||
r'share_title\s*=\s*(["\'])(?P<title>[^\1]+?)\1',
|
|
||||||
webpage, 'title', group='title')
|
|
||||||
description = self._search_regex(
|
|
||||||
r'share_description\s*=\s*(["\'])(?P<description>[^\'"]+?)\1',
|
|
||||||
webpage, 'description', default=None, group='description')
|
|
||||||
thumbnail = self._search_regex(
|
|
||||||
r'poster\s*=\s*(["\'])(?P<thumbnail>[^\1]+?)\1',
|
|
||||||
webpage, 'thumbnail', default=False, group='thumbnail')
|
|
||||||
|
|
||||||
uploader_id = self._search_regex(
|
|
||||||
r'twitter_handle\s*=\s*(["\'])(?P<uploader_id>[^\1]+?)\1',
|
|
||||||
webpage, 'uploader id', fatal=False, group='uploader_id')
|
|
||||||
uploader = self._search_regex(
|
|
||||||
r'window\.channelName\s*=\s*(["\'])Embedded:(?P<uploader>[^\1]+?)\1',
|
|
||||||
webpage, 'uploader', default=False, group='uploader')
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': description,
|
'thumbnail': video_data.get('poster_url'),
|
||||||
'thumbnail': thumbnail,
|
'uploader': video_data.get('channel_name'),
|
||||||
'uploader': uploader,
|
'uploader_id': video_data.get('channel_slug'),
|
||||||
'uploader_id': uploader_id,
|
'duration': float_or_none(video_data.get('duration', 1000)),
|
||||||
|
'tags': video_data.get('tags'),
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
}
|
}
|
||||||
|
@ -516,9 +516,14 @@ class PBSIE(InfoExtractor):
|
|||||||
# https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
|
# https://projects.pbs.org/confluence/display/coveapi/COVE+Video+Specifications
|
||||||
if not bitrate or bitrate not in ('400k', '800k', '1200k', '2500k'):
|
if not bitrate or bitrate not in ('400k', '800k', '1200k', '2500k'):
|
||||||
continue
|
continue
|
||||||
|
f_url = re.sub(r'\d+k|baseline', bitrate, http_url)
|
||||||
|
# This may produce invalid links sometimes (e.g.
|
||||||
|
# http://www.pbs.org/wgbh/frontline/film/suicide-plan)
|
||||||
|
if not self._is_valid_url(f_url, display_id, 'http-%s video' % bitrate):
|
||||||
|
continue
|
||||||
f = m3u8_format.copy()
|
f = m3u8_format.copy()
|
||||||
f.update({
|
f.update({
|
||||||
'url': re.sub(r'\d+k|baseline', bitrate, http_url),
|
'url': f_url,
|
||||||
'format_id': m3u8_format['format_id'].replace('hls', 'http'),
|
'format_id': m3u8_format['format_id'].replace('hls', 'http'),
|
||||||
'protocol': 'http',
|
'protocol': 'http',
|
||||||
})
|
})
|
||||||
|
@ -120,9 +120,12 @@ class PeriscopeUserIE(InfoExtractor):
|
|||||||
title = user.get('display_name') or user.get('username')
|
title = user.get('display_name') or user.get('username')
|
||||||
description = user.get('description')
|
description = user.get('description')
|
||||||
|
|
||||||
|
broadcast_ids = (data_store.get('UserBroadcastHistory', {}).get('broadcastIds') or
|
||||||
|
data_store.get('BroadcastCache', {}).get('broadcastIds', []))
|
||||||
|
|
||||||
entries = [
|
entries = [
|
||||||
self.url_result(
|
self.url_result(
|
||||||
'https://www.periscope.tv/%s/%s' % (user_id, broadcast['id']))
|
'https://www.periscope.tv/%s/%s' % (user_id, broadcast_id))
|
||||||
for broadcast in data_store.get('UserBroadcastHistory', {}).get('broadcasts', [])]
|
for broadcast_id in broadcast_ids]
|
||||||
|
|
||||||
return self.playlist_result(entries, user_id, title, description)
|
return self.playlist_result(entries, user_id, title, description)
|
||||||
|
@ -49,7 +49,7 @@ class PladformIE(InfoExtractor):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _extract_url(webpage):
|
def _extract_url(webpage):
|
||||||
mobj = re.search(
|
mobj = re.search(
|
||||||
r'<iframe[^>]+src="(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)"', webpage)
|
r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//out\.pladform\.ru/player\?.+?)\1', webpage)
|
||||||
if mobj:
|
if mobj:
|
||||||
return mobj.group('url')
|
return mobj.group('url')
|
||||||
|
|
||||||
|
95
youtube_dl/extractor/polskieradio.py
Normal file
95
youtube_dl/extractor/polskieradio.py
Normal file
@ -0,0 +1,95 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..compat import (
|
||||||
|
compat_str,
|
||||||
|
compat_urllib_parse_unquote,
|
||||||
|
)
|
||||||
|
from ..utils import (
|
||||||
|
int_or_none,
|
||||||
|
strip_or_none,
|
||||||
|
unified_timestamp,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PolskieRadioIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1587943',
|
||||||
|
'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
|
||||||
|
'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
|
||||||
|
},
|
||||||
|
'playlist': [{
|
||||||
|
'md5': '2984ee6ce9046d91fc233bc1a864a09a',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1540576',
|
||||||
|
'ext': 'mp3',
|
||||||
|
'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
|
||||||
|
'timestamp': 1456594200,
|
||||||
|
'upload_date': '20160227',
|
||||||
|
'duration': 2364,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.polskieradio.pl/265/5217/Artykul/1635803,Euro-2016-nie-ma-miejsca-na-blad-Polacy-graja-ze-Szwajcaria-o-cwiercfinal',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1635803',
|
||||||
|
'title': 'Euro 2016: nie ma miejsca na błąd. Polacy grają ze Szwajcarią o ćwierćfinał',
|
||||||
|
'description': 'md5:01cb7d0cad58664095d72b51a1ebada2',
|
||||||
|
},
|
||||||
|
'playlist_mincount': 12,
|
||||||
|
}, {
|
||||||
|
'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# with mp4 video
|
||||||
|
'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
playlist_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, playlist_id)
|
||||||
|
|
||||||
|
content = self._search_regex(
|
||||||
|
r'(?s)<div[^>]+class="audio atarticle"[^>]*>(.+?)<script>',
|
||||||
|
webpage, 'content')
|
||||||
|
|
||||||
|
timestamp = unified_timestamp(self._html_search_regex(
|
||||||
|
r'(?s)<span[^>]+id="datetime2"[^>]*>(.+?)</span>',
|
||||||
|
webpage, 'timestamp', fatal=False))
|
||||||
|
|
||||||
|
entries = []
|
||||||
|
|
||||||
|
media_urls = set()
|
||||||
|
|
||||||
|
for data_media in re.findall(r'<[^>]+data-media=({[^>]+})', content):
|
||||||
|
media = self._parse_json(data_media, playlist_id, fatal=False)
|
||||||
|
if not media.get('file') or not media.get('desc'):
|
||||||
|
continue
|
||||||
|
media_url = self._proto_relative_url(media['file'], 'http:')
|
||||||
|
if media_url in media_urls:
|
||||||
|
continue
|
||||||
|
media_urls.add(media_url)
|
||||||
|
entries.append({
|
||||||
|
'id': compat_str(media['id']),
|
||||||
|
'url': media_url,
|
||||||
|
'title': compat_urllib_parse_unquote(media['desc']),
|
||||||
|
'duration': int_or_none(media.get('length')),
|
||||||
|
'vcodec': 'none' if media.get('provider') == 'audio' else None,
|
||||||
|
'timestamp': timestamp,
|
||||||
|
})
|
||||||
|
|
||||||
|
title = self._og_search_title(webpage).strip()
|
||||||
|
description = strip_or_none(self._og_search_description(webpage))
|
||||||
|
|
||||||
|
return self.playlist_result(entries, playlist_id, title, description)
|
@ -1,19 +1,32 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import json
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
|
ExtractorError,
|
||||||
int_or_none,
|
int_or_none,
|
||||||
js_to_json,
|
js_to_json,
|
||||||
qualities,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class PornHdIE(InfoExtractor):
|
class PornHdIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
|
_VALID_URL = r'https?://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
|
'url': 'http://www.pornhd.com/videos/9864/selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
|
||||||
|
'md5': 'c8b964b1f0a4b5f7f28ae3a5c9f86ad5',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '9864',
|
||||||
|
'display_id': 'selfie-restroom-masturbation-fun-with-chubby-cutie-hd-porn-video',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Restroom selfie masturbation',
|
||||||
|
'description': 'md5:3748420395e03e31ac96857a8f125b2b',
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg',
|
||||||
|
'view_count': int,
|
||||||
|
'age_limit': 18,
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
# removed video
|
||||||
'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
|
'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
|
||||||
'md5': '956b8ca569f7f4d8ec563e2c41598441',
|
'md5': '956b8ca569f7f4d8ec563e2c41598441',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
@ -25,8 +38,9 @@ class PornHdIE(InfoExtractor):
|
|||||||
'thumbnail': 're:^https?://.*\.jpg',
|
'thumbnail': 're:^https?://.*\.jpg',
|
||||||
'view_count': int,
|
'view_count': int,
|
||||||
'age_limit': 18,
|
'age_limit': 18,
|
||||||
}
|
},
|
||||||
}
|
'skip': 'Not available anymore',
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
@ -38,28 +52,38 @@ class PornHdIE(InfoExtractor):
|
|||||||
title = self._html_search_regex(
|
title = self._html_search_regex(
|
||||||
[r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
|
[r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
|
||||||
r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
|
r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
|
||||||
description = self._html_search_regex(
|
|
||||||
r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
|
|
||||||
view_count = int_or_none(self._html_search_regex(
|
|
||||||
r'(\d+) views\s*</span>', webpage, 'view count', fatal=False))
|
|
||||||
thumbnail = self._search_regex(
|
|
||||||
r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
|
|
||||||
|
|
||||||
quality = qualities(['sd', 'hd'])
|
sources = self._parse_json(js_to_json(self._search_regex(
|
||||||
sources = json.loads(js_to_json(self._search_regex(
|
|
||||||
r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
|
r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
|
||||||
webpage, 'sources')))
|
webpage, 'sources', default='{}')), video_id)
|
||||||
|
|
||||||
|
if not sources:
|
||||||
|
message = self._html_search_regex(
|
||||||
|
r'(?s)<(div|p)[^>]+class="no-video"[^>]*>(?P<value>.+?)</\1',
|
||||||
|
webpage, 'error message', group='value')
|
||||||
|
raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for qname, video_url in sources.items():
|
for format_id, video_url in sources.items():
|
||||||
if not video_url:
|
if not video_url:
|
||||||
continue
|
continue
|
||||||
|
height = int_or_none(self._search_regex(
|
||||||
|
r'^(\d+)[pP]', format_id, 'height', default=None))
|
||||||
formats.append({
|
formats.append({
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'format_id': qname,
|
'format_id': format_id,
|
||||||
'quality': quality(qname),
|
'height': height,
|
||||||
})
|
})
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
description = self._html_search_regex(
|
||||||
|
r'<(div|p)[^>]+class="description"[^>]*>(?P<value>[^<]+)</\1',
|
||||||
|
webpage, 'description', fatal=False, group='value')
|
||||||
|
view_count = int_or_none(self._html_search_regex(
|
||||||
|
r'(\d+) views\s*<', webpage, 'view count', fatal=False))
|
||||||
|
thumbnail = self._search_regex(
|
||||||
|
r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import itertools
|
import itertools
|
||||||
@ -24,7 +25,15 @@ from ..aes import (
|
|||||||
|
|
||||||
|
|
||||||
class PornHubIE(InfoExtractor):
|
class PornHubIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)(?P<id>[0-9a-z]+)'
|
IE_DESC = 'PornHub and Thumbzilla'
|
||||||
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://
|
||||||
|
(?:
|
||||||
|
(?:[a-z]+\.)?pornhub\.com/(?:view_video\.php\?viewkey=|embed/)|
|
||||||
|
(?:www\.)?thumbzilla\.com/video/
|
||||||
|
)
|
||||||
|
(?P<id>[0-9a-z]+)
|
||||||
|
'''
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
|
||||||
'md5': '1e19b41231a02eba417839222ac9d58e',
|
'md5': '1e19b41231a02eba417839222ac9d58e',
|
||||||
@ -39,13 +48,47 @@ class PornHubIE(InfoExtractor):
|
|||||||
'dislike_count': int,
|
'dislike_count': int,
|
||||||
'comment_count': int,
|
'comment_count': int,
|
||||||
'age_limit': 18,
|
'age_limit': 18,
|
||||||
}
|
},
|
||||||
|
}, {
|
||||||
|
# non-ASCII title
|
||||||
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=1331683002',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1331683002',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': '重庆婷婷女王足交',
|
||||||
|
'uploader': 'cj397186295',
|
||||||
|
'duration': 1753,
|
||||||
|
'view_count': int,
|
||||||
|
'like_count': int,
|
||||||
|
'dislike_count': int,
|
||||||
|
'comment_count': int,
|
||||||
|
'age_limit': 18,
|
||||||
|
},
|
||||||
|
'params': {
|
||||||
|
'skip_download': True,
|
||||||
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
}, {
|
}, {
|
||||||
|
# removed at the request of cam4.com
|
||||||
'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
|
'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
|
||||||
'only_matching': True,
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# removed at the request of the copyright owner
|
||||||
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=788152859',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# removed by uploader
|
||||||
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
# private video
|
||||||
|
'url': 'http://www.pornhub.com/view_video.php?viewkey=ph56fd731fce6b7',
|
||||||
|
'only_matching': True,
|
||||||
|
}, {
|
||||||
|
'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
|
||||||
|
'only_matching': True,
|
||||||
}]
|
}]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -68,27 +111,33 @@ class PornHubIE(InfoExtractor):
|
|||||||
webpage = self._download_webpage(req, video_id)
|
webpage = self._download_webpage(req, video_id)
|
||||||
|
|
||||||
error_msg = self._html_search_regex(
|
error_msg = self._html_search_regex(
|
||||||
r'(?s)<div class="userMessageSection[^"]*".*?>(.*?)</div>',
|
r'(?s)<div[^>]+class=(["\']).*?\b(?:removed|userMessageSection)\b.*?\1[^>]*>(?P<error>.+?)</div>',
|
||||||
webpage, 'error message', default=None)
|
webpage, 'error message', default=None, group='error')
|
||||||
if error_msg:
|
if error_msg:
|
||||||
error_msg = re.sub(r'\s+', ' ', error_msg)
|
error_msg = re.sub(r'\s+', ' ', error_msg)
|
||||||
raise ExtractorError(
|
raise ExtractorError(
|
||||||
'PornHub said: %s' % error_msg,
|
'PornHub said: %s' % error_msg,
|
||||||
expected=True, video_id=video_id)
|
expected=True, video_id=video_id)
|
||||||
|
|
||||||
|
# video_title from flashvars contains whitespace instead of non-ASCII (see
|
||||||
|
# http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
|
||||||
|
# on that anymore.
|
||||||
|
title = self._html_search_meta(
|
||||||
|
'twitter:title', webpage, default=None) or self._search_regex(
|
||||||
|
(r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)',
|
||||||
|
r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1',
|
||||||
|
r'shareTitle\s*=\s*(["\'])(?P<title>.+?)\1'),
|
||||||
|
webpage, 'title', group='title')
|
||||||
|
|
||||||
flashvars = self._parse_json(
|
flashvars = self._parse_json(
|
||||||
self._search_regex(
|
self._search_regex(
|
||||||
r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
|
r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
|
||||||
video_id)
|
video_id)
|
||||||
if flashvars:
|
if flashvars:
|
||||||
video_title = flashvars.get('video_title')
|
|
||||||
thumbnail = flashvars.get('image_url')
|
thumbnail = flashvars.get('image_url')
|
||||||
duration = int_or_none(flashvars.get('video_duration'))
|
duration = int_or_none(flashvars.get('video_duration'))
|
||||||
else:
|
else:
|
||||||
video_title, thumbnail, duration = [None] * 3
|
title, thumbnail, duration = [None] * 3
|
||||||
|
|
||||||
if not video_title:
|
|
||||||
video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
|
|
||||||
|
|
||||||
video_uploader = self._html_search_regex(
|
video_uploader = self._html_search_regex(
|
||||||
r'(?s)From: .+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
|
r'(?s)From: .+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
|
||||||
@ -137,7 +186,7 @@ class PornHubIE(InfoExtractor):
|
|||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'uploader': video_uploader,
|
'uploader': video_uploader,
|
||||||
'title': video_title,
|
'title': title,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
'view_count': view_count,
|
'view_count': view_count,
|
||||||
|
@ -5,7 +5,7 @@ import re
|
|||||||
|
|
||||||
from hashlib import sha1
|
from hashlib import sha1
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urllib_parse_urlencode
|
from ..compat import compat_str
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
determine_ext,
|
determine_ext,
|
||||||
@ -71,6 +71,7 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'This video is unavailable',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
|
'url': 'http://www.sixx.de/stars-style/video/sexy-laufen-in-ugg-boots-clip',
|
||||||
@ -86,6 +87,7 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'This video is unavailable',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
|
'url': 'http://www.sat1.de/film/der-ruecktritt/video/im-interview-kai-wiesinger-clip',
|
||||||
@ -101,6 +103,7 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'This video is unavailable',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
|
'url': 'http://www.kabeleins.de/tv/rosins-restaurants/videos/jagd-auf-fertigkost-im-elsthal-teil-2-ganze-folge',
|
||||||
@ -116,6 +119,7 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'This video is unavailable',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
|
'url': 'http://www.ran.de/fussball/bundesliga/video/schalke-toennies-moechte-raul-zurueck-ganze-folge',
|
||||||
@ -131,6 +135,7 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
# rtmp download
|
# rtmp download
|
||||||
'skip_download': True,
|
'skip_download': True,
|
||||||
},
|
},
|
||||||
|
'skip': 'This video is unavailable',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
|
'url': 'http://www.the-voice-of-germany.de/video/31-andreas-kuemmert-rocket-man-clip',
|
||||||
@ -227,70 +232,42 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
]
|
]
|
||||||
|
|
||||||
def _extract_clip(self, url, webpage):
|
def _extract_clip(self, url, webpage):
|
||||||
clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
|
clip_id = self._html_search_regex(
|
||||||
|
self._CLIPID_REGEXES, webpage, 'clip id')
|
||||||
|
|
||||||
access_token = 'prosieben'
|
access_token = 'prosieben'
|
||||||
client_name = 'kolibri-2.0.19-splec4'
|
client_name = 'kolibri-2.0.19-splec4'
|
||||||
client_location = url
|
client_location = url
|
||||||
|
|
||||||
videos_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos?%s' % compat_urllib_parse_urlencode({
|
video = self._download_json(
|
||||||
'access_token': access_token,
|
'http://vas.sim-technik.de/vas/live/v2/videos',
|
||||||
'client_location': client_location,
|
clip_id, 'Downloading videos JSON', query={
|
||||||
'client_name': client_name,
|
'access_token': access_token,
|
||||||
'ids': clip_id,
|
'client_location': client_location,
|
||||||
})
|
'client_name': client_name,
|
||||||
|
'ids': clip_id,
|
||||||
video = self._download_json(videos_api_url, clip_id, 'Downloading videos JSON')[0]
|
})[0]
|
||||||
|
|
||||||
if video.get('is_protected') is True:
|
if video.get('is_protected') is True:
|
||||||
raise ExtractorError('This video is DRM protected.', expected=True)
|
raise ExtractorError('This video is DRM protected.', expected=True)
|
||||||
|
|
||||||
duration = float_or_none(video.get('duration'))
|
duration = float_or_none(video.get('duration'))
|
||||||
source_ids = [source['id'] for source in video['sources']]
|
source_ids = [compat_str(source['id']) for source in video['sources']]
|
||||||
source_ids_str = ','.join(map(str, source_ids))
|
|
||||||
|
|
||||||
g = '01!8d8F_)r9]4s[qeuXfP%'
|
g = '01!8d8F_)r9]4s[qeuXfP%'
|
||||||
|
client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name]).encode('utf-8')).hexdigest()
|
||||||
|
|
||||||
client_id = g[:2] + sha1(''.join([clip_id, g, access_token, client_location, g, client_name])
|
sources = self._download_json(
|
||||||
.encode('utf-8')).hexdigest()
|
'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources' % clip_id,
|
||||||
|
clip_id, 'Downloading sources JSON', query={
|
||||||
sources_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources?%s' % (clip_id, compat_urllib_parse_urlencode({
|
'access_token': access_token,
|
||||||
'access_token': access_token,
|
'client_id': client_id,
|
||||||
'client_id': client_id,
|
'client_location': client_location,
|
||||||
'client_location': client_location,
|
'client_name': client_name,
|
||||||
'client_name': client_name,
|
})
|
||||||
}))
|
|
||||||
|
|
||||||
sources = self._download_json(sources_api_url, clip_id, 'Downloading sources JSON')
|
|
||||||
server_id = sources['server_id']
|
server_id = sources['server_id']
|
||||||
|
|
||||||
client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id,
|
|
||||||
client_location, source_ids_str, g, client_name])
|
|
||||||
.encode('utf-8')).hexdigest()
|
|
||||||
|
|
||||||
url_api_url = 'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url?%s' % (clip_id, compat_urllib_parse_urlencode({
|
|
||||||
'access_token': access_token,
|
|
||||||
'client_id': client_id,
|
|
||||||
'client_location': client_location,
|
|
||||||
'client_name': client_name,
|
|
||||||
'server_id': server_id,
|
|
||||||
'source_ids': source_ids_str,
|
|
||||||
}))
|
|
||||||
|
|
||||||
urls = self._download_json(url_api_url, clip_id, 'Downloading urls JSON')
|
|
||||||
|
|
||||||
title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
|
title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
|
||||||
description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
|
|
||||||
thumbnail = self._og_search_thumbnail(webpage)
|
|
||||||
|
|
||||||
upload_date = unified_strdate(self._html_search_regex(
|
|
||||||
self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
|
|
||||||
|
|
||||||
formats = []
|
|
||||||
|
|
||||||
urls_sources = urls['sources']
|
|
||||||
if isinstance(urls_sources, dict):
|
|
||||||
urls_sources = urls_sources.values()
|
|
||||||
|
|
||||||
def fix_bitrate(bitrate):
|
def fix_bitrate(bitrate):
|
||||||
bitrate = int_or_none(bitrate)
|
bitrate = int_or_none(bitrate)
|
||||||
@ -298,37 +275,73 @@ class ProSiebenSat1IE(InfoExtractor):
|
|||||||
return None
|
return None
|
||||||
return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
|
return (bitrate // 1000) if bitrate % 1000 == 0 else bitrate
|
||||||
|
|
||||||
for source in urls_sources:
|
formats = []
|
||||||
protocol = source['protocol']
|
for source_id in source_ids:
|
||||||
source_url = source['url']
|
client_id = g[:2] + sha1(''.join([g, clip_id, access_token, server_id, client_location, source_id, g, client_name]).encode('utf-8')).hexdigest()
|
||||||
if protocol == 'rtmp' or protocol == 'rtmpe':
|
urls = self._download_json(
|
||||||
mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
|
'http://vas.sim-technik.de/vas/live/v2/videos/%s/sources/url' % clip_id,
|
||||||
if not mobj:
|
clip_id, 'Downloading urls JSON', fatal=False, query={
|
||||||
|
'access_token': access_token,
|
||||||
|
'client_id': client_id,
|
||||||
|
'client_location': client_location,
|
||||||
|
'client_name': client_name,
|
||||||
|
'server_id': server_id,
|
||||||
|
'source_ids': source_id,
|
||||||
|
})
|
||||||
|
if not urls:
|
||||||
|
continue
|
||||||
|
if urls.get('status_code') != 0:
|
||||||
|
raise ExtractorError('This video is unavailable', expected=True)
|
||||||
|
urls_sources = urls['sources']
|
||||||
|
if isinstance(urls_sources, dict):
|
||||||
|
urls_sources = urls_sources.values()
|
||||||
|
for source in urls_sources:
|
||||||
|
source_url = source.get('url')
|
||||||
|
if not source_url:
|
||||||
continue
|
continue
|
||||||
path = mobj.group('path')
|
protocol = source.get('protocol')
|
||||||
mp4colon_index = path.rfind('mp4:')
|
mimetype = source.get('mimetype')
|
||||||
app = path[:mp4colon_index]
|
if mimetype == 'application/f4m+xml' or 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
|
||||||
play_path = path[mp4colon_index:]
|
formats.extend(self._extract_f4m_formats(
|
||||||
formats.append({
|
source_url, clip_id, f4m_id='hds', fatal=False))
|
||||||
'url': '%s/%s' % (mobj.group('url'), app),
|
elif mimetype == 'application/x-mpegURL':
|
||||||
'app': app,
|
formats.extend(self._extract_m3u8_formats(
|
||||||
'play_path': play_path,
|
source_url, clip_id, 'mp4', 'm3u8_native',
|
||||||
'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
|
m3u8_id='hls', fatal=False))
|
||||||
'page_url': 'http://www.prosieben.de',
|
else:
|
||||||
'vbr': fix_bitrate(source['bitrate']),
|
tbr = fix_bitrate(source['bitrate'])
|
||||||
'ext': 'mp4',
|
if protocol in ('rtmp', 'rtmpe'):
|
||||||
'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
|
mobj = re.search(r'^(?P<url>rtmpe?://[^/]+)/(?P<path>.+)$', source_url)
|
||||||
})
|
if not mobj:
|
||||||
elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
|
continue
|
||||||
formats.extend(self._extract_f4m_formats(source_url, clip_id))
|
path = mobj.group('path')
|
||||||
else:
|
mp4colon_index = path.rfind('mp4:')
|
||||||
formats.append({
|
app = path[:mp4colon_index]
|
||||||
'url': source_url,
|
play_path = path[mp4colon_index:]
|
||||||
'vbr': fix_bitrate(source['bitrate']),
|
formats.append({
|
||||||
})
|
'url': '%s/%s' % (mobj.group('url'), app),
|
||||||
|
'app': app,
|
||||||
|
'play_path': play_path,
|
||||||
|
'player_url': 'http://livepassdl.conviva.com/hf/ver/2.79.0.17083/LivePassModuleMain.swf',
|
||||||
|
'page_url': 'http://www.prosieben.de',
|
||||||
|
'tbr': tbr,
|
||||||
|
'ext': 'flv',
|
||||||
|
'format_id': 'rtmp%s' % ('-%d' % tbr if tbr else ''),
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
formats.append({
|
||||||
|
'url': source_url,
|
||||||
|
'tbr': tbr,
|
||||||
|
'format_id': 'http%s' % ('-%d' % tbr if tbr else ''),
|
||||||
|
})
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
description = self._html_search_regex(
|
||||||
|
self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
|
||||||
|
thumbnail = self._og_search_thumbnail(webpage)
|
||||||
|
upload_date = unified_strdate(self._html_search_regex(
|
||||||
|
self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': clip_id,
|
'id': clip_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
|
@ -2,22 +2,19 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import int_or_none
|
||||||
js_to_json,
|
|
||||||
unescapeHTML,
|
|
||||||
int_or_none,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class R7IE(InfoExtractor):
|
class R7IE(InfoExtractor):
|
||||||
_VALID_URL = r'''(?x)https?://
|
_VALID_URL = r'''(?x)
|
||||||
|
https?://
|
||||||
(?:
|
(?:
|
||||||
(?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
|
(?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
|
||||||
noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
|
noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
|
||||||
player\.r7\.com/video/i/
|
player\.r7\.com/video/i/
|
||||||
)
|
)
|
||||||
(?P<id>[\da-f]{24})
|
(?P<id>[\da-f]{24})
|
||||||
'''
|
'''
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
|
'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
|
||||||
'md5': '403c4e393617e8e8ddc748978ee8efde',
|
'md5': '403c4e393617e8e8ddc748978ee8efde',
|
||||||
@ -25,6 +22,7 @@ class R7IE(InfoExtractor):
|
|||||||
'id': '54e7050b0cf2ff57e0279389',
|
'id': '54e7050b0cf2ff57e0279389',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
|
'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
|
||||||
|
'description': 'md5:01812008664be76a6479aa58ec865b72',
|
||||||
'thumbnail': 're:^https?://.*\.jpg$',
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
'duration': 98,
|
'duration': 98,
|
||||||
'like_count': int,
|
'like_count': int,
|
||||||
@ -44,45 +42,72 @@ class R7IE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
webpage = self._download_webpage(
|
video = self._download_json(
|
||||||
'http://player.r7.com/video/i/%s' % video_id, video_id)
|
'http://player-api.r7.com/video/i/%s' % video_id, video_id)
|
||||||
|
|
||||||
item = self._parse_json(js_to_json(self._search_regex(
|
title = video['title']
|
||||||
r'(?s)var\s+item\s*=\s*({.+?});', webpage, 'player')), video_id)
|
|
||||||
|
|
||||||
title = unescapeHTML(item['title'])
|
|
||||||
thumbnail = item.get('init', {}).get('thumbUri')
|
|
||||||
duration = None
|
|
||||||
|
|
||||||
statistics = item.get('statistics', {})
|
|
||||||
like_count = int_or_none(statistics.get('likes'))
|
|
||||||
view_count = int_or_none(statistics.get('views'))
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for format_key, format_dict in item['playlist'][0].items():
|
media_url_hls = video.get('media_url_hls')
|
||||||
src = format_dict.get('src')
|
if media_url_hls:
|
||||||
if not src:
|
formats.extend(self._extract_m3u8_formats(
|
||||||
continue
|
media_url_hls, video_id, 'mp4', entry_protocol='m3u8_native',
|
||||||
format_id = format_dict.get('format') or format_key
|
m3u8_id='hls', fatal=False))
|
||||||
if duration is None:
|
media_url = video.get('media_url')
|
||||||
duration = format_dict.get('duration')
|
if media_url:
|
||||||
if '.f4m' in src:
|
f = {
|
||||||
formats.extend(self._extract_f4m_formats(src, video_id, preference=-1))
|
'url': media_url,
|
||||||
elif src.endswith('.m3u8'):
|
'format_id': 'http',
|
||||||
formats.extend(self._extract_m3u8_formats(src, video_id, 'mp4', preference=-2))
|
}
|
||||||
else:
|
# m3u8 format always matches the http format, let's copy metadata from
|
||||||
formats.append({
|
# one to another
|
||||||
'url': src,
|
m3u8_formats = list(filter(
|
||||||
'format_id': format_id,
|
lambda f: f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
|
||||||
})
|
formats))
|
||||||
|
if len(m3u8_formats) == 1:
|
||||||
|
f_copy = m3u8_formats[0].copy()
|
||||||
|
f_copy.update(f)
|
||||||
|
f_copy['protocol'] = 'http'
|
||||||
|
f = f_copy
|
||||||
|
formats.append(f)
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
description = video.get('description')
|
||||||
|
thumbnail = video.get('thumb')
|
||||||
|
duration = int_or_none(video.get('media_duration'))
|
||||||
|
like_count = int_or_none(video.get('likes'))
|
||||||
|
view_count = int_or_none(video.get('views'))
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
|
'description': description,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
'like_count': like_count,
|
'like_count': like_count,
|
||||||
'view_count': view_count,
|
'view_count': view_count,
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class R7ArticleIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:[a-zA-Z]+)\.r7\.com/(?:[^/]+/)+[^/?#&]+-(?P<id>\d+)'
|
||||||
|
_TEST = {
|
||||||
|
'url': 'http://tv.r7.com/record-play/balanco-geral/videos/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-16102015',
|
||||||
|
'only_matching': True,
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def suitable(cls, url):
|
||||||
|
return False if R7IE.suitable(url) else super(R7ArticleIE, cls).suitable(url)
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
display_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
|
video_id = self._search_regex(
|
||||||
|
r'<div[^>]+(?:id=["\']player-|class=["\']embed["\'][^>]+id=["\'])([\da-f]{24})',
|
||||||
|
webpage, 'video id')
|
||||||
|
|
||||||
|
return self.url_result('http://player.r7.com/video/i/%s' % video_id, R7IE.ie_key())
|
||||||
|
@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import(
|
from ..utils import (
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
str_to_int,
|
str_to_int,
|
||||||
)
|
)
|
||||||
|
@ -1,47 +1,141 @@
|
|||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import (
|
from ..compat import compat_urlparse
|
||||||
compat_urllib_parse,
|
|
||||||
compat_urlparse,
|
|
||||||
)
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
|
||||||
determine_ext,
|
determine_ext,
|
||||||
|
ExtractorError,
|
||||||
|
find_xpath_attr,
|
||||||
|
fix_xml_ampersands,
|
||||||
|
int_or_none,
|
||||||
parse_duration,
|
parse_duration,
|
||||||
unified_strdate,
|
unified_strdate,
|
||||||
int_or_none,
|
update_url_query,
|
||||||
xpath_text,
|
xpath_text,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class RaiTVIE(InfoExtractor):
|
class RaiBaseIE(InfoExtractor):
|
||||||
_VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+media/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
|
def _extract_relinker_formats(self, relinker_url, video_id):
|
||||||
|
formats = []
|
||||||
|
|
||||||
|
for platform in ('mon', 'flash', 'native'):
|
||||||
|
relinker = self._download_xml(
|
||||||
|
relinker_url, video_id,
|
||||||
|
note='Downloading XML metadata for platform %s' % platform,
|
||||||
|
transform_source=fix_xml_ampersands,
|
||||||
|
query={'output': 45, 'pl': platform},
|
||||||
|
headers=self.geo_verification_headers())
|
||||||
|
|
||||||
|
media_url = find_xpath_attr(relinker, './url', 'type', 'content').text
|
||||||
|
if media_url == 'http://download.rai.it/video_no_available.mp4':
|
||||||
|
self.raise_geo_restricted()
|
||||||
|
|
||||||
|
ext = determine_ext(media_url)
|
||||||
|
if (ext == 'm3u8' and platform != 'mon') or (ext == 'f4m' and platform != 'flash'):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if ext == 'm3u8':
|
||||||
|
formats.extend(self._extract_m3u8_formats(
|
||||||
|
media_url, video_id, 'mp4', 'm3u8_native',
|
||||||
|
m3u8_id='hls', fatal=False))
|
||||||
|
elif ext == 'f4m':
|
||||||
|
manifest_url = update_url_query(
|
||||||
|
media_url.replace('manifest#live_hds.f4m', 'manifest.f4m'),
|
||||||
|
{'hdcore': '3.7.0', 'plugin': 'aasp-3.7.0.39.44'})
|
||||||
|
formats.extend(self._extract_f4m_formats(
|
||||||
|
manifest_url, video_id, f4m_id='hds', fatal=False))
|
||||||
|
else:
|
||||||
|
bitrate = int_or_none(xpath_text(relinker, 'bitrate'))
|
||||||
|
formats.append({
|
||||||
|
'url': media_url,
|
||||||
|
'tbr': bitrate if bitrate > 0 else None,
|
||||||
|
'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
|
||||||
|
})
|
||||||
|
|
||||||
|
return formats
|
||||||
|
|
||||||
|
def _extract_from_content_id(self, content_id, base_url):
|
||||||
|
media = self._download_json(
|
||||||
|
'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % content_id,
|
||||||
|
content_id, 'Downloading video JSON')
|
||||||
|
|
||||||
|
thumbnails = []
|
||||||
|
for image_type in ('image', 'image_medium', 'image_300'):
|
||||||
|
thumbnail_url = media.get(image_type)
|
||||||
|
if thumbnail_url:
|
||||||
|
thumbnails.append({
|
||||||
|
'url': compat_urlparse.urljoin(base_url, thumbnail_url),
|
||||||
|
})
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
media_type = media['type']
|
||||||
|
if 'Audio' in media_type:
|
||||||
|
formats.append({
|
||||||
|
'format_id': media.get('formatoAudio'),
|
||||||
|
'url': media['audioUrl'],
|
||||||
|
'ext': media.get('formatoAudio'),
|
||||||
|
})
|
||||||
|
elif 'Video' in media_type:
|
||||||
|
formats.extend(self._extract_relinker_formats(media['mediaUri'], content_id))
|
||||||
|
self._sort_formats(formats)
|
||||||
|
else:
|
||||||
|
raise ExtractorError('not a media file')
|
||||||
|
|
||||||
|
subtitles = {}
|
||||||
|
captions = media.get('subtitlesUrl')
|
||||||
|
if captions:
|
||||||
|
STL_EXT = '.stl'
|
||||||
|
SRT_EXT = '.srt'
|
||||||
|
if captions.endswith(STL_EXT):
|
||||||
|
captions = captions[:-len(STL_EXT)] + SRT_EXT
|
||||||
|
subtitles['it'] = [{
|
||||||
|
'ext': 'srt',
|
||||||
|
'url': captions,
|
||||||
|
}]
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': content_id,
|
||||||
|
'title': media['name'],
|
||||||
|
'description': media.get('desc'),
|
||||||
|
'thumbnails': thumbnails,
|
||||||
|
'uploader': media.get('author'),
|
||||||
|
'upload_date': unified_strdate(media.get('date')),
|
||||||
|
'duration': parse_duration(media.get('length')),
|
||||||
|
'formats': formats,
|
||||||
|
'subtitles': subtitles,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class RaiTVIE(RaiBaseIE):
|
||||||
|
_VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/(?:[^/]+/)+(?:media|ondemand)/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
{
|
{
|
||||||
'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
|
'url': 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html',
|
||||||
'md5': '96382709b61dd64a6b88e0f791e6df4c',
|
'md5': '8970abf8caf8aef4696e7b1f2adfc696',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
|
'id': 'cb27157f-9dd0-4aee-b788-b1f67643a391',
|
||||||
'ext': 'flv',
|
'ext': 'mp4',
|
||||||
'title': 'Report del 07/04/2014',
|
'title': 'Report del 07/04/2014',
|
||||||
'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
|
'description': 'md5:f27c544694cacb46a078db84ec35d2d9',
|
||||||
'upload_date': '20140407',
|
'upload_date': '20140407',
|
||||||
'duration': 6160,
|
'duration': 6160,
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
# no m3u8 stream
|
||||||
'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
|
'url': 'http://www.raisport.rai.it/dl/raiSport/media/rassegna-stampa-04a9f4bd-b563-40cf-82a6-aad3529cb4a9.html',
|
||||||
'md5': 'd9751b78eac9710d62c2447b224dea39',
|
# HDS download, MD5 is unstable
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
|
'id': '04a9f4bd-b563-40cf-82a6-aad3529cb4a9',
|
||||||
'ext': 'flv',
|
'ext': 'flv',
|
||||||
'title': 'TG PRIMO TEMPO',
|
'title': 'TG PRIMO TEMPO',
|
||||||
'upload_date': '20140612',
|
'upload_date': '20140612',
|
||||||
'duration': 1758,
|
'duration': 1758,
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
},
|
},
|
||||||
|
'skip': 'Geo-restricted to Italy',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
|
'url': 'http://www.rainews.it/dl/rainews/media/state-of-the-net-Antonella-La-Carpia-regole-virali-7aafdea9-0e5d-49d5-88a6-7e65da67ae13.html',
|
||||||
@ -67,127 +161,70 @@ class RaiTVIE(InfoExtractor):
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
|
'url': 'http://www.ilcandidato.rai.it/dl/ray/media/Il-Candidato---Primo-episodio-Le-Primarie-28e5525a-b495-45e8-a7c3-bc48ba45d2b6.html',
|
||||||
'md5': '496ab63e420574447f70d02578333437',
|
'md5': 'e57493e1cb8bc7c564663f363b171847',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
|
'id': '28e5525a-b495-45e8-a7c3-bc48ba45d2b6',
|
||||||
'ext': 'flv',
|
'ext': 'mp4',
|
||||||
'title': 'Il Candidato - Primo episodio: "Le Primarie"',
|
'title': 'Il Candidato - Primo episodio: "Le Primarie"',
|
||||||
'description': 'md5:364b604f7db50594678f483353164fb8',
|
'description': 'md5:364b604f7db50594678f483353164fb8',
|
||||||
'upload_date': '20140923',
|
'upload_date': '20140923',
|
||||||
'duration': 386,
|
'duration': 386,
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
media = self._download_json(
|
|
||||||
'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-%s.html?json' % video_id,
|
|
||||||
video_id, 'Downloading video JSON')
|
|
||||||
|
|
||||||
thumbnails = []
|
return self._extract_from_content_id(video_id, url)
|
||||||
for image_type in ('image', 'image_medium', 'image_300'):
|
|
||||||
thumbnail_url = media.get(image_type)
|
|
||||||
if thumbnail_url:
|
|
||||||
thumbnails.append({
|
|
||||||
'url': thumbnail_url,
|
|
||||||
})
|
|
||||||
|
|
||||||
subtitles = []
|
|
||||||
formats = []
|
|
||||||
media_type = media['type']
|
|
||||||
if 'Audio' in media_type:
|
|
||||||
formats.append({
|
|
||||||
'format_id': media.get('formatoAudio'),
|
|
||||||
'url': media['audioUrl'],
|
|
||||||
'ext': media.get('formatoAudio'),
|
|
||||||
})
|
|
||||||
elif 'Video' in media_type:
|
|
||||||
def fix_xml(xml):
|
|
||||||
return xml.replace(' tag elementi', '').replace('>/', '</')
|
|
||||||
|
|
||||||
relinker = self._download_xml(
|
|
||||||
media['mediaUri'] + '&output=43',
|
|
||||||
video_id, transform_source=fix_xml)
|
|
||||||
|
|
||||||
has_subtitle = False
|
|
||||||
|
|
||||||
for element in relinker.findall('element'):
|
|
||||||
media_url = xpath_text(element, 'url')
|
|
||||||
ext = determine_ext(media_url)
|
|
||||||
content_type = xpath_text(element, 'content-type')
|
|
||||||
if ext == 'm3u8':
|
|
||||||
formats.extend(self._extract_m3u8_formats(
|
|
||||||
media_url, video_id, 'mp4', 'm3u8_native',
|
|
||||||
m3u8_id='hls', fatal=False))
|
|
||||||
elif ext == 'f4m':
|
|
||||||
formats.extend(self._extract_f4m_formats(
|
|
||||||
media_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44',
|
|
||||||
video_id, f4m_id='hds', fatal=False))
|
|
||||||
elif ext == 'stl':
|
|
||||||
has_subtitle = True
|
|
||||||
elif content_type.startswith('video/'):
|
|
||||||
bitrate = int_or_none(xpath_text(element, 'bitrate'))
|
|
||||||
formats.append({
|
|
||||||
'url': media_url,
|
|
||||||
'tbr': bitrate if bitrate > 0 else None,
|
|
||||||
'format_id': 'http-%d' % bitrate if bitrate > 0 else 'http',
|
|
||||||
})
|
|
||||||
elif content_type.startswith('image/'):
|
|
||||||
thumbnails.append({
|
|
||||||
'url': media_url,
|
|
||||||
})
|
|
||||||
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
if has_subtitle:
|
|
||||||
webpage = self._download_webpage(url, video_id)
|
|
||||||
subtitles = self._get_subtitles(video_id, webpage)
|
|
||||||
else:
|
|
||||||
raise ExtractorError('not a media file')
|
|
||||||
|
|
||||||
return {
|
|
||||||
'id': video_id,
|
|
||||||
'title': media['name'],
|
|
||||||
'description': media.get('desc'),
|
|
||||||
'thumbnails': thumbnails,
|
|
||||||
'uploader': media.get('author'),
|
|
||||||
'upload_date': unified_strdate(media.get('date')),
|
|
||||||
'duration': parse_duration(media.get('length')),
|
|
||||||
'formats': formats,
|
|
||||||
'subtitles': subtitles,
|
|
||||||
}
|
|
||||||
|
|
||||||
def _get_subtitles(self, video_id, webpage):
|
|
||||||
subtitles = {}
|
|
||||||
m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
|
|
||||||
if m:
|
|
||||||
captions = m.group('captions')
|
|
||||||
STL_EXT = '.stl'
|
|
||||||
SRT_EXT = '.srt'
|
|
||||||
if captions.endswith(STL_EXT):
|
|
||||||
captions = captions[:-len(STL_EXT)] + SRT_EXT
|
|
||||||
subtitles['it'] = [{
|
|
||||||
'ext': 'srt',
|
|
||||||
'url': 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions),
|
|
||||||
}]
|
|
||||||
return subtitles
|
|
||||||
|
|
||||||
|
|
||||||
class RaiIE(InfoExtractor):
|
class RaiIE(RaiBaseIE):
|
||||||
_VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
|
_VALID_URL = r'https?://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html'
|
||||||
_TESTS = [
|
_TESTS = [
|
||||||
{
|
{
|
||||||
'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
|
'url': 'http://www.report.rai.it/dl/Report/puntata/ContentItem-0c7a664b-d0f4-4b2c-8835-3f82e46f433e.html',
|
||||||
'md5': 'e0e7a8a131e249d1aa0ebf270d1d8db7',
|
'md5': '2dd727e61114e1ee9c47f0da6914e178',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
|
'id': '59d69d28-6bb6-409d-a4b5-ed44096560af',
|
||||||
'ext': 'flv',
|
'ext': 'mp4',
|
||||||
'title': 'Il pacco',
|
'title': 'Il pacco',
|
||||||
'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
|
'description': 'md5:4b1afae1364115ce5d78ed83cd2e5b3a',
|
||||||
'upload_date': '20141221',
|
'upload_date': '20141221',
|
||||||
},
|
},
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
# Direct relinker URL
|
||||||
|
'url': 'http://www.rai.tv/dl/RaiTV/dirette/PublishingBlock-1912dbbf-3f96-44c3-b4cf-523681fbacbc.html?channel=EuroNews',
|
||||||
|
# HDS live stream, MD5 is unstable
|
||||||
|
'info_dict': {
|
||||||
|
'id': '1912dbbf-3f96-44c3-b4cf-523681fbacbc',
|
||||||
|
'ext': 'flv',
|
||||||
|
'title': 'EuroNews',
|
||||||
|
},
|
||||||
|
'skip': 'Geo-restricted to Italy',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Embedded content item ID
|
||||||
|
'url': 'http://www.tg1.rai.it/dl/tg1/2010/edizioni/ContentSet-9b6e0cba-4bef-4aef-8cf0-9f7f665b7dfb-tg1.html?item=undefined',
|
||||||
|
'md5': '84c1135ce960e8822ae63cec34441d63',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '0960e765-62c8-474a-ac4b-7eb3e2be39c8',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'TG1 ore 20:00 del 02/07/2016',
|
||||||
|
'upload_date': '20160702',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'url': 'http://www.rainews.it/dl/rainews/live/ContentItem-3156f2f2-dc70-4953-8e2f-70d7489d4ce9.html',
|
||||||
|
# HDS live stream, MD5 is unstable
|
||||||
|
'info_dict': {
|
||||||
|
'id': '3156f2f2-dc70-4953-8e2f-70d7489d4ce9',
|
||||||
|
'ext': 'flv',
|
||||||
|
'title': 'La diretta di Rainews24',
|
||||||
|
},
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -201,7 +238,30 @@ class RaiIE(InfoExtractor):
|
|||||||
iframe_url = self._search_regex(
|
iframe_url = self._search_regex(
|
||||||
[r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
|
[r'<iframe[^>]+src="([^"]*/dl/[^"]+\?iframe\b[^"]*)"',
|
||||||
r'drawMediaRaiTV\(["\'](.+?)["\']'],
|
r'drawMediaRaiTV\(["\'](.+?)["\']'],
|
||||||
webpage, 'iframe')
|
webpage, 'iframe', default=None)
|
||||||
if not iframe_url.startswith('http'):
|
if iframe_url:
|
||||||
iframe_url = compat_urlparse.urljoin(url, iframe_url)
|
if not iframe_url.startswith('http'):
|
||||||
return self.url_result(iframe_url)
|
iframe_url = compat_urlparse.urljoin(url, iframe_url)
|
||||||
|
return self.url_result(iframe_url)
|
||||||
|
|
||||||
|
content_item_id = self._search_regex(
|
||||||
|
r'initEdizione\((?P<q1>[\'"])ContentItem-(?P<content_id>[^\'"]+)(?P=q1)',
|
||||||
|
webpage, 'content item ID', group='content_id', default=None)
|
||||||
|
if content_item_id:
|
||||||
|
return self._extract_from_content_id(content_item_id, url)
|
||||||
|
|
||||||
|
relinker_url = compat_urlparse.urljoin(url, self._search_regex(
|
||||||
|
r'(?:var\s+videoURL|mediaInfo\.mediaUri)\s*=\s*(?P<q1>[\'"])(?P<url>(https?:)?//mediapolis\.rai\.it/relinker/relinkerServlet\.htm\?cont=\d+)(?P=q1)',
|
||||||
|
webpage, 'relinker URL', group='url'))
|
||||||
|
formats = self._extract_relinker_formats(relinker_url, video_id)
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
title = self._search_regex(
|
||||||
|
r'var\s+videoTitolo\s*=\s*([\'"])(?P<title>[^\'"]+)\1',
|
||||||
|
webpage, 'title', group='title', default=None) or self._og_search_title(webpage)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': title,
|
||||||
|
'formats': formats,
|
||||||
|
}
|
||||||
|
@ -1,23 +1,23 @@
|
|||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
parse_duration,
|
parse_duration,
|
||||||
parse_iso8601,
|
parse_iso8601,
|
||||||
|
js_to_json,
|
||||||
)
|
)
|
||||||
|
from ..compat import compat_str
|
||||||
|
|
||||||
|
|
||||||
class RDSIE(InfoExtractor):
|
class RDSIE(InfoExtractor):
|
||||||
IE_DESC = 'RDS.ca'
|
IE_DESC = 'RDS.ca'
|
||||||
_VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<display_id>[^/]+)-(?P<id>\d+\.\d+)'
|
_VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<id>[^/]+)-\d+\.\d+'
|
||||||
|
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
|
'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '3.1132799',
|
'id': '604333',
|
||||||
'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
|
'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Fowler Jr. prend la direction de Jacksonville',
|
'title': 'Fowler Jr. prend la direction de Jacksonville',
|
||||||
@ -33,22 +33,17 @@ class RDSIE(InfoExtractor):
|
|||||||
}]
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
mobj = re.match(self._VALID_URL, url)
|
display_id = self._match_id(url)
|
||||||
video_id = mobj.group('id')
|
|
||||||
display_id = mobj.group('display_id')
|
|
||||||
|
|
||||||
webpage = self._download_webpage(url, display_id)
|
webpage = self._download_webpage(url, display_id)
|
||||||
|
|
||||||
# TODO: extract f4m from 9c9media.com
|
item = self._parse_json(self._search_regex(r'(?s)itemToPush\s*=\s*({.+?});', webpage, 'item'), display_id, js_to_json)
|
||||||
video_url = self._search_regex(
|
video_id = compat_str(item['id'])
|
||||||
r'<span[^>]+itemprop="contentURL"[^>]+content="([^"]+)"',
|
title = item.get('title') or self._og_search_title(webpage) or self._html_search_meta(
|
||||||
webpage, 'video url')
|
|
||||||
|
|
||||||
title = self._og_search_title(webpage) or self._html_search_meta(
|
|
||||||
'title', webpage, 'title', fatal=True)
|
'title', webpage, 'title', fatal=True)
|
||||||
description = self._og_search_description(webpage) or self._html_search_meta(
|
description = self._og_search_description(webpage) or self._html_search_meta(
|
||||||
'description', webpage, 'description')
|
'description', webpage, 'description')
|
||||||
thumbnail = self._og_search_thumbnail(webpage) or self._search_regex(
|
thumbnail = item.get('urlImageBig') or self._og_search_thumbnail(webpage) or self._search_regex(
|
||||||
[r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
|
[r'<link[^>]+itemprop="thumbnailUrl"[^>]+href="([^"]+)"',
|
||||||
r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
|
r'<span[^>]+itemprop="thumbnailUrl"[^>]+content="([^"]+)"'],
|
||||||
webpage, 'thumbnail', fatal=False)
|
webpage, 'thumbnail', fatal=False)
|
||||||
@ -61,13 +56,15 @@ class RDSIE(InfoExtractor):
|
|||||||
age_limit = self._family_friendly_search(webpage)
|
age_limit = self._family_friendly_search(webpage)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'display_id': display_id,
|
'display_id': display_id,
|
||||||
'url': video_url,
|
'url': '9c9media:rds_web:%s' % video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': description,
|
'description': description,
|
||||||
'thumbnail': thumbnail,
|
'thumbnail': thumbnail,
|
||||||
'timestamp': timestamp,
|
'timestamp': timestamp,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
'age_limit': age_limit,
|
'age_limit': age_limit,
|
||||||
|
'ie_key': 'NineCNineMedia',
|
||||||
}
|
}
|
||||||
|
69
youtube_dl/extractor/rockstargames.py
Normal file
69
youtube_dl/extractor/rockstargames.py
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
int_or_none,
|
||||||
|
parse_iso8601,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class RockstarGamesIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?rockstargames\.com/videos(?:/video/|#?/?\?.*\bvideo=)(?P<id>\d+)'
|
||||||
|
_TESTS = [{
|
||||||
|
'url': 'https://www.rockstargames.com/videos/video/11544/',
|
||||||
|
'md5': '03b5caa6e357a4bd50e3143fc03e5733',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '11544',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Further Adventures in Finance and Felony Trailer',
|
||||||
|
'description': 'md5:6d31f55f30cb101b5476c4a379e324a3',
|
||||||
|
'thumbnail': 're:^https?://.*\.jpg$',
|
||||||
|
'timestamp': 1464876000,
|
||||||
|
'upload_date': '20160602',
|
||||||
|
}
|
||||||
|
}, {
|
||||||
|
'url': 'http://www.rockstargames.com/videos#/?video=48',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
video = self._download_json(
|
||||||
|
'https://www.rockstargames.com/videoplayer/videos/get-video.json',
|
||||||
|
video_id, query={
|
||||||
|
'id': video_id,
|
||||||
|
'locale': 'en_us',
|
||||||
|
})['video']
|
||||||
|
|
||||||
|
title = video['title']
|
||||||
|
|
||||||
|
formats = []
|
||||||
|
for video in video['files_processed']['video/mp4']:
|
||||||
|
if not video.get('src'):
|
||||||
|
continue
|
||||||
|
resolution = video.get('resolution')
|
||||||
|
height = int_or_none(self._search_regex(
|
||||||
|
r'^(\d+)[pP]$', resolution or '', 'height', default=None))
|
||||||
|
formats.append({
|
||||||
|
'url': self._proto_relative_url(video['src']),
|
||||||
|
'format_id': resolution,
|
||||||
|
'height': height,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not formats:
|
||||||
|
youtube_id = video.get('youtube_id')
|
||||||
|
if youtube_id:
|
||||||
|
return self.url_result(youtube_id, 'Youtube')
|
||||||
|
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': title,
|
||||||
|
'description': video.get('description'),
|
||||||
|
'thumbnail': self._proto_relative_url(video.get('screencap')),
|
||||||
|
'timestamp': parse_iso8601(video.get('created')),
|
||||||
|
'formats': formats,
|
||||||
|
}
|
@ -9,7 +9,7 @@ class RTVNHIE(InfoExtractor):
|
|||||||
_VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
|
_VALID_URL = r'https?://(?:www\.)?rtvnh\.nl/video/(?P<id>[0-9]+)'
|
||||||
_TEST = {
|
_TEST = {
|
||||||
'url': 'http://www.rtvnh.nl/video/131946',
|
'url': 'http://www.rtvnh.nl/video/131946',
|
||||||
'md5': '6e1d0ab079e2a00b6161442d3ceacfc1',
|
'md5': 'cdbec9f44550763c8afc96050fa747dc',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
'id': '131946',
|
'id': '131946',
|
||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
@ -29,15 +29,29 @@ class RTVNHIE(InfoExtractor):
|
|||||||
raise ExtractorError(
|
raise ExtractorError(
|
||||||
'%s returned error code %d' % (self.IE_NAME, status), expected=True)
|
'%s returned error code %d' % (self.IE_NAME, status), expected=True)
|
||||||
|
|
||||||
formats = self._extract_smil_formats(
|
formats = []
|
||||||
'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id, fatal=False)
|
rtmp_formats = self._extract_smil_formats(
|
||||||
|
'http://www.rtvnh.nl/video/smil?m=' + video_id, video_id)
|
||||||
|
formats.extend(rtmp_formats)
|
||||||
|
|
||||||
for item in meta['source']['fb']:
|
for rtmp_format in rtmp_formats:
|
||||||
if item.get('type') == 'hls':
|
rtmp_url = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
|
||||||
formats.extend(self._extract_m3u8_formats(
|
rtsp_format = rtmp_format.copy()
|
||||||
item['file'], video_id, ext='mp4', entry_protocol='m3u8_native'))
|
del rtsp_format['play_path']
|
||||||
elif item.get('type') == '':
|
del rtsp_format['ext']
|
||||||
formats.append({'url': item['file']})
|
rtsp_format.update({
|
||||||
|
'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
|
||||||
|
'url': rtmp_url.replace('rtmp://', 'rtsp://'),
|
||||||
|
'protocol': 'rtsp',
|
||||||
|
})
|
||||||
|
formats.append(rtsp_format)
|
||||||
|
http_base_url = rtmp_url.replace('rtmp://', 'http://')
|
||||||
|
formats.extend(self._extract_m3u8_formats(
|
||||||
|
http_base_url + '/playlist.m3u8', video_id, 'mp4',
|
||||||
|
'm3u8_native', m3u8_id='hls', fatal=False))
|
||||||
|
formats.extend(self._extract_f4m_formats(
|
||||||
|
http_base_url + '/manifest.f4m',
|
||||||
|
video_id, f4m_id='hds', fatal=False))
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
@ -1,18 +1,12 @@
|
|||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
import itertools
|
|
||||||
import json
|
import json
|
||||||
import re
|
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urlparse
|
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
int_or_none,
|
int_or_none,
|
||||||
js_to_json,
|
|
||||||
mimetype2ext,
|
mimetype2ext,
|
||||||
sanitized_Request,
|
|
||||||
unified_strdate,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -27,7 +21,8 @@ class SandiaIE(InfoExtractor):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'title': 'Xyce Software Training - Section 1',
|
'title': 'Xyce Software Training - Section 1',
|
||||||
'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
|
'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
|
||||||
'upload_date': '20120904',
|
'upload_date': '20120409',
|
||||||
|
'timestamp': 1333983600,
|
||||||
'duration': 7794,
|
'duration': 7794,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -35,81 +30,36 @@ class SandiaIE(InfoExtractor):
|
|||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
req = sanitized_Request(url)
|
presentation_data = self._download_json(
|
||||||
req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
|
'http://digitalops.sandia.gov/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
|
||||||
webpage = self._download_webpage(req, video_id)
|
video_id, data=json.dumps({
|
||||||
|
'getPlayerOptionsRequest': {
|
||||||
|
'ResourceId': video_id,
|
||||||
|
'QueryString': '',
|
||||||
|
}
|
||||||
|
}), headers={
|
||||||
|
'Content-Type': 'application/json; charset=utf-8',
|
||||||
|
})['d']['Presentation']
|
||||||
|
|
||||||
js_path = self._search_regex(
|
title = presentation_data['Title']
|
||||||
r'<script type="text/javascript" src="(/Mediasite/FileServer/Presentation/[^"]+)"',
|
|
||||||
webpage, 'JS code URL')
|
|
||||||
js_url = compat_urlparse.urljoin(url, js_path)
|
|
||||||
|
|
||||||
js_code = self._download_webpage(
|
|
||||||
js_url, video_id, note='Downloading player')
|
|
||||||
|
|
||||||
def extract_str(key, **args):
|
|
||||||
return self._search_regex(
|
|
||||||
r'Mediasite\.PlaybackManifest\.%s\s*=\s*(.+);\s*?\n' % re.escape(key),
|
|
||||||
js_code, key, **args)
|
|
||||||
|
|
||||||
def extract_data(key, **args):
|
|
||||||
data_json = extract_str(key, **args)
|
|
||||||
if data_json is None:
|
|
||||||
return data_json
|
|
||||||
return self._parse_json(
|
|
||||||
data_json, video_id, transform_source=js_to_json)
|
|
||||||
|
|
||||||
formats = []
|
formats = []
|
||||||
for i in itertools.count():
|
for stream in presentation_data.get('Streams', []):
|
||||||
fd = extract_data('VideoUrls[%d]' % i, default=None)
|
for fd in stream.get('VideoUrls', []):
|
||||||
if fd is None:
|
formats.append({
|
||||||
break
|
'format_id': fd['MediaType'],
|
||||||
formats.append({
|
'format_note': fd['MimeType'].partition('/')[2],
|
||||||
'format_id': '%s' % i,
|
'ext': mimetype2ext(fd['MimeType']),
|
||||||
'format_note': fd['MimeType'].partition('/')[2],
|
'url': fd['Location'],
|
||||||
'ext': mimetype2ext(fd['MimeType']),
|
'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
|
||||||
'url': fd['Location'],
|
})
|
||||||
'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
|
|
||||||
})
|
|
||||||
self._sort_formats(formats)
|
self._sort_formats(formats)
|
||||||
|
|
||||||
slide_baseurl = compat_urlparse.urljoin(
|
|
||||||
url, extract_data('SlideBaseUrl'))
|
|
||||||
slide_template = slide_baseurl + re.sub(
|
|
||||||
r'\{0:D?([0-9+])\}', r'%0\1d', extract_data('SlideImageFileNameTemplate'))
|
|
||||||
slides = []
|
|
||||||
last_slide_time = 0
|
|
||||||
for i in itertools.count(1):
|
|
||||||
sd = extract_str('Slides[%d]' % i, default=None)
|
|
||||||
if sd is None:
|
|
||||||
break
|
|
||||||
timestamp = int_or_none(self._search_regex(
|
|
||||||
r'^Mediasite\.PlaybackManifest\.CreateSlide\("[^"]*"\s*,\s*([0-9]+),',
|
|
||||||
sd, 'slide %s timestamp' % i, fatal=False))
|
|
||||||
slides.append({
|
|
||||||
'url': slide_template % i,
|
|
||||||
'duration': timestamp - last_slide_time,
|
|
||||||
})
|
|
||||||
last_slide_time = timestamp
|
|
||||||
formats.append({
|
|
||||||
'format_id': 'slides',
|
|
||||||
'protocol': 'slideshow',
|
|
||||||
'url': json.dumps(slides),
|
|
||||||
'preference': -10000, # Downloader not yet written
|
|
||||||
})
|
|
||||||
self._sort_formats(formats)
|
|
||||||
|
|
||||||
title = extract_data('Title')
|
|
||||||
description = extract_data('Description', fatal=False)
|
|
||||||
duration = int_or_none(extract_data(
|
|
||||||
'Duration', fatal=False), scale=1000)
|
|
||||||
upload_date = unified_strdate(extract_data('AirDate', fatal=False))
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': description,
|
'description': presentation_data.get('Description'),
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
'upload_date': upload_date,
|
'timestamp': int_or_none(presentation_data.get('UnixTime'), 1000),
|
||||||
'duration': duration,
|
'duration': int_or_none(presentation_data.get('Duration'), 1000),
|
||||||
}
|
}
|
||||||
|
60
youtube_dl/extractor/sixplay.py
Normal file
60
youtube_dl/extractor/sixplay.py
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
from ..utils import (
|
||||||
|
qualities,
|
||||||
|
int_or_none,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SixPlayIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'(?:6play:|https?://(?:www\.)?6play\.fr/.+?-c_)(?P<id>[0-9]+)'
|
||||||
|
_TEST = {
|
||||||
|
'url': 'http://www.6play.fr/jamel-et-ses-amis-au-marrakech-du-rire-p_1316/jamel-et-ses-amis-au-marrakech-du-rire-2015-c_11495320',
|
||||||
|
'md5': '42310bffe4ba3982db112b9cd3467328',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '11495320',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'Jamel et ses amis au Marrakech du rire 2015',
|
||||||
|
'description': 'md5:ba2149d5c321d5201b78070ee839d872',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
clip_data = self._download_json(
|
||||||
|
'https://player.m6web.fr/v2/video/config/6play-auth/FR/%s.json' % video_id,
|
||||||
|
video_id)
|
||||||
|
video_data = clip_data['videoInfo']
|
||||||
|
|
||||||
|
quality_key = qualities(['lq', 'sd', 'hq', 'hd'])
|
||||||
|
formats = []
|
||||||
|
for source in clip_data['sources']:
|
||||||
|
source_type, source_url = source.get('type'), source.get('src')
|
||||||
|
if not source_url or source_type == 'hls/primetime':
|
||||||
|
continue
|
||||||
|
if source_type == 'application/vnd.apple.mpegURL':
|
||||||
|
formats.extend(self._extract_m3u8_formats(
|
||||||
|
source_url, video_id, 'mp4', 'm3u8_native',
|
||||||
|
m3u8_id='hls', fatal=False))
|
||||||
|
formats.extend(self._extract_f4m_formats(
|
||||||
|
source_url.replace('.m3u8', '.f4m'),
|
||||||
|
video_id, f4m_id='hds', fatal=False))
|
||||||
|
elif source_type == 'video/mp4':
|
||||||
|
quality = source.get('quality')
|
||||||
|
formats.append({
|
||||||
|
'url': source_url,
|
||||||
|
'format_id': quality,
|
||||||
|
'quality': quality_key(quality),
|
||||||
|
})
|
||||||
|
self._sort_formats(formats)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'id': video_id,
|
||||||
|
'title': video_data['title'].strip(),
|
||||||
|
'description': video_data.get('description'),
|
||||||
|
'duration': int_or_none(video_data.get('duration')),
|
||||||
|
'series': video_data.get('titlePgm'),
|
||||||
|
'formats': formats,
|
||||||
|
}
|
@ -67,7 +67,7 @@ class SkyNewsArabiaIE(SkyNewsArabiaBaseIE):
|
|||||||
|
|
||||||
|
|
||||||
class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE):
|
class SkyNewsArabiaArticleIE(SkyNewsArabiaBaseIE):
|
||||||
IE_NAME = 'skynewsarabia:video'
|
IE_NAME = 'skynewsarabia:article'
|
||||||
_VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
|
_VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
|
||||||
_TESTS = [{
|
_TESTS = [{
|
||||||
'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
|
'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
|
||||||
|
33
youtube_dl/extractor/skysports.py
Normal file
33
youtube_dl/extractor/skysports.py
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .common import InfoExtractor
|
||||||
|
|
||||||
|
|
||||||
|
class SkySportsIE(InfoExtractor):
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/(?P<id>[0-9]+)'
|
||||||
|
_TEST = {
|
||||||
|
'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
|
||||||
|
'md5': 'c44a1db29f27daf9a0003e010af82100',
|
||||||
|
'info_dict': {
|
||||||
|
'id': '10328419',
|
||||||
|
'ext': 'flv',
|
||||||
|
'title': 'Bale: Its our time to shine',
|
||||||
|
'description': 'md5:9fd1de3614d525f5addda32ac3c482c9',
|
||||||
|
},
|
||||||
|
'add_ie': ['Ooyala'],
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'_type': 'url_transparent',
|
||||||
|
'id': video_id,
|
||||||
|
'url': 'ooyala:%s' % self._search_regex(
|
||||||
|
r'data-video-id="([^"]+)"', webpage, 'ooyala id'),
|
||||||
|
'title': self._og_search_title(webpage),
|
||||||
|
'description': self._og_search_description(webpage),
|
||||||
|
'ie_key': 'Ooyala',
|
||||||
|
}
|
@ -9,6 +9,7 @@ from ..compat import (
|
|||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
ExtractorError,
|
ExtractorError,
|
||||||
|
get_element_by_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -40,7 +41,7 @@ class SlideshareIE(InfoExtractor):
|
|||||||
bucket = info['jsplayer']['video_bucket']
|
bucket = info['jsplayer']['video_bucket']
|
||||||
ext = info['jsplayer']['video_extension']
|
ext = info['jsplayer']['video_extension']
|
||||||
video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
|
video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
|
||||||
description = self._html_search_regex(
|
description = get_element_by_id('slideshow-description-paragraph', webpage) or self._html_search_regex(
|
||||||
r'(?s)<p[^>]+itemprop="description"[^>]*>(.+?)</p>', webpage,
|
r'(?s)<p[^>]+itemprop="description"[^>]*>(.+?)</p>', webpage,
|
||||||
'description', fatal=False)
|
'description', fatal=False)
|
||||||
|
|
||||||
@ -51,5 +52,5 @@ class SlideshareIE(InfoExtractor):
|
|||||||
'ext': ext,
|
'ext': ext,
|
||||||
'url': video_url,
|
'url': video_url,
|
||||||
'thumbnail': info['slideshow']['pin_image_url'],
|
'thumbnail': info['slideshow']['pin_image_url'],
|
||||||
'description': description,
|
'description': description.strip() if description else None,
|
||||||
}
|
}
|
||||||
|
@ -8,10 +8,7 @@ from ..compat import (
|
|||||||
compat_str,
|
compat_str,
|
||||||
compat_urllib_parse_urlencode,
|
compat_urllib_parse_urlencode,
|
||||||
)
|
)
|
||||||
from ..utils import (
|
from ..utils import ExtractorError
|
||||||
ExtractorError,
|
|
||||||
sanitized_Request,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class SohuIE(InfoExtractor):
|
class SohuIE(InfoExtractor):
|
||||||
@ -96,15 +93,10 @@ class SohuIE(InfoExtractor):
|
|||||||
else:
|
else:
|
||||||
base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
|
base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
|
||||||
|
|
||||||
req = sanitized_Request(base_data_url + vid_id)
|
|
||||||
|
|
||||||
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
|
|
||||||
if cn_verification_proxy:
|
|
||||||
req.add_header('Ytdl-request-proxy', cn_verification_proxy)
|
|
||||||
|
|
||||||
return self._download_json(
|
return self._download_json(
|
||||||
req, video_id,
|
base_data_url + vid_id, video_id,
|
||||||
'Downloading JSON data for %s' % vid_id)
|
'Downloading JSON data for %s' % vid_id,
|
||||||
|
headers=self.geo_verification_headers())
|
||||||
|
|
||||||
mobj = re.match(self._VALID_URL, url)
|
mobj = re.match(self._VALID_URL, url)
|
||||||
video_id = mobj.group('id')
|
video_id = mobj.group('id')
|
||||||
|
@ -4,8 +4,13 @@ from __future__ import unicode_literals
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..compat import compat_urlparse
|
|
||||||
from .spiegeltv import SpiegeltvIE
|
from .spiegeltv import SpiegeltvIE
|
||||||
|
from ..compat import compat_urlparse
|
||||||
|
from ..utils import (
|
||||||
|
extract_attributes,
|
||||||
|
unified_strdate,
|
||||||
|
get_element_by_attribute,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class SpiegelIE(InfoExtractor):
|
class SpiegelIE(InfoExtractor):
|
||||||
@ -19,6 +24,7 @@ class SpiegelIE(InfoExtractor):
|
|||||||
'title': 'Vulkanausbruch in Ecuador: Der "Feuerschlund" ist wieder aktiv',
|
'title': 'Vulkanausbruch in Ecuador: Der "Feuerschlund" ist wieder aktiv',
|
||||||
'description': 'md5:8029d8310232196eb235d27575a8b9f4',
|
'description': 'md5:8029d8310232196eb235d27575a8b9f4',
|
||||||
'duration': 49,
|
'duration': 49,
|
||||||
|
'upload_date': '20130311',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
|
'url': 'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
|
||||||
@ -29,6 +35,7 @@ class SpiegelIE(InfoExtractor):
|
|||||||
'title': 'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers',
|
'title': 'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers',
|
||||||
'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
|
'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
|
||||||
'duration': 983,
|
'duration': 983,
|
||||||
|
'upload_date': '20131115',
|
||||||
},
|
},
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
|
'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
|
||||||
@ -38,6 +45,7 @@ class SpiegelIE(InfoExtractor):
|
|||||||
'ext': 'mp4',
|
'ext': 'mp4',
|
||||||
'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
|
'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
|
||||||
'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
|
'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
|
||||||
|
'upload_date': '20140904',
|
||||||
}
|
}
|
||||||
}, {
|
}, {
|
||||||
'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
|
'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-iframe.html',
|
||||||
@ -52,10 +60,10 @@ class SpiegelIE(InfoExtractor):
|
|||||||
if SpiegeltvIE.suitable(handle.geturl()):
|
if SpiegeltvIE.suitable(handle.geturl()):
|
||||||
return self.url_result(handle.geturl(), 'Spiegeltv')
|
return self.url_result(handle.geturl(), 'Spiegeltv')
|
||||||
|
|
||||||
title = re.sub(r'\s+', ' ', self._html_search_regex(
|
video_data = extract_attributes(self._search_regex(r'(<div[^>]+id="spVideoElements"[^>]+>)', webpage, 'video element', default=''))
|
||||||
r'(?s)<(?:h1|div) class="module-title"[^>]*>(.*?)</(?:h1|div)>',
|
|
||||||
webpage, 'title'))
|
title = video_data.get('data-video-title') or get_element_by_attribute('class', 'module-title', webpage)
|
||||||
description = self._html_search_meta('description', webpage, 'description')
|
description = video_data.get('data-video-teaser') or self._html_search_meta('description', webpage, 'description')
|
||||||
|
|
||||||
base_url = self._search_regex(
|
base_url = self._search_regex(
|
||||||
[r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
|
[r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
|
||||||
@ -87,8 +95,9 @@ class SpiegelIE(InfoExtractor):
|
|||||||
return {
|
return {
|
||||||
'id': video_id,
|
'id': video_id,
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': description,
|
'description': description.strip() if description else None,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
|
'upload_date': unified_strdate(video_data.get('data-video-date')),
|
||||||
'formats': formats,
|
'formats': formats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
38
youtube_dl/extractor/sportschau.py
Normal file
38
youtube_dl/extractor/sportschau.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .wdr import WDRBaseIE
|
||||||
|
from ..utils import get_element_by_attribute
|
||||||
|
|
||||||
|
|
||||||
|
class SportschauIE(WDRBaseIE):
|
||||||
|
IE_NAME = 'Sportschau'
|
||||||
|
_VALID_URL = r'https?://(?:www\.)?sportschau\.de/(?:[^/]+/)+video-?(?P<id>[^/#?]+)\.html'
|
||||||
|
_TEST = {
|
||||||
|
'url': 'http://www.sportschau.de/uefaeuro2016/videos/video-dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100.html',
|
||||||
|
'info_dict': {
|
||||||
|
'id': 'mdb-1140188',
|
||||||
|
'display_id': 'dfb-team-geht-gut-gelaunt-ins-spiel-gegen-polen-100',
|
||||||
|
'ext': 'mp4',
|
||||||
|
'title': 'DFB-Team geht gut gelaunt ins Spiel gegen Polen',
|
||||||
|
'description': 'Vor dem zweiten Gruppenspiel gegen Polen herrscht gute Stimmung im deutschen Team. Insbesondere Bastian Schweinsteiger strotzt vor Optimismus nach seinem Tor gegen die Ukraine.',
|
||||||
|
'upload_date': '20160615',
|
||||||
|
},
|
||||||
|
'skip': 'Geo-restricted to Germany',
|
||||||
|
}
|
||||||
|
|
||||||
|
def _real_extract(self, url):
|
||||||
|
video_id = self._match_id(url)
|
||||||
|
|
||||||
|
webpage = self._download_webpage(url, video_id)
|
||||||
|
title = get_element_by_attribute('class', 'headline', webpage)
|
||||||
|
description = self._html_search_meta('description', webpage, 'description')
|
||||||
|
|
||||||
|
info = self._extract_wdr_video(webpage, video_id)
|
||||||
|
|
||||||
|
info.update({
|
||||||
|
'title': title,
|
||||||
|
'description': description,
|
||||||
|
})
|
||||||
|
|
||||||
|
return info
|
@ -9,6 +9,7 @@ from ..utils import (
|
|||||||
|
|
||||||
|
|
||||||
class SRMediathekIE(ARDMediathekIE):
|
class SRMediathekIE(ARDMediathekIE):
|
||||||
|
IE_NAME = 'sr:mediathek'
|
||||||
IE_DESC = 'Saarländischer Rundfunk'
|
IE_DESC = 'Saarländischer Rundfunk'
|
||||||
_VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
|
_VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
|
||||||
|
|
||||||
|
@ -56,7 +56,7 @@ class StitcherIE(InfoExtractor):
|
|||||||
|
|
||||||
episode = self._parse_json(
|
episode = self._parse_json(
|
||||||
js_to_json(self._search_regex(
|
js_to_json(self._search_regex(
|
||||||
r'(?s)var\s+stitcher\s*=\s*({.+?});\n', webpage, 'episode config')),
|
r'(?s)var\s+stitcher(?:Config)?\s*=\s*({.+?});\n', webpage, 'episode config')),
|
||||||
display_id)['config']['episode']
|
display_id)['config']['episode']
|
||||||
|
|
||||||
title = unescapeHTML(episode['title'])
|
title = unescapeHTML(episode['title'])
|
||||||
|
@ -5,7 +5,7 @@ import re
|
|||||||
|
|
||||||
from .common import InfoExtractor
|
from .common import InfoExtractor
|
||||||
from ..utils import (
|
from ..utils import (
|
||||||
sanitized_Request,
|
ExtractorError,
|
||||||
urlencode_postdata,
|
urlencode_postdata,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -14,7 +14,7 @@ class StreamcloudIE(InfoExtractor):
|
|||||||
IE_NAME = 'streamcloud.eu'
|
IE_NAME = 'streamcloud.eu'
|
||||||
_VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'
|
_VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'
|
||||||
|
|
||||||
_TEST = {
|
_TESTS = [{
|
||||||
'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
|
'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
|
||||||
'md5': '6bea4c7fa5daaacc2a946b7146286686',
|
'md5': '6bea4c7fa5daaacc2a946b7146286686',
|
||||||
'info_dict': {
|
'info_dict': {
|
||||||
@ -23,7 +23,10 @@ class StreamcloudIE(InfoExtractor):
|
|||||||
'title': 'youtube-dl test video \'/\\ ä ↭',
|
'title': 'youtube-dl test video \'/\\ ä ↭',
|
||||||
},
|
},
|
||||||
'skip': 'Only available from the EU'
|
'skip': 'Only available from the EU'
|
||||||
}
|
}, {
|
||||||
|
'url': 'http://streamcloud.eu/ua8cmfh1nbe6/NSHIP-148--KUC-NG--H264-.mp4.html',
|
||||||
|
'only_matching': True,
|
||||||
|
}]
|
||||||
|
|
||||||
def _real_extract(self, url):
|
def _real_extract(self, url):
|
||||||
video_id = self._match_id(url)
|
video_id = self._match_id(url)
|
||||||
@ -31,26 +34,36 @@ class StreamcloudIE(InfoExtractor):
|
|||||||
|
|
||||||
orig_webpage = self._download_webpage(url, video_id)
|
orig_webpage = self._download_webpage(url, video_id)
|
||||||
|
|
||||||
|
if '>File Not Found<' in orig_webpage:
|
||||||
|
raise ExtractorError(
|
||||||
|
'Video %s does not exist' % video_id, expected=True)
|
||||||
|
|
||||||
fields = re.findall(r'''(?x)<input\s+
|
fields = re.findall(r'''(?x)<input\s+
|
||||||
type="(?:hidden|submit)"\s+
|
type="(?:hidden|submit)"\s+
|
||||||
name="([^"]+)"\s+
|
name="([^"]+)"\s+
|
||||||
(?:id="[^"]+"\s+)?
|
(?:id="[^"]+"\s+)?
|
||||||
value="([^"]*)"
|
value="([^"]*)"
|
||||||
''', orig_webpage)
|
''', orig_webpage)
|
||||||
post = urlencode_postdata(fields)
|
|
||||||
|
|
||||||
self._sleep(12, video_id)
|
self._sleep(12, video_id)
|
||||||
headers = {
|
|
||||||
b'Content-Type': b'application/x-www-form-urlencoded',
|
|
||||||
}
|
|
||||||
req = sanitized_Request(url, post, headers)
|
|
||||||
|
|
||||||
webpage = self._download_webpage(
|
webpage = self._download_webpage(
|
||||||
req, video_id, note='Downloading video page ...')
|
url, video_id, data=urlencode_postdata(fields), headers={
|
||||||
title = self._html_search_regex(
|
b'Content-Type': b'application/x-www-form-urlencoded',
|
||||||
r'<h1[^>]*>([^<]+)<', webpage, 'title')
|
})
|
||||||
video_url = self._search_regex(
|
|
||||||
r'file:\s*"([^"]+)"', webpage, 'video URL')
|
try:
|
||||||
|
title = self._html_search_regex(
|
||||||
|
r'<h1[^>]*>([^<]+)<', webpage, 'title')
|
||||||
|
video_url = self._search_regex(
|
||||||
|
r'file:\s*"([^"]+)"', webpage, 'video URL')
|
||||||
|
except ExtractorError:
|
||||||
|
message = self._html_search_regex(
|
||||||
|
r'(?s)<div[^>]+class=(["\']).*?msgboxinfo.*?\1[^>]*>(?P<message>.+?)</div>',
|
||||||
|
webpage, 'message', default=None, group='message')
|
||||||
|
if message:
|
||||||
|
raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True)
|
||||||
|
raise
|
||||||
thumbnail = self._search_regex(
|
thumbnail = self._search_regex(
|
||||||
r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)
|
r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False)
|
||||||
|
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user