2016-10-02 04:39:18 -07:00
# coding: utf-8
2014-01-27 15:56:09 -08:00
from __future__ import unicode_literals
2013-10-16 07:57:40 -07:00
import re
2014-01-27 15:56:09 -08:00
import itertools
2013-10-16 07:57:40 -07:00
from . common import InfoExtractor
2014-12-13 03:24:42 -08:00
from . . compat import (
2013-10-16 07:57:40 -07:00
compat_str ,
2017-07-04 02:26:02 -07:00
compat_parse_qs ,
compat_urllib_parse_urlparse ,
2014-12-13 03:24:42 -08:00
)
from . . utils import (
2015-11-21 04:02:52 -08:00
determine_ext ,
2017-09-10 05:09:27 -07:00
bool_or_none ,
2017-07-04 02:26:02 -07:00
int_or_none ,
2017-09-10 05:09:27 -07:00
try_get ,
unified_timestamp ,
2018-07-21 05:08:28 -07:00
url_or_none ,
2013-10-16 07:57:40 -07:00
)
2017-09-10 04:39:13 -07:00
class RutubeBaseIE ( InfoExtractor ) :
def _extract_video ( self , video , video_id = None , require_title = True ) :
title = video [ ' title ' ] if require_title else video . get ( ' title ' )
age_limit = video . get ( ' is_adult ' )
if age_limit is not None :
age_limit = 18 if age_limit is True else 0
uploader_id = try_get ( video , lambda x : x [ ' author ' ] [ ' id ' ] )
category = try_get ( video , lambda x : x [ ' category ' ] [ ' name ' ] )
return {
' id ' : video . get ( ' id ' ) or video_id ,
' title ' : title ,
' description ' : video . get ( ' description ' ) ,
' thumbnail ' : video . get ( ' thumbnail_url ' ) ,
' duration ' : int_or_none ( video . get ( ' duration ' ) ) ,
' uploader ' : try_get ( video , lambda x : x [ ' author ' ] [ ' name ' ] ) ,
' uploader_id ' : compat_str ( uploader_id ) if uploader_id else None ,
' timestamp ' : unified_timestamp ( video . get ( ' created_ts ' ) ) ,
' category ' : [ category ] if category else None ,
' age_limit ' : age_limit ,
' view_count ' : int_or_none ( video . get ( ' hits ' ) ) ,
' comment_count ' : int_or_none ( video . get ( ' comments_count ' ) ) ,
2017-09-10 05:09:27 -07:00
' is_live ' : bool_or_none ( video . get ( ' is_livestream ' ) ) ,
2017-09-10 04:39:13 -07:00
}
class RutubeIE ( RutubeBaseIE ) :
2014-01-27 15:56:09 -08:00
IE_NAME = ' rutube '
2014-01-27 18:32:07 -08:00
IE_DESC = ' Rutube videos '
2017-03-03 09:42:51 -08:00
_VALID_URL = r ' https?://rutube \ .ru/(?:video|(?:play/)?embed)/(?P<id>[ \ da-z] {32} ) '
2013-10-16 07:57:40 -07:00
2015-12-15 14:44:17 -08:00
_TESTS = [ {
2014-01-27 15:56:09 -08:00
' url ' : ' http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/ ' ,
2017-09-10 04:39:13 -07:00
' md5 ' : ' 79938ade01294ef7e27574890d0d3769 ' ,
2014-01-27 15:56:09 -08:00
' info_dict ' : {
2014-03-30 01:35:07 -07:00
' id ' : ' 3eac3b4561676c17df9132a9a1e62e3e ' ,
2017-09-10 04:39:13 -07:00
' ext ' : ' flv ' ,
2014-01-27 15:56:09 -08:00
' title ' : ' Раненный кенгуру забежал в аптеку ' ,
2014-01-27 18:31:14 -08:00
' description ' : ' http://www.ntdtv.ru ' ,
' duration ' : 80 ,
2014-01-27 15:56:09 -08:00
' uploader ' : ' NTDRussian ' ,
' uploader_id ' : ' 29790 ' ,
2017-09-10 04:39:13 -07:00
' timestamp ' : 1381943602 ,
2014-01-27 18:31:14 -08:00
' upload_date ' : ' 20131016 ' ,
2015-08-08 09:11:06 -07:00
' age_limit ' : 0 ,
2013-10-16 07:57:40 -07:00
} ,
2015-12-15 14:44:17 -08:00
} , {
' url ' : ' http://rutube.ru/play/embed/a10e53b86e8f349080f718582ce4c661 ' ,
' only_matching ' : True ,
2017-03-03 09:42:51 -08:00
} , {
' url ' : ' http://rutube.ru/embed/a10e53b86e8f349080f718582ce4c661 ' ,
' only_matching ' : True ,
2017-07-04 02:26:02 -07:00
} , {
' url ' : ' http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/?pl_id=4252 ' ,
' only_matching ' : True ,
2017-09-10 04:39:13 -07:00
} , {
' url ' : ' https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_type=source ' ,
' only_matching ' : True ,
2015-12-15 14:44:17 -08:00
} ]
2013-10-16 07:57:40 -07:00
2017-07-04 02:26:02 -07:00
@classmethod
def suitable ( cls , url ) :
2017-09-10 04:39:13 -07:00
return False if RutubePlaylistIE . suitable ( url ) else super ( RutubeIE , cls ) . suitable ( url )
2017-07-04 02:26:02 -07:00
2017-03-03 09:46:33 -08:00
@staticmethod
def _extract_urls ( webpage ) :
return [ mobj . group ( ' url ' ) for mobj in re . finditer (
r ' <iframe[^>]+?src=([ " \' ])(?P<url>(?:https?:)?//rutube \ .ru/embed/[ \ da-z] {32} .*?) \ 1 ' ,
webpage ) ]
2013-10-16 07:57:40 -07:00
def _real_extract ( self , url ) :
2014-12-13 03:24:42 -08:00
video_id = self . _match_id ( url )
2017-09-10 04:39:13 -07:00
2014-03-30 02:26:35 -07:00
video = self . _download_json (
2014-03-30 01:35:07 -07:00
' http://rutube.ru/api/video/ %s /?format=json ' % video_id ,
video_id , ' Downloading video JSON ' )
2014-03-30 02:26:35 -07:00
2017-09-10 04:39:13 -07:00
info = self . _extract_video ( video , video_id )
2014-04-19 01:59:12 -07:00
options = self . _download_json (
2014-04-21 05:04:11 -07:00
' http://rutube.ru/api/play/options/ %s /?format=json ' % video_id ,
2014-04-19 01:59:12 -07:00
video_id , ' Downloading options JSON ' )
2015-11-21 04:02:52 -08:00
formats = [ ]
for format_id , format_url in options [ ' video_balancer ' ] . items ( ) :
ext = determine_ext ( format_url )
if ext == ' m3u8 ' :
2015-12-28 10:58:24 -08:00
formats . extend ( self . _extract_m3u8_formats (
format_url , video_id , ' mp4 ' , m3u8_id = format_id , fatal = False ) )
2015-11-21 04:02:52 -08:00
elif ext == ' f4m ' :
2015-12-28 10:58:24 -08:00
formats . extend ( self . _extract_f4m_formats (
format_url , video_id , f4m_id = format_id , fatal = False ) )
2015-11-21 04:02:52 -08:00
else :
formats . append ( {
' url ' : format_url ,
' format_id ' : format_id ,
} )
self . _sort_formats ( formats )
2013-10-16 07:57:40 -07:00
2017-09-10 04:39:13 -07:00
info [ ' formats ' ] = formats
return info
2014-01-27 15:56:09 -08:00
2015-01-03 18:15:27 -08:00
class RutubeEmbedIE ( InfoExtractor ) :
IE_NAME = ' rutube:embed '
IE_DESC = ' Rutube embedded videos '
2016-09-08 03:04:57 -07:00
_VALID_URL = r ' https?://rutube \ .ru/(?:video|play)/embed/(?P<id>[0-9]+) '
2015-01-03 18:15:27 -08:00
2015-11-21 03:39:24 -08:00
_TESTS = [ {
2015-01-03 18:15:27 -08:00
' url ' : ' http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38= ' ,
' info_dict ' : {
' id ' : ' a10e53b86e8f349080f718582ce4c661 ' ,
2017-09-10 04:39:13 -07:00
' ext ' : ' flv ' ,
' timestamp ' : 1387830582 ,
2015-01-03 18:15:27 -08:00
' upload_date ' : ' 20131223 ' ,
' uploader_id ' : ' 297833 ' ,
' description ' : ' Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix<br/><br/> восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89 ' ,
' uploader ' : ' subziro89 ILya ' ,
' title ' : ' Мистический городок Эйри в Индиан 5 серия озвучка subziro89 ' ,
} ,
' params ' : {
2017-09-10 04:39:13 -07:00
' skip_download ' : True ,
2015-01-03 18:15:27 -08:00
} ,
2015-11-21 03:39:24 -08:00
} , {
' url ' : ' http://rutube.ru/play/embed/8083783 ' ,
' only_matching ' : True ,
} ]
2015-01-03 18:15:27 -08:00
def _real_extract ( self , url ) :
embed_id = self . _match_id ( url )
webpage = self . _download_webpage ( url , embed_id )
canonical_url = self . _html_search_regex (
r ' <link \ s+rel= " canonical " \ s+href= " ([^ " ]+?) " ' , webpage ,
' Canonical URL ' )
2017-09-10 04:39:13 -07:00
return self . url_result ( canonical_url , RutubeIE . ie_key ( ) )
2015-01-03 18:15:27 -08:00
2017-09-10 04:39:13 -07:00
class RutubePlaylistBaseIE ( RutubeBaseIE ) :
def _next_page_url ( self , page_num , playlist_id , * args , * * kwargs ) :
return self . _PAGE_TEMPLATE % ( playlist_id , page_num )
def _entries ( self , playlist_id , * args , * * kwargs ) :
next_page_url = None
for pagenum in itertools . count ( 1 ) :
page = self . _download_json (
next_page_url or self . _next_page_url (
pagenum , playlist_id , * args , * * kwargs ) ,
playlist_id , ' Downloading page %s ' % pagenum )
results = page . get ( ' results ' )
if not results or not isinstance ( results , list ) :
break
for result in results :
2018-07-21 05:08:28 -07:00
video_url = url_or_none ( result . get ( ' video_url ' ) )
if not video_url :
2017-09-10 04:39:13 -07:00
continue
entry = self . _extract_video ( result , require_title = False )
entry . update ( {
' _type ' : ' url ' ,
' url ' : video_url ,
' ie_key ' : RutubeIE . ie_key ( ) ,
} )
yield entry
next_page_url = page . get ( ' next ' )
if not next_page_url or not page . get ( ' has_next ' ) :
break
def _extract_playlist ( self , playlist_id , * args , * * kwargs ) :
return self . playlist_result (
self . _entries ( playlist_id , * args , * * kwargs ) ,
playlist_id , kwargs . get ( ' playlist_name ' ) )
def _real_extract ( self , url ) :
return self . _extract_playlist ( self . _match_id ( url ) )
class RutubeChannelIE ( RutubePlaylistBaseIE ) :
2014-01-27 15:56:09 -08:00
IE_NAME = ' rutube:channel '
2014-01-27 18:32:07 -08:00
IE_DESC = ' Rutube channels '
2016-03-21 08:36:32 -07:00
_VALID_URL = r ' https?://rutube \ .ru/tags/video/(?P<id> \ d+) '
2014-08-27 15:58:24 -07:00
_TESTS = [ {
' url ' : ' http://rutube.ru/tags/video/1800/ ' ,
' info_dict ' : {
' id ' : ' 1800 ' ,
} ,
' playlist_mincount ' : 68 ,
} ]
2014-01-27 15:56:09 -08:00
_PAGE_TEMPLATE = ' http://rutube.ru/api/tags/video/ %s /?page= %s &format=json '
2017-09-10 04:39:13 -07:00
class RutubeMovieIE ( RutubePlaylistBaseIE ) :
2014-01-27 15:56:09 -08:00
IE_NAME = ' rutube:movie '
2014-01-27 18:32:07 -08:00
IE_DESC = ' Rutube movies '
2016-03-21 08:36:32 -07:00
_VALID_URL = r ' https?://rutube \ .ru/metainfo/tv/(?P<id> \ d+) '
2014-08-27 15:58:24 -07:00
_TESTS = [ ]
2014-01-27 15:56:09 -08:00
_MOVIE_TEMPLATE = ' http://rutube.ru/api/metainfo/tv/ %s /?format=json '
_PAGE_TEMPLATE = ' http://rutube.ru/api/metainfo/tv/ %s /video?page= %s &format=json '
def _real_extract ( self , url ) :
2014-12-13 03:24:42 -08:00
movie_id = self . _match_id ( url )
2014-03-30 02:26:35 -07:00
movie = self . _download_json (
2014-01-27 18:32:07 -08:00
self . _MOVIE_TEMPLATE % movie_id , movie_id ,
' Downloading movie JSON ' )
2017-09-10 04:39:13 -07:00
return self . _extract_playlist (
movie_id , playlist_name = movie . get ( ' name ' ) )
2014-01-27 17:47:17 -08:00
2017-09-10 04:39:13 -07:00
class RutubePersonIE ( RutubePlaylistBaseIE ) :
2014-01-27 17:47:17 -08:00
IE_NAME = ' rutube:person '
IE_DESC = ' Rutube person videos '
2016-03-21 08:36:32 -07:00
_VALID_URL = r ' https?://rutube \ .ru/video/person/(?P<id> \ d+) '
2014-08-27 15:58:24 -07:00
_TESTS = [ {
' url ' : ' http://rutube.ru/video/person/313878/ ' ,
' info_dict ' : {
' id ' : ' 313878 ' ,
} ,
' playlist_mincount ' : 37 ,
} ]
2014-01-27 17:47:17 -08:00
2014-01-27 18:32:07 -08:00
_PAGE_TEMPLATE = ' http://rutube.ru/api/video/person/ %s /?page= %s &format=json '
2017-07-04 02:26:02 -07:00
2017-09-10 04:39:13 -07:00
class RutubePlaylistIE ( RutubePlaylistBaseIE ) :
2017-07-04 02:26:02 -07:00
IE_NAME = ' rutube:playlist '
IE_DESC = ' Rutube playlists '
2017-09-10 04:39:13 -07:00
_VALID_URL = r ' https?://rutube \ .ru/(?:video|(?:play/)?embed)/[ \ da-z] {32} / \ ?.*? \ bpl_id=(?P<id> \ d+) '
2017-07-04 02:26:02 -07:00
_TESTS = [ {
2017-09-10 04:39:13 -07:00
' url ' : ' https://rutube.ru/video/cecd58ed7d531fc0f3d795d51cee9026/?pl_id=3097&pl_type=tag ' ,
2017-07-04 02:26:02 -07:00
' info_dict ' : {
2017-09-10 04:39:13 -07:00
' id ' : ' 3097 ' ,
2017-07-04 02:26:02 -07:00
} ,
2017-09-10 04:39:13 -07:00
' playlist_count ' : 27 ,
} , {
' url ' : ' https://rutube.ru/video/10b3a03fc01d5bbcc632a2f3514e8aab/?pl_id=4252&pl_type=source ' ,
' only_matching ' : True ,
2017-07-04 02:26:02 -07:00
} ]
2017-09-10 04:39:13 -07:00
_PAGE_TEMPLATE = ' http://rutube.ru/api/playlist/ %s / %s /?page= %s &format=json '
2017-07-04 02:26:02 -07:00
2017-09-10 13:22:27 -07:00
@classmethod
def suitable ( cls , url ) :
if not super ( RutubePlaylistIE , cls ) . suitable ( url ) :
return False
2017-07-04 02:26:02 -07:00
params = compat_parse_qs ( compat_urllib_parse_urlparse ( url ) . query )
2017-09-10 04:39:13 -07:00
return params . get ( ' pl_type ' , [ None ] ) [ 0 ] and int_or_none ( params . get ( ' pl_id ' , [ None ] ) [ 0 ] )
2017-07-04 02:26:02 -07:00
2017-09-10 04:39:13 -07:00
def _next_page_url ( self , page_num , playlist_id , item_kind ) :
return self . _PAGE_TEMPLATE % ( item_kind , playlist_id , page_num )
2017-07-04 02:26:02 -07:00
2017-09-10 04:39:13 -07:00
def _real_extract ( self , url ) :
qs = compat_parse_qs ( compat_urllib_parse_urlparse ( url ) . query )
playlist_kind = qs [ ' pl_type ' ] [ 0 ]
playlist_id = qs [ ' pl_id ' ] [ 0 ]
return self . _extract_playlist ( playlist_id , item_kind = playlist_kind )