release 2017.01.22

Add audio format argument dependency warning

.github/ISSUE_TEMPLATE.md

@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.02*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.02**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2017.01.22*. If it's not read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2017.01.22**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through [README](https://github.com/rg3/youtube-dl/blob/master/README.md) and **most notably** [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -35,7 +35,7 @@ $ youtube-dl -v <your command line>
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2017.01.02
+[debug] youtube-dl version 2017.01.22
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}

CONTRIBUTING.md

@@ -124,7 +124,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 'id': '42',
                 'ext': 'mp4',
                 'title': 'Video title goes here',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 # TODO more properties, either as:
                 # * A value
                 # * MD5 checksum; start the string with md5:

ChangeLog

@@ -1,3 +1,104 @@
+version 2017.01.22
+
+Extractors
++ [pornflip] Add support for pornflip.com (#11556, #11795)
+* [chaturbate] Fix extraction (#11797, #11802)
++ [azmedien] Add support for AZ Medien sites (#11784, #11785)
++ [nextmedia] Support redirected URLs
++ [vimeo:channel] Extract videos' titles for playlist entries (#11796)
++ [youtube] Extract episode metadata (#9695, #11774)
++ [cspan] Support Ustream embedded videos (#11547)
++ [1tv] Add support for HLS videos (#11786)
+* [uol] Fix extraction (#11770)
+* [mtv] Relax triforce feed regular expression (#11766)
+
+
+version 2017.01.18
+
+Extractors
+* [bilibili] Fix extraction (#11077)
++ [canalplus] Add fallback for video id (#11764)
+* [20min] Fix extraction (#11683, #11751)
+* [imdb] Extend URL regular expression (#11744)
++ [naver] Add support for tv.naver.com links (#11743)
+
+
+version 2017.01.16
+
+Core
+* [options] Apply custom config to final composite configuration (#11741)
+* [YoutubeDL] Improve protocol auto determining (#11720)
+
+Extractors
+* [xiami] Relax URL regular expressions
+* [xiami] Improve track metadata extraction (#11699)
++ [limelight] Check hand-make direct HTTP links
++ [limelight] Add support for direct HTTP links at video.llnw.net (#11737)
++ [brightcove] Recognize another player ID pattern (#11688)
++ [niconico] Support login via cookies (#7968)
+* [yourupload] Fix extraction (#11601)
++ [beam:live] Add support for beam.pro live streams (#10702, #11596)
+* [vevo] Improve geo restriction detection
++ [dramafever] Add support for URLs with language code (#11714)
+* [cbc] Improve playlist support (#11704)
+
+
+version 2017.01.14
+
+Core
++ [common] Add ability to customize akamai manifest host
++ [utils] Add more date formats
+
+Extractors
+- [mtv] Eliminate _transform_rtmp_url
+* [mtv] Generalize triforce mgid extraction
++ [cmt] Add support for full episodes and video clips (#11623)
++ [mitele] Extract DASH formats
++ [ooyala] Add support for videos with embedToken (#11684)
+* [mixcloud] Fix extraction (#11674)
+* [openload] Fix extraction (#10408)
+* [tv4] Improve extraction (#11698)
+* [freesound] Fix and improve extraction (#11602)
++ [nick] Add support for beta.nick.com (#11655)
+* [mtv,cc] Use HLS by default with native HLS downloader (#11641)
+* [mtv] Fix non-HLS extraction
+
+
+version 2017.01.10
+
+Extractors
+* [youtube] Fix extraction (#11663, #11664)
++ [inc] Add support for inc.com (#11277, #11647)
++ [youtube] Add itag 212 (#11575)
++ [egghead:course] Add support for egghead.io courses
+
+
+version 2017.01.08
+
+Core
+* Fix "invalid escape sequence" errors under Python 3.6 (#11581)
+
+Extractors
++ [hitrecord] Add support for hitrecord.org (#10867, #11626)
+- [videott] Remove extractor
+* [swrmediathek] Improve extraction
+- [sharesix] Remove extractor
+- [aol:features] Remove extractor
+* [sendtonews] Improve info extraction
+* [3sat,phoenix] Fix extraction (#11619)
+* [comedycentral/mtv] Add support for HLS videos (#11600)
+* [discoverygo] Fix JSON data parsing (#11219, #11522)
+
+
+version 2017.01.05
+
+Extractors
++ [zdf] Fix extraction (#11055, #11063)
+* [pornhub:playlist] Improve extraction (#11594)
++ [cctv] Add support for ncpa-classic.com (#11591)
++ [tunein] Add support for embeds (#11579)
+
+
 version 2017.01.02
 
 Extractors

README.md

@@ -374,7 +374,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                      avprobe)
     --audio-format FORMAT            Specify audio format: "best", "aac",
                                      "vorbis", "mp3", "m4a", "opus", or "wav";
-                                     "best" by default
+                                     "best" by default; No effect without -x
     --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
                                      a value between 0 (better) and 9 (worse)
                                      for VBR or a specific bitrate like 128K
@@ -446,6 +446,8 @@ Note that options in configuration file are just the same options aka switches u
 
 You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
+You can also use `--config-location` if you want to use custom configuration file for a particular youtube-dl run.
+
 ### Authentication with `.netrc` file
 
 You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](http://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you:
@@ -839,7 +841,7 @@ Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`.
 
 In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [Export Cookies](https://addons.mozilla.org/en-US/firefox/addon/export-cookies/) (for Firefox).
 
-Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows, `LF` (`\n`) for Linux and `CR` (`\r`) for Mac OS. `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
+Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, Mac OS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 
 Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare).
 
@@ -961,7 +963,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 'id': '42',
                 'ext': 'mp4',
                 'title': 'Video title goes here',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 # TODO more properties, either as:
                 # * A value
                 # * MD5 checksum; start the string with md5:

docs/supportedsites.md

@@ -74,6 +74,8 @@
  - **awaan:live**
  - **awaan:season**
  - **awaan:video**
+ - **AZMedien**: AZ Medien videos
+ - **AZMedienShow**: AZ Medien shows
  - **Azubu**
  - **AzubuLive**
  - **BaiduVideo**: 百度视频
@@ -86,6 +88,7 @@
  - **bbc.co.uk:article**: BBC articles
  - **bbc.co.uk:iplayer:playlist**
  - **bbc.co.uk:playlist**
+ - **Beam:live**
  - **Beatport**
  - **Beeg**
  - **BehindKink**
@@ -214,6 +217,7 @@
  - **EaglePlatform**
  - **EbaumsWorld**
  - **EchoMsk**
+ - **egghead:course**: egghead.io course
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
@@ -240,7 +244,6 @@
  - **fc2**
  - **fc2:embed**
  - **Fczenit**
- - **features.aol.com**
  - **fernsehkritik.tv**
  - **Firstpost**
  - **FiveTV**
@@ -304,6 +307,7 @@
  - **history:topic**: History.com Topic
  - **hitbox**
  - **hitbox:live**
+ - **HitRecord**
  - **HornBunny**
  - **HotNewHipHop**
  - **HotStar**
@@ -321,6 +325,7 @@
  - **Imgur**
  - **ImgurAlbum**
  - **Ina**
+ - **Inc**
  - **Indavideo**
  - **IndavideoEmbed**
  - **InfoQ**
@@ -569,6 +574,7 @@
  - **PolskieRadio**
  - **PolskieRadioCategory**
  - **PornCom**
+ - **PornFlip**
  - **PornHd**
  - **PornHub**: PornHub and Thumbzilla
  - **PornHubPlaylist**
@@ -650,7 +656,6 @@
  - **screen.yahoo:search**: Yahoo screen search
  - **Screencast**
  - **ScreencastOMatic**
- - **ScreenJunkies**
  - **Seeker**
  - **SenateISVP**
  - **SendtoNews**
@@ -658,7 +663,6 @@
  - **Sexu**
  - **Shahid**
  - **Shared**: shared.sx
- - **ShareSix**
  - **ShowRoomLive**
  - **Sina**
  - **SixPlay**
@@ -845,7 +849,6 @@
  - **videomore:season**
  - **videomore:video**
  - **VideoPremium**
- - **VideoTt**: video.tt - Your True Tube (Currently broken)
  - **videoweed**: VideoWeed
  - **Vidio**
  - **vidme**

test/test_utils.py

@@ -295,6 +295,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('27.02.2016 17:30'), '20160227')
         self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
         self.assertEqual(unified_strdate('Feb 7, 2016 at 6:35 pm'), '20160207')
+        self.assertEqual(unified_strdate('July 15th, 2013'), '20130715')
+        self.assertEqual(unified_strdate('September 1st, 2013'), '20130901')
+        self.assertEqual(unified_strdate('Sep 2nd, 2013'), '20130902')
 
     def test_unified_timestamps(self):
         self.assertEqual(unified_timestamp('December 21, 2010'), 1292889600)

youtube_dl/YoutubeDL.py

@@ -1339,7 +1339,7 @@ class YoutubeDL(object):
                 format['format_id'] = compat_str(i)
             else:
                 # Sanitize format_id from characters used in format selector expression
-                format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
+                format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
             format_id = format['format_id']
             if format_id not in formats_dict:
                 formats_dict[format_id] = []
@@ -1363,7 +1363,7 @@ class YoutubeDL(object):
                 format['ext'] = determine_ext(format['url']).lower()
             # Automatically determine protocol if missing (useful for format
             # selection purposes)
-            if 'protocol' not in format:
+            if format.get('protocol') is None:
                 format['protocol'] = determine_protocol(format)
             # Add HTTP headers, so that external programs can use them from the
             # json output

youtube_dl/compat.py

@@ -2344,7 +2344,7 @@ try:
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
 except ImportError:  # Python 2
     _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
-                else re.compile('([\x00-\x7f]+)'))
+                else re.compile(r'([\x00-\x7f]+)'))
 
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version

youtube_dl/extractor/abcnews.py

@@ -23,7 +23,7 @@ class AbcNewsVideoIE(AMPIE):
             'title': '\'This Week\' Exclusive: Iran\'s Foreign Minister Zarif',
             'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
             'duration': 180,
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
         'params': {
             # m3u8 download
@@ -59,7 +59,7 @@ class AbcNewsIE(InfoExtractor):
             'display_id': 'dramatic-video-rare-death-job-america',
             'title': 'Occupational Hazards',
             'description': 'Nightline investigates the dangers that lurk at various jobs.',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20100428',
             'timestamp': 1272412800,
         },

youtube_dl/extractor/abcotvs.py

@@ -23,7 +23,7 @@ class ABCOTVSIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'East Bay museum celebrates vintage synthesizers',
                 'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'timestamp': 1421123075,
                 'upload_date': '20150113',
                 'uploader': 'Jonathan Bloom',

youtube_dl/extractor/adobetv.py

@@ -30,7 +30,7 @@ class AdobeTVIE(AdobeTVBaseIE):
             'ext': 'mp4',
             'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
             'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
-            'thumbnail': 're:https?://.*\.jpg$',
+            'thumbnail': r're:https?://.*\.jpg$',
             'upload_date': '20110914',
             'duration': 60,
             'view_count': int,

youtube_dl/extractor/airmozilla.py

@@ -20,7 +20,7 @@ class AirMozillaIE(InfoExtractor):
             'id': '6x4q2w',
             'ext': 'mp4',
             'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
-            'thumbnail': 're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
+            'thumbnail': r're:https?://vid\.ly/(?P<id>[0-9a-z-]+)/poster',
             'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
             'timestamp': 1422487800,
             'upload_date': '20150128',

youtube_dl/extractor/allocine.py

@@ -21,7 +21,7 @@ class AllocineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Astérix - Le Domaine des Dieux Teaser VF',
             'description': 'md5:4a754271d9c6f16c72629a8a993ee884',
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
         },
     }, {
         'url': 'http://www.allocine.fr/video/player_gen_cmedia=19540403&cfilm=222257.html',
@@ -32,7 +32,7 @@ class AllocineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Planes 2 Bande-annonce VF',
             'description': 'Regardez la bande annonce du film Planes 2 (Planes 2 Bande-annonce VF). Planes 2, un film de Roberts Gannaway',
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
         },
     }, {
         'url': 'http://www.allocine.fr/video/player_gen_cmedia=19544709&cfilm=181290.html',
@@ -43,7 +43,7 @@ class AllocineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Dragons 2 - Bande annonce finale VF',
             'description': 'md5:6cdd2d7c2687d4c6aafe80a35e17267a',
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
         },
     }, {
         'url': 'http://www.allocine.fr/video/video-19550147/',
@@ -53,7 +53,7 @@ class AllocineIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Faux Raccord N°123 - Les gaffes de Cliffhanger',
             'description': 'md5:bc734b83ffa2d8a12188d9eb48bb6354',
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
         },
     }]
 

youtube_dl/extractor/alphaporno.py

@@ -19,7 +19,7 @@ class AlphaPornoIE(InfoExtractor):
             'display_id': 'sensual-striptease-porn-with-samantha-alexandra',
             'ext': 'mp4',
             'title': 'Sensual striptease porn with Samantha Alexandra',
-            'thumbnail': 're:https?://.*\.jpg$',
+            'thumbnail': r're:https?://.*\.jpg$',
             'timestamp': 1418694611,
             'upload_date': '20141216',
             'duration': 387,

youtube_dl/extractor/aol.py

@@ -12,7 +12,7 @@ from ..utils import (
 
 class AolIE(InfoExtractor):
     IE_NAME = 'on.aol.com'
-    _VALID_URL = r'(?:aol-video:|https?://on\.aol\.com/(?:[^/]+/)*(?:[^/?#&]+-)?)(?P<id>[^/?#&]+)'
+    _VALID_URL = r'(?:aol-video:|https?://(?:(?:www|on)\.)?aol\.com/(?:[^/]+/)*(?:[^/?#&]+-)?)(?P<id>[^/?#&]+)'
 
     _TESTS = [{
         # video with 5min ID
@@ -33,7 +33,7 @@ class AolIE(InfoExtractor):
         }
     }, {
         # video with vidible ID
-        'url': 'http://on.aol.com/video/netflix-is-raising-rates-5707d6b8e4b090497b04f706?context=PC:homepage:PL1944:1460189336183',
+        'url': 'http://www.aol.com/video/view/netflix-is-raising-rates/5707d6b8e4b090497b04f706/',
         'info_dict': {
             'id': '5707d6b8e4b090497b04f706',
             'ext': 'mp4',
@@ -108,30 +108,3 @@ class AolIE(InfoExtractor):
             'uploader': video_data.get('videoOwner'),
             'formats': formats,
         }
-
-
-class AolFeaturesIE(InfoExtractor):
-    IE_NAME = 'features.aol.com'
-    _VALID_URL = r'https?://features\.aol\.com/video/(?P<id>[^/?#]+)'
-
-    _TESTS = [{
-        'url': 'http://features.aol.com/video/behind-secret-second-careers-late-night-talk-show-hosts',
-        'md5': '7db483bb0c09c85e241f84a34238cc75',
-        'info_dict': {
-            'id': '519507715',
-            'ext': 'mp4',
-            'title': 'What To Watch - February 17, 2016',
-        },
-        'add_ie': ['FiveMin'],
-        'params': {
-            # encrypted m3u8 download
-            'skip_download': True,
-        },
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        return self.url_result(self._search_regex(
-            r'<script type="text/javascript" src="(https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js[^"]+)"',
-            webpage, '5min embed url'), 'FiveMin')

youtube_dl/extractor/ard.py

@@ -253,7 +253,7 @@ class ARDIE(InfoExtractor):
             'duration': 2600,
             'title': 'Die Story im Ersten: Mission unter falscher Flagge',
             'upload_date': '20140804',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
         'skip': 'HTTP Error 404: Not Found',
     }

youtube_dl/extractor/atresplayer.py

@@ -30,7 +30,7 @@ class AtresPlayerIE(InfoExtractor):
                 'title': 'Especial Solidario de Nochebuena',
                 'description': 'md5:e2d52ff12214fa937107d21064075bf1',
                 'duration': 5527.6,
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
             },
             'skip': 'This video is only available for registered users'
         },
@@ -43,7 +43,7 @@ class AtresPlayerIE(InfoExtractor):
                 'title': 'David Bustamante',
                 'description': 'md5:f33f1c0a05be57f6708d4dd83a3b81c6',
                 'duration': 1439.0,
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
         {

youtube_dl/extractor/atttechchannel.py

@@ -14,7 +14,7 @@ class ATTTechChannelIE(InfoExtractor):
             'ext': 'flv',
             'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
             'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20140127',
         },
         'params': {

youtube_dl/extractor/audioboom.py

@@ -17,7 +17,7 @@ class AudioBoomIE(InfoExtractor):
             'description': 'Guest:   Nate Davis - NFL free agency,   Guest:   Stan Gans',
             'duration': 2245.72,
             'uploader': 'Steve Czaban',
-            'uploader_url': 're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
+            'uploader_url': r're:https?://(?:www\.)?audioboom\.com/channel/steveczabanyahoosportsradio',
         }
     }, {
         'url': 'https://audioboom.com/posts/4279833-3-09-2016-czaban-hour-3?t=0',

youtube_dl/extractor/azmedien.py

@@ -0,0 +1,132 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from .kaltura import KalturaIE
+from ..utils import (
+    get_element_by_class,
+    strip_or_none,
+)
+
+
+class AZMedienBaseIE(InfoExtractor):
+    def _kaltura_video(self, partner_id, entry_id):
+        return self.url_result(
+            'kaltura:%s:%s' % (partner_id, entry_id), ie=KalturaIE.ie_key(),
+            video_id=entry_id)
+
+
+class AZMedienIE(AZMedienBaseIE):
+    IE_DESC = 'AZ Medien videos'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:www\.)?
+                        (?:
+                            telezueri\.ch|
+                            telebaern\.tv|
+                            telem1\.ch
+                        )/
+                        [0-9]+-show-[^/\#]+
+                        (?:
+                            /[0-9]+-episode-[^/\#]+
+                            (?:
+                                /[0-9]+-segment-(?:[^/\#]+\#)?|
+                                \#
+                            )|
+                            \#
+                        )
+                        (?P<id>[^\#]+)
+                    '''
+
+    _TESTS = [{
+        # URL with 'segment'
+        'url': 'http://www.telezueri.ch/62-show-zuerinews/13772-episode-sonntag-18-dezember-2016/32419-segment-massenabweisungen-beim-hiltl-club-wegen-pelzboom',
+        'info_dict': {
+            'id': '1_2444peh4',
+            'ext': 'mov',
+            'title': 'Massenabweisungen beim Hiltl Club wegen Pelzboom',
+            'description': 'md5:9ea9dd1b159ad65b36ddcf7f0d7c76a8',
+            'uploader_id': 'TeleZ?ri',
+            'upload_date': '20161218',
+            'timestamp': 1482084490,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # URL with 'segment' and fragment:
+        'url': 'http://www.telebaern.tv/118-show-news/14240-episode-dienstag-17-januar-2017/33666-segment-achtung-gefahr#zu-wenig-pflegerinnen-und-pfleger',
+        'only_matching': True
+    }, {
+        # URL with 'episode' and fragment:
+        'url': 'http://www.telem1.ch/47-show-sonntalk/13986-episode-soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz#soldaten-fuer-grenzschutz-energiestrategie-obama-bilanz',
+        'only_matching': True
+    }, {
+        # URL with 'show' and fragment:
+        'url': 'http://www.telezueri.ch/66-show-sonntalk#burka-plakate-trump-putin-china-besuch',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        partner_id = self._search_regex(
+            r'<script[^>]+src=["\'](?:https?:)?//(?:[^/]+\.)?kaltura\.com(?:/[^/]+)*/(?:p|partner_id)/([0-9]+)',
+            webpage, 'kaltura partner id')
+        entry_id = self._html_search_regex(
+            r'<a[^>]+data-id=(["\'])(?P<id>(?:(?!\1).)+)\1[^>]+data-slug=["\']%s'
+            % re.escape(video_id), webpage, 'kaltura entry id', group='id')
+
+        return self._kaltura_video(partner_id, entry_id)
+
+
+class AZMedienShowIE(AZMedienBaseIE):
+    IE_DESC = 'AZ Medien shows'
+    _VALID_URL = r'https?://(?:www\.)?(?:telezueri\.ch|telebaern\.tv|telem1\.ch)/(?P<id>[0-9]+-show-[^/#]+(?:/[0-9]+-episode-[^/#]+)?)$'
+
+    _TESTS = [{
+        # URL with 'episode'
+        'url': 'http://www.telebaern.tv/118-show-news/13735-episode-donnerstag-15-dezember-2016',
+        'info_dict': {
+            'id': '118-show-news/13735-episode-donnerstag-15-dezember-2016',
+            'title': 'News - Donnerstag, 15. Dezember 2016',
+        },
+        'playlist_count': 9,
+    }, {
+        # URL with 'show' only
+        'url': 'http://www.telezueri.ch/86-show-talktaeglich',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        webpage = self._download_webpage(url, show_id)
+
+        entries = []
+
+        partner_id = self._search_regex(
+            r'src=["\'](?:https?:)?//(?:[^/]+\.)kaltura\.com/(?:[^/]+/)*(?:p|partner_id)/(\d+)',
+            webpage, 'kaltura partner id', default=None)
+
+        if partner_id:
+            entries = [
+                self._kaltura_video(partner_id, m.group('id'))
+                for m in re.finditer(
+                    r'data-id=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage)]
+
+        if not entries:
+            entries = [
+                self.url_result(m.group('url'), ie=AZMedienIE.ie_key())
+                for m in re.finditer(
+                    r'<a[^>]+data-real=(["\'])(?P<url>http.+?)\1', webpage)]
+
+        title = self._search_regex(
+            r'episodeShareTitle\s*=\s*(["\'])(?P<title>(?:(?!\1).)+)\1',
+            webpage, 'title',
+            default=strip_or_none(get_element_by_class(
+                'title-block-cell', webpage)), group='title')
+
+        return self.playlist_result(entries, show_id, title)

youtube_dl/extractor/azubu.py

@@ -21,7 +21,7 @@ class AzubuIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
                 'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
-                'thumbnail': 're:^https?://.*\.jpe?g',
+                'thumbnail': r're:^https?://.*\.jpe?g',
                 'timestamp': 1417523507.334,
                 'upload_date': '20141202',
                 'duration': 9988.7,
@@ -38,7 +38,7 @@ class AzubuIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
                 'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
-                'thumbnail': 're:^https?://.*\.jpe?g',
+                'thumbnail': r're:^https?://.*\.jpe?g',
                 'timestamp': 1410530893.320,
                 'upload_date': '20140912',
                 'duration': 172.385,

youtube_dl/extractor/beampro.py

@@ -0,0 +1,73 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    compat_str,
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
+
+
+class BeamProLiveIE(InfoExtractor):
+    IE_NAME = 'Beam:live'
+    _VALID_URL = r'https?://(?:\w+\.)?beam\.pro/(?P<id>[^/?#&]+)'
+    _RATINGS = {'family': 0, 'teen': 13, '18+': 18}
+    _TEST = {
+        'url': 'http://www.beam.pro/niterhayven',
+        'info_dict': {
+            'id': '261562',
+            'ext': 'mp4',
+            'title': 'Introducing The Witcher 3 //  The Grind Starts Now!',
+            'description': 'md5:0b161ac080f15fe05d18a07adb44a74d',
+            'thumbnail': r're:https://.*\.jpg$',
+            'timestamp': 1483477281,
+            'upload_date': '20170103',
+            'uploader': 'niterhayven',
+            'uploader_id': '373396',
+            'age_limit': 18,
+            'is_live': True,
+            'view_count': int,
+        },
+        'skip': 'niterhayven is offline',
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_name = self._match_id(url)
+
+        chan = self._download_json(
+            'https://beam.pro/api/v1/channels/%s' % channel_name, channel_name)
+
+        if chan.get('online') is False:
+            raise ExtractorError(
+                '{0} is offline'.format(channel_name), expected=True)
+
+        channel_id = chan['id']
+
+        formats = self._extract_m3u8_formats(
+            'https://beam.pro/api/v1/channels/%s/manifest.m3u8' % channel_id,
+            channel_name, ext='mp4', m3u8_id='hls', fatal=False)
+        self._sort_formats(formats)
+
+        user_id = chan.get('userId') or try_get(chan, lambda x: x['user']['id'])
+
+        return {
+            'id': compat_str(chan.get('id') or channel_name),
+            'title': self._live_title(chan.get('name') or channel_name),
+            'description': clean_html(chan.get('description')),
+            'thumbnail': try_get(chan, lambda x: x['thumbnail']['url'], compat_str),
+            'timestamp': parse_iso8601(chan.get('updatedAt')),
+            'uploader': chan.get('token') or try_get(
+                chan, lambda x: x['user']['username'], compat_str),
+            'uploader_id': compat_str(user_id) if user_id else None,
+            'age_limit': self._RATINGS.get(chan.get('audience')),
+            'is_live': True,
+            'view_count': int_or_none(chan.get('viewersTotal')),
+            'formats': formats,
+        }

youtube_dl/extractor/bet.py

@@ -17,7 +17,7 @@ class BetIE(MTVServicesInfoExtractor):
                 'description': 'President Obama urges persistence in confronting racism and bias.',
                 'duration': 1534,
                 'upload_date': '20141208',
-                'thumbnail': 're:(?i)^https?://.*\.jpg$',
+                'thumbnail': r're:(?i)^https?://.*\.jpg$',
                 'subtitles': {
                     'en': 'mincount:2',
                 }
@@ -37,7 +37,7 @@ class BetIE(MTVServicesInfoExtractor):
                 'description': 'A BET News special.',
                 'duration': 1696,
                 'upload_date': '20141125',
-                'thumbnail': 're:(?i)^https?://.*\.jpg$',
+                'thumbnail': r're:(?i)^https?://.*\.jpg$',
                 'subtitles': {
                     'en': 'mincount:2',
                 }

youtube_dl/extractor/bild.py

@@ -19,7 +19,7 @@ class BildIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Das können die  neuen iPads',
             'description': 'md5:a4058c4fa2a804ab59c00d7244bbf62f',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 196,
         }
     }

youtube_dl/extractor/bilibili.py

@@ -28,14 +28,14 @@ class BiliBiliIE(InfoExtractor):
             'duration': 308.315,
             'timestamp': 1398012660,
             'upload_date': '20140420',
-            'thumbnail': 're:^https?://.+\.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',
             'uploader': '菊子桑',
             'uploader_id': '156160',
         },
     }
 
-    _APP_KEY = '6f90a59ac58a4123'
-    _BILIBILI_KEY = '0bfd84cc3940035173f35e6777508326'
+    _APP_KEY = '84956560bc028eb7'
+    _BILIBILI_KEY = '94aba54af9065f71de72f5508f1cd42e'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

youtube_dl/extractor/biobiochiletv.py

@@ -19,7 +19,7 @@ class BioBioChileTVIE(InfoExtractor):
             'id': 'sobre-camaras-y-camarillas-parlamentarias',
             'ext': 'mp4',
             'title': 'Sobre Cámaras y camarillas parlamentarias',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Fernando Atria',
         },
         'skip': 'URL expired and redirected to http://www.biobiochile.cl/portada/bbtv/index.html',
@@ -31,7 +31,7 @@ class BioBioChileTVIE(InfoExtractor):
             'id': 'natalia-valdebenito-repasa-a-diputado-hasbun-paso-a-la-categoria-de-hablar-brutalidades',
             'ext': 'mp4',
             'title': 'Natalia Valdebenito repasa a diputado Hasbún: Pasó a la categoría de hablar brutalidades',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Piangella Obrador',
         },
         'params': {

youtube_dl/extractor/breakcom.py

@@ -1,9 +1,9 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     parse_age_limit,
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class BreakIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<site>break|screenjunkies)\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
         'info_dict': {
@@ -20,45 +20,124 @@ class BreakIE(InfoExtractor):
             'title': 'When Girls Act Like D-Bags',
             'age_limit': 13,
         }
+    }, {
+        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
+        'md5': '5c2b686bec3d43de42bde9ec047536b0',
+        'info_dict': {
+            'id': '2841915',
+            'display_id': 'best-quentin-tarantino-movie',
+            'ext': 'mp4',
+            'title': 'Best Quentin Tarantino Movie',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 3671,
+            'age_limit': 13,
+            'tags': list,
+        },
+    }, {
+        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
+        'info_dict': {
+            'id': '2348808',
+            'display_id': 'honest-trailers-the-dark-knight',
+            'ext': 'mp4',
+            'title': 'Honest Trailers - The Dark Knight',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'age_limit': 10,
+            'tags': list,
+        },
+    }, {
+        # requires subscription but worked around
+        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
+        'info_dict': {
+            'id': '3003285',
+            'display_id': 'knocking-dead-ep-1-the-show-so-far',
+            'ext': 'mp4',
+            'title': 'State of The Dead Recap: Knocking Dead Pilot',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 3307,
+            'age_limit': 13,
+            'tags': list,
+        },
     }, {
         'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            'http://www.break.com/embed/%s' % video_id, video_id)
-        info = json.loads(self._search_regex(
-            r'var embedVars = ({.*})\s*?</script>',
-            webpage, 'info json', flags=re.DOTALL))
+    _DEFAULT_BITRATES = (48, 150, 320, 496, 864, 2240, 3264)
 
-        youtube_id = info.get('youtubeId')
+    def _real_extract(self, url):
+        site, display_id, video_id = re.match(self._VALID_URL, url).groups()
+
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._search_regex(
+                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
+                webpage, 'video id')
+
+        webpage = self._download_webpage(
+            'http://www.%s.com/embed/%s' % (site, video_id),
+            display_id, 'Downloading video embed page')
+        embed_vars = self._parse_json(
+            self._search_regex(
+                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
+            display_id)
+
+        youtube_id = embed_vars.get('youtubeId')
         if youtube_id:
             return self.url_result(youtube_id, 'Youtube')
 
-        formats = [{
-            'url': media['uri'] + '?' + info['AuthToken'],
-            'tbr': media['bitRate'],
-            'width': media['width'],
-            'height': media['height'],
-        } for media in info['media'] if media.get('mediaPurpose') == 'play']
+        title = embed_vars['contentName']
 
-        if not formats:
+        formats = []
+        bitrates = []
+        for f in embed_vars.get('media', []):
+            if not f.get('uri') or f.get('mediaPurpose') != 'play':
+                continue
+            bitrate = int_or_none(f.get('bitRate'))
+            if bitrate:
+                bitrates.append(bitrate)
             formats.append({
-                'url': info['videoUri']
+                'url': f['uri'],
+                'format_id': 'http-%d' % bitrate if bitrate else 'http',
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'tbr': bitrate,
+                'format': 'mp4',
             })
 
-        self._sort_formats(formats)
+        if not bitrates:
+            # When subscriptionLevel > 0, i.e. plus subscription is required
+            # media list will be empty. However, hds and hls uris are still
+            # available. We can grab them assuming bitrates to be default.
+            bitrates = self._DEFAULT_BITRATES
 
-        duration = int_or_none(info.get('videoLengthInSeconds'))
-        age_limit = parse_age_limit(info.get('audienceRating'))
+        auth_token = embed_vars.get('AuthToken')
+
+        def construct_manifest_url(base_url, ext):
+            pieces = [base_url]
+            pieces.extend([compat_str(b) for b in bitrates])
+            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
+            return ','.join(pieces)
+
+        if bitrates and auth_token:
+            hds_url = embed_vars.get('hdsUri')
+            if hds_url:
+                formats.extend(self._extract_f4m_formats(
+                    construct_manifest_url(hds_url, 'f4m'),
+                    display_id, f4m_id='hds', fatal=False))
+            hls_url = embed_vars.get('hlsUri')
+            if hls_url:
+                formats.extend(self._extract_m3u8_formats(
+                    construct_manifest_url(hls_url, 'm3u8'),
+                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': info['contentName'],
-            'thumbnail': info['thumbUri'],
-            'duration': duration,
-            'age_limit': age_limit,
+            'display_id': display_id,
+            'title': title,
+            'thumbnail': embed_vars.get('thumbUri'),
+            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
+            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
+            'tags': embed_vars.get('tags', '').split(','),
             'formats': formats,
         }

youtube_dl/extractor/brightcove.py

@@ -179,7 +179,7 @@ class BrightcoveLegacyIE(InfoExtractor):
 
         params = {}
 
-        playerID = find_param('playerID')
+        playerID = find_param('playerID') or find_param('playerId')
         if playerID is None:
             raise ExtractorError('Cannot find player ID')
         params['playerID'] = playerID
@@ -204,7 +204,7 @@ class BrightcoveLegacyIE(InfoExtractor):
         #   // build Brightcove <object /> XML
         # }
         m = re.search(
-            r'''(?x)customBC.\createVideo\(
+            r'''(?x)customBC\.createVideo\(
                 .*?                                                  # skipping width and height
                 ["\'](?P<playerID>\d+)["\']\s*,\s*                   # playerID
                 ["\'](?P<playerKey>AQ[^"\']{48})[^"\']*["\']\s*,\s*  # playerKey begins with AQ and is 50 characters

youtube_dl/extractor/byutv.py

@@ -16,7 +16,7 @@ class BYUtvIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Season 5 Episode 5',
             'description': 'md5:e07269172baff037f8e8bf9956bc9747',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 1486.486,
         },
         'params': {

youtube_dl/extractor/camdemy.py

@@ -26,7 +26,7 @@ class CamdemyIE(InfoExtractor):
             'id': '5181',
             'ext': 'mp4',
             'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'creator': 'ss11spring',
             'duration': 1591,
             'upload_date': '20130114',
@@ -41,7 +41,7 @@ class CamdemyIE(InfoExtractor):
             'id': '13885',
             'ext': 'mp4',
             'title': 'EverCam + Camdemy QuickStart',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'description': 'md5:2a9f989c2b153a2342acee579c6e7db6',
             'creator': 'evercam',
             'duration': 318,

youtube_dl/extractor/canalplus.py

@@ -107,7 +107,7 @@ class CanalplusIE(InfoExtractor):
             [r'<canal:player[^>]+?videoId=(["\'])(?P<id>\d+)',
              r'id=["\']canal_video_player(?P<id>\d+)',
              r'data-video=["\'](?P<id>\d+)'],
-            webpage, 'video id', group='id')
+            webpage, 'video id', default=mobj.group('vid'), group='id')
 
         info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         video_data = self._download_json(info_url, video_id, 'Downloading video JSON')

youtube_dl/extractor/canvas.py

@@ -17,7 +17,7 @@ class CanvasIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'De afspraak veilt voor de Warmste Week',
             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 49.02,
         }
     }, {
@@ -29,7 +29,7 @@ class CanvasIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Pieter 0167',
             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 2553.08,
             'subtitles': {
                 'nl': [{
@@ -48,7 +48,7 @@ class CanvasIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Herbekijk Sorry voor alles',
             'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 3788.06,
         },
         'params': {

youtube_dl/extractor/carambatv.py

@@ -21,7 +21,7 @@ class CarambaTVIE(InfoExtractor):
             'id': '191910501',
             'ext': 'mp4',
             'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 2678.31,
         },
     }, {
@@ -69,7 +69,7 @@ class CarambaTVPageIE(InfoExtractor):
             'id': '475222',
             'ext': 'flv',
             'title': '[BadComedian] - Разборка в Маниле (Абсолютный обзор)',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             # duration reported by videomore is incorrect
             'duration': int,
         },

youtube_dl/extractor/cbc.py

@@ -90,36 +90,49 @@ class CBCIE(InfoExtractor):
             },
         }],
         'skip': 'Geo-restricted to Canada',
+    }, {
+        # multiple CBC.APP.Caffeine.initInstance(...)
+        'url': 'http://www.cbc.ca/news/canada/calgary/dog-indoor-exercise-winter-1.3928238',
+        'info_dict': {
+            'title': 'Keep Rover active during the deep freeze with doggie pushups and other fun indoor tasks',
+            'id': 'dog-indoor-exercise-winter-1.3928238',
+        },
+        'playlist_mincount': 6,
     }]
 
     @classmethod
     def suitable(cls, url):
         return False if CBCPlayerIE.suitable(url) else super(CBCIE, cls).suitable(url)
 
+    def _extract_player_init(self, player_init, display_id):
+        player_info = self._parse_json(player_init, display_id, js_to_json)
+        media_id = player_info.get('mediaId')
+        if not media_id:
+            clip_id = player_info['clipId']
+            feed = self._download_json(
+                'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
+                clip_id, fatal=False)
+            if feed:
+                media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
+            if not media_id:
+                media_id = self._download_json(
+                    'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
+                    clip_id)['entries'][0]['id'].split('/')[-1]
+        return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        player_init = self._search_regex(
-            r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage, 'player init',
-            default=None)
-        if player_init:
-            player_info = self._parse_json(player_init, display_id, js_to_json)
-            media_id = player_info.get('mediaId')
-            if not media_id:
-                clip_id = player_info['clipId']
-                feed = self._download_json(
-                    'http://tpfeed.cbc.ca/f/ExhSPC/vms_5akSXx4Ng_Zn?byCustomValue={:mpsReleases}{%s}' % clip_id,
-                    clip_id, fatal=False)
-                if feed:
-                    media_id = try_get(feed, lambda x: x['entries'][0]['guid'], compat_str)
-                if not media_id:
-                    media_id = self._download_json(
-                        'http://feed.theplatform.com/f/h9dtGB/punlNGjMlc1F?fields=id&byContent=byReleases%3DbyId%253D' + clip_id,
-                        clip_id)['entries'][0]['id'].split('/')[-1]
-            return self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
-        else:
-            entries = [self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id) for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)]
-            return self.playlist_result(entries)
+        entries = [
+            self._extract_player_init(player_init, display_id)
+            for player_init in re.findall(r'CBC\.APP\.Caffeine\.initInstance\(({.+?})\);', webpage)]
+        entries.extend([
+            self.url_result('cbcplayer:%s' % media_id, 'CBCPlayer', media_id)
+            for media_id in re.findall(r'<iframe[^>]+src="[^"]+?mediaId=(\d+)"', webpage)])
+        return self.playlist_result(
+            entries, display_id,
+            self._og_search_title(webpage, fatal=False),
+            self._og_search_description(webpage))
 
 
 class CBCPlayerIE(InfoExtractor):

youtube_dl/extractor/cbsnews.py

@@ -39,7 +39,7 @@ class CBSNewsIE(CBSIE):
                 'upload_date': '20140404',
                 'timestamp': 1396650660,
                 'uploader': 'CBSI-NEW',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 205,
                 'subtitles': {
                     'en': [{

youtube_dl/extractor/ccc.py

@@ -19,7 +19,7 @@ class CCCIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Introduction to Processor Design',
             'description': 'md5:df55f6d073d4ceae55aae6f2fd98a0ac',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20131228',
             'timestamp': 1388188800,
             'duration': 3710,
@@ -32,7 +32,7 @@ class CCCIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        event_id = self._search_regex("data-id='(\d+)'", webpage, 'event id')
+        event_id = self._search_regex(r"data-id='(\d+)'", webpage, 'event id')
         event_data = self._download_json('https://media.ccc.de/public/events/%s' % event_id, event_id)
 
         formats = []

youtube_dl/extractor/cctv.py

@@ -14,8 +14,9 @@ from ..utils import (
 
 class CCTVIE(InfoExtractor):
     IE_DESC = '央视网'
-    _VALID_URL = r'https?://(?:[^/]+)\.(?:cntv|cctv)\.(?:com|cn)/(?:[^/]+/)*?(?P<id>[^/?#&]+?)(?:/index)?(?:\.s?html|[?#&]|$)'
+    _VALID_URL = r'https?://(?:(?:[^/]+)\.(?:cntv|cctv)\.(?:com|cn)|(?:www\.)?ncpa-classic\.com)/(?:[^/]+/)*?(?P<id>[^/?#&]+?)(?:/index)?(?:\.s?html|[?#&]|$)'
     _TESTS = [{
+        # fo.addVariable("videoCenterId","id")
         'url': 'http://sports.cntv.cn/2016/02/12/ARTIaBRxv4rTT1yWf1frW2wi160212.shtml',
         'md5': 'd61ec00a493e09da810bf406a078f691',
         'info_dict': {
@@ -29,6 +30,7 @@ class CCTVIE(InfoExtractor):
             'upload_date': '20160212',
         },
     }, {
+        # var guid = "id"
         'url': 'http://tv.cctv.com/2016/02/05/VIDEUS7apq3lKrHG9Dncm03B160205.shtml',
         'info_dict': {
             'id': 'efc5d49e5b3b4ab2b34f3a502b73d3ae',
@@ -44,6 +46,7 @@ class CCTVIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
+        # changePlayer('id')
         'url': 'http://english.cntv.cn/special/four_comprehensives/index.shtml',
         'info_dict': {
             'id': '4bb9bb4db7a6471ba85fdeda5af0381e',
@@ -59,6 +62,7 @@ class CCTVIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
+        # loadvideo('id')
         'url': 'http://cctv.cntv.cn/lm/tvseries_russian/yilugesanghua/index.shtml',
         'info_dict': {
             'id': 'b15f009ff45c43968b9af583fc2e04b2',
@@ -73,6 +77,37 @@ class CCTVIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # var initMyAray = 'id'
+        'url': 'http://www.ncpa-classic.com/2013/05/22/VIDE1369219508996867.shtml',
+        'info_dict': {
+            'id': 'a194cfa7f18c426b823d876668325946',
+            'ext': 'mp4',
+            'title': '小泽征尔音乐塾 音乐梦想无国界',
+            'duration': 2173,
+            'timestamp': 1369248264,
+            'upload_date': '20130522',
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        # var ids = ["id"]
+        'url': 'http://www.ncpa-classic.com/clt/more/416/index.shtml',
+        'info_dict': {
+            'id': 'a8606119a4884588a79d81c02abecc16',
+            'ext': 'mp3',
+            'title': '来自维也纳的新年贺礼',
+            'description': 'md5:f13764ae8dd484e84dd4b39d5bcba2a7',
+            'duration': 1578,
+            'uploader': 'djy',
+            'timestamp': 1482942419,
+            'upload_date': '20161228',
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'http://ent.cntv.cn/2016/01/18/ARTIjprSSJH8DryTVr5Bx8Wb160118.shtml',
         'only_matching': True,
@@ -87,7 +122,7 @@ class CCTVIE(InfoExtractor):
         'only_matching': True,
     }, {
         'url': 'http://tv.cntv.cn/video/C39296/95cfac44cabd3ddc4a9438780a4e5c44',
-        'only_matching': True
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -97,8 +132,10 @@ class CCTVIE(InfoExtractor):
         video_id = self._search_regex(
             [r'var\s+guid\s*=\s*["\']([\da-fA-F]+)',
              r'videoCenterId["\']\s*,\s*["\']([\da-fA-F]+)',
-             r'"changePlayer\s*\(\s*["\']([\da-fA-F]+)',
-             r'"load[Vv]ideo\s*\(\s*["\']([\da-fA-F]+)'],
+             r'changePlayer\s*\(\s*["\']([\da-fA-F]+)',
+             r'load[Vv]ideo\s*\(\s*["\']([\da-fA-F]+)',
+             r'var\s+initMyAray\s*=\s*["\']([\da-fA-F]+)',
+             r'var\s+ids\s*=\s*\[["\']([\da-fA-F]+)'],
             webpage, 'video id')
 
         data = self._download_json(
@@ -138,7 +175,8 @@ class CCTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         uploader = data.get('editer_name')
-        description = self._html_search_meta('description', webpage)
+        description = self._html_search_meta(
+            'description', webpage, default=None)
         timestamp = unified_timestamp(data.get('f_pgmtime'))
         duration = float_or_none(try_get(video, lambda x: x['totalLength']))
 

youtube_dl/extractor/cda.py

@@ -24,7 +24,7 @@ class CDAIE(InfoExtractor):
             'height': 720,
             'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
             'description': 'md5:269ccd135d550da90d1662651fcb9772',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'average_rating': float,
             'duration': 39
         }
@@ -36,7 +36,7 @@ class CDAIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Lądowanie na lotnisku na Maderze',
             'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'crash404',
             'view_count': int,
             'average_rating': float,

youtube_dl/extractor/ceskatelevize.py

@@ -25,7 +25,7 @@ class CeskaTelevizeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Hyde Park Civilizace',
             'description': 'md5:fe93f6eda372d150759d11644ebbfb4a',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 3350,
         },
         'params': {
@@ -39,7 +39,7 @@ class CeskaTelevizeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Hyde Park Civilizace: Bonus 01 - En',
             'description': 'English Subtittles',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 81.3,
         },
         'params': {
@@ -52,7 +52,7 @@ class CeskaTelevizeIE(InfoExtractor):
         'info_dict': {
             'id': 402,
             'ext': 'mp4',
-            'title': 're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'is_live': True,
         },
         'params': {
@@ -80,7 +80,7 @@ class CeskaTelevizeIE(InfoExtractor):
                 'id': '61924494877068022',
                 'ext': 'mp4',
                 'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': 're:^https?://.*\.jpg',
+                'thumbnail': r're:^https?://.*\.jpg',
                 'duration': 1558.3,
             },
         }],

youtube_dl/extractor/channel9.py

@@ -31,7 +31,7 @@ class Channel9IE(InfoExtractor):
             'title': 'Developer Kick-Off Session: Stuff We Love',
             'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
             'duration': 4576,
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
             'session_code': 'KOS002',
             'session_day': 'Day 1',
             'session_room': 'Arena 1A',
@@ -47,7 +47,7 @@ class Channel9IE(InfoExtractor):
             'title': 'Self-service BI with Power BI - nuclear testing',
             'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
             'duration': 1540,
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
             'authors': ['Mike Wilmot'],
         },
     }, {
@@ -59,7 +59,7 @@ class Channel9IE(InfoExtractor):
             'title': 'Ranges for the Standard Library',
             'description': 'md5:2e6b4917677af3728c5f6d63784c4c5d',
             'duration': 5646,
-            'thumbnail': 're:http://.*\.jpg',
+            'thumbnail': r're:http://.*\.jpg',
         },
         'params': {
             'skip_download': True,

youtube_dl/extractor/charlierose.py

@@ -13,7 +13,7 @@ class CharlieRoseIE(InfoExtractor):
             'id': '27996',
             'ext': 'mp4',
             'title': 'Remembering Zaha Hadid',
-            'thumbnail': 're:^https?://.*\.jpg\?\d+',
+            'thumbnail': r're:^https?://.*\.jpg\?\d+',
             'description': 'We revisit past conversations with Zaha Hadid, in memory of the world renowned Iraqi architect.',
             'subtitles': {
                 'en': [{

youtube_dl/extractor/chaturbate.py

@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
@@ -31,30 +33,35 @@ class ChaturbateIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        m3u8_url = self._search_regex(
-            r'src=(["\'])(?P<url>http.+?\.m3u8.*?)\1', webpage,
-            'playlist', default=None, group='url')
+        m3u8_formats = [(m.group('id').lower(), m.group('url')) for m in re.finditer(
+            r'hlsSource(?P<id>.+?)\s*=\s*(?P<q>["\'])(?P<url>http.+?)(?P=q)', webpage)]
 
-        if not m3u8_url:
+        if not m3u8_formats:
             error = self._search_regex(
                 [r'<span[^>]+class=(["\'])desc_span\1[^>]*>(?P<error>[^<]+)</span>',
                  r'<div[^>]+id=(["\'])defchat\1[^>]*>\s*<p><strong>(?P<error>[^<]+)<'],
                 webpage, 'error', group='error', default=None)
             if not error:
-                if any(p not in webpage for p in (
+                if any(p in webpage for p in (
                         self._ROOM_OFFLINE, 'offline_tipping', 'tip_offline')):
                     error = self._ROOM_OFFLINE
             if error:
                 raise ExtractorError(error, expected=True)
             raise ExtractorError('Unable to find stream URL')
 
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+        formats = []
+        for m3u8_id, m3u8_url in m3u8_formats:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, ext='mp4',
+                # ffmpeg skips segments for fast m3u8
+                preference=-10 if m3u8_id == 'fast' else None,
+                m3u8_id=m3u8_id, fatal=False, live=True))
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': self._live_title(video_id),
-            'thumbnail': 'https://cdn-s.highwebmedia.com/uHK3McUtGCG3SMFcd4ZJsRv8/roomimage/%s.jpg' % video_id,
+            'thumbnail': 'https://roomimg.stream.highwebmedia.com/ri/%s.jpg' % video_id,
             'age_limit': self._rta_search(webpage),
             'is_live': True,
             'formats': formats,

youtube_dl/extractor/cliphunter.py

@@ -30,7 +30,7 @@ class CliphunterIE(InfoExtractor):
             'id': '1012420',
             'ext': 'flv',
             'title': 'Fun Jynx Maze solo',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'age_limit': 18,
         },
         'skip': 'Video gone',
@@ -41,7 +41,7 @@ class CliphunterIE(InfoExtractor):
             'id': '2019449',
             'ext': 'mp4',
             'title': 'ShesNew - My booty girlfriend, Victoria Paradice\'s pussy filled with jizz',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'age_limit': 18,
         },
     }]

youtube_dl/extractor/clipsyndicate.py

@@ -18,7 +18,7 @@ class ClipsyndicateIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Brick Briscoe',
             'duration': 612,
-            'thumbnail': 're:^https?://.+\.jpg',
+            'thumbnail': r're:^https?://.+\.jpg',
         },
     }, {
         'url': 'http://chic.clipsyndicate.com/video/play/5844117/shark_attack',

youtube_dl/extractor/clubic.py

@@ -19,7 +19,7 @@ class ClubicIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Clubic Week 2.0 : le FBI se lance dans la photo d\u0092identité',
             'description': 're:Gueule de bois chez Nokia. Le constructeur a indiqué cette.*',
-            'thumbnail': 're:^http://img\.clubic\.com/.*\.jpg$',
+            'thumbnail': r're:^http://img\.clubic\.com/.*\.jpg$',
         }
     }, {
         'url': 'http://www.clubic.com/video/video-clubic-week-2-0-apple-iphone-6s-et-plus-mais-surtout-le-pencil-469792.html',

youtube_dl/extractor/cmt.py

@@ -1,13 +1,11 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVIE
-from ..utils import ExtractorError
 
 
 class CMTIE(MTVIE):
     IE_NAME = 'cmt.com'
-    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
-    _FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
+    _VALID_URL = r'https?://(?:www\.)?cmt\.com/(?:videos|shows|full-episodes|video-clips)/(?P<id>[^/]+)'
 
     _TESTS = [{
         'url': 'http://www.cmt.com/videos/garth-brooks/989124/the-call-featuring-trisha-yearwood.jhtml#artist=30061',
@@ -33,17 +31,24 @@ class CMTIE(MTVIE):
     }, {
         'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
         'only_matching': True,
+    }, {
+        'url': 'http://www.cmt.com/full-episodes/537qb3/nashville-the-wayfaring-stranger-season-5-ep-501',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.cmt.com/video-clips/t9e4ci/nashville-juliette-in-2-minutes',
+        'only_matching': True,
     }]
 
-    @classmethod
-    def _transform_rtmp_url(cls, rtmp_video_url):
-        if 'error_not_available.swf' in rtmp_video_url:
-            raise ExtractorError(
-                '%s said: video is not available' % cls.IE_NAME, expected=True)
-
-        return super(CMTIE, cls)._transform_rtmp_url(rtmp_video_url)
-
     def _extract_mgid(self, webpage):
-        return self._search_regex(
+        mgid = self._search_regex(
             r'MTVN\.VIDEO\.contentUri\s*=\s*([\'"])(?P<mgid>.+?)\1',
-            webpage, 'mgid', group='mgid')
+            webpage, 'mgid', group='mgid', default=None)
+        if not mgid:
+            mgid = self._extract_triforce_mgid(webpage)
+        return mgid
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        mgid = self._extract_mgid(webpage)
+        return self.url_result('http://media.mtvnservices.com/embed/%s' % mgid)

youtube_dl/extractor/collegerama.py

@@ -21,7 +21,7 @@ class CollegeRamaIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
                 'description': '',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 7713.088,
                 'timestamp': 1413309600,
                 'upload_date': '20141014',

youtube_dl/extractor/comedycentral.py

@@ -48,15 +48,7 @@ class ComedyCentralFullEpisodesIE(MTVServicesInfoExtractor):
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
-
-        feed_json = self._search_regex(r'var triforceManifestFeed\s*=\s*(\{.+?\});\n', webpage, 'triforce feeed')
-        feed = self._parse_json(feed_json, playlist_id)
-        zones = feed['manifest']['zones']
-
-        video_zone = zones['t2_lc_promo1']
-        feed = self._download_json(video_zone['feed'], playlist_id)
-        mgid = feed['result']['data']['id']
-
+        mgid = self._extract_triforce_mgid(webpage, data_zone='t2_lc_promo1')
         videos_info = self._get_videos_info(mgid)
         return videos_info
 
@@ -79,7 +71,7 @@ class ToshIE(MTVServicesInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Tosh.0|June 9, 2077|2|211|Twitter Users Share Summer Plans',
                 'description': 'Tosh asked fans to share their summer plans.',
-                'thumbnail': 're:^https?://.*\.jpg',
+                'thumbnail': r're:^https?://.*\.jpg',
                 # It's really reported to be published on year 2077
                 'upload_date': '20770610',
                 'timestamp': 3390510600,
@@ -93,12 +85,6 @@ class ToshIE(MTVServicesInfoExtractor):
         'only_matching': True,
     }]
 
-    @classmethod
-    def _transform_rtmp_url(cls, rtmp_video_url):
-        new_urls = super(ToshIE, cls)._transform_rtmp_url(rtmp_video_url)
-        new_urls['rtmp'] = rtmp_video_url.replace('viacomccstrm', 'viacommtvstrm')
-        return new_urls
-
 
 class ComedyCentralTVIE(MTVServicesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/(?:staffeln|shows)/(?P<id>[^/?#&]+)'

youtube_dl/extractor/common.py

@@ -1967,10 +1967,13 @@ class InfoExtractor(object):
                 entries.append(media_info)
         return entries
 
-    def _extract_akamai_formats(self, manifest_url, video_id):
+    def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
         formats = []
         hdcore_sign = 'hdcore=3.7.0'
-        f4m_url = re.sub(r'(https?://.+?)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        f4m_url = re.sub(r'(https?://[^/+])/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
+        hds_host = hosts.get('hds')
+        if hds_host:
+            f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)
         if 'hdcore=' not in f4m_url:
             f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
         f4m_formats = self._extract_f4m_formats(
@@ -1978,7 +1981,10 @@ class InfoExtractor(object):
         for entry in f4m_formats:
             entry.update({'extra_param_to_segment_url': hdcore_sign})
         formats.extend(f4m_formats)
-        m3u8_url = re.sub(r'(https?://.+?)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
+        m3u8_url = re.sub(r'(https?://[^/]+)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
+        hls_host = hosts.get('hls')
+        if hls_host:
+            m3u8_url = re.sub(r'(https?://)[^/]+', r'\1' + hls_host, m3u8_url)
         formats.extend(self._extract_m3u8_formats(
             m3u8_url, video_id, 'mp4', 'm3u8_native',
             m3u8_id='hls', fatal=False))

youtube_dl/extractor/coub.py

@@ -20,7 +20,7 @@ class CoubIE(InfoExtractor):
             'id': '5u5n1',
             'ext': 'mp4',
             'title': 'The Matrix Moonwalk',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 4.6,
             'timestamp': 1428527772,
             'upload_date': '20150408',

youtube_dl/extractor/crackle.py

@@ -14,7 +14,7 @@ class CrackleIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Everybody Respects A Bloody Nose',
             'description': 'Jerry is kaffeeklatsching in L.A. with funnyman J.B. Smoove (Saturday Night Live, Real Husbands of Hollywood). They’re headed for brew at 10 Speed Coffee in a 1964 Studebaker Avanti.',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 906,
             'series': 'Comedians In Cars Getting Coffee',
             'season_number': 8,

youtube_dl/extractor/criterion.py

@@ -14,7 +14,7 @@ class CriterionIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Le Samouraï',
             'description': 'md5:a2b4b116326558149bef81f76dcbb93f',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         }
     }
 

youtube_dl/extractor/crooksandliars.py

@@ -16,7 +16,7 @@ class CrooksAndLiarsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!',
             'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1428207000,
             'upload_date': '20150405',
             'uploader': 'Heather',

youtube_dl/extractor/crunchyroll.py

@@ -142,7 +142,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'ext': 'flv',
             'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
             'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Danny Choo Network',
             'upload_date': '20120213',
         },
@@ -158,7 +158,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
             'ext': 'mp4',
             'title': 'Re:ZERO -Starting Life in Another World- Episode 5 – The Morning of Our Promise Is Still Distant',
             'description': 'md5:97664de1ab24bbf77a9c01918cb7dca9',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'TV TOKYO',
             'upload_date': '20160508',
         },

youtube_dl/extractor/cspan.py

@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
 )
 from .senateisvp import SenateISVPIE
+from .ustream import UstreamIE
 
 
 class CSpanIE(InfoExtractor):
@@ -22,14 +23,13 @@ class CSpanIE(InfoExtractor):
         'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
         'info_dict': {
             'id': '315139',
-            'ext': 'mp4',
             'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
-            'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
         },
+        'playlist_mincount': 2,
         'skip': 'Regularly fails on travis, for unknown reasons',
     }, {
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
-        'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
+        # md5 is unstable
         'info_dict': {
             'id': 'c4486943',
             'ext': 'mp4',
@@ -38,14 +38,11 @@ class CSpanIE(InfoExtractor):
         }
     }, {
         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
-        'md5': '2ae5051559169baadba13fc35345ae74',
         'info_dict': {
             'id': '342759',
-            'ext': 'mp4',
             'title': 'General Motors Ignition Switch Recall',
-            'duration': 14848,
-            'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
         },
+        'playlist_mincount': 6,
     }, {
         # Video from senate.gov
         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
@@ -57,12 +54,30 @@ class CSpanIE(InfoExtractor):
         'params': {
             'skip_download': True,  # m3u8 downloads
         }
+    }, {
+        # Ustream embedded video
+        'url': 'https://www.c-span.org/video/?114917-1/armed-services',
+        'info_dict': {
+            'id': '58428542',
+            'ext': 'flv',
+            'title': 'USHR07 Armed Services Committee',
+            'description': 'hsas00-2118-20150204-1000et-07\n\n\nUSHR07 Armed Services Committee',
+            'timestamp': 1423060374,
+            'upload_date': '20150204',
+            'uploader': 'HouseCommittee',
+            'uploader_id': '12987475',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_type = None
         webpage = self._download_webpage(url, video_id)
+
+        ustream_url = UstreamIE._extract_url(webpage)
+        if ustream_url:
+            return self.url_result(ustream_url, UstreamIE.ie_key())
+
         # We first look for clipid, because clipprog always appears before
         patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
         results = list(filter(None, (re.search(p, webpage) for p in patterns)))

youtube_dl/extractor/ctsnews.py

@@ -28,7 +28,7 @@ class CtsNewsIE(InfoExtractor):
             'ext': 'mp4',
             'title': '韓國31歲童顏男 貌如十多歲小孩',
             'description': '越有年紀的人，越希望看起來年輕一點，而南韓卻有一位31歲的男子，看起來像是11、12歲的小孩，身...',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'timestamp': 1378205880,
             'upload_date': '20130903',
         }
@@ -41,7 +41,7 @@ class CtsNewsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'iPhone6熱銷 蘋果財報亮眼',
             'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20150128',
             'uploader_id': 'TBSCTS',
             'uploader': '中華電視公司',

youtube_dl/extractor/cultureunplugged.py

@@ -21,7 +21,7 @@ class CultureUnpluggedIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'The Next, Best West',
             'description': 'md5:0423cd00833dea1519cf014e9d0903b1',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'creator': 'Coldstream Creative',
             'duration': 2203,
             'view_count': int,

youtube_dl/extractor/dailymotion.py

@@ -58,7 +58,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
                 'description': 'Several come bundled with the Steam Controller.',
-                'thumbnail': 're:^https?:.*\.(?:jpg|png)$',
+                'thumbnail': r're:^https?:.*\.(?:jpg|png)$',
                 'duration': 74,
                 'timestamp': 1425657362,
                 'upload_date': '20150306',

youtube_dl/extractor/daum.py

@@ -32,7 +32,7 @@ class DaumIE(InfoExtractor):
             'title': '마크 헌트 vs 안토니오 실바',
             'description': 'Mark Hunt vs Antonio Silva',
             'upload_date': '20131217',
-            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
             'duration': 2117,
             'view_count': int,
             'comment_count': int,
@@ -45,7 +45,7 @@ class DaumIE(InfoExtractor):
             'title': '1297회, \'아빠 아들로 태어나길 잘 했어\' 민수, 감동의 눈물[아빠 어디가] 20150118',
             'description': 'md5:79794514261164ff27e36a21ad229fc5',
             'upload_date': '20150604',
-            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
             'duration': 154,
             'view_count': int,
             'comment_count': int,
@@ -61,7 +61,7 @@ class DaumIE(InfoExtractor):
             'title': '01-Korean War ( Trouble on the horizon )',
             'description': '\nKorean War 01\nTrouble on the horizon\n전쟁의 먹구름',
             'upload_date': '20080223',
-            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
             'duration': 249,
             'view_count': int,
             'comment_count': int,
@@ -139,7 +139,7 @@ class DaumClipIE(InfoExtractor):
             'title': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'description': 'DOTA 2GETHER 시즌2 6회 - 2부',
             'upload_date': '20130831',
-            'thumbnail': 're:^https?://.*\.(?:jpg|png)',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
             'duration': 3868,
             'view_count': int,
         },

youtube_dl/extractor/dbtv.py

@@ -17,7 +17,7 @@ class DBTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Skulle teste ut fornøyelsespark, men kollegaen var bare opptatt av bikinikroppen',
             'description': 'md5:1504a54606c4dde3e4e61fc97aa857e0',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'timestamp': 1404039863,
             'upload_date': '20140629',
             'duration': 69.544,

youtube_dl/extractor/dctp.py

@@ -17,7 +17,7 @@ class DctpTvIE(InfoExtractor):
             'title': 'Videoinstallation für eine Kaufhausfassade',
             'description': 'Kurzfilm',
             'upload_date': '20110407',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
     }
 

youtube_dl/extractor/deezer.py

@@ -19,7 +19,7 @@ class DeezerPlaylistIE(InfoExtractor):
             'id': '176747451',
             'title': 'Best!',
             'uploader': 'Anonymous',
-            'thumbnail': 're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$',
+            'thumbnail': r're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$',
         },
         'playlist_count': 30,
         'skip': 'Only available in .de',

youtube_dl/extractor/dhm.py

@@ -17,7 +17,7 @@ class DHMIE(InfoExtractor):
             'title': 'MARSHALL PLAN AT WORK IN WESTERN GERMANY, THE',
             'description': 'md5:1fabd480c153f97b07add61c44407c82',
             'duration': 660,
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
         'url': 'http://www.dhm.de/filmarchiv/02-mapping-the-wall/peter-g/rolle-1/',
@@ -26,7 +26,7 @@ class DHMIE(InfoExtractor):
             'id': 'rolle-1',
             'ext': 'flv',
             'title': 'ROLLE 1',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
     }]
 

youtube_dl/extractor/digiteka.py

@@ -36,7 +36,7 @@ class DigitekaIE(InfoExtractor):
             'id': 's8uk0r',
             'ext': 'mp4',
             'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 74,
             'upload_date': '20150317',
             'timestamp': 1426604939,
@@ -50,7 +50,7 @@ class DigitekaIE(InfoExtractor):
             'id': 'xvpfp8',
             'ext': 'mp4',
             'title': 'Two - C\'est La Vie (clip)',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'duration': 233,
             'upload_date': '20150224',
             'timestamp': 1424760500,

youtube_dl/extractor/discoverygo.py

@@ -6,7 +6,6 @@ from ..utils import (
     extract_attributes,
     int_or_none,
     parse_age_limit,
-    unescapeHTML,
     ExtractorError,
 )
 
@@ -49,7 +48,7 @@ class DiscoveryGoIE(InfoExtractor):
                 webpage, 'video container'))
 
         video = self._parse_json(
-            unescapeHTML(container.get('data-video') or container.get('data-json')),
+            container.get('data-video') or container.get('data-json'),
             display_id)
 
         title = video['name']

youtube_dl/extractor/douyutv.py

@@ -26,8 +26,8 @@ class DouyuTVIE(InfoExtractor):
             'display_id': 'iseven',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 're:.*m7show@163\.com.*',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': r're:.*m7show@163\.com.*',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'is_live': True,
         },
@@ -42,7 +42,7 @@ class DouyuTVIE(InfoExtractor):
             'ext': 'flv',
             'title': 're:^小漠从零单排记！——CSOL2躲猫猫 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:746a2f7a253966a06755a912f0acc0d2',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'douyu小漠',
             'is_live': True,
         },
@@ -57,8 +57,8 @@ class DouyuTVIE(InfoExtractor):
             'display_id': '17732',
             'ext': 'flv',
             'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
-            'description': 're:.*m7show@163\.com.*',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': r're:.*m7show@163\.com.*',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': '7师傅',
             'is_live': True,
         },

youtube_dl/extractor/dramafever.py

@@ -66,7 +66,7 @@ class DramaFeverBaseIE(AMPIE):
 
 class DramaFeverIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+/[0-9]+)(?:/|$)'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/1/Cooking_with_Shin/',
         'info_dict': {
@@ -76,7 +76,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             'description': 'md5:a8eec7942e1664a6896fcd5e1287bfd0',
             'episode': 'Episode 1',
             'episode_number': 1,
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1404336058,
             'upload_date': '20140702',
             'duration': 343,
@@ -94,7 +94,7 @@ class DramaFeverIE(DramaFeverBaseIE):
             'description': 'md5:3ff2ee8fedaef86e076791c909cf2e91',
             'episode': 'Mnet Asian Music Awards 2015 - Part 3',
             'episode_number': 4,
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1450213200,
             'upload_date': '20151215',
             'duration': 5602,
@@ -103,6 +103,9 @@ class DramaFeverIE(DramaFeverBaseIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.dramafever.com/zh-cn/drama/4972/15/Doctor_Romantic/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -148,7 +151,7 @@ class DramaFeverIE(DramaFeverBaseIE):
 
 class DramaFeverSeriesIE(DramaFeverBaseIE):
     IE_NAME = 'dramafever:series'
-    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
+    _VALID_URL = r'https?://(?:www\.)?dramafever\.com/(?:[^/]+/)?drama/(?P<id>[0-9]+)(?:/(?:(?!\d+(?:/|$)).+)?)?$'
     _TESTS = [{
         'url': 'http://www.dramafever.com/drama/4512/Cooking_with_Shin/',
         'info_dict': {

youtube_dl/extractor/drbonanza.py

@@ -20,7 +20,7 @@ class DRBonanzaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Talkshowet - Leonard Cohen',
             'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
-            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
+            'thumbnail': r're:^https?://.*\.(?:gif|jpg)$',
             'timestamp': 1295537932,
             'upload_date': '20110120',
             'duration': 3664,
@@ -36,7 +36,7 @@ class DRBonanzaIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
             'description': 'md5:501e5a195749480552e214fbbed16c4e',
-            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
+            'thumbnail': r're:^https?://.*\.(?:gif|jpg)$',
             'timestamp': 1223274900,
             'upload_date': '20081006',
             'duration': 7369,

youtube_dl/extractor/dreisat.py

@@ -2,10 +2,19 @@ from __future__ import unicode_literals
 
 import re
 
-from .zdf import ZDFIE
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+    xpath_text,
+    determine_ext,
+    qualities,
+    float_or_none,
+    ExtractorError,
+)
 
 
-class DreiSatIE(ZDFIE):
+class DreiSatIE(InfoExtractor):
     IE_NAME = '3sat'
     _VALID_URL = r'(?:https?://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
     _TESTS = [
@@ -31,6 +40,163 @@ class DreiSatIE(ZDFIE):
         },
     ]
 
+    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
+        param_groups = {}
+        for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)):
+            group_id = param_group.attrib.get(self._xpath_ns('id', 'http://www.w3.org/XML/1998/namespace'))
+            params = {}
+            for param in param_group:
+                params[param.get('name')] = param.get('value')
+            param_groups[group_id] = params
+
+        formats = []
+        for video in smil.findall(self._xpath_ns('.//video', namespace)):
+            src = video.get('src')
+            if not src:
+                continue
+            bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            group_id = video.get('paramGroup')
+            param_group = param_groups[group_id]
+            for proto in param_group['protocols'].split(','):
+                formats.append({
+                    'url': '%s://%s' % (proto, param_group['host']),
+                    'app': param_group['app'],
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': '%s-%d' % (proto, bitrate),
+                    'tbr': bitrate,
+                })
+        self._sort_formats(formats)
+        return formats
+
+    def extract_from_xml_url(self, video_id, xml_url):
+        doc = self._download_xml(
+            xml_url, video_id,
+            note='Downloading video info',
+            errnote='Failed to download video info')
+
+        status_code = doc.find('./status/statuscode')
+        if status_code is not None and status_code.text != 'ok':
+            code = status_code.text
+            if code == 'notVisibleAnymore':
+                message = 'Video %s is not available' % video_id
+            else:
+                message = '%s returned error: %s' % (self.IE_NAME, code)
+            raise ExtractorError(message, expected=True)
+
+        title = doc.find('.//information/title').text
+        description = xpath_text(doc, './/information/detail', 'description')
+        duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
+        uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
+        uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
+        upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
+
+        def xml_to_thumbnails(fnode):
+            thumbnails = []
+            for node in fnode:
+                thumbnail_url = node.text
+                if not thumbnail_url:
+                    continue
+                thumbnail = {
+                    'url': thumbnail_url,
+                }
+                if 'key' in node.attrib:
+                    m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
+                    if m:
+                        thumbnail['width'] = int(m.group(1))
+                        thumbnail['height'] = int(m.group(2))
+                thumbnails.append(thumbnail)
+            return thumbnails
+
+        thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
+
+        format_nodes = doc.findall('.//formitaeten/formitaet')
+        quality = qualities(['veryhigh', 'high', 'med', 'low'])
+
+        def get_quality(elem):
+            return quality(xpath_text(elem, 'quality'))
+        format_nodes.sort(key=get_quality)
+        format_ids = []
+        formats = []
+        for fnode in format_nodes:
+            video_url = fnode.find('url').text
+            is_available = 'http://www.metafilegenerator' not in video_url
+            if not is_available:
+                continue
+            format_id = fnode.attrib['basetype']
+            quality = xpath_text(fnode, './quality', 'quality')
+            format_m = re.match(r'''(?x)
+                (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
+                (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
+            ''', format_id)
+
+            ext = determine_ext(video_url, None) or format_m.group('container')
+            if ext not in ('smil', 'f4m', 'm3u8'):
+                format_id = format_id + '-' + quality
+            if format_id in format_ids:
+                continue
+
+            if ext == 'meta':
+                continue
+            elif ext == 'smil':
+                formats.extend(self._extract_smil_formats(
+                    video_url, video_id, fatal=False))
+            elif ext == 'm3u8':
+                # the certificates are misconfigured (see
+                # https://github.com/rg3/youtube-dl/issues/8665)
+                if video_url.startswith('https://'):
+                    continue
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    video_url, video_id, f4m_id=format_id, fatal=False))
+            else:
+                proto = format_m.group('proto').lower()
+
+                abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
+                vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
+
+                width = int_or_none(xpath_text(fnode, './width', 'width'))
+                height = int_or_none(xpath_text(fnode, './height', 'height'))
+
+                filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
+
+                format_note = ''
+                if not format_note:
+                    format_note = None
+
+                formats.append({
+                    'format_id': format_id,
+                    'url': video_url,
+                    'ext': ext,
+                    'acodec': format_m.group('acodec'),
+                    'vcodec': format_m.group('vcodec'),
+                    'abr': abr,
+                    'vbr': vbr,
+                    'width': width,
+                    'height': height,
+                    'filesize': filesize,
+                    'format_note': format_note,
+                    'protocol': proto,
+                    '_available': is_available,
+                })
+            format_ids.append(format_id)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')

youtube_dl/extractor/drtuber.py

@@ -22,7 +22,7 @@ class DrTuberIE(InfoExtractor):
             'like_count': int,
             'comment_count': int,
             'categories': ['Babe', 'Blonde', 'Erotic', 'Outdoor', 'Softcore', 'Solo'],
-            'thumbnail': 're:https?://.*\.jpg$',
+            'thumbnail': r're:https?://.*\.jpg$',
             'age_limit': 18,
         }
     }, {

youtube_dl/extractor/dumpert.py

@@ -21,7 +21,7 @@ class DumpertIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ik heb nieuws voor je',
             'description': 'Niet schrikken hoor',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         }
     }, {
         'url': 'http://www.dumpert.nl/embed/6675421/dc440fe7/',

youtube_dl/extractor/eagleplatform.py

@@ -31,7 +31,7 @@ class EaglePlatformIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Навальный вышел на свободу',
             'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 87,
             'view_count': int,
             'age_limit': 0,
@@ -45,7 +45,7 @@ class EaglePlatformIE(InfoExtractor):
             'id': '12820',
             'ext': 'mp4',
             'title': "'O Sole Mio",
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 216,
             'view_count': int,
         },

youtube_dl/extractor/egghead.py

@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class EggheadCourseIE(InfoExtractor):
+    IE_DESC = 'egghead.io course'
+    IE_NAME = 'egghead:course'
+    _VALID_URL = r'https://egghead\.io/courses/(?P<id>[a-zA-Z_0-9-]+)'
+    _TEST = {
+        'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
+        'playlist_count': 29,
+        'info_dict': {
+            'id': 'professor-frisby-introduces-composable-functional-javascript',
+            'title': 'Professor Frisby Introduces Composable Functional JavaScript',
+            'description': 're:(?s)^This course teaches the ubiquitous.*You\'ll start composing functionality before you know it.$',
+        },
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        title = self._html_search_regex(r'<h1 class="title">([^<]+)</h1>', webpage, 'title')
+        ul = self._search_regex(r'(?s)<ul class="series-lessons-list">(.*?)</ul>', webpage, 'session list')
+
+        found = re.findall(r'(?s)<a class="[^"]*"\s*href="([^"]+)">\s*<li class="item', ul)
+        entries = [self.url_result(m) for m in found]
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': title,
+            'description': self._og_search_description(webpage),
+            'entries': entries,
+        }

youtube_dl/extractor/einthusan.py

@@ -19,7 +19,7 @@ class EinthusanIE(InfoExtractor):
                 'id': '2447',
                 'ext': 'mp4',
                 'title': 'Ek Villain',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'description': 'md5:9d29fc91a7abadd4591fb862fa560d93',
             }
         },
@@ -30,7 +30,7 @@ class EinthusanIE(InfoExtractor):
                 'id': '1671',
                 'ext': 'mp4',
                 'title': 'Soodhu Kavvuum',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'description': 'md5:b40f2bf7320b4f9414f3780817b2af8c',
             }
         },

youtube_dl/extractor/eroprofile.py

@@ -22,7 +22,7 @@ class EroProfileIE(InfoExtractor):
             'display_id': 'sexy-babe-softcore',
             'ext': 'm4v',
             'title': 'sexy babe softcore',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
         }
     }, {
@@ -32,7 +32,7 @@ class EroProfileIE(InfoExtractor):
             'id': '1133519',
             'ext': 'm4v',
             'title': 'Try It On Pee_cut_2.wmv - 4shared.com - file sharing - download movie file',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
         },
         'skip': 'Requires login',

youtube_dl/extractor/escapist.py

@@ -45,7 +45,7 @@ class EscapistIE(InfoExtractor):
             'ext': 'mp4',
             'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
             'title': "Breaking Down Baldur's Gate",
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 264,
             'uploader': 'The Escapist',
         }
@@ -57,7 +57,7 @@ class EscapistIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'This week, Zero Punctuation reviews Evolve.',
             'title': 'Evolve - One vs Multiplayer',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 304,
             'uploader': 'The Escapist',
         }

youtube_dl/extractor/esri.py

@@ -22,7 +22,7 @@ class EsriVideoIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'ArcGIS Online - Developing Applications',
             'description': 'Jeremy Bartley demonstrates how to develop applications with ArcGIS Online.',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 185,
             'upload_date': '20120419',
         }

youtube_dl/extractor/europa.py

@@ -23,7 +23,7 @@ class EuropaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'TRADE - Wikileaks on TTIP',
             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20150811',
             'duration': 34,
             'view_count': int,

youtube_dl/extractor/expotv.py

@@ -17,7 +17,7 @@ class ExpoTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'NYX Butter Lipstick Little Susie',
             'description': 'Goes on like butter, but looks better!',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Stephanie S.',
             'upload_date': '20150520',
             'view_count': int,

youtube_dl/extractor/extractors.py

@@ -38,10 +38,7 @@ from .amcnetworks import AMCNetworksIE
 from .animeondemand import AnimeOnDemandIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
-from .aol import (
-    AolIE,
-    AolFeaturesIE,
-)
+from .aol import AolIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE
@@ -80,6 +77,10 @@ from .awaan import (
     AWAANLiveIE,
     AWAANSeasonIE,
 )
+from .azmedien import (
+    AZMedienIE,
+    AZMedienShowIE,
+)
 from .azubu import AzubuIE, AzubuLiveIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
@@ -91,6 +92,7 @@ from .bbc import (
     BBCCoUkPlaylistIE,
     BBCIE,
 )
+from .beampro import BeamProLiveIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .bellmedia import BellMediaIE
@@ -255,6 +257,7 @@ from .dw import (
 from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
+from .egghead import EggheadCourseIE
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
@@ -369,6 +372,7 @@ from .hgtv import (
 )
 from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
+from .hitrecord import HitRecordIE
 from .hornbunny import HornBunnyIE
 from .hotnewhiphop import HotNewHipHopIE
 from .hotstar import HotStarIE
@@ -396,6 +400,7 @@ from .imgur import (
     ImgurAlbumIE,
 )
 from .ina import InaIE
+from .inc import IncIE
 from .indavideo import (
     IndavideoIE,
     IndavideoEmbedIE,
@@ -719,6 +724,7 @@ from .polskieradio import (
 )
 from .porn91 import Porn91IE
 from .porncom import PornComIE
+from .pornflip import PornFlipIE
 from .pornhd import PornHdIE
 from .pornhub import (
     PornHubIE,
@@ -813,7 +819,6 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .screencastomatic import ScreencastOMaticIE
-from .screenjunkies import ScreenJunkiesIE
 from .seeker import SeekerIE
 from .senateisvp import SenateISVPIE
 from .sendtonews import SendtoNewsIE
@@ -824,7 +829,6 @@ from .shared import (
     SharedIE,
     VivoIE,
 )
-from .sharesix import ShareSixIE
 from .showroomlive import ShowRoomLiveIE
 from .sina import SinaIE
 from .sixplay import SixPlayIE
@@ -1075,7 +1079,6 @@ from .videomore import (
     VideomoreSeasonIE,
 )
 from .videopremium import VideoPremiumIE
-from .videott import VideoTtIE
 from .vidio import VidioIE
 from .vidme import (
     VidmeIE,

youtube_dl/extractor/fc2.py

@@ -133,7 +133,7 @@ class FC2EmbedIE(InfoExtractor):
             'id': '201403223kCqB3Ez',
             'ext': 'flv',
             'title': 'プリズン･ブレイク S1-01 マイケル 【吹替】',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
     }
 

youtube_dl/extractor/firsttv.py

@@ -26,7 +26,7 @@ class FirstTVIE(InfoExtractor):
             'id': '40049',
             'ext': 'mp4',
             'title': 'Гость Людмила Сенчина. Наедине со всеми. Выпуск от 12.02.2015',
-            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$',
             'upload_date': '20150212',
             'duration': 2694,
         },
@@ -37,7 +37,7 @@ class FirstTVIE(InfoExtractor):
             'id': '364746',
             'ext': 'mp4',
             'title': 'Весенняя аллергия. Доброе утро. Фрагмент выпуска от 07.04.2016',
-            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'thumbnail': r're:^https?://.*\.(?:jpg|JPG)$',
             'upload_date': '20160407',
             'duration': 179,
             'formats': 'mincount:3',
@@ -86,18 +86,43 @@ class FirstTVIE(InfoExtractor):
             title = item['title']
             quality = qualities(QUALITIES)
             formats = []
+            path = None
             for f in item.get('mbr', []):
                 src = f.get('src')
                 if not src or not isinstance(src, compat_str):
                     continue
                 tbr = int_or_none(self._search_regex(
                     r'_(\d{3,})\.mp4', src, 'tbr', default=None))
+                if not path:
+                    path = self._search_regex(
+                        r'//[^/]+/(.+?)_\d+\.mp4', src,
+                        'm3u8 path', default=None)
                 formats.append({
                     'url': src,
                     'format_id': f.get('name'),
                     'tbr': tbr,
-                    'quality': quality(f.get('name')),
+                    'source_preference': quality(f.get('name')),
                 })
+            # m3u8 URL format is reverse engineered from [1] (search for
+            # master.m3u8). dashEdges (that is currently balancer-vod.1tv.ru)
+            # is taken from [2].
+            # 1. http://static.1tv.ru/player/eump1tv-current/eump-1tv.all.min.js?rnd=9097422834:formatted
+            # 2. http://static.1tv.ru/player/eump1tv-config/config-main.js?rnd=9097422834
+            if not path and len(formats) == 1:
+                path = self._search_regex(
+                    r'//[^/]+/(.+?$)', formats[0]['url'],
+                    'm3u8 path', default=None)
+            if path:
+                if len(formats) == 1:
+                    m3u8_path = ','
+                else:
+                    tbrs = [compat_str(t) for t in sorted(f['tbr'] for f in formats)]
+                    m3u8_path = '_,%s,%s' % (','.join(tbrs), '.mp4')
+                formats.extend(self._extract_m3u8_formats(
+                    'http://balancer-vod.1tv.ru/%s%s.urlset/master.m3u8'
+                    % (path, m3u8_path),
+                    display_id, 'mp4',
+                    entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
             self._sort_formats(formats)
 
             thumbnail = item.get('poster') or self._og_search_thumbnail(webpage)

youtube_dl/extractor/fivetv.py

@@ -25,7 +25,7 @@ class FiveTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Россияне выбрали имя для общенациональной платежной системы',
             'description': 'md5:a8aa13e2b7ad36789e9f77a74b6de660',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 180,
         },
     }, {
@@ -35,7 +35,7 @@ class FiveTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': '3D принтер',
             'description': 'md5:d76c736d29ef7ec5c0cf7d7c65ffcb41',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 180,
         },
     }, {
@@ -44,7 +44,7 @@ class FiveTVIE(InfoExtractor):
             'id': 'glavnoe',
             'ext': 'mp4',
             'title': 'Итоги недели с 8 по 14 июня 2015 года',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
         'url': 'http://www.5-tv.ru/glavnoe/broadcasts/508645/',

youtube_dl/extractor/fktv.py

@@ -19,7 +19,7 @@ class FKTVIE(InfoExtractor):
             'id': '1',
             'ext': 'mp4',
             'title': 'Folge 1 vom 10. April 2007',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
     }
 

youtube_dl/extractor/flipagram.py

@@ -81,7 +81,7 @@ class FlipagramIE(InfoExtractor):
             'filesize': int_or_none(cover.get('size')),
         } for cover in flipagram.get('covers', []) if cover.get('url')]
 
-        # Note that this only retrieves comments that are initally loaded.
+        # Note that this only retrieves comments that are initially loaded.
         # For videos with large amounts of comments, most won't be retrieved.
         comments = []
         for comment in video_data.get('comments', {}).get(video_id, {}).get('items', []):

youtube_dl/extractor/foxgay.py

@@ -20,7 +20,7 @@ class FoxgayIE(InfoExtractor):
             'title': 'Fuck Turkish-style',
             'description': 'md5:6ae2d9486921891efe89231ace13ffdf',
             'age_limit': 18,
-            'thumbnail': 're:https?://.*\.jpg$',
+            'thumbnail': r're:https?://.*\.jpg$',
         },
     }
 

youtube_dl/extractor/foxnews.py

@@ -22,7 +22,7 @@ class FoxNewsIE(AMPIE):
                 'duration': 265,
                 'timestamp': 1304411491,
                 'upload_date': '20110503',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
         {
@@ -36,7 +36,7 @@ class FoxNewsIE(AMPIE):
                 'duration': 292,
                 'timestamp': 1417662047,
                 'upload_date': '20141204',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
             },
             'params': {
                 # m3u8 download
@@ -111,7 +111,7 @@ class FoxNewsInsiderIE(InfoExtractor):
             'description': 'Is campus censorship getting out of control?',
             'timestamp': 1472168725,
             'upload_date': '20160825',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         },
         'params': {
             # m3u8 download

youtube_dl/extractor/franceculture.py

@@ -17,7 +17,7 @@ class FranceCultureIE(InfoExtractor):
             'display_id': 'rendez-vous-au-pays-des-geeks',
             'ext': 'mp3',
             'title': 'Rendez-vous au pays des geeks',
-            'thumbnail': 're:^https?://.*\\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20140301',
             'vcodec': 'none',
         }

youtube_dl/extractor/francetv.py

@@ -168,7 +168,7 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'id': 'NI_173343',
             'ext': 'mp4',
             'title': 'Les entreprises familiales : le secret de la réussite',
-            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
             'timestamp': 1433273139,
             'upload_date': '20150602',
         },
@@ -184,7 +184,7 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
             'ext': 'mp4',
             'title': 'Olivier Monthus, réalisateur de "Bretagne, le choix de l’Armor"',
             'description': 'md5:a3264114c9d29aeca11ced113c37b16c',
-            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'thumbnail': r're:^https?://.*\.jpe?g$',
             'timestamp': 1458300695,
             'upload_date': '20160318',
         },

youtube_dl/extractor/freesound.py

@@ -3,10 +3,16 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    get_element_by_class,
+    get_element_by_id,
+    unified_strdate,
+)
 
 
 class FreesoundIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?freesound\.org/people/([^/]+)/sounds/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?freesound\.org/people/[^/]+/sounds/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.freesound.org/people/miklovan/sounds/194503/',
         'md5': '12280ceb42c81f19a515c745eae07650',
@@ -14,26 +20,60 @@ class FreesoundIE(InfoExtractor):
             'id': '194503',
             'ext': 'mp3',
             'title': 'gulls in the city.wav',
-            'uploader': 'miklovan',
             'description': 'the sounds of seagulls in the city',
+            'duration': 130.233,
+            'uploader': 'miklovan',
+            'upload_date': '20130715',
+            'tags': list,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        music_id = mobj.group('id')
-        webpage = self._download_webpage(url, music_id)
-        title = self._html_search_regex(
-            r'<div id="single_sample_header">.*?<a href="#">(.+?)</a>',
-            webpage, 'music title', flags=re.DOTALL)
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        audio_url = self._og_search_property('audio', webpage, 'song url')
+        title = self._og_search_property('audio:title', webpage, 'song title')
+
         description = self._html_search_regex(
-            r'<div id="sound_description">(.*?)</div>', webpage, 'description',
-            fatal=False, flags=re.DOTALL)
+            r'(?s)id=["\']sound_description["\'][^>]*>(.+?)</div>',
+            webpage, 'description', fatal=False)
+
+        duration = float_or_none(
+            get_element_by_class('duration', webpage), scale=1000)
+
+        upload_date = unified_strdate(get_element_by_id('sound_date', webpage))
+        uploader = self._og_search_property(
+            'audio:artist', webpage, 'uploader', fatal=False)
+
+        channels = self._html_search_regex(
+            r'Channels</dt><dd>(.+?)</dd>', webpage,
+            'channels info', fatal=False)
+
+        tags_str = get_element_by_class('tags', webpage)
+        tags = re.findall(r'<a[^>]+>([^<]+)', tags_str) if tags_str else None
+
+        audio_urls = [audio_url]
+
+        LQ_FORMAT = '-lq.mp3'
+        if LQ_FORMAT in audio_url:
+            audio_urls.append(audio_url.replace(LQ_FORMAT, '-hq.mp3'))
+
+        formats = [{
+            'url': format_url,
+            'format_note': channels,
+            'quality': quality,
+        } for quality, format_url in enumerate(audio_urls)]
+        self._sort_formats(formats)
 
         return {
-            'id': music_id,
+            'id': audio_id,
             'title': title,
-            'url': self._og_search_property('audio', webpage, 'music url'),
-            'uploader': self._og_search_property('audio:artist', webpage, 'music uploader'),
             'description': description,
+            'duration': duration,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'tags': tags,
+            'formats': formats,
         }

youtube_dl/extractor/funimation.py

@@ -29,7 +29,7 @@ class FunimationIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Air - 1 - Breeze',
             'description': 'md5:1769f43cd5fc130ace8fd87232207892',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
         },
         'skip': 'Access without user interaction is forbidden by CloudFlare, and video removed',
     }, {
@@ -40,7 +40,7 @@ class FunimationIE(InfoExtractor):
             'ext': 'mp4',
             'title': '.hack//SIGN - 1 - Role Play',
             'description': 'md5:b602bdc15eef4c9bbb201bb6e6a4a2dd',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
         },
         'skip': 'Access without user interaction is forbidden by CloudFlare',
     }, {
@@ -51,7 +51,7 @@ class FunimationIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Attack on Titan: Junior High - Broadcast Dub Preview',
             'description': 'md5:f8ec49c0aff702a7832cd81b8a44f803',
-            'thumbnail': 're:https?://.*\.(?:jpg|png)',
+            'thumbnail': r're:https?://.*\.(?:jpg|png)',
         },
         'skip': 'Access without user interaction is forbidden by CloudFlare',
     }]

youtube_dl/extractor/funnyordie.py

@@ -17,7 +17,7 @@ class FunnyOrDieIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Heart-Shaped Box: Literal Video Version',
             'description': 'md5:ea09a01bc9a1c46d9ab696c01747c338',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': r're:^http:.*\.jpg$',
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
@@ -26,7 +26,7 @@ class FunnyOrDieIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Please Use This Song (Jon Lajoie)',
             'description': 'Please use this to sell something.  www.jonlajoie.com',
-            'thumbnail': 're:^http:.*\.jpg$',
+            'thumbnail': r're:^http:.*\.jpg$',
         },
         'params': {
             'skip_download': True,

youtube_dl/extractor/gamersyde.py

@@ -20,7 +20,7 @@ class GamersydeIE(InfoExtractor):
             'ext': 'mp4',
             'duration': 372,
             'title': 'Bloodborne - Birth of a hero',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         }
     }
 

youtube_dl/extractor/gamespot.py

@@ -63,7 +63,7 @@ class GameSpotIE(OnceIE):
             streams, ('progressive_hd', 'progressive_high', 'progressive_low'))
         if progressive_url and manifest_url:
             qualities_basename = self._search_regex(
-                '/([^/]+)\.csmil/',
+                r'/([^/]+)\.csmil/',
                 manifest_url, 'qualities basename', default=None)
             if qualities_basename:
                 QUALITIES_RE = r'((,\d+)+,?)'

youtube_dl/extractor/gamestar.py

@@ -18,7 +18,7 @@ class GameStarIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil',
             'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den...',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'timestamp': 1406542020,
             'upload_date': '20140728',
             'duration': 17

youtube_dl/extractor/gazeta.py

@@ -16,7 +16,7 @@ class GazetaIE(InfoExtractor):
             'ext': 'mp4',
             'title': '«70–80 процентов гражданских в Донецке на грани голода»',
             'description': 'md5:38617526050bd17b234728e7f9620a71',
-            'thumbnail': 're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://.*\.jpg',
         },
         'skip': 'video not found',
     }, {

youtube_dl/extractor/generic.py

@@ -73,10 +73,13 @@ from .kaltura import KalturaIE
 from .eagleplatform import EaglePlatformIE
 from .facebook import FacebookIE
 from .soundcloud import SoundcloudIE
+from .tunein import TuneInBaseIE
 from .vbox7 import Vbox7IE
 from .dbtv import DBTVIE
 from .piksel import PikselIE
 from .videa import VideaIE
+from .twentymin import TwentyMinutenIE
+from .ustream import UstreamIE
 
 
 class GenericIE(InfoExtractor):
@@ -238,7 +241,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Tikibad ontruimd wegens brand',
                 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 33,
             },
             'params': {
@@ -299,7 +302,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20130224',
                 'uploader_id': 'TheVerge',
-                'description': 're:^Chris Ziegler takes a look at the\.*',
+                'description': r're:^Chris Ziegler takes a look at the\.*',
                 'uploader': 'The Verge',
                 'title': 'First Firefox OS phones side-by-side',
             },
@@ -421,6 +424,26 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,  # m3u8 download
             },
         },
+        {
+            # Brightcove with alternative playerID key
+            'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',
+            'info_dict': {
+                'id': 'nmeth.2062_SV1',
+                'title': 'Simultaneous multiview imaging of the Drosophila syncytial blastoderm : Quantitative high-speed imaging of entire developing embryos with simultaneous multiview light-sheet microscopy : Nature Methods : Nature Research',
+            },
+            'playlist': [{
+                'info_dict': {
+                    'id': '2228375078001',
+                    'ext': 'mp4',
+                    'title': 'nmeth.2062-sv1',
+                    'description': 'nmeth.2062-sv1',
+                    'timestamp': 1363357591,
+                    'upload_date': '20130315',
+                    'uploader': 'Nature Publishing Group',
+                    'uploader_id': '1964492299001',
+                },
+            }],
+        },
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
@@ -538,7 +561,7 @@ class GenericIE(InfoExtractor):
                 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
                 'ext': 'mp4',
                 'title': 'Ужастики, русский трейлер (2015)',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 153,
             }
         },
@@ -566,17 +589,6 @@ class GenericIE(InfoExtractor):
                 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
             }
         },
-        # Embedded Ustream video
-        {
-            'url': 'http://www.american.edu/spa/pti/nsa-privacy-janus-2014.cfm',
-            'md5': '27b99cdb639c9b12a79bca876a073417',
-            'info_dict': {
-                'id': '45734260',
-                'ext': 'flv',
-                'uploader': 'AU SPA:  The NSA and Privacy',
-                'title': 'NSA and Privacy Forum Debate featuring General Hayden and Barton Gellman'
-            }
-        },
         # nowvideo embed hidden behind percent encoding
         {
             'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
@@ -758,7 +770,7 @@ class GenericIE(InfoExtractor):
                 'duration': 48,
                 'timestamp': 1401537900,
                 'upload_date': '20140531',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
         # Wistia embed
@@ -828,6 +840,21 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 7,
         },
+        # TuneIn station embed
+        {
+            'url': 'http://radiocnrv.com/promouvoir-radio-cnrv/',
+            'info_dict': {
+                'id': '204146',
+                'ext': 'mp3',
+                'title': 'CNRV',
+                'location': 'Paris, France',
+                'is_live': True,
+            },
+            'params': {
+                # Live stream
+                'skip_download': True,
+            },
+        },
         # Livestream embed
         {
             'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
@@ -1015,7 +1042,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Навальный вышел на свободу',
                 'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 87,
                 'view_count': int,
                 'age_limit': 0,
@@ -1029,7 +1056,7 @@ class GenericIE(InfoExtractor):
                 'id': '12820',
                 'ext': 'mp4',
                 'title': "'O Sole Mio",
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 216,
                 'view_count': int,
             },
@@ -1042,7 +1069,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
                 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
                 'duration': 694,
                 'age_limit': 0,
             },
@@ -1054,7 +1081,7 @@ class GenericIE(InfoExtractor):
                 'id': '3519514',
                 'ext': 'mp4',
                 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
-                'thumbnail': 're:^https?://.*\.png$',
+                'thumbnail': r're:^https?://.*\.png$',
                 'duration': 45.115,
             },
         },
@@ -1137,7 +1164,7 @@ class GenericIE(InfoExtractor):
                 'id': '300346',
                 'ext': 'mp4',
                 'title': '中一中男師變性 全校師生力挺',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'thumbnail': r're:^https?://.*\.jpg$',
             },
             'params': {
                 # m3u8 download
@@ -1183,7 +1210,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Sauvons les abeilles ! - Le débat',
                 'description': 'md5:d9082128b1c5277987825d684939ca26',
-                'thumbnail': 're:^https?://.*\.jpe?g$',
+                'thumbnail': r're:^https?://.*\.jpe?g$',
                 'timestamp': 1434970506,
                 'upload_date': '20150622',
                 'uploader': 'Public Sénat',
@@ -1197,7 +1224,7 @@ class GenericIE(InfoExtractor):
                 'id': '2855',
                 'ext': 'mp4',
                 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
-                'thumbnail': 're:^https?://.*\.jpe?g$',
+                'thumbnail': r're:^https?://.*\.jpe?g$',
                 'uploader': 'ClickHole',
                 'uploader_id': 'clickhole',
             }
@@ -1432,6 +1459,20 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 2,
         },
+        {
+            # 20 minuten embed
+            'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
+            'info_dict': {
+                'id': '523629',
+                'ext': 'mp4',
+                'title': 'So kommen Sie bei Eis und Schnee sicher an',
+                'description': 'md5:117c212f64b25e3d95747e5276863f7d',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': [TwentyMinutenIE.ie_key()],
+        }
         # {
         #     # TODO: find another test
         #     # http://schema.org/VideoObject
@@ -1923,7 +1964,14 @@ class GenericIE(InfoExtractor):
                 re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage) or
                 re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
-            return OoyalaIE._build_url_result(smuggle_url(mobj.group('ec'), {'domain': url}))
+            embed_token = self._search_regex(
+                r'embedToken[\'"]?\s*:\s*[\'"]([^\'"]+)',
+                webpage, 'ooyala embed token', default=None)
+            return OoyalaIE._build_url_result(smuggle_url(
+                mobj.group('ec'), {
+                    'domain': url,
+                    'embed_token': embed_token,
+                }))
 
         # Look for multiple Ooyala embeds on SBN network websites
         mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
@@ -2054,10 +2102,9 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'TED')
 
         # Look for embedded Ustream videos
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>http://www\.ustream\.tv/embed/.+?)\1', webpage)
-        if mobj is not None:
-            return self.url_result(mobj.group('url'), 'Ustream')
+        ustream_url = UstreamIE._extract_url(webpage)
+        if ustream_url:
+            return self.url_result(ustream_url, UstreamIE.ie_key())
 
         # Look for embedded arte.tv player
         mobj = re.search(
@@ -2088,6 +2135,11 @@ class GenericIE(InfoExtractor):
         if soundcloud_urls:
             return _playlist_from_matches(soundcloud_urls, getter=unescapeHTML, ie=SoundcloudIE.ie_key())
 
+        # Look for tunein player
+        tunein_urls = TuneInBaseIE._extract_urls(webpage)
+        if tunein_urls:
+            return _playlist_from_matches(tunein_urls)
+
         # Look for embedded mtvservices player
         mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
         if mtvservices_url:
@@ -2373,6 +2425,12 @@ class GenericIE(InfoExtractor):
         if videa_urls:
             return _playlist_from_matches(videa_urls, ie=VideaIE.ie_key())
 
+        # Look for 20 minuten embeds
+        twentymin_urls = TwentyMinutenIE._extract_urls(webpage)
+        if twentymin_urls:
+            return _playlist_from_matches(
+                twentymin_urls, ie=TwentyMinutenIE.ie_key())
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')

youtube_dl/extractor/giantbomb.py

@@ -23,7 +23,7 @@ class GiantBombIE(InfoExtractor):
             'title': 'Quick Look: Destiny: The Dark Below',
             'description': 'md5:0aa3aaf2772a41b91d44c63f30dfad24',
             'duration': 2399,
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
         }
     }
 

youtube_dl/extractor/giga.py

@@ -24,7 +24,7 @@ class GigaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Anime Awesome: Chihiros Reise ins Zauberland – Das Beste kommt zum Schluss',
             'description': 'md5:afdf5862241aded4718a30dff6a57baf',
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 578,
             'timestamp': 1414749706,
             'upload_date': '20141031',

youtube_dl/extractor/glide.py

@@ -14,7 +14,7 @@ class GlideIE(InfoExtractor):
             'id': 'UZF8zlmuQbe4mr+7dCiQ0w==',
             'ext': 'mp4',
             'title': "Damon's Glide message",
-            'thumbnail': 're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
+            'thumbnail': r're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
         }
     }