1 from __future__ 
import unicode_literals
 
   5 from .common 
import InfoExtractor
 
  22 class NPOBaseIE(InfoExtractor
): 
  23     def _get_token(self
, video_id
): 
  24         return self
._download
_json
( 
  25             'http://ida.omroep.nl/app.php/auth', video_id
, 
  26             note
='Downloading token')['token'] 
  29 class NPOIE(NPOBaseIE
): 
  31     IE_DESC 
= 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl' 
  38                                 npo\.nl/(?!(?:live|radio)/)(?:[^/]+/){2}| 
  39                                 ntr\.nl/(?:[^/]+/){2,}| 
  40                                 omroepwnl\.nl/video/fragment/[^/]+__| 
  41                                 (?:zapp|npo3)\.nl/(?:[^/]+/){2} 
  48         'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719', 
  49         'md5': '4b3f9c429157ec4775f2c9cb7b911016', 
  51             'id': 'VPWON_1220719', 
  54             'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.', 
  55             'upload_date': '20140622', 
  58         'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800', 
  59         'md5': 'da50a5787dbfc1603c4ad80f31c5120b', 
  61             'id': 'VARA_101191800', 
  63             'title': 'De Mega Mike & Mega Thomas show: The best of.', 
  64             'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4', 
  65             'upload_date': '20090227', 
  69         'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289', 
  70         'md5': 'f8065e4e5a7824068ed3c7e783178f2c', 
  72             'id': 'VPWON_1169289', 
  74             'title': 'Tegenlicht: Zwart geld. De toekomst komt uit Afrika', 
  75             'description': 'md5:52cf4eefbc96fffcbdc06d024147abea', 
  76             'upload_date': '20130225', 
  80         'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706', 
  82             'id': 'WO_VPRO_043706', 
  84             'title': 'De nieuwe mens - Deel 1', 
  85             'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b', 
  89             'skip_download': True, 
  93         'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771', 
  95             'id': 'WO_NOS_762771', 
  97             'title': 'Hoe gaat Europa verder na Parijs?', 
 100             'skip_download': True, 
 103         'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content', 
 105             'id': 'VPWON_1233944', 
 107             'title': 'Aap, poot, pies', 
 108             'description': 'md5:c9c8005d1869ae65b858e82c01a91fde', 
 109             'upload_date': '20150508', 
 113             'skip_download': True, 
 116         'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698', 
 118             'id': 'POW_00996502', 
 120             'title': '''"Dit is wel een 'landslide'..."''', 
 121             'description': 'md5:f8d66d537dfb641380226e31ca57b8e8', 
 122             'upload_date': '20150508', 
 126             'skip_download': True, 
 130         'url': 'http://www.npo.nl/jouw-stad-rotterdam/29-01-2017/RBX_FUNX_6683215/RBX_FUNX_7601437', 
 132             'id': 'RBX_FUNX_6683215', 
 134             'title': 'Jouw Stad Rotterdam', 
 135             'description': 'md5:db251505244f097717ec59fabc372d9f', 
 138             'skip_download': True, 
 141         'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547', 
 142         'only_matching': True, 
 144         'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118', 
 145         'only_matching': True, 
 147         'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990', 
 148         'only_matching': True, 
 150         'url': 'https://www.npo3.nl/3onderzoekt/16-09-2015/VPWON_1239870', 
 151         'only_matching': True, 
 154         'url': 'npo:LI_NL1_4188102', 
 155         'only_matching': True, 
 157         'url': 'http://www.npo.nl/radio-gaga/13-06-2017/BNN_101383373', 
 158         'only_matching': True, 
 161     def _real_extract(self
, url
): 
 162         video_id 
= self
._match
_id
(url
) 
 163         return self
._get
_info
(video_id
) 
 165     def _get_info(self
, video_id
): 
 166         metadata 
= self
._download
_json
( 
 167             'http://e.omroep.nl/metadata/%s' % video_id
, 
 169             # We have to remove the javascript callback 
 170             transform_source
=strip_jsonp
, 
 173         # For some videos actual video id (prid) is different (e.g. for 
 174         # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698 
 175         # video id is POMS_WNL_853698 but prid is POW_00996502) 
 176         video_id 
= metadata
.get('prid') or video_id
 
 178         # titel is too generic in some cases so utilize aflevering_titel as well 
 179         # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html) 
 180         title 
= metadata
['titel'] 
 181         sub_title 
= metadata
.get('aflevering_titel') 
 182         if sub_title 
and sub_title 
!= title
: 
 183             title 
+= ': %s' % sub_title
 
 185         token 
= self
._get
_token
(video_id
) 
 190         quality 
= qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std']) 
 191         items 
= self
._download
_json
( 
 192             'http://ida.omroep.nl/app.php/%s' % video_id
, video_id
, 
 193             'Downloading formats JSON', query
={ 
 197         for num
, item 
in enumerate(items
): 
 198             item_url 
= item
.get('url') 
 199             if not item_url 
or item_url 
in urls
: 
 202             format_id 
= self
._search
_regex
( 
 203                 r
'video/ida/([^/]+)', item_url
, 'format id', 
 206             def add_format_url(format_url
): 
 209                     'format_id': format_id
, 
 210                     'quality': quality(format_id
), 
 213             # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706 
 214             if item
.get('contentType') in ('url', 'audio'): 
 215                 add_format_url(item_url
) 
 219                 stream_info 
= self
._download
_json
( 
 220                     item_url 
+ '&type=json', video_id
, 
 221                     'Downloading %s stream JSON' 
 222                     % item
.get('label') or item
.get('format') or format_id 
or num
) 
 223             except ExtractorError 
as ee
: 
 224                 if isinstance(ee
.cause
, compat_HTTPError
) and ee
.cause
.code 
== 404: 
 225                     error 
= (self
._parse
_json
( 
 226                         ee
.cause
.read().decode(), video_id
, 
 227                         fatal
=False) or {}).get('errorstring') 
 229                         raise ExtractorError(error
, expected
=True) 
 231             # Stream URL instead of JSON, example: npo:LI_NL1_4188102 
 232             if isinstance(stream_info
, compat_str
): 
 233                 if not stream_info
.startswith('http'): 
 235                 video_url 
= stream_info
 
 238                 video_url 
= stream_info
.get('url') 
 239             if not video_url 
or video_url 
in urls
: 
 242             if determine_ext(video_url
) == 'm3u8': 
 243                 formats
.extend(self
._extract
_m
3u8_formats
( 
 244                     video_url
, video_id
, ext
='mp4', 
 245                     entry_protocol
='m3u8_native', m3u8_id
='hls', fatal
=False)) 
 247                 add_format_url(video_url
) 
 249         is_live 
= metadata
.get('medium') == 'live' 
 252             for num
, stream 
in enumerate(metadata
.get('streams', [])): 
 253                 stream_url 
= stream
.get('url') 
 254                 if not stream_url 
or stream_url 
in urls
: 
 257                 # smooth streaming is not supported 
 258                 stream_type 
= stream
.get('type', '').lower() 
 259                 if stream_type 
in ['ss', 'ms']: 
 261                 if stream_type 
== 'hds': 
 262                     f4m_formats 
= self
._extract
_f
4m
_formats
( 
 263                         stream_url
, video_id
, fatal
=False) 
 264                     # f4m downloader downloads only piece of live stream 
 265                     for f4m_format 
in f4m_formats
: 
 266                         f4m_format
['preference'] = -1 
 267                     formats
.extend(f4m_formats
) 
 268                 elif stream_type 
== 'hls': 
 269                     formats
.extend(self
._extract
_m
3u8_formats
( 
 270                         stream_url
, video_id
, ext
='mp4', fatal
=False)) 
 271                 # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706 
 272                 elif '.asf' in stream_url
: 
 273                     asx 
= self
._download
_xml
( 
 274                         stream_url
, video_id
, 
 275                         'Downloading stream %d ASX playlist' % num
, 
 276                         transform_source
=fix_xml_ampersands
, fatal
=False) 
 279                     ref 
= asx
.find('./ENTRY/Ref') 
 282                     video_url 
= ref
.get('href') 
 283                     if not video_url 
or video_url 
in urls
: 
 288                         'ext': stream
.get('formaat', 'asf'), 
 289                         'quality': stream
.get('kwaliteit'), 
 295                         'quality': stream
.get('kwaliteit'), 
 298         self
._sort
_formats
(formats
) 
 301         if metadata
.get('tt888') == 'ja': 
 304                 'url': 'http://tt888.omroep.nl/tt888/%s' % video_id
, 
 309             'title': self
._live
_title
(title
) if is_live 
else title
, 
 310             'description': metadata
.get('info'), 
 311             'thumbnail': metadata
.get('images', [{'url': None}])[-1]['url'], 
 312             'upload_date': unified_strdate(metadata
.get('gidsdatum')), 
 313             'duration': parse_duration(metadata
.get('tijdsduur')), 
 315             'subtitles': subtitles
, 
 320 class NPOLiveIE(NPOBaseIE
): 
 321     IE_NAME 
= 'npo.nl:live' 
 322     _VALID_URL 
= r
'https?://(?:www\.)?npo\.nl/live(?:/(?P<id>[^/?#&]+))?' 
 325         'url': 'http://www.npo.nl/live/npo-1', 
 327             'id': 'LI_NL1_4188102', 
 328             'display_id': 'npo-1', 
 330             'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 
 334             'skip_download': True, 
 337         'url': 'http://www.npo.nl/live', 
 338         'only_matching': True, 
 341     def _real_extract(self
, url
): 
 342         display_id 
= self
._match
_id
(url
) or 'npo-1' 
 344         webpage 
= self
._download
_webpage
(url
, display_id
) 
 346         live_id 
= self
._search
_regex
( 
 347             [r
'media-id="([^"]+)"', r
'data-prid="([^"]+)"'], webpage
, 'live id') 
 350             '_type': 'url_transparent', 
 351             'url': 'npo:%s' % live_id
, 
 352             'ie_key': NPOIE
.ie_key(), 
 354             'display_id': display_id
, 
 358 class NPORadioIE(InfoExtractor
): 
 359     IE_NAME 
= 'npo.nl:radio' 
 360     _VALID_URL 
= r
'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)/?$' 
 363         'url': 'http://www.npo.nl/radio/radio-1', 
 367             'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 
 371             'skip_download': True, 
 376     def _html_get_attribute_regex(attribute
): 
 377         return r
'{0}\s*=\s*\'([^
\']+)\''.format(attribute) 
 379     def _real_extract(self, url): 
 380         video_id = self._match_id(url) 
 382         webpage = self._download_webpage(url, video_id) 
 384         title = self._html_search_regex( 
 385             self._html_get_attribute_regex('data
-channel
'), webpage, 'title
') 
 387         stream = self._parse_json( 
 388             self._html_search_regex(self._html_get_attribute_regex('data
-streams
'), webpage, 'data
-streams
'), 
 391         codec = stream.get('codec
') 
 395             'url
': stream['url
'], 
 396             'title
': self._live_title(title), 
 403 class NPORadioFragmentIE(InfoExtractor): 
 404     IE_NAME = 'npo
.nl
:radio
:fragment
' 
 405     _VALID_URL = r'https?
://(?
:www\
.)?npo\
.nl
/radio
/[^
/]+/fragment
/(?P
<id>\d
+)' 
 408         'url
': 'http
://www
.npo
.nl
/radio
/radio
-5/fragment
/174356', 
 409         'md5
': 'dd8cc470dad764d0fdc70a9a1e2d18c2
', 
 413             'title
': 'Jubileumconcert Willeke Alberti
', 
 417     def _real_extract(self, url): 
 418         audio_id = self._match_id(url) 
 420         webpage = self._download_webpage(url, audio_id) 
 422         title = self._html_search_regex( 
 423             r'href
="/radio/[^/]+/fragment/%s" title
="([^"]+)"' % audio_id, 
 426         audio_url = self._search_regex( 
 427             r"data
-streams
='([^']+)'", webpage, 'audio url
') 
 436 class NPODataMidEmbedIE(InfoExtractor): 
 437     def _real_extract(self, url): 
 438         display_id = self._match_id(url) 
 439         webpage = self._download_webpage(url, display_id) 
 440         video_id = self._search_regex( 
 441             r'data
-mid
=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id') 
 443             '_type': 'url_transparent', 
 445             'url': 'npo:%s' % video_id, 
 446             'display_id': display_id 
 450 class SchoolTVIE(NPODataMidEmbedIE): 
 452     _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)' 
 455         'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/', 
 457             'id': 'WO_NTR_429477', 
 458             'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam', 
 459             'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?', 
 461             'description': 'md5:abfa0ff690adb73fd0297fd033aaa631' 
 464             # Skip because of m3u8 download 
 465             'skip_download': True 
 470 class HetKlokhuisIE(NPODataMidEmbedIE): 
 471     IE_NAME = 'hetklokhuis' 
 472     _VALID_URL = r'https?://(?:www\.)?hetklokhuis\.nl/[^/]+/\d+/(?P<id>[^/?#&]+)' 
 475         'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven', 
 477             'id': 'VPWON_1260528', 
 478             'display_id': 'Zwaartekrachtsgolven', 
 480             'title': 'Het Klokhuis: Zwaartekrachtsgolven', 
 481             'description': 'md5:c94f31fb930d76c2efa4a4a71651dd48', 
 482             'upload_date': '20170223', 
 485             'skip_download': True 
 490 class NPOPlaylistBaseIE(NPOIE): 
 491     def _real_extract(self, url): 
 492         playlist_id = self._match_id(url) 
 494         webpage = self._download_webpage(url, playlist_id) 
 497             self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id) 
 498             for video_id in orderedSet(re.findall(self._PLAYLIST_ENTRY_RE, webpage)) 
 501         playlist_title = self._html_search_regex( 
 502             self._PLAYLIST_TITLE_RE, webpage, 'playlist title', 
 503             default=None) or self._og_search_title(webpage) 
 505         return self.playlist_result(entries, playlist_id, playlist_title) 
 508 class VPROIE(NPOPlaylistBaseIE): 
 510     _VALID_URL = r'https?://(?:www\.)?(?:(?:tegenlicht\.)?vpro|2doc)\.nl/(?:[^/]+/)*(?P<id>[^/]+)\.html' 
 511     _PLAYLIST_TITLE_RE = (r'<h1[^>]+class=["\'].*?
\bmedia
-platform
-title
\b.*?
["\'][^>]*>([^<]+)', 
 512                           r'<h5[^>]+class=["\'].*?
\bmedia
-platform
-subtitle
\b.*?
["\'][^>]*>([^<]+)') 
 513     _PLAYLIST_ENTRY_RE = r'data-media-id="([^
"]+)"' 
 517             'url
': 'http
://tegenlicht
.vpro
.nl
/afleveringen
/2012-2013/de
-toekomst
-komt
-uit
-afrika
.html
', 
 518             'md5
': 'f8065e4e5a7824068ed3c7e783178f2c
', 
 520                 'id': 'VPWON_1169289
', 
 522                 'title
': 'De toekomst komt uit Afrika
', 
 523                 'description
': 'md5
:52cf4eefbc96fffcbdc06d024147abea
', 
 524                 'upload_date
': '20130225', 
 526             'skip
': 'Video gone
', 
 529             'url
': 'http
://www
.vpro
.nl
/programmas
/2doc
/2015/sergio
-herman
.html
', 
 531                 'id': 'sergio
-herman
', 
 532                 'title
': 'sergio herman
: fucking perfect
', 
 537             # playlist with youtube embed 
 538             'url
': 'http
://www
.vpro
.nl
/programmas
/2doc
/2015/education
-education
.html
', 
 540                 'id': 'education
-education
', 
 541                 'title
': 'education education
', 
 546             'url
': 'http
://www
.2doc
.nl
/documentaires
/series
/2doc
/2015/oktober
/de
-tegenprestatie
.html
', 
 548                 'id': 'de
-tegenprestatie
', 
 549                 'title
': 'De Tegenprestatie
', 
 553             'url
': 'http
://www
.2doc
.nl
/speel~VARA_101375237~mh17
-het
-verdriet
-van
-nederland~
.html
', 
 555                 'id': 'VARA_101375237
', 
 557                 'title
': 'MH17
: Het verdriet van Nederland
', 
 558                 'description
': 'md5
:09e1a37c1fdb144621e22479691a9f18
', 
 559                 'upload_date
': '20150716', 
 562                 # Skip because of m3u8 download 
 563                 'skip_download
': True 
 569 class WNLIE(NPOPlaylistBaseIE): 
 571     _VALID_URL = r'https?
://(?
:www\
.)?omroepwnl\
.nl
/video
/detail
/(?P
<id>[^
/]+)__\d
+' 
 572     _PLAYLIST_TITLE_RE = r'(?s
)<h1
[^
>]+class="subject"[^
>]*>(.+?
)</h1
>' 
 573     _PLAYLIST_ENTRY_RE = r'<a
[^
>]+href
="([^"]+)"[^>]+class="js
-mid
"[^>]*>Deel \d+' 
 576         'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515', 
 578             'id': 'vandaag-de-dag-6-mei', 
 579             'title': 'Vandaag de Dag 6 mei', 
 585 class AndereTijdenIE(NPOPlaylistBaseIE): 
 586     IE_NAME = 'anderetijden' 
 587     _VALID_URL = r'https?://(?:www\.)?anderetijden\.nl/programma/(?:[^/]+/)+(?P<id>[^/?#&]+)' 
 588     _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class=["\'].*?
\bpage
-title
\b.*?
["\'][^>]*>(.+?)</h1>' 
 589     _PLAYLIST_ENTRY_RE = r'<figure[^>]+class=["\']episode
-container episode
-page
["\'][^>]+data-prid=["\'](.+?
)["\']' 
 592         'url': 'http://anderetijden.nl/programma/1/Andere-Tijden/aflevering/676/Duitse-soldaten-over-de-Slag-bij-Arnhem', 
 594             'id': 'Duitse-soldaten-over-de-Slag-bij-Arnhem', 
 595             'title': 'Duitse soldaten over de Slag bij Arnhem',