1 from __future__ 
import unicode_literals
 
   5 from .common 
import InfoExtractor
 
  23 class NPOBaseIE(InfoExtractor
): 
  24     def _get_token(self
, video_id
): 
  25         return self
._download
_json
( 
  26             'http://ida.omroep.nl/app.php/auth', video_id
, 
  27             note
='Downloading token')['token'] 
  30 class NPOIE(NPOBaseIE
): 
  32     IE_DESC 
= 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl' 
  40                                 (?:ntr|npostart)\.nl/(?:[^/]+/){2,}| 
  41                                 omroepwnl\.nl/video/fragment/[^/]+__| 
  42                                 (?:zapp|npo3)\.nl/(?:[^/]+/){2,} 
  49         'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719', 
  50         'md5': '4b3f9c429157ec4775f2c9cb7b911016', 
  52             'id': 'VPWON_1220719', 
  55             'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.', 
  56             'upload_date': '20140622', 
  59         'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800', 
  60         'md5': 'da50a5787dbfc1603c4ad80f31c5120b', 
  62             'id': 'VARA_101191800', 
  64             'title': 'De Mega Mike & Mega Thomas show: The best of.', 
  65             'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4', 
  66             'upload_date': '20090227', 
  70         'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289', 
  71         'md5': 'f8065e4e5a7824068ed3c7e783178f2c', 
  73             'id': 'VPWON_1169289', 
  75             'title': 'Tegenlicht: Zwart geld. De toekomst komt uit Afrika', 
  76             'description': 'md5:52cf4eefbc96fffcbdc06d024147abea', 
  77             'upload_date': '20130225', 
  81         'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706', 
  83             'id': 'WO_VPRO_043706', 
  85             'title': 'De nieuwe mens - Deel 1', 
  86             'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b', 
  90             'skip_download': True, 
  94         'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771', 
  96             'id': 'WO_NOS_762771', 
  98             'title': 'Hoe gaat Europa verder na Parijs?', 
 101             'skip_download': True, 
 104         'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content', 
 106             'id': 'VPWON_1233944', 
 108             'title': 'Aap, poot, pies', 
 109             'description': 'md5:c9c8005d1869ae65b858e82c01a91fde', 
 110             'upload_date': '20150508', 
 114             'skip_download': True, 
 117         'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698', 
 119             'id': 'POW_00996502', 
 121             'title': '''"Dit is wel een 'landslide'..."''', 
 122             'description': 'md5:f8d66d537dfb641380226e31ca57b8e8', 
 123             'upload_date': '20150508', 
 127             'skip_download': True, 
 131         'url': 'http://www.npo.nl/jouw-stad-rotterdam/29-01-2017/RBX_FUNX_6683215/RBX_FUNX_7601437', 
 133             'id': 'RBX_FUNX_6683215', 
 135             'title': 'Jouw Stad Rotterdam', 
 136             'description': 'md5:db251505244f097717ec59fabc372d9f', 
 139             'skip_download': True, 
 142         'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547', 
 143         'only_matching': True, 
 145         'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118', 
 146         'only_matching': True, 
 148         'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990', 
 149         'only_matching': True, 
 151         'url': 'https://www.npo3.nl/3onderzoekt/16-09-2015/VPWON_1239870', 
 152         'only_matching': True, 
 155         'url': 'npo:LI_NL1_4188102', 
 156         'only_matching': True, 
 158         'url': 'http://www.npo.nl/radio-gaga/13-06-2017/BNN_101383373', 
 159         'only_matching': True, 
 161         'url': 'https://www.zapp.nl/1803-skelterlab/instructie-video-s/740-instructievideo-s/POMS_AT_11736927', 
 162         'only_matching': True, 
 164         'url': 'https://www.npostart.nl/broodje-gezond-ei/28-05-2018/KN_1698996', 
 165         'only_matching': True, 
 167         'url': 'https://npo.nl/KN_1698996', 
 168         'only_matching': True, 
 172     def suitable(cls
, url
): 
 173         return (False if any(ie
.suitable(url
) 
 174                 for ie 
in (NPOLiveIE
, NPORadioIE
, NPORadioFragmentIE
)) 
 175                 else super(NPOIE
, cls
).suitable(url
)) 
 177     def _real_extract(self
, url
): 
 178         video_id 
= self
._match
_id
(url
) 
 179         return self
._get
_info
(video_id
) 
 181     def _get_info(self
, video_id
): 
 182         metadata 
= self
._download
_json
( 
 183             'http://e.omroep.nl/metadata/%s' % video_id
, 
 185             # We have to remove the javascript callback 
 186             transform_source
=strip_jsonp
, 
 189         error 
= metadata
.get('error') 
 191             raise ExtractorError(error
, expected
=True) 
 193         # For some videos actual video id (prid) is different (e.g. for 
 194         # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698 
 195         # video id is POMS_WNL_853698 but prid is POW_00996502) 
 196         video_id 
= metadata
.get('prid') or video_id
 
 198         # titel is too generic in some cases so utilize aflevering_titel as well 
 199         # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html) 
 200         title 
= metadata
['titel'] 
 201         sub_title 
= metadata
.get('aflevering_titel') 
 202         if sub_title 
and sub_title 
!= title
: 
 203             title 
+= ': %s' % sub_title
 
 205         token 
= self
._get
_token
(video_id
) 
 210         def is_legal_url(format_url
): 
 211             return format_url 
and format_url 
not in urls 
and re
.match( 
 212                 r
'^(?:https?:)?//', format_url
) 
 214         QUALITY_LABELS 
= ('Laag', 'Normaal', 'Hoog') 
 215         QUALITY_FORMATS 
= ('adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std') 
 217         quality_from_label 
= qualities(QUALITY_LABELS
) 
 218         quality_from_format_id 
= qualities(QUALITY_FORMATS
) 
 219         items 
= self
._download
_json
( 
 220             'http://ida.omroep.nl/app.php/%s' % video_id
, video_id
, 
 221             'Downloading formats JSON', query
={ 
 225         for num
, item 
in enumerate(items
): 
 226             item_url 
= item
.get('url') 
 227             if not is_legal_url(item_url
): 
 230             format_id 
= self
._search
_regex
( 
 231                 r
'video/ida/([^/]+)', item_url
, 'format id', 
 234             item_label 
= item
.get('label') 
 236             def add_format_url(format_url
): 
 237                 width 
= int_or_none(self
._search
_regex
( 
 238                     r
'(\d+)[xX]\d+', format_url
, 'width', default
=None)) 
 239                 height 
= int_or_none(self
._search
_regex
( 
 240                     r
'\d+[xX](\d+)', format_url
, 'height', default
=None)) 
 241                 if item_label 
in QUALITY_LABELS
: 
 242                     quality 
= quality_from_label(item_label
) 
 244                 elif item_label 
in QUALITY_FORMATS
: 
 245                     quality 
= quality_from_format_id(format_id
) 
 248                     quality
, f_id 
= [None] * 2 
 257             # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706 
 258             if item
.get('contentType') in ('url', 'audio'): 
 259                 add_format_url(item_url
) 
 263                 stream_info 
= self
._download
_json
( 
 264                     item_url 
+ '&type=json', video_id
, 
 265                     'Downloading %s stream JSON' 
 266                     % item_label 
or item
.get('format') or format_id 
or num
) 
 267             except ExtractorError 
as ee
: 
 268                 if isinstance(ee
.cause
, compat_HTTPError
) and ee
.cause
.code 
== 404: 
 269                     error 
= (self
._parse
_json
( 
 270                         ee
.cause
.read().decode(), video_id
, 
 271                         fatal
=False) or {}).get('errorstring') 
 273                         raise ExtractorError(error
, expected
=True) 
 275             # Stream URL instead of JSON, example: npo:LI_NL1_4188102 
 276             if isinstance(stream_info
, compat_str
): 
 277                 if not stream_info
.startswith('http'): 
 279                 video_url 
= stream_info
 
 282                 video_url 
= stream_info
.get('url') 
 283             if not video_url 
or video_url 
in urls
: 
 286             if determine_ext(video_url
) == 'm3u8': 
 287                 formats
.extend(self
._extract
_m
3u8_formats
( 
 288                     video_url
, video_id
, ext
='mp4', 
 289                     entry_protocol
='m3u8_native', m3u8_id
='hls', fatal
=False)) 
 291                 add_format_url(video_url
) 
 293         is_live 
= metadata
.get('medium') == 'live' 
 296             for num
, stream 
in enumerate(metadata
.get('streams', [])): 
 297                 stream_url 
= stream
.get('url') 
 298                 if not is_legal_url(stream_url
): 
 301                 # smooth streaming is not supported 
 302                 stream_type 
= stream
.get('type', '').lower() 
 303                 if stream_type 
in ['ss', 'ms']: 
 305                 if stream_type 
== 'hds': 
 306                     f4m_formats 
= self
._extract
_f
4m
_formats
( 
 307                         stream_url
, video_id
, fatal
=False) 
 308                     # f4m downloader downloads only piece of live stream 
 309                     for f4m_format 
in f4m_formats
: 
 310                         f4m_format
['preference'] = -1 
 311                     formats
.extend(f4m_formats
) 
 312                 elif stream_type 
== 'hls': 
 313                     formats
.extend(self
._extract
_m
3u8_formats
( 
 314                         stream_url
, video_id
, ext
='mp4', fatal
=False)) 
 315                 # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706 
 316                 elif '.asf' in stream_url
: 
 317                     asx 
= self
._download
_xml
( 
 318                         stream_url
, video_id
, 
 319                         'Downloading stream %d ASX playlist' % num
, 
 320                         transform_source
=fix_xml_ampersands
, fatal
=False) 
 323                     ref 
= asx
.find('./ENTRY/Ref') 
 326                     video_url 
= ref
.get('href') 
 327                     if not video_url 
or video_url 
in urls
: 
 332                         'ext': stream
.get('formaat', 'asf'), 
 333                         'quality': stream
.get('kwaliteit'), 
 339                         'quality': stream
.get('kwaliteit'), 
 342         self
._sort
_formats
(formats
) 
 345         if metadata
.get('tt888') == 'ja': 
 348                 'url': 'http://tt888.omroep.nl/tt888/%s' % video_id
, 
 353             'title': self
._live
_title
(title
) if is_live 
else title
, 
 354             'description': metadata
.get('info'), 
 355             'thumbnail': metadata
.get('images', [{'url': None}])[-1]['url'], 
 356             'upload_date': unified_strdate(metadata
.get('gidsdatum')), 
 357             'duration': parse_duration(metadata
.get('tijdsduur')), 
 359             'subtitles': subtitles
, 
 364 class NPOLiveIE(NPOBaseIE
): 
 365     IE_NAME 
= 'npo.nl:live' 
 366     _VALID_URL 
= r
'https?://(?:www\.)?npo\.nl/live(?:/(?P<id>[^/?#&]+))?' 
 369         'url': 'http://www.npo.nl/live/npo-1', 
 371             'id': 'LI_NL1_4188102', 
 372             'display_id': 'npo-1', 
 374             'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 
 378             'skip_download': True, 
 381         'url': 'http://www.npo.nl/live', 
 382         'only_matching': True, 
 385     def _real_extract(self
, url
): 
 386         display_id 
= self
._match
_id
(url
) or 'npo-1' 
 388         webpage 
= self
._download
_webpage
(url
, display_id
) 
 390         live_id 
= self
._search
_regex
( 
 391             [r
'media-id="([^"]+)"', r
'data-prid="([^"]+)"'], webpage
, 'live id') 
 394             '_type': 'url_transparent', 
 395             'url': 'npo:%s' % live_id
, 
 396             'ie_key': NPOIE
.ie_key(), 
 398             'display_id': display_id
, 
 402 class NPORadioIE(InfoExtractor
): 
 403     IE_NAME 
= 'npo.nl:radio' 
 404     _VALID_URL 
= r
'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)' 
 407         'url': 'http://www.npo.nl/radio/radio-1', 
 411             'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 
 415             'skip_download': True, 
 420     def suitable(cls
, url
): 
 421         return False if NPORadioFragmentIE
.suitable(url
) else super(NPORadioIE
, cls
).suitable(url
) 
 424     def _html_get_attribute_regex(attribute
): 
 425         return r
'{0}\s*=\s*\'([^
\']+)\''.format(attribute) 
 427     def _real_extract(self, url): 
 428         video_id = self._match_id(url) 
 430         webpage = self._download_webpage(url, video_id) 
 432         title = self._html_search_regex( 
 433             self._html_get_attribute_regex('data
-channel
'), webpage, 'title
') 
 435         stream = self._parse_json( 
 436             self._html_search_regex(self._html_get_attribute_regex('data
-streams
'), webpage, 'data
-streams
'), 
 439         codec = stream.get('codec
') 
 443             'url
': stream['url
'], 
 444             'title
': self._live_title(title), 
 451 class NPORadioFragmentIE(InfoExtractor): 
 452     IE_NAME = 'npo
.nl
:radio
:fragment
' 
 453     _VALID_URL = r'https?
://(?
:www\
.)?npo\
.nl
/radio
/[^
/]+/fragment
/(?P
<id>\d
+)' 
 456         'url
': 'http
://www
.npo
.nl
/radio
/radio
-5/fragment
/174356', 
 457         'md5
': 'dd8cc470dad764d0fdc70a9a1e2d18c2
', 
 461             'title
': 'Jubileumconcert Willeke Alberti
', 
 465     def _real_extract(self, url): 
 466         audio_id = self._match_id(url) 
 468         webpage = self._download_webpage(url, audio_id) 
 470         title = self._html_search_regex( 
 471             r'href
="/radio/[^/]+/fragment/%s" title
="([^"]+)"' % audio_id, 
 474         audio_url = self._search_regex( 
 475             r"data
-streams
='([^']+)'", webpage, 'audio url
') 
 484 class NPODataMidEmbedIE(InfoExtractor): 
 485     def _real_extract(self, url): 
 486         display_id = self._match_id(url) 
 487         webpage = self._download_webpage(url, display_id) 
 488         video_id = self._search_regex( 
 489             r'data
-mid
=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id') 
 491             '_type': 'url_transparent', 
 493             'url': 'npo:%s' % video_id, 
 494             'display_id': display_id 
 498 class SchoolTVIE(NPODataMidEmbedIE): 
 500     _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)' 
 503         'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/', 
 505             'id': 'WO_NTR_429477', 
 506             'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam', 
 507             'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?', 
 509             'description': 'md5:abfa0ff690adb73fd0297fd033aaa631' 
 512             # Skip because of m3u8 download 
 513             'skip_download': True 
 518 class HetKlokhuisIE(NPODataMidEmbedIE): 
 519     IE_NAME = 'hetklokhuis' 
 520     _VALID_URL = r'https?://(?:www\.)?hetklokhuis\.nl/[^/]+/\d+/(?P<id>[^/?#&]+)' 
 523         'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven', 
 525             'id': 'VPWON_1260528', 
 526             'display_id': 'Zwaartekrachtsgolven', 
 528             'title': 'Het Klokhuis: Zwaartekrachtsgolven', 
 529             'description': 'md5:c94f31fb930d76c2efa4a4a71651dd48', 
 530             'upload_date': '20170223', 
 533             'skip_download': True 
 538 class NPOPlaylistBaseIE(NPOIE): 
 539     def _real_extract(self, url): 
 540         playlist_id = self._match_id(url) 
 542         webpage = self._download_webpage(url, playlist_id) 
 545             self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id) 
 546             for video_id in orderedSet(re.findall(self._PLAYLIST_ENTRY_RE, webpage)) 
 549         playlist_title = self._html_search_regex( 
 550             self._PLAYLIST_TITLE_RE, webpage, 'playlist title', 
 551             default=None) or self._og_search_title(webpage) 
 553         return self.playlist_result(entries, playlist_id, playlist_title) 
 556 class VPROIE(NPOPlaylistBaseIE): 
 558     _VALID_URL = r'https?://(?:www\.)?(?:(?:tegenlicht\.)?vpro|2doc)\.nl/(?:[^/]+/)*(?P<id>[^/]+)\.html' 
 559     _PLAYLIST_TITLE_RE = (r'<h1[^>]+class=["\'].*?
\bmedia
-platform
-title
\b.*?
["\'][^>]*>([^<]+)', 
 560                           r'<h5[^>]+class=["\'].*?
\bmedia
-platform
-subtitle
\b.*?
["\'][^>]*>([^<]+)') 
 561     _PLAYLIST_ENTRY_RE = r'data-media-id="([^
"]+)"' 
 565             'url
': 'http
://tegenlicht
.vpro
.nl
/afleveringen
/2012-2013/de
-toekomst
-komt
-uit
-afrika
.html
', 
 566             'md5
': 'f8065e4e5a7824068ed3c7e783178f2c
', 
 568                 'id': 'VPWON_1169289
', 
 570                 'title
': 'De toekomst komt uit Afrika
', 
 571                 'description
': 'md5
:52cf4eefbc96fffcbdc06d024147abea
', 
 572                 'upload_date
': '20130225', 
 574             'skip
': 'Video gone
', 
 577             'url
': 'http
://www
.vpro
.nl
/programmas
/2doc
/2015/sergio
-herman
.html
', 
 579                 'id': 'sergio
-herman
', 
 580                 'title
': 'sergio herman
: fucking perfect
', 
 585             # playlist with youtube embed 
 586             'url
': 'http
://www
.vpro
.nl
/programmas
/2doc
/2015/education
-education
.html
', 
 588                 'id': 'education
-education
', 
 589                 'title
': 'education education
', 
 594             'url
': 'http
://www
.2doc
.nl
/documentaires
/series
/2doc
/2015/oktober
/de
-tegenprestatie
.html
', 
 596                 'id': 'de
-tegenprestatie
', 
 597                 'title
': 'De Tegenprestatie
', 
 601             'url
': 'http
://www
.2doc
.nl
/speel~VARA_101375237~mh17
-het
-verdriet
-van
-nederland~
.html
', 
 603                 'id': 'VARA_101375237
', 
 605                 'title
': 'MH17
: Het verdriet van Nederland
', 
 606                 'description
': 'md5
:09e1a37c1fdb144621e22479691a9f18
', 
 607                 'upload_date
': '20150716', 
 610                 # Skip because of m3u8 download 
 611                 'skip_download
': True 
 617 class WNLIE(NPOPlaylistBaseIE): 
 619     _VALID_URL = r'https?
://(?
:www\
.)?omroepwnl\
.nl
/video
/detail
/(?P
<id>[^
/]+)__\d
+' 
 620     _PLAYLIST_TITLE_RE = r'(?s
)<h1
[^
>]+class="subject"[^
>]*>(.+?
)</h1
>' 
 621     _PLAYLIST_ENTRY_RE = r'<a
[^
>]+href
="([^"]+)"[^>]+class="js
-mid
"[^>]*>Deel \d+' 
 624         'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515', 
 626             'id': 'vandaag-de-dag-6-mei', 
 627             'title': 'Vandaag de Dag 6 mei', 
 633 class AndereTijdenIE(NPOPlaylistBaseIE): 
 634     IE_NAME = 'anderetijden' 
 635     _VALID_URL = r'https?://(?:www\.)?anderetijden\.nl/programma/(?:[^/]+/)+(?P<id>[^/?#&]+)' 
 636     _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class=["\'].*?
\bpage
-title
\b.*?
["\'][^>]*>(.+?)</h1>' 
 637     _PLAYLIST_ENTRY_RE = r'<figure[^>]+class=["\']episode
-container episode
-page
["\'][^>]+data-prid=["\'](.+?
)["\']' 
 640         'url': 'http://anderetijden.nl/programma/1/Andere-Tijden/aflevering/676/Duitse-soldaten-over-de-Slag-bij-Arnhem', 
 642             'id': 'Duitse-soldaten-over-de-Slag-bij-Arnhem', 
 643             'title': 'Duitse soldaten over de Slag bij Arnhem',