1 from __future__ 
import unicode_literals
 
   5 from .common 
import InfoExtractor
 
  23 class NPOBaseIE(InfoExtractor
): 
  24     def _get_token(self
, video_id
): 
  25         return self
._download
_json
( 
  26             'http://ida.omroep.nl/app.php/auth', video_id
, 
  27             note
='Downloading token')['token'] 
  30 class NPOIE(NPOBaseIE
): 
  32     IE_DESC 
= 'npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl' 
  39                                 npo\.nl/(?!(?:live|radio)/)(?:[^/]+/){2}| 
  40                                 ntr\.nl/(?:[^/]+/){2,}| 
  41                                 omroepwnl\.nl/video/fragment/[^/]+__| 
  42                                 (?:zapp|npo3)\.nl/(?:[^/]+/){2,} 
  49         'url': 'http://www.npo.nl/nieuwsuur/22-06-2014/VPWON_1220719', 
  50         'md5': '4b3f9c429157ec4775f2c9cb7b911016', 
  52             'id': 'VPWON_1220719', 
  55             'description': 'Dagelijks tussen tien en elf: nieuws, sport en achtergronden.', 
  56             'upload_date': '20140622', 
  59         'url': 'http://www.npo.nl/de-mega-mike-mega-thomas-show/27-02-2009/VARA_101191800', 
  60         'md5': 'da50a5787dbfc1603c4ad80f31c5120b', 
  62             'id': 'VARA_101191800', 
  64             'title': 'De Mega Mike & Mega Thomas show: The best of.', 
  65             'description': 'md5:3b74c97fc9d6901d5a665aac0e5400f4', 
  66             'upload_date': '20090227', 
  70         'url': 'http://www.npo.nl/tegenlicht/25-02-2013/VPWON_1169289', 
  71         'md5': 'f8065e4e5a7824068ed3c7e783178f2c', 
  73             'id': 'VPWON_1169289', 
  75             'title': 'Tegenlicht: Zwart geld. De toekomst komt uit Afrika', 
  76             'description': 'md5:52cf4eefbc96fffcbdc06d024147abea', 
  77             'upload_date': '20130225', 
  81         'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706', 
  83             'id': 'WO_VPRO_043706', 
  85             'title': 'De nieuwe mens - Deel 1', 
  86             'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b', 
  90             'skip_download': True, 
  94         'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771', 
  96             'id': 'WO_NOS_762771', 
  98             'title': 'Hoe gaat Europa verder na Parijs?', 
 101             'skip_download': True, 
 104         'url': 'http://www.ntr.nl/Aap-Poot-Pies/27/detail/Aap-poot-pies/VPWON_1233944#content', 
 106             'id': 'VPWON_1233944', 
 108             'title': 'Aap, poot, pies', 
 109             'description': 'md5:c9c8005d1869ae65b858e82c01a91fde', 
 110             'upload_date': '20150508', 
 114             'skip_download': True, 
 117         'url': 'http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698', 
 119             'id': 'POW_00996502', 
 121             'title': '''"Dit is wel een 'landslide'..."''', 
 122             'description': 'md5:f8d66d537dfb641380226e31ca57b8e8', 
 123             'upload_date': '20150508', 
 127             'skip_download': True, 
 131         'url': 'http://www.npo.nl/jouw-stad-rotterdam/29-01-2017/RBX_FUNX_6683215/RBX_FUNX_7601437', 
 133             'id': 'RBX_FUNX_6683215', 
 135             'title': 'Jouw Stad Rotterdam', 
 136             'description': 'md5:db251505244f097717ec59fabc372d9f', 
 139             'skip_download': True, 
 142         'url': 'http://www.zapp.nl/de-bzt-show/gemist/KN_1687547', 
 143         'only_matching': True, 
 145         'url': 'http://www.zapp.nl/de-bzt-show/filmpjes/POMS_KN_7315118', 
 146         'only_matching': True, 
 148         'url': 'http://www.zapp.nl/beste-vrienden-quiz/extra-video-s/WO_NTR_1067990', 
 149         'only_matching': True, 
 151         'url': 'https://www.npo3.nl/3onderzoekt/16-09-2015/VPWON_1239870', 
 152         'only_matching': True, 
 155         'url': 'npo:LI_NL1_4188102', 
 156         'only_matching': True, 
 158         'url': 'http://www.npo.nl/radio-gaga/13-06-2017/BNN_101383373', 
 159         'only_matching': True, 
 161         'url': 'https://www.zapp.nl/1803-skelterlab/instructie-video-s/740-instructievideo-s/POMS_AT_11736927', 
 162         'only_matching': True, 
 165     def _real_extract(self
, url
): 
 166         video_id 
= self
._match
_id
(url
) 
 167         return self
._get
_info
(video_id
) 
 169     def _get_info(self
, video_id
): 
 170         metadata 
= self
._download
_json
( 
 171             'http://e.omroep.nl/metadata/%s' % video_id
, 
 173             # We have to remove the javascript callback 
 174             transform_source
=strip_jsonp
, 
 177         error 
= metadata
.get('error') 
 179             raise ExtractorError(error
, expected
=True) 
 181         # For some videos actual video id (prid) is different (e.g. for 
 182         # http://www.omroepwnl.nl/video/fragment/vandaag-de-dag-verkiezingen__POMS_WNL_853698 
 183         # video id is POMS_WNL_853698 but prid is POW_00996502) 
 184         video_id 
= metadata
.get('prid') or video_id
 
 186         # titel is too generic in some cases so utilize aflevering_titel as well 
 187         # when available (e.g. http://tegenlicht.vpro.nl/afleveringen/2014-2015/access-to-africa.html) 
 188         title 
= metadata
['titel'] 
 189         sub_title 
= metadata
.get('aflevering_titel') 
 190         if sub_title 
and sub_title 
!= title
: 
 191             title 
+= ': %s' % sub_title
 
 193         token 
= self
._get
_token
(video_id
) 
 198         def is_legal_url(format_url
): 
 199             return format_url 
and format_url 
not in urls 
and re
.match( 
 200                 r
'^(?:https?:)?//', format_url
) 
 202         QUALITY_LABELS 
= ('Laag', 'Normaal', 'Hoog') 
 203         QUALITY_FORMATS 
= ('adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std') 
 205         quality_from_label 
= qualities(QUALITY_LABELS
) 
 206         quality_from_format_id 
= qualities(QUALITY_FORMATS
) 
 207         items 
= self
._download
_json
( 
 208             'http://ida.omroep.nl/app.php/%s' % video_id
, video_id
, 
 209             'Downloading formats JSON', query
={ 
 213         for num
, item 
in enumerate(items
): 
 214             item_url 
= item
.get('url') 
 215             if not is_legal_url(item_url
): 
 218             format_id 
= self
._search
_regex
( 
 219                 r
'video/ida/([^/]+)', item_url
, 'format id', 
 222             item_label 
= item
.get('label') 
 224             def add_format_url(format_url
): 
 225                 width 
= int_or_none(self
._search
_regex
( 
 226                     r
'(\d+)[xX]\d+', format_url
, 'width', default
=None)) 
 227                 height 
= int_or_none(self
._search
_regex
( 
 228                     r
'\d+[xX](\d+)', format_url
, 'height', default
=None)) 
 229                 if item_label 
in QUALITY_LABELS
: 
 230                     quality 
= quality_from_label(item_label
) 
 232                 elif item_label 
in QUALITY_FORMATS
: 
 233                     quality 
= quality_from_format_id(format_id
) 
 236                     quality
, f_id 
= [None] * 2 
 245             # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706 
 246             if item
.get('contentType') in ('url', 'audio'): 
 247                 add_format_url(item_url
) 
 251                 stream_info 
= self
._download
_json
( 
 252                     item_url 
+ '&type=json', video_id
, 
 253                     'Downloading %s stream JSON' 
 254                     % item_label 
or item
.get('format') or format_id 
or num
) 
 255             except ExtractorError 
as ee
: 
 256                 if isinstance(ee
.cause
, compat_HTTPError
) and ee
.cause
.code 
== 404: 
 257                     error 
= (self
._parse
_json
( 
 258                         ee
.cause
.read().decode(), video_id
, 
 259                         fatal
=False) or {}).get('errorstring') 
 261                         raise ExtractorError(error
, expected
=True) 
 263             # Stream URL instead of JSON, example: npo:LI_NL1_4188102 
 264             if isinstance(stream_info
, compat_str
): 
 265                 if not stream_info
.startswith('http'): 
 267                 video_url 
= stream_info
 
 270                 video_url 
= stream_info
.get('url') 
 271             if not video_url 
or video_url 
in urls
: 
 274             if determine_ext(video_url
) == 'm3u8': 
 275                 formats
.extend(self
._extract
_m
3u8_formats
( 
 276                     video_url
, video_id
, ext
='mp4', 
 277                     entry_protocol
='m3u8_native', m3u8_id
='hls', fatal
=False)) 
 279                 add_format_url(video_url
) 
 281         is_live 
= metadata
.get('medium') == 'live' 
 284             for num
, stream 
in enumerate(metadata
.get('streams', [])): 
 285                 stream_url 
= stream
.get('url') 
 286                 if not is_legal_url(stream_url
): 
 289                 # smooth streaming is not supported 
 290                 stream_type 
= stream
.get('type', '').lower() 
 291                 if stream_type 
in ['ss', 'ms']: 
 293                 if stream_type 
== 'hds': 
 294                     f4m_formats 
= self
._extract
_f
4m
_formats
( 
 295                         stream_url
, video_id
, fatal
=False) 
 296                     # f4m downloader downloads only piece of live stream 
 297                     for f4m_format 
in f4m_formats
: 
 298                         f4m_format
['preference'] = -1 
 299                     formats
.extend(f4m_formats
) 
 300                 elif stream_type 
== 'hls': 
 301                     formats
.extend(self
._extract
_m
3u8_formats
( 
 302                         stream_url
, video_id
, ext
='mp4', fatal
=False)) 
 303                 # Example: http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706 
 304                 elif '.asf' in stream_url
: 
 305                     asx 
= self
._download
_xml
( 
 306                         stream_url
, video_id
, 
 307                         'Downloading stream %d ASX playlist' % num
, 
 308                         transform_source
=fix_xml_ampersands
, fatal
=False) 
 311                     ref 
= asx
.find('./ENTRY/Ref') 
 314                     video_url 
= ref
.get('href') 
 315                     if not video_url 
or video_url 
in urls
: 
 320                         'ext': stream
.get('formaat', 'asf'), 
 321                         'quality': stream
.get('kwaliteit'), 
 327                         'quality': stream
.get('kwaliteit'), 
 330         self
._sort
_formats
(formats
) 
 333         if metadata
.get('tt888') == 'ja': 
 336                 'url': 'http://tt888.omroep.nl/tt888/%s' % video_id
, 
 341             'title': self
._live
_title
(title
) if is_live 
else title
, 
 342             'description': metadata
.get('info'), 
 343             'thumbnail': metadata
.get('images', [{'url': None}])[-1]['url'], 
 344             'upload_date': unified_strdate(metadata
.get('gidsdatum')), 
 345             'duration': parse_duration(metadata
.get('tijdsduur')), 
 347             'subtitles': subtitles
, 
 352 class NPOLiveIE(NPOBaseIE
): 
 353     IE_NAME 
= 'npo.nl:live' 
 354     _VALID_URL 
= r
'https?://(?:www\.)?npo\.nl/live(?:/(?P<id>[^/?#&]+))?' 
 357         'url': 'http://www.npo.nl/live/npo-1', 
 359             'id': 'LI_NL1_4188102', 
 360             'display_id': 'npo-1', 
 362             'title': 're:^NPO 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 
 366             'skip_download': True, 
 369         'url': 'http://www.npo.nl/live', 
 370         'only_matching': True, 
 373     def _real_extract(self
, url
): 
 374         display_id 
= self
._match
_id
(url
) or 'npo-1' 
 376         webpage 
= self
._download
_webpage
(url
, display_id
) 
 378         live_id 
= self
._search
_regex
( 
 379             [r
'media-id="([^"]+)"', r
'data-prid="([^"]+)"'], webpage
, 'live id') 
 382             '_type': 'url_transparent', 
 383             'url': 'npo:%s' % live_id
, 
 384             'ie_key': NPOIE
.ie_key(), 
 386             'display_id': display_id
, 
 390 class NPORadioIE(InfoExtractor
): 
 391     IE_NAME 
= 'npo.nl:radio' 
 392     _VALID_URL 
= r
'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)/?$' 
 395         'url': 'http://www.npo.nl/radio/radio-1', 
 399             'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$', 
 403             'skip_download': True, 
 408     def _html_get_attribute_regex(attribute
): 
 409         return r
'{0}\s*=\s*\'([^
\']+)\''.format(attribute) 
 411     def _real_extract(self, url): 
 412         video_id = self._match_id(url) 
 414         webpage = self._download_webpage(url, video_id) 
 416         title = self._html_search_regex( 
 417             self._html_get_attribute_regex('data
-channel
'), webpage, 'title
') 
 419         stream = self._parse_json( 
 420             self._html_search_regex(self._html_get_attribute_regex('data
-streams
'), webpage, 'data
-streams
'), 
 423         codec = stream.get('codec
') 
 427             'url
': stream['url
'], 
 428             'title
': self._live_title(title), 
 435 class NPORadioFragmentIE(InfoExtractor): 
 436     IE_NAME = 'npo
.nl
:radio
:fragment
' 
 437     _VALID_URL = r'https?
://(?
:www\
.)?npo\
.nl
/radio
/[^
/]+/fragment
/(?P
<id>\d
+)' 
 440         'url
': 'http
://www
.npo
.nl
/radio
/radio
-5/fragment
/174356', 
 441         'md5
': 'dd8cc470dad764d0fdc70a9a1e2d18c2
', 
 445             'title
': 'Jubileumconcert Willeke Alberti
', 
 449     def _real_extract(self, url): 
 450         audio_id = self._match_id(url) 
 452         webpage = self._download_webpage(url, audio_id) 
 454         title = self._html_search_regex( 
 455             r'href
="/radio/[^/]+/fragment/%s" title
="([^"]+)"' % audio_id, 
 458         audio_url = self._search_regex( 
 459             r"data
-streams
='([^']+)'", webpage, 'audio url
') 
 468 class NPODataMidEmbedIE(InfoExtractor): 
 469     def _real_extract(self, url): 
 470         display_id = self._match_id(url) 
 471         webpage = self._download_webpage(url, display_id) 
 472         video_id = self._search_regex( 
 473             r'data
-mid
=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video_id', group='id') 
 475             '_type': 'url_transparent', 
 477             'url': 'npo:%s' % video_id, 
 478             'display_id': display_id 
 482 class SchoolTVIE(NPODataMidEmbedIE): 
 484     _VALID_URL = r'https?://(?:www\.)?schooltv\.nl/video/(?P<id>[^/?#&]+)' 
 487         'url': 'http://www.schooltv.nl/video/ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam/', 
 489             'id': 'WO_NTR_429477', 
 490             'display_id': 'ademhaling-de-hele-dag-haal-je-adem-maar-wat-gebeurt-er-dan-eigenlijk-in-je-lichaam', 
 491             'title': 'Ademhaling: De hele dag haal je adem. Maar wat gebeurt er dan eigenlijk in je lichaam?', 
 493             'description': 'md5:abfa0ff690adb73fd0297fd033aaa631' 
 496             # Skip because of m3u8 download 
 497             'skip_download': True 
 502 class HetKlokhuisIE(NPODataMidEmbedIE): 
 503     IE_NAME = 'hetklokhuis' 
 504     _VALID_URL = r'https?://(?:www\.)?hetklokhuis\.nl/[^/]+/\d+/(?P<id>[^/?#&]+)' 
 507         'url': 'http://hetklokhuis.nl/tv-uitzending/3471/Zwaartekrachtsgolven', 
 509             'id': 'VPWON_1260528', 
 510             'display_id': 'Zwaartekrachtsgolven', 
 512             'title': 'Het Klokhuis: Zwaartekrachtsgolven', 
 513             'description': 'md5:c94f31fb930d76c2efa4a4a71651dd48', 
 514             'upload_date': '20170223', 
 517             'skip_download': True 
 522 class NPOPlaylistBaseIE(NPOIE): 
 523     def _real_extract(self, url): 
 524         playlist_id = self._match_id(url) 
 526         webpage = self._download_webpage(url, playlist_id) 
 529             self.url_result('npo:%s' % video_id if not video_id.startswith('http') else video_id) 
 530             for video_id in orderedSet(re.findall(self._PLAYLIST_ENTRY_RE, webpage)) 
 533         playlist_title = self._html_search_regex( 
 534             self._PLAYLIST_TITLE_RE, webpage, 'playlist title', 
 535             default=None) or self._og_search_title(webpage) 
 537         return self.playlist_result(entries, playlist_id, playlist_title) 
 540 class VPROIE(NPOPlaylistBaseIE): 
 542     _VALID_URL = r'https?://(?:www\.)?(?:(?:tegenlicht\.)?vpro|2doc)\.nl/(?:[^/]+/)*(?P<id>[^/]+)\.html' 
 543     _PLAYLIST_TITLE_RE = (r'<h1[^>]+class=["\'].*?
\bmedia
-platform
-title
\b.*?
["\'][^>]*>([^<]+)', 
 544                           r'<h5[^>]+class=["\'].*?
\bmedia
-platform
-subtitle
\b.*?
["\'][^>]*>([^<]+)') 
 545     _PLAYLIST_ENTRY_RE = r'data-media-id="([^
"]+)"' 
 549             'url
': 'http
://tegenlicht
.vpro
.nl
/afleveringen
/2012-2013/de
-toekomst
-komt
-uit
-afrika
.html
', 
 550             'md5
': 'f8065e4e5a7824068ed3c7e783178f2c
', 
 552                 'id': 'VPWON_1169289
', 
 554                 'title
': 'De toekomst komt uit Afrika
', 
 555                 'description
': 'md5
:52cf4eefbc96fffcbdc06d024147abea
', 
 556                 'upload_date
': '20130225', 
 558             'skip
': 'Video gone
', 
 561             'url
': 'http
://www
.vpro
.nl
/programmas
/2doc
/2015/sergio
-herman
.html
', 
 563                 'id': 'sergio
-herman
', 
 564                 'title
': 'sergio herman
: fucking perfect
', 
 569             # playlist with youtube embed 
 570             'url
': 'http
://www
.vpro
.nl
/programmas
/2doc
/2015/education
-education
.html
', 
 572                 'id': 'education
-education
', 
 573                 'title
': 'education education
', 
 578             'url
': 'http
://www
.2doc
.nl
/documentaires
/series
/2doc
/2015/oktober
/de
-tegenprestatie
.html
', 
 580                 'id': 'de
-tegenprestatie
', 
 581                 'title
': 'De Tegenprestatie
', 
 585             'url
': 'http
://www
.2doc
.nl
/speel~VARA_101375237~mh17
-het
-verdriet
-van
-nederland~
.html
', 
 587                 'id': 'VARA_101375237
', 
 589                 'title
': 'MH17
: Het verdriet van Nederland
', 
 590                 'description
': 'md5
:09e1a37c1fdb144621e22479691a9f18
', 
 591                 'upload_date
': '20150716', 
 594                 # Skip because of m3u8 download 
 595                 'skip_download
': True 
 601 class WNLIE(NPOPlaylistBaseIE): 
 603     _VALID_URL = r'https?
://(?
:www\
.)?omroepwnl\
.nl
/video
/detail
/(?P
<id>[^
/]+)__\d
+' 
 604     _PLAYLIST_TITLE_RE = r'(?s
)<h1
[^
>]+class="subject"[^
>]*>(.+?
)</h1
>' 
 605     _PLAYLIST_ENTRY_RE = r'<a
[^
>]+href
="([^"]+)"[^>]+class="js
-mid
"[^>]*>Deel \d+' 
 608         'url': 'http://www.omroepwnl.nl/video/detail/vandaag-de-dag-6-mei__060515', 
 610             'id': 'vandaag-de-dag-6-mei', 
 611             'title': 'Vandaag de Dag 6 mei', 
 617 class AndereTijdenIE(NPOPlaylistBaseIE): 
 618     IE_NAME = 'anderetijden' 
 619     _VALID_URL = r'https?://(?:www\.)?anderetijden\.nl/programma/(?:[^/]+/)+(?P<id>[^/?#&]+)' 
 620     _PLAYLIST_TITLE_RE = r'(?s)<h1[^>]+class=["\'].*?
\bpage
-title
\b.*?
["\'][^>]*>(.+?)</h1>' 
 621     _PLAYLIST_ENTRY_RE = r'<figure[^>]+class=["\']episode
-container episode
-page
["\'][^>]+data-prid=["\'](.+?
)["\']' 
 624         'url': 'http://anderetijden.nl/programma/1/Andere-Tijden/aflevering/676/Duitse-soldaten-over-de-Slag-bij-Arnhem', 
 626             'id': 'Duitse-soldaten-over-de-Slag-bij-Arnhem', 
 627             'title': 'Duitse soldaten over de Slag bij Arnhem',