]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/nrk.py
New upstream version 2018.04.25
[youtubedl] / youtube_dl / extractor / nrk.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_urllib_parse_unquote
8 from ..utils import (
9 ExtractorError,
10 int_or_none,
11 parse_age_limit,
12 parse_duration,
13 )
14
15
16 class NRKBaseIE(InfoExtractor):
17 _GEO_COUNTRIES = ['NO']
18
19 def _real_extract(self, url):
20 video_id = self._match_id(url)
21
22 data = self._download_json(
23 'http://%s/mediaelement/%s' % (self._API_HOST, video_id),
24 video_id, 'Downloading mediaelement JSON')
25
26 title = data.get('fullTitle') or data.get('mainTitle') or data['title']
27 video_id = data.get('id') or video_id
28
29 entries = []
30
31 conviva = data.get('convivaStatistics') or {}
32 live = (data.get('mediaElementType') == 'Live' or
33 data.get('isLive') is True or conviva.get('isLive'))
34
35 def make_title(t):
36 return self._live_title(t) if live else t
37
38 media_assets = data.get('mediaAssets')
39 if media_assets and isinstance(media_assets, list):
40 def video_id_and_title(idx):
41 return ((video_id, title) if len(media_assets) == 1
42 else ('%s-%d' % (video_id, idx), '%s (Part %d)' % (title, idx)))
43 for num, asset in enumerate(media_assets, 1):
44 asset_url = asset.get('url')
45 if not asset_url:
46 continue
47 formats = self._extract_akamai_formats(asset_url, video_id)
48 if not formats:
49 continue
50 self._sort_formats(formats)
51
52 # Some f4m streams may not work with hdcore in fragments' URLs
53 for f in formats:
54 extra_param = f.get('extra_param_to_segment_url')
55 if extra_param and 'hdcore' in extra_param:
56 del f['extra_param_to_segment_url']
57
58 entry_id, entry_title = video_id_and_title(num)
59 duration = parse_duration(asset.get('duration'))
60 subtitles = {}
61 for subtitle in ('webVtt', 'timedText'):
62 subtitle_url = asset.get('%sSubtitlesUrl' % subtitle)
63 if subtitle_url:
64 subtitles.setdefault('no', []).append({
65 'url': compat_urllib_parse_unquote(subtitle_url)
66 })
67 entries.append({
68 'id': asset.get('carrierId') or entry_id,
69 'title': make_title(entry_title),
70 'duration': duration,
71 'subtitles': subtitles,
72 'formats': formats,
73 })
74
75 if not entries:
76 media_url = data.get('mediaUrl')
77 if media_url:
78 formats = self._extract_akamai_formats(media_url, video_id)
79 self._sort_formats(formats)
80 duration = parse_duration(data.get('duration'))
81 entries = [{
82 'id': video_id,
83 'title': make_title(title),
84 'duration': duration,
85 'formats': formats,
86 }]
87
88 if not entries:
89 MESSAGES = {
90 'ProgramRightsAreNotReady': 'Du kan dessverre ikke se eller høre programmet',
91 'ProgramRightsHasExpired': 'Programmet har gått ut',
92 'ProgramIsGeoBlocked': 'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
93 }
94 message_type = data.get('messageType', '')
95 # Can be ProgramIsGeoBlocked or ChannelIsGeoBlocked*
96 if 'IsGeoBlocked' in message_type:
97 self.raise_geo_restricted(
98 msg=MESSAGES.get('ProgramIsGeoBlocked'),
99 countries=self._GEO_COUNTRIES)
100 raise ExtractorError(
101 '%s said: %s' % (self.IE_NAME, MESSAGES.get(
102 message_type, message_type)),
103 expected=True)
104
105 series = conviva.get('seriesName') or data.get('seriesTitle')
106 episode = conviva.get('episodeName') or data.get('episodeNumberOrDate')
107
108 season_number = None
109 episode_number = None
110 if data.get('mediaElementType') == 'Episode':
111 _season_episode = data.get('scoresStatistics', {}).get('springStreamStream') or \
112 data.get('relativeOriginUrl', '')
113 EPISODENUM_RE = [
114 r'/s(?P<season>\d{,2})e(?P<episode>\d{,2})\.',
115 r'/sesong-(?P<season>\d{,2})/episode-(?P<episode>\d{,2})',
116 ]
117 season_number = int_or_none(self._search_regex(
118 EPISODENUM_RE, _season_episode, 'season number',
119 default=None, group='season'))
120 episode_number = int_or_none(self._search_regex(
121 EPISODENUM_RE, _season_episode, 'episode number',
122 default=None, group='episode'))
123
124 thumbnails = None
125 images = data.get('images')
126 if images and isinstance(images, dict):
127 web_images = images.get('webImages')
128 if isinstance(web_images, list):
129 thumbnails = [{
130 'url': image['imageUrl'],
131 'width': int_or_none(image.get('width')),
132 'height': int_or_none(image.get('height')),
133 } for image in web_images if image.get('imageUrl')]
134
135 description = data.get('description')
136 category = data.get('mediaAnalytics', {}).get('category')
137
138 common_info = {
139 'description': description,
140 'series': series,
141 'episode': episode,
142 'season_number': season_number,
143 'episode_number': episode_number,
144 'categories': [category] if category else None,
145 'age_limit': parse_age_limit(data.get('legalAge')),
146 'thumbnails': thumbnails,
147 }
148
149 vcodec = 'none' if data.get('mediaType') == 'Audio' else None
150
151 for entry in entries:
152 entry.update(common_info)
153 for f in entry['formats']:
154 f['vcodec'] = vcodec
155
156 points = data.get('shortIndexPoints')
157 if isinstance(points, list):
158 chapters = []
159 for next_num, point in enumerate(points, start=1):
160 if not isinstance(point, dict):
161 continue
162 start_time = parse_duration(point.get('startPoint'))
163 if start_time is None:
164 continue
165 end_time = parse_duration(
166 data.get('duration')
167 if next_num == len(points)
168 else points[next_num].get('startPoint'))
169 if end_time is None:
170 continue
171 chapters.append({
172 'start_time': start_time,
173 'end_time': end_time,
174 'title': point.get('title'),
175 })
176 if chapters and len(entries) == 1:
177 entries[0]['chapters'] = chapters
178
179 return self.playlist_result(entries, video_id, title, description)
180
181
182 class NRKIE(NRKBaseIE):
183 _VALID_URL = r'''(?x)
184 (?:
185 nrk:|
186 https?://
187 (?:
188 (?:www\.)?nrk\.no/video/PS\*|
189 v8[-.]psapi\.nrk\.no/mediaelement/
190 )
191 )
192 (?P<id>[^?#&]+)
193 '''
194 _API_HOST = 'v8-psapi.nrk.no'
195 _TESTS = [{
196 # video
197 'url': 'http://www.nrk.no/video/PS*150533',
198 'md5': '2f7f6eeb2aacdd99885f355428715cfa',
199 'info_dict': {
200 'id': '150533',
201 'ext': 'mp4',
202 'title': 'Dompap og andre fugler i Piip-Show',
203 'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
204 'duration': 263,
205 }
206 }, {
207 # audio
208 'url': 'http://www.nrk.no/video/PS*154915',
209 # MD5 is unstable
210 'info_dict': {
211 'id': '154915',
212 'ext': 'flv',
213 'title': 'Slik høres internett ut når du er blind',
214 'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
215 'duration': 20,
216 }
217 }, {
218 'url': 'nrk:ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
219 'only_matching': True,
220 }, {
221 'url': 'nrk:clip/7707d5a3-ebe7-434a-87d5-a3ebe7a34a70',
222 'only_matching': True,
223 }, {
224 'url': 'https://v8-psapi.nrk.no/mediaelement/ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
225 'only_matching': True,
226 }]
227
228
229 class NRKTVIE(NRKBaseIE):
230 IE_DESC = 'NRK TV and NRK Radio'
231 _EPISODE_RE = r'(?P<id>[a-zA-Z]{4}\d{8})'
232 _VALID_URL = r'''(?x)
233 https?://
234 (?:tv|radio)\.nrk(?:super)?\.no/
235 (?:serie/[^/]+|program)/
236 (?![Ee]pisodes)%s
237 (?:/\d{2}-\d{2}-\d{4})?
238 (?:\#del=(?P<part_id>\d+))?
239 ''' % _EPISODE_RE
240 _API_HOST = 'psapi-ne.nrk.no'
241
242 _TESTS = [{
243 'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
244 'md5': '4e9ca6629f09e588ed240fb11619922a',
245 'info_dict': {
246 'id': 'MUHH48000314AA',
247 'ext': 'mp4',
248 'title': '20 spørsmål 23.05.2014',
249 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
250 'duration': 1741,
251 'series': '20 spørsmål - TV',
252 'episode': '23.05.2014',
253 },
254 }, {
255 'url': 'https://tv.nrk.no/program/mdfp15000514',
256 'info_dict': {
257 'id': 'MDFP15000514CA',
258 'ext': 'mp4',
259 'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
260 'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
261 'duration': 4605,
262 'series': 'Kunnskapskanalen',
263 'episode': '24.05.2014',
264 },
265 'params': {
266 'skip_download': True,
267 },
268 }, {
269 # single playlist video
270 'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
271 'info_dict': {
272 'id': 'MSPO40010515-part2',
273 'ext': 'flv',
274 'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
275 'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
276 },
277 'params': {
278 'skip_download': True,
279 },
280 'expected_warnings': ['Video is geo restricted'],
281 'skip': 'particular part is not supported currently',
282 }, {
283 'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
284 'playlist': [{
285 'info_dict': {
286 'id': 'MSPO40010515AH',
287 'ext': 'mp4',
288 'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 1)',
289 'description': 'md5:c03aba1e917561eface5214020551b7a',
290 'duration': 772,
291 'series': 'Tour de Ski',
292 'episode': '06.01.2015',
293 },
294 'params': {
295 'skip_download': True,
296 },
297 }, {
298 'info_dict': {
299 'id': 'MSPO40010515BH',
300 'ext': 'mp4',
301 'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015 (Part 2)',
302 'description': 'md5:c03aba1e917561eface5214020551b7a',
303 'duration': 6175,
304 'series': 'Tour de Ski',
305 'episode': '06.01.2015',
306 },
307 'params': {
308 'skip_download': True,
309 },
310 }],
311 'info_dict': {
312 'id': 'MSPO40010515',
313 'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
314 'description': 'md5:c03aba1e917561eface5214020551b7a',
315 },
316 'expected_warnings': ['Video is geo restricted'],
317 }, {
318 'url': 'https://tv.nrk.no/serie/anno/KMTE50001317/sesong-3/episode-13',
319 'info_dict': {
320 'id': 'KMTE50001317AA',
321 'ext': 'mp4',
322 'title': 'Anno 13:30',
323 'description': 'md5:11d9613661a8dbe6f9bef54e3a4cbbfa',
324 'duration': 2340,
325 'series': 'Anno',
326 'episode': '13:30',
327 'season_number': 3,
328 'episode_number': 13,
329 },
330 'params': {
331 'skip_download': True,
332 },
333 }, {
334 'url': 'https://tv.nrk.no/serie/nytt-paa-nytt/MUHH46000317/27-01-2017',
335 'info_dict': {
336 'id': 'MUHH46000317AA',
337 'ext': 'mp4',
338 'title': 'Nytt på Nytt 27.01.2017',
339 'description': 'md5:5358d6388fba0ea6f0b6d11c48b9eb4b',
340 'duration': 1796,
341 'series': 'Nytt på nytt',
342 'episode': '27.01.2017',
343 },
344 'params': {
345 'skip_download': True,
346 },
347 }, {
348 'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
349 'only_matching': True,
350 }]
351
352
353 class NRKTVDirekteIE(NRKTVIE):
354 IE_DESC = 'NRK TV Direkte and NRK Radio Direkte'
355 _VALID_URL = r'https?://(?:tv|radio)\.nrk\.no/direkte/(?P<id>[^/?#&]+)'
356
357 _TESTS = [{
358 'url': 'https://tv.nrk.no/direkte/nrk1',
359 'only_matching': True,
360 }, {
361 'url': 'https://radio.nrk.no/direkte/p1_oslo_akershus',
362 'only_matching': True,
363 }]
364
365
366 class NRKPlaylistBaseIE(InfoExtractor):
367 def _extract_description(self, webpage):
368 pass
369
370 def _real_extract(self, url):
371 playlist_id = self._match_id(url)
372
373 webpage = self._download_webpage(url, playlist_id)
374
375 entries = [
376 self.url_result('nrk:%s' % video_id, NRKIE.ie_key())
377 for video_id in re.findall(self._ITEM_RE, webpage)
378 ]
379
380 playlist_title = self. _extract_title(webpage)
381 playlist_description = self._extract_description(webpage)
382
383 return self.playlist_result(
384 entries, playlist_id, playlist_title, playlist_description)
385
386
387 class NRKPlaylistIE(NRKPlaylistBaseIE):
388 _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'
389 _ITEM_RE = r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"'
390 _TESTS = [{
391 'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
392 'info_dict': {
393 'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
394 'title': 'Gjenopplev den historiske solformørkelsen',
395 'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
396 },
397 'playlist_count': 2,
398 }, {
399 'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
400 'info_dict': {
401 'id': 'rivertonprisen-til-karin-fossum-1.12266449',
402 'title': 'Rivertonprisen til Karin Fossum',
403 'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
404 },
405 'playlist_count': 5,
406 }]
407
408 def _extract_title(self, webpage):
409 return self._og_search_title(webpage, fatal=False)
410
411 def _extract_description(self, webpage):
412 return self._og_search_description(webpage)
413
414
415 class NRKTVEpisodesIE(NRKPlaylistBaseIE):
416 _VALID_URL = r'https?://tv\.nrk\.no/program/[Ee]pisodes/[^/]+/(?P<id>\d+)'
417 _ITEM_RE = r'data-episode=["\']%s' % NRKTVIE._EPISODE_RE
418 _TESTS = [{
419 'url': 'https://tv.nrk.no/program/episodes/nytt-paa-nytt/69031',
420 'info_dict': {
421 'id': '69031',
422 'title': 'Nytt på nytt, sesong: 201210',
423 },
424 'playlist_count': 4,
425 }]
426
427 def _extract_title(self, webpage):
428 return self._html_search_regex(
429 r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
430
431
432 class NRKTVSeriesIE(InfoExtractor):
433 _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/serie/(?P<id>[^/]+)'
434 _ITEM_RE = r'(?:data-season=["\']|id=["\']season-)(?P<id>\d+)'
435 _TESTS = [{
436 'url': 'https://tv.nrk.no/serie/groenn-glede',
437 'info_dict': {
438 'id': 'groenn-glede',
439 'title': 'Grønn glede',
440 'description': 'md5:7576e92ae7f65da6993cf90ee29e4608',
441 },
442 'playlist_mincount': 9,
443 }, {
444 'url': 'http://tv.nrksuper.no/serie/labyrint',
445 'info_dict': {
446 'id': 'labyrint',
447 'title': 'Labyrint',
448 'description': 'md5:58afd450974c89e27d5a19212eee7115',
449 },
450 'playlist_mincount': 3,
451 }, {
452 'url': 'https://tv.nrk.no/serie/broedrene-dal-og-spektralsteinene',
453 'only_matching': True,
454 }, {
455 'url': 'https://tv.nrk.no/serie/saving-the-human-race',
456 'only_matching': True,
457 }, {
458 'url': 'https://tv.nrk.no/serie/postmann-pat',
459 'only_matching': True,
460 }]
461
462 @classmethod
463 def suitable(cls, url):
464 return False if NRKTVIE.suitable(url) else super(NRKTVSeriesIE, cls).suitable(url)
465
466 def _real_extract(self, url):
467 series_id = self._match_id(url)
468
469 webpage = self._download_webpage(url, series_id)
470
471 entries = [
472 self.url_result(
473 'https://tv.nrk.no/program/Episodes/{series}/{season}'.format(
474 series=series_id, season=season_id))
475 for season_id in re.findall(self._ITEM_RE, webpage)
476 ]
477
478 title = self._html_search_meta(
479 'seriestitle', webpage,
480 'title', default=None) or self._og_search_title(
481 webpage, fatal=False)
482
483 description = self._html_search_meta(
484 'series_description', webpage,
485 'description', default=None) or self._og_search_description(webpage)
486
487 return self.playlist_result(entries, series_id, title, description)
488
489
490 class NRKSkoleIE(InfoExtractor):
491 IE_DESC = 'NRK Skole'
492 _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/?\?.*\bmediaId=(?P<id>\d+)'
493
494 _TESTS = [{
495 'url': 'https://www.nrk.no/skole/?page=search&q=&mediaId=14099',
496 'md5': '6bc936b01f9dd8ed45bc58b252b2d9b6',
497 'info_dict': {
498 'id': '6021',
499 'ext': 'mp4',
500 'title': 'Genetikk og eneggede tvillinger',
501 'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
502 'duration': 399,
503 },
504 }, {
505 'url': 'https://www.nrk.no/skole/?page=objectives&subject=naturfag&objective=K15114&mediaId=19355',
506 'only_matching': True,
507 }]
508
509 def _real_extract(self, url):
510 video_id = self._match_id(url)
511
512 webpage = self._download_webpage(
513 'https://mimir.nrk.no/plugin/1.0/static?mediaId=%s' % video_id,
514 video_id)
515
516 nrk_id = self._parse_json(
517 self._search_regex(
518 r'<script[^>]+type=["\']application/json["\'][^>]*>({.+?})</script>',
519 webpage, 'application json'),
520 video_id)['activeMedia']['psId']
521
522 return self.url_result('nrk:%s' % nrk_id)