]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ndr.py
Update changelog.
[youtubedl] / youtube_dl / extractor / ndr.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..utils import (
8 determine_ext,
9 int_or_none,
10 merge_dicts,
11 parse_iso8601,
12 qualities,
13 try_get,
14 urljoin,
15 )
16
17
18 class NDRBaseIE(InfoExtractor):
19 def _real_extract(self, url):
20 mobj = re.match(self._VALID_URL, url)
21 display_id = next(group for group in mobj.groups() if group)
22 webpage = self._download_webpage(url, display_id)
23 return self._extract_embed(webpage, display_id)
24
25
26 class NDRIE(NDRBaseIE):
27 IE_NAME = 'ndr'
28 IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
29 _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
30 _TESTS = [{
31 # httpVideo, same content id
32 'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
33 'md5': '6515bc255dc5c5f8c85bbc38e035a659',
34 'info_dict': {
35 'id': 'hafengeburtstag988',
36 'display_id': 'Party-Poette-und-Parade',
37 'ext': 'mp4',
38 'title': 'Party, Pötte und Parade',
39 'description': 'md5:ad14f9d2f91d3040b6930c697e5f6b4c',
40 'uploader': 'ndrtv',
41 'timestamp': 1431108900,
42 'upload_date': '20150510',
43 'duration': 3498,
44 },
45 'params': {
46 'skip_download': True,
47 },
48 }, {
49 # httpVideo, different content id
50 'url': 'http://www.ndr.de/sport/fussball/40-Osnabrueck-spielt-sich-in-einen-Rausch,osna270.html',
51 'md5': '1043ff203eab307f0c51702ec49e9a71',
52 'info_dict': {
53 'id': 'osna272',
54 'display_id': '40-Osnabrueck-spielt-sich-in-einen-Rausch',
55 'ext': 'mp4',
56 'title': 'Osnabrück - Wehen Wiesbaden: Die Highlights',
57 'description': 'md5:32e9b800b3d2d4008103752682d5dc01',
58 'uploader': 'ndrtv',
59 'timestamp': 1442059200,
60 'upload_date': '20150912',
61 'duration': 510,
62 },
63 'params': {
64 'skip_download': True,
65 },
66 }, {
67 # httpAudio, same content id
68 'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
69 'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
70 'info_dict': {
71 'id': 'audio51535',
72 'display_id': 'La-Valette-entgeht-der-Hinrichtung',
73 'ext': 'mp3',
74 'title': 'La Valette entgeht der Hinrichtung',
75 'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
76 'uploader': 'ndrinfo',
77 'timestamp': 1290626100,
78 'upload_date': '20140729',
79 'duration': 884,
80 },
81 'params': {
82 'skip_download': True,
83 },
84 }, {
85 'url': 'https://www.ndr.de/Fettes-Brot-Ferris-MC-und-Thees-Uhlmann-live-on-stage,festivalsommer116.html',
86 'only_matching': True,
87 }]
88
89 def _extract_embed(self, webpage, display_id):
90 embed_url = self._html_search_meta(
91 'embedURL', webpage, 'embed URL',
92 default=None) or self._search_regex(
93 r'\bembedUrl["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
94 'embed URL', group='url')
95 description = self._search_regex(
96 r'<p[^>]+itemprop="description">([^<]+)</p>',
97 webpage, 'description', default=None) or self._og_search_description(webpage)
98 timestamp = parse_iso8601(
99 self._search_regex(
100 r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
101 webpage, 'upload date', default=None))
102 info = self._search_json_ld(webpage, display_id, default={})
103 return merge_dicts({
104 '_type': 'url_transparent',
105 'url': embed_url,
106 'display_id': display_id,
107 'description': description,
108 'timestamp': timestamp,
109 }, info)
110
111
112 class NJoyIE(NDRBaseIE):
113 IE_NAME = 'njoy'
114 IE_DESC = 'N-JOY'
115 _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
116 _TESTS = [{
117 # httpVideo, same content id
118 'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
119 'md5': 'cb63be60cd6f9dd75218803146d8dc67',
120 'info_dict': {
121 'id': 'comedycontest2480',
122 'display_id': 'Benaissa-beim-NDR-Comedy-Contest',
123 'ext': 'mp4',
124 'title': 'Benaissa beim NDR Comedy Contest',
125 'description': 'md5:f057a6c4e1c728b10d33b5ffd36ddc39',
126 'uploader': 'ndrtv',
127 'upload_date': '20141129',
128 'duration': 654,
129 },
130 'params': {
131 'skip_download': True,
132 },
133 }, {
134 # httpVideo, different content id
135 'url': 'http://www.n-joy.de/musik/Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-,felixjaehn168.html',
136 'md5': '417660fffa90e6df2fda19f1b40a64d8',
137 'info_dict': {
138 'id': 'dockville882',
139 'display_id': 'Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-',
140 'ext': 'mp4',
141 'title': '"Ich hab noch nie" mit Felix Jaehn',
142 'description': 'md5:85dd312d53be1b99e1f998a16452a2f3',
143 'uploader': 'njoy',
144 'upload_date': '20150822',
145 'duration': 211,
146 },
147 'params': {
148 'skip_download': True,
149 },
150 }, {
151 'url': 'http://www.n-joy.de/radio/webradio/morningshow209.html',
152 'only_matching': True,
153 }]
154
155 def _extract_embed(self, webpage, display_id):
156 video_id = self._search_regex(
157 r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
158 description = self._search_regex(
159 r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
160 webpage, 'description', fatal=False)
161 return {
162 '_type': 'url_transparent',
163 'ie_key': 'NDREmbedBase',
164 'url': 'ndr:%s' % video_id,
165 'display_id': display_id,
166 'description': description,
167 }
168
169
170 class NDREmbedBaseIE(InfoExtractor):
171 IE_NAME = 'ndr:embed:base'
172 _VALID_URL = r'(?:ndr:(?P<id_s>[\da-z]+)|https?://www\.ndr\.de/(?P<id>[\da-z]+)-ppjson\.json)'
173 _TESTS = [{
174 'url': 'ndr:soundcheck3366',
175 'only_matching': True,
176 }, {
177 'url': 'http://www.ndr.de/soundcheck3366-ppjson.json',
178 'only_matching': True,
179 }]
180
181 def _real_extract(self, url):
182 mobj = re.match(self._VALID_URL, url)
183 video_id = mobj.group('id') or mobj.group('id_s')
184
185 ppjson = self._download_json(
186 'http://www.ndr.de/%s-ppjson.json' % video_id, video_id)
187
188 playlist = ppjson['playlist']
189
190 formats = []
191 quality_key = qualities(('xs', 's', 'm', 'l', 'xl'))
192
193 for format_id, f in playlist.items():
194 src = f.get('src')
195 if not src:
196 continue
197 ext = determine_ext(src, None)
198 if ext == 'f4m':
199 formats.extend(self._extract_f4m_formats(
200 src + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
201 f4m_id='hds', fatal=False))
202 elif ext == 'm3u8':
203 formats.extend(self._extract_m3u8_formats(
204 src, video_id, 'mp4', m3u8_id='hls',
205 entry_protocol='m3u8_native', fatal=False))
206 else:
207 quality = f.get('quality')
208 ff = {
209 'url': src,
210 'format_id': quality or format_id,
211 'quality': quality_key(quality),
212 }
213 type_ = f.get('type')
214 if type_ and type_.split('/')[0] == 'audio':
215 ff['vcodec'] = 'none'
216 ff['ext'] = ext or 'mp3'
217 formats.append(ff)
218 self._sort_formats(formats)
219
220 config = playlist['config']
221
222 live = playlist.get('config', {}).get('streamType') in ['httpVideoLive', 'httpAudioLive']
223 title = config['title']
224 if live:
225 title = self._live_title(title)
226 uploader = ppjson.get('config', {}).get('branding')
227 upload_date = ppjson.get('config', {}).get('publicationDate')
228 duration = int_or_none(config.get('duration'))
229
230 thumbnails = []
231 poster = try_get(config, lambda x: x['poster'], dict) or {}
232 for thumbnail_id, thumbnail in poster.items():
233 thumbnail_url = urljoin(url, thumbnail.get('src'))
234 if not thumbnail_url:
235 continue
236 thumbnails.append({
237 'id': thumbnail.get('quality') or thumbnail_id,
238 'url': thumbnail_url,
239 'preference': quality_key(thumbnail.get('quality')),
240 })
241
242 return {
243 'id': video_id,
244 'title': title,
245 'is_live': live,
246 'uploader': uploader if uploader != '-' else None,
247 'upload_date': upload_date[0:8] if upload_date else None,
248 'duration': duration,
249 'thumbnails': thumbnails,
250 'formats': formats,
251 }
252
253
254 class NDREmbedIE(NDREmbedBaseIE):
255 IE_NAME = 'ndr:embed'
256 _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
257 _TESTS = [{
258 'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
259 'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
260 'info_dict': {
261 'id': 'ndraktuell28488',
262 'ext': 'mp4',
263 'title': 'Norddeutschland begrüßt Flüchtlinge',
264 'is_live': False,
265 'uploader': 'ndrtv',
266 'upload_date': '20150907',
267 'duration': 132,
268 },
269 }, {
270 'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html',
271 'md5': '002085c44bae38802d94ae5802a36e78',
272 'info_dict': {
273 'id': 'soundcheck3366',
274 'ext': 'mp4',
275 'title': 'Ella Henderson braucht Vergleiche nicht zu scheuen',
276 'is_live': False,
277 'uploader': 'ndr2',
278 'upload_date': '20150912',
279 'duration': 3554,
280 },
281 'params': {
282 'skip_download': True,
283 },
284 }, {
285 'url': 'http://www.ndr.de/info/audio51535-player.html',
286 'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
287 'info_dict': {
288 'id': 'audio51535',
289 'ext': 'mp3',
290 'title': 'La Valette entgeht der Hinrichtung',
291 'is_live': False,
292 'uploader': 'ndrinfo',
293 'upload_date': '20140729',
294 'duration': 884,
295 },
296 'params': {
297 'skip_download': True,
298 },
299 }, {
300 'url': 'http://www.ndr.de/fernsehen/sendungen/visite/visite11010-externalPlayer.html',
301 'md5': 'ae57f80511c1e1f2fd0d0d3d31aeae7c',
302 'info_dict': {
303 'id': 'visite11010',
304 'ext': 'mp4',
305 'title': 'Visite - die ganze Sendung',
306 'is_live': False,
307 'uploader': 'ndrtv',
308 'upload_date': '20150902',
309 'duration': 3525,
310 },
311 'params': {
312 'skip_download': True,
313 },
314 }, {
315 # httpVideoLive
316 'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html',
317 'info_dict': {
318 'id': 'livestream217',
319 'ext': 'flv',
320 'title': r're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
321 'is_live': True,
322 'upload_date': '20150910',
323 },
324 'params': {
325 'skip_download': True,
326 },
327 }, {
328 'url': 'http://www.ndr.de/ndrkultur/audio255020-player.html',
329 'only_matching': True,
330 }, {
331 'url': 'http://www.ndr.de/fernsehen/sendungen/nordtour/nordtour7124-player.html',
332 'only_matching': True,
333 }, {
334 'url': 'http://www.ndr.de/kultur/film/videos/videoimport10424-player.html',
335 'only_matching': True,
336 }, {
337 'url': 'http://www.ndr.de/fernsehen/sendungen/hamburg_journal/hamj43006-player.html',
338 'only_matching': True,
339 }, {
340 'url': 'http://www.ndr.de/fernsehen/sendungen/weltbilder/weltbilder4518-player.html',
341 'only_matching': True,
342 }, {
343 'url': 'http://www.ndr.de/fernsehen/doku952-player.html',
344 'only_matching': True,
345 }]
346
347
348 class NJoyEmbedIE(NDREmbedBaseIE):
349 IE_NAME = 'njoy:embed'
350 _VALID_URL = r'https?://(?:www\.)?n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
351 _TESTS = [{
352 # httpVideo
353 'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
354 'md5': '8483cbfe2320bd4d28a349d62d88bd74',
355 'info_dict': {
356 'id': 'doku948',
357 'ext': 'mp4',
358 'title': 'Zehn Jahre Reeperbahn Festival - die Doku',
359 'is_live': False,
360 'upload_date': '20150807',
361 'duration': 1011,
362 },
363 }, {
364 # httpAudio
365 'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html',
366 'md5': 'd989f80f28ac954430f7b8a48197188a',
367 'info_dict': {
368 'id': 'stefanrichter100',
369 'ext': 'mp3',
370 'title': 'Interview mit einem Augenzeugen',
371 'is_live': False,
372 'uploader': 'njoy',
373 'upload_date': '20150909',
374 'duration': 140,
375 },
376 'params': {
377 'skip_download': True,
378 },
379 }, {
380 # httpAudioLive, no explicit ext
381 'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html',
382 'info_dict': {
383 'id': 'webradioweltweit100',
384 'ext': 'mp3',
385 'title': r're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
386 'is_live': True,
387 'uploader': 'njoy',
388 'upload_date': '20150810',
389 },
390 'params': {
391 'skip_download': True,
392 },
393 }, {
394 'url': 'http://www.n-joy.de/musik/dockville882-player_image-3905259e-0803-4764-ac72-8b7de077d80a_theme-n-joy.html',
395 'only_matching': True,
396 }, {
397 'url': 'http://www.n-joy.de/radio/sendungen/morningshow/urlaubsfotos190-player_image-066a5df1-5c95-49ec-a323-941d848718db_theme-n-joy.html',
398 'only_matching': True,
399 }, {
400 'url': 'http://www.n-joy.de/entertainment/comedy/krudetv290-player_image-ab261bfe-51bf-4bf3-87ba-c5122ee35b3d_theme-n-joy.html',
401 'only_matching': True,
402 }]