]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/orf.py
d/p/skip-style-checks-for-flake8: Add metadata to please lintian.
[youtubedl] / youtube_dl / extractor / orf.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9 clean_html,
10 determine_ext,
11 float_or_none,
12 HEADRequest,
13 int_or_none,
14 orderedSet,
15 remove_end,
16 str_or_none,
17 strip_jsonp,
18 unescapeHTML,
19 unified_strdate,
20 url_or_none,
21 )
22
23
24 class ORFTVthekIE(InfoExtractor):
25 IE_NAME = 'orf:tvthek'
26 IE_DESC = 'ORF TVthek'
27 _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
28
29 _TESTS = [{
30 'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
31 'playlist': [{
32 'md5': '2942210346ed779588f428a92db88712',
33 'info_dict': {
34 'id': '8896777',
35 'ext': 'mp4',
36 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
37 'description': 'md5:c1272f0245537812d4e36419c207b67d',
38 'duration': 2668,
39 'upload_date': '20141208',
40 },
41 }],
42 'skip': 'Blocked outside of Austria / Germany',
43 }, {
44 'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
45 'info_dict': {
46 'id': '7982259',
47 'ext': 'mp4',
48 'title': 'Best of Ingrid Thurnher',
49 'upload_date': '20140527',
50 'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
51 },
52 'params': {
53 'skip_download': True, # rtsp downloads
54 },
55 'skip': 'Blocked outside of Austria / Germany',
56 }, {
57 'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
58 'only_matching': True,
59 }, {
60 'url': 'http://tvthek.orf.at/profile/Universum/35429',
61 'only_matching': True,
62 }]
63
64 def _real_extract(self, url):
65 playlist_id = self._match_id(url)
66 webpage = self._download_webpage(url, playlist_id)
67
68 data_jsb = self._parse_json(
69 self._search_regex(
70 r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
71 webpage, 'playlist', group='json'),
72 playlist_id, transform_source=unescapeHTML)['playlist']['videos']
73
74 entries = []
75 for sd in data_jsb:
76 video_id, title = sd.get('id'), sd.get('title')
77 if not video_id or not title:
78 continue
79 video_id = compat_str(video_id)
80 formats = []
81 for fd in sd['sources']:
82 src = url_or_none(fd.get('src'))
83 if not src:
84 continue
85 format_id_list = []
86 for key in ('delivery', 'quality', 'quality_string'):
87 value = fd.get(key)
88 if value:
89 format_id_list.append(value)
90 format_id = '-'.join(format_id_list)
91 ext = determine_ext(src)
92 if ext == 'm3u8':
93 m3u8_formats = self._extract_m3u8_formats(
94 src, video_id, 'mp4', m3u8_id=format_id, fatal=False)
95 if any('/geoprotection' in f['url'] for f in m3u8_formats):
96 self.raise_geo_restricted()
97 formats.extend(m3u8_formats)
98 elif ext == 'f4m':
99 formats.extend(self._extract_f4m_formats(
100 src, video_id, f4m_id=format_id, fatal=False))
101 else:
102 formats.append({
103 'format_id': format_id,
104 'url': src,
105 'protocol': fd.get('protocol'),
106 })
107
108 # Check for geoblocking.
109 # There is a property is_geoprotection, but that's always false
110 geo_str = sd.get('geoprotection_string')
111 if geo_str:
112 try:
113 http_url = next(
114 f['url']
115 for f in formats
116 if re.match(r'^https?://.*\.mp4$', f['url']))
117 except StopIteration:
118 pass
119 else:
120 req = HEADRequest(http_url)
121 self._request_webpage(
122 req, video_id,
123 note='Testing for geoblocking',
124 errnote=((
125 'This video seems to be blocked outside of %s. '
126 'You may want to try the streaming-* formats.')
127 % geo_str),
128 fatal=False)
129
130 self._check_formats(formats, video_id)
131 self._sort_formats(formats)
132
133 subtitles = {}
134 for sub in sd.get('subtitles', []):
135 sub_src = sub.get('src')
136 if not sub_src:
137 continue
138 subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
139 'url': sub_src,
140 })
141
142 upload_date = unified_strdate(sd.get('created_date'))
143 entries.append({
144 '_type': 'video',
145 'id': video_id,
146 'title': title,
147 'formats': formats,
148 'subtitles': subtitles,
149 'description': sd.get('description'),
150 'duration': int_or_none(sd.get('duration_in_seconds')),
151 'upload_date': upload_date,
152 'thumbnail': sd.get('image_full_url'),
153 })
154
155 return {
156 '_type': 'playlist',
157 'entries': entries,
158 'id': playlist_id,
159 }
160
161
162 class ORFRadioIE(InfoExtractor):
163 def _real_extract(self, url):
164 mobj = re.match(self._VALID_URL, url)
165 station = mobj.group('station')
166 show_date = mobj.group('date')
167 show_id = mobj.group('show')
168
169 data = self._download_json(
170 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s'
171 % (station, show_id, show_date), show_id)
172
173 entries = []
174 for info in data['streams']:
175 loop_stream_id = str_or_none(info.get('loopStreamId'))
176 if not loop_stream_id:
177 continue
178 title = str_or_none(data.get('title'))
179 if not title:
180 continue
181 start = int_or_none(info.get('start'), scale=1000)
182 end = int_or_none(info.get('end'), scale=1000)
183 duration = end - start if end and start else None
184 entries.append({
185 'id': loop_stream_id.replace('.mp3', ''),
186 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, loop_stream_id),
187 'title': title,
188 'description': clean_html(data.get('subtitle')),
189 'duration': duration,
190 'timestamp': start,
191 'ext': 'mp3',
192 'series': data.get('programTitle'),
193 })
194
195 return {
196 '_type': 'playlist',
197 'id': show_id,
198 'title': data.get('title'),
199 'description': clean_html(data.get('subtitle')),
200 'entries': entries,
201 }
202
203
204 class ORFFM4IE(ORFRadioIE):
205 IE_NAME = 'orf:fm4'
206 IE_DESC = 'radio FM4'
207 _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>4\w+)'
208
209 _TEST = {
210 'url': 'http://fm4.orf.at/player/20170107/4CC',
211 'md5': '2b0be47375432a7ef104453432a19212',
212 'info_dict': {
213 'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
214 'ext': 'mp3',
215 'title': 'Solid Steel Radioshow',
216 'description': 'Die Mixshow von Coldcut und Ninja Tune.',
217 'duration': 3599,
218 'timestamp': 1483819257,
219 'upload_date': '20170107',
220 },
221 'skip': 'Shows from ORF radios are only available for 7 days.',
222 'only_matching': True,
223 }
224
225
226 class ORFOE1IE(ORFRadioIE):
227 IE_NAME = 'orf:oe1'
228 IE_DESC = 'Radio Österreich 1'
229 _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
230
231 _TEST = {
232 'url': 'http://oe1.orf.at/player/20170108/456544',
233 'md5': '34d8a6e67ea888293741c86a099b745b',
234 'info_dict': {
235 'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
236 'ext': 'mp3',
237 'title': 'Morgenjournal',
238 'duration': 609,
239 'timestamp': 1483858796,
240 'upload_date': '20170108',
241 },
242 'skip': 'Shows from ORF radios are only available for 7 days.'
243 }
244
245
246 class ORFIPTVIE(InfoExtractor):
247 IE_NAME = 'orf:iptv'
248 IE_DESC = 'iptv.ORF.at'
249 _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
250
251 _TEST = {
252 'url': 'http://iptv.orf.at/stories/2275236/',
253 'md5': 'c8b22af4718a4b4af58342529453e3e5',
254 'info_dict': {
255 'id': '350612',
256 'ext': 'flv',
257 'title': 'Weitere Evakuierungen um Vulkan Calbuco',
258 'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
259 'duration': 68.197,
260 'thumbnail': r're:^https?://.*\.jpg$',
261 'upload_date': '20150425',
262 },
263 }
264
265 def _real_extract(self, url):
266 story_id = self._match_id(url)
267
268 webpage = self._download_webpage(
269 'http://iptv.orf.at/stories/%s' % story_id, story_id)
270
271 video_id = self._search_regex(
272 r'data-video(?:id)?="(\d+)"', webpage, 'video id')
273
274 data = self._download_json(
275 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
276 video_id)[0]
277
278 duration = float_or_none(data['duration'], 1000)
279
280 video = data['sources']['default']
281 load_balancer_url = video['loadBalancerUrl']
282 abr = int_or_none(video.get('audioBitrate'))
283 vbr = int_or_none(video.get('bitrate'))
284 fps = int_or_none(video.get('videoFps'))
285 width = int_or_none(video.get('videoWidth'))
286 height = int_or_none(video.get('videoHeight'))
287 thumbnail = video.get('preview')
288
289 rendition = self._download_json(
290 load_balancer_url, video_id, transform_source=strip_jsonp)
291
292 f = {
293 'abr': abr,
294 'vbr': vbr,
295 'fps': fps,
296 'width': width,
297 'height': height,
298 }
299
300 formats = []
301 for format_id, format_url in rendition['redirect'].items():
302 if format_id == 'rtmp':
303 ff = f.copy()
304 ff.update({
305 'url': format_url,
306 'format_id': format_id,
307 })
308 formats.append(ff)
309 elif determine_ext(format_url) == 'f4m':
310 formats.extend(self._extract_f4m_formats(
311 format_url, video_id, f4m_id=format_id))
312 elif determine_ext(format_url) == 'm3u8':
313 formats.extend(self._extract_m3u8_formats(
314 format_url, video_id, 'mp4', m3u8_id=format_id))
315 else:
316 continue
317 self._sort_formats(formats)
318
319 title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
320 description = self._og_search_description(webpage)
321 upload_date = unified_strdate(self._html_search_meta(
322 'dc.date', webpage, 'upload date'))
323
324 return {
325 'id': video_id,
326 'title': title,
327 'description': description,
328 'duration': duration,
329 'thumbnail': thumbnail,
330 'upload_date': upload_date,
331 'formats': formats,
332 }
333
334
335 class ORFFM4StoryIE(InfoExtractor):
336 IE_NAME = 'orf:fm4:story'
337 IE_DESC = 'fm4.orf.at stories'
338 _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
339
340 _TEST = {
341 'url': 'http://fm4.orf.at/stories/2865738/',
342 'playlist': [{
343 'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
344 'info_dict': {
345 'id': '547792',
346 'ext': 'flv',
347 'title': 'Manu Delago und Inner Tongue live',
348 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
349 'duration': 1748.52,
350 'thumbnail': r're:^https?://.*\.jpg$',
351 'upload_date': '20170913',
352 },
353 }, {
354 'md5': 'c6dd2179731f86f4f55a7b49899d515f',
355 'info_dict': {
356 'id': '547798',
357 'ext': 'flv',
358 'title': 'Manu Delago und Inner Tongue live (2)',
359 'duration': 1504.08,
360 'thumbnail': r're:^https?://.*\.jpg$',
361 'upload_date': '20170913',
362 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
363 },
364 }],
365 }
366
367 def _real_extract(self, url):
368 story_id = self._match_id(url)
369 webpage = self._download_webpage(url, story_id)
370
371 entries = []
372 all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
373 for idx, video_id in enumerate(all_ids):
374 data = self._download_json(
375 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
376 video_id)[0]
377
378 duration = float_or_none(data['duration'], 1000)
379
380 video = data['sources']['q8c']
381 load_balancer_url = video['loadBalancerUrl']
382 abr = int_or_none(video.get('audioBitrate'))
383 vbr = int_or_none(video.get('bitrate'))
384 fps = int_or_none(video.get('videoFps'))
385 width = int_or_none(video.get('videoWidth'))
386 height = int_or_none(video.get('videoHeight'))
387 thumbnail = video.get('preview')
388
389 rendition = self._download_json(
390 load_balancer_url, video_id, transform_source=strip_jsonp)
391
392 f = {
393 'abr': abr,
394 'vbr': vbr,
395 'fps': fps,
396 'width': width,
397 'height': height,
398 }
399
400 formats = []
401 for format_id, format_url in rendition['redirect'].items():
402 if format_id == 'rtmp':
403 ff = f.copy()
404 ff.update({
405 'url': format_url,
406 'format_id': format_id,
407 })
408 formats.append(ff)
409 elif determine_ext(format_url) == 'f4m':
410 formats.extend(self._extract_f4m_formats(
411 format_url, video_id, f4m_id=format_id))
412 elif determine_ext(format_url) == 'm3u8':
413 formats.extend(self._extract_m3u8_formats(
414 format_url, video_id, 'mp4', m3u8_id=format_id))
415 else:
416 continue
417 self._sort_formats(formats)
418
419 title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
420 if idx >= 1:
421 # Titles are duplicates, make them unique
422 title += ' (' + str(idx + 1) + ')'
423 description = self._og_search_description(webpage)
424 upload_date = unified_strdate(self._html_search_meta(
425 'dc.date', webpage, 'upload date'))
426
427 entries.append({
428 'id': video_id,
429 'title': title,
430 'description': description,
431 'duration': duration,
432 'thumbnail': thumbnail,
433 'upload_date': upload_date,
434 'formats': formats,
435 })
436
437 return self.playlist_result(entries)