]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/orf.py
debian/patches: Refresh patches to apply cleanly.
[youtubedl] / youtube_dl / extractor / orf.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9 determine_ext,
10 float_or_none,
11 HEADRequest,
12 int_or_none,
13 orderedSet,
14 remove_end,
15 strip_jsonp,
16 unescapeHTML,
17 unified_strdate,
18 url_or_none,
19 )
20
21
22 class ORFTVthekIE(InfoExtractor):
23 IE_NAME = 'orf:tvthek'
24 IE_DESC = 'ORF TVthek'
25 _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
26
27 _TESTS = [{
28 'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
29 'playlist': [{
30 'md5': '2942210346ed779588f428a92db88712',
31 'info_dict': {
32 'id': '8896777',
33 'ext': 'mp4',
34 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
35 'description': 'md5:c1272f0245537812d4e36419c207b67d',
36 'duration': 2668,
37 'upload_date': '20141208',
38 },
39 }],
40 'skip': 'Blocked outside of Austria / Germany',
41 }, {
42 'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
43 'info_dict': {
44 'id': '7982259',
45 'ext': 'mp4',
46 'title': 'Best of Ingrid Thurnher',
47 'upload_date': '20140527',
48 'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
49 },
50 'params': {
51 'skip_download': True, # rtsp downloads
52 },
53 'skip': 'Blocked outside of Austria / Germany',
54 }, {
55 'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
56 'only_matching': True,
57 }, {
58 'url': 'http://tvthek.orf.at/profile/Universum/35429',
59 'only_matching': True,
60 }]
61
62 def _real_extract(self, url):
63 playlist_id = self._match_id(url)
64 webpage = self._download_webpage(url, playlist_id)
65
66 data_jsb = self._parse_json(
67 self._search_regex(
68 r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
69 webpage, 'playlist', group='json'),
70 playlist_id, transform_source=unescapeHTML)['playlist']['videos']
71
72 entries = []
73 for sd in data_jsb:
74 video_id, title = sd.get('id'), sd.get('title')
75 if not video_id or not title:
76 continue
77 video_id = compat_str(video_id)
78 formats = []
79 for fd in sd['sources']:
80 src = url_or_none(fd.get('src'))
81 if not src:
82 continue
83 format_id_list = []
84 for key in ('delivery', 'quality', 'quality_string'):
85 value = fd.get(key)
86 if value:
87 format_id_list.append(value)
88 format_id = '-'.join(format_id_list)
89 if determine_ext(fd['src']) == 'm3u8':
90 formats.extend(self._extract_m3u8_formats(
91 fd['src'], video_id, 'mp4', m3u8_id=format_id))
92 elif determine_ext(fd['src']) == 'f4m':
93 formats.extend(self._extract_f4m_formats(
94 fd['src'], video_id, f4m_id=format_id))
95 else:
96 formats.append({
97 'format_id': format_id,
98 'url': src,
99 'protocol': fd.get('protocol'),
100 })
101
102 # Check for geoblocking.
103 # There is a property is_geoprotection, but that's always false
104 geo_str = sd.get('geoprotection_string')
105 if geo_str:
106 try:
107 http_url = next(
108 f['url']
109 for f in formats
110 if re.match(r'^https?://.*\.mp4$', f['url']))
111 except StopIteration:
112 pass
113 else:
114 req = HEADRequest(http_url)
115 self._request_webpage(
116 req, video_id,
117 note='Testing for geoblocking',
118 errnote=((
119 'This video seems to be blocked outside of %s. '
120 'You may want to try the streaming-* formats.')
121 % geo_str),
122 fatal=False)
123
124 self._check_formats(formats, video_id)
125 self._sort_formats(formats)
126
127 subtitles = {}
128 for sub in sd.get('subtitles', []):
129 sub_src = sub.get('src')
130 if not sub_src:
131 continue
132 subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
133 'url': sub_src,
134 })
135
136 upload_date = unified_strdate(sd.get('created_date'))
137 entries.append({
138 '_type': 'video',
139 'id': video_id,
140 'title': title,
141 'formats': formats,
142 'subtitles': subtitles,
143 'description': sd.get('description'),
144 'duration': int_or_none(sd.get('duration_in_seconds')),
145 'upload_date': upload_date,
146 'thumbnail': sd.get('image_full_url'),
147 })
148
149 return {
150 '_type': 'playlist',
151 'entries': entries,
152 'id': playlist_id,
153 }
154
155
156 class ORFRadioIE(InfoExtractor):
157 def _real_extract(self, url):
158 mobj = re.match(self._VALID_URL, url)
159 station = mobj.group('station')
160 show_date = mobj.group('date')
161 show_id = mobj.group('show')
162
163 if station == 'fm4':
164 show_id = '4%s' % show_id
165
166 data = self._download_json(
167 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date),
168 show_id
169 )
170
171 def extract_entry_dict(info, title, subtitle):
172 return {
173 'id': info['loopStreamId'].replace('.mp3', ''),
174 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']),
175 'title': title,
176 'description': subtitle,
177 'duration': (info['end'] - info['start']) / 1000,
178 'timestamp': info['start'] / 1000,
179 'ext': 'mp3',
180 'series': data.get('programTitle')
181 }
182
183 entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]
184
185 return {
186 '_type': 'playlist',
187 'id': show_id,
188 'title': data['title'],
189 'description': data['subtitle'],
190 'entries': entries
191 }
192
193
194 class ORFFM4IE(ORFRadioIE):
195 IE_NAME = 'orf:fm4'
196 IE_DESC = 'radio FM4'
197 _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
198
199 _TEST = {
200 'url': 'http://fm4.orf.at/player/20170107/CC',
201 'md5': '2b0be47375432a7ef104453432a19212',
202 'info_dict': {
203 'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
204 'ext': 'mp3',
205 'title': 'Solid Steel Radioshow',
206 'description': 'Die Mixshow von Coldcut und Ninja Tune.',
207 'duration': 3599,
208 'timestamp': 1483819257,
209 'upload_date': '20170107',
210 },
211 'skip': 'Shows from ORF radios are only available for 7 days.'
212 }
213
214
215 class ORFOE1IE(ORFRadioIE):
216 IE_NAME = 'orf:oe1'
217 IE_DESC = 'Radio Österreich 1'
218 _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
219
220 _TEST = {
221 'url': 'http://oe1.orf.at/player/20170108/456544',
222 'md5': '34d8a6e67ea888293741c86a099b745b',
223 'info_dict': {
224 'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
225 'ext': 'mp3',
226 'title': 'Morgenjournal',
227 'duration': 609,
228 'timestamp': 1483858796,
229 'upload_date': '20170108',
230 },
231 'skip': 'Shows from ORF radios are only available for 7 days.'
232 }
233
234
235 class ORFIPTVIE(InfoExtractor):
236 IE_NAME = 'orf:iptv'
237 IE_DESC = 'iptv.ORF.at'
238 _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
239
240 _TEST = {
241 'url': 'http://iptv.orf.at/stories/2275236/',
242 'md5': 'c8b22af4718a4b4af58342529453e3e5',
243 'info_dict': {
244 'id': '350612',
245 'ext': 'flv',
246 'title': 'Weitere Evakuierungen um Vulkan Calbuco',
247 'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
248 'duration': 68.197,
249 'thumbnail': r're:^https?://.*\.jpg$',
250 'upload_date': '20150425',
251 },
252 }
253
254 def _real_extract(self, url):
255 story_id = self._match_id(url)
256
257 webpage = self._download_webpage(
258 'http://iptv.orf.at/stories/%s' % story_id, story_id)
259
260 video_id = self._search_regex(
261 r'data-video(?:id)?="(\d+)"', webpage, 'video id')
262
263 data = self._download_json(
264 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
265 video_id)[0]
266
267 duration = float_or_none(data['duration'], 1000)
268
269 video = data['sources']['default']
270 load_balancer_url = video['loadBalancerUrl']
271 abr = int_or_none(video.get('audioBitrate'))
272 vbr = int_or_none(video.get('bitrate'))
273 fps = int_or_none(video.get('videoFps'))
274 width = int_or_none(video.get('videoWidth'))
275 height = int_or_none(video.get('videoHeight'))
276 thumbnail = video.get('preview')
277
278 rendition = self._download_json(
279 load_balancer_url, video_id, transform_source=strip_jsonp)
280
281 f = {
282 'abr': abr,
283 'vbr': vbr,
284 'fps': fps,
285 'width': width,
286 'height': height,
287 }
288
289 formats = []
290 for format_id, format_url in rendition['redirect'].items():
291 if format_id == 'rtmp':
292 ff = f.copy()
293 ff.update({
294 'url': format_url,
295 'format_id': format_id,
296 })
297 formats.append(ff)
298 elif determine_ext(format_url) == 'f4m':
299 formats.extend(self._extract_f4m_formats(
300 format_url, video_id, f4m_id=format_id))
301 elif determine_ext(format_url) == 'm3u8':
302 formats.extend(self._extract_m3u8_formats(
303 format_url, video_id, 'mp4', m3u8_id=format_id))
304 else:
305 continue
306 self._sort_formats(formats)
307
308 title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
309 description = self._og_search_description(webpage)
310 upload_date = unified_strdate(self._html_search_meta(
311 'dc.date', webpage, 'upload date'))
312
313 return {
314 'id': video_id,
315 'title': title,
316 'description': description,
317 'duration': duration,
318 'thumbnail': thumbnail,
319 'upload_date': upload_date,
320 'formats': formats,
321 }
322
323
324 class ORFFM4StoryIE(InfoExtractor):
325 IE_NAME = 'orf:fm4:story'
326 IE_DESC = 'fm4.orf.at stories'
327 _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
328
329 _TEST = {
330 'url': 'http://fm4.orf.at/stories/2865738/',
331 'playlist': [{
332 'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
333 'info_dict': {
334 'id': '547792',
335 'ext': 'flv',
336 'title': 'Manu Delago und Inner Tongue live',
337 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
338 'duration': 1748.52,
339 'thumbnail': r're:^https?://.*\.jpg$',
340 'upload_date': '20170913',
341 },
342 }, {
343 'md5': 'c6dd2179731f86f4f55a7b49899d515f',
344 'info_dict': {
345 'id': '547798',
346 'ext': 'flv',
347 'title': 'Manu Delago und Inner Tongue live (2)',
348 'duration': 1504.08,
349 'thumbnail': r're:^https?://.*\.jpg$',
350 'upload_date': '20170913',
351 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
352 },
353 }],
354 }
355
356 def _real_extract(self, url):
357 story_id = self._match_id(url)
358 webpage = self._download_webpage(url, story_id)
359
360 entries = []
361 all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
362 for idx, video_id in enumerate(all_ids):
363 data = self._download_json(
364 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
365 video_id)[0]
366
367 duration = float_or_none(data['duration'], 1000)
368
369 video = data['sources']['q8c']
370 load_balancer_url = video['loadBalancerUrl']
371 abr = int_or_none(video.get('audioBitrate'))
372 vbr = int_or_none(video.get('bitrate'))
373 fps = int_or_none(video.get('videoFps'))
374 width = int_or_none(video.get('videoWidth'))
375 height = int_or_none(video.get('videoHeight'))
376 thumbnail = video.get('preview')
377
378 rendition = self._download_json(
379 load_balancer_url, video_id, transform_source=strip_jsonp)
380
381 f = {
382 'abr': abr,
383 'vbr': vbr,
384 'fps': fps,
385 'width': width,
386 'height': height,
387 }
388
389 formats = []
390 for format_id, format_url in rendition['redirect'].items():
391 if format_id == 'rtmp':
392 ff = f.copy()
393 ff.update({
394 'url': format_url,
395 'format_id': format_id,
396 })
397 formats.append(ff)
398 elif determine_ext(format_url) == 'f4m':
399 formats.extend(self._extract_f4m_formats(
400 format_url, video_id, f4m_id=format_id))
401 elif determine_ext(format_url) == 'm3u8':
402 formats.extend(self._extract_m3u8_formats(
403 format_url, video_id, 'mp4', m3u8_id=format_id))
404 else:
405 continue
406 self._sort_formats(formats)
407
408 title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
409 if idx >= 1:
410 # Titles are duplicates, make them unique
411 title += ' (' + str(idx + 1) + ')'
412 description = self._og_search_description(webpage)
413 upload_date = unified_strdate(self._html_search_meta(
414 'dc.date', webpage, 'upload date'))
415
416 entries.append({
417 'id': video_id,
418 'title': title,
419 'description': description,
420 'duration': duration,
421 'thumbnail': thumbnail,
422 'upload_date': upload_date,
423 'formats': formats,
424 })
425
426 return self.playlist_result(entries)