]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/orf.py
Update changelog.
[youtubedl] / youtube_dl / extractor / orf.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9 determine_ext,
10 float_or_none,
11 HEADRequest,
12 int_or_none,
13 orderedSet,
14 remove_end,
15 strip_jsonp,
16 unescapeHTML,
17 unified_strdate,
18 url_or_none,
19 )
20
21
22 class ORFTVthekIE(InfoExtractor):
23 IE_NAME = 'orf:tvthek'
24 IE_DESC = 'ORF TVthek'
25 _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
26
27 _TESTS = [{
28 'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
29 'playlist': [{
30 'md5': '2942210346ed779588f428a92db88712',
31 'info_dict': {
32 'id': '8896777',
33 'ext': 'mp4',
34 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
35 'description': 'md5:c1272f0245537812d4e36419c207b67d',
36 'duration': 2668,
37 'upload_date': '20141208',
38 },
39 }],
40 'skip': 'Blocked outside of Austria / Germany',
41 }, {
42 'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
43 'info_dict': {
44 'id': '7982259',
45 'ext': 'mp4',
46 'title': 'Best of Ingrid Thurnher',
47 'upload_date': '20140527',
48 'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
49 },
50 'params': {
51 'skip_download': True, # rtsp downloads
52 },
53 'skip': 'Blocked outside of Austria / Germany',
54 }, {
55 'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
56 'only_matching': True,
57 }, {
58 'url': 'http://tvthek.orf.at/profile/Universum/35429',
59 'only_matching': True,
60 }]
61
62 def _real_extract(self, url):
63 playlist_id = self._match_id(url)
64 webpage = self._download_webpage(url, playlist_id)
65
66 data_jsb = self._parse_json(
67 self._search_regex(
68 r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
69 webpage, 'playlist', group='json'),
70 playlist_id, transform_source=unescapeHTML)['playlist']['videos']
71
72 entries = []
73 for sd in data_jsb:
74 video_id, title = sd.get('id'), sd.get('title')
75 if not video_id or not title:
76 continue
77 video_id = compat_str(video_id)
78 formats = []
79 for fd in sd['sources']:
80 src = url_or_none(fd.get('src'))
81 if not src:
82 continue
83 format_id_list = []
84 for key in ('delivery', 'quality', 'quality_string'):
85 value = fd.get(key)
86 if value:
87 format_id_list.append(value)
88 format_id = '-'.join(format_id_list)
89 if determine_ext(fd['src']) == 'm3u8':
90 formats.extend(self._extract_m3u8_formats(
91 fd['src'], video_id, 'mp4', m3u8_id=format_id))
92 elif determine_ext(fd['src']) == 'f4m':
93 formats.extend(self._extract_f4m_formats(
94 fd['src'], video_id, f4m_id=format_id))
95 else:
96 formats.append({
97 'format_id': format_id,
98 'url': src,
99 'protocol': fd.get('protocol'),
100 })
101
102 # Check for geoblocking.
103 # There is a property is_geoprotection, but that's always false
104 geo_str = sd.get('geoprotection_string')
105 if geo_str:
106 try:
107 http_url = next(
108 f['url']
109 for f in formats
110 if re.match(r'^https?://.*\.mp4$', f['url']))
111 except StopIteration:
112 pass
113 else:
114 req = HEADRequest(http_url)
115 self._request_webpage(
116 req, video_id,
117 note='Testing for geoblocking',
118 errnote=((
119 'This video seems to be blocked outside of %s. '
120 'You may want to try the streaming-* formats.')
121 % geo_str),
122 fatal=False)
123
124 self._check_formats(formats, video_id)
125 self._sort_formats(formats)
126
127 subtitles = {}
128 for sub in sd.get('subtitles', []):
129 sub_src = sub.get('src')
130 if not sub_src:
131 continue
132 subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
133 'url': sub_src,
134 })
135
136 upload_date = unified_strdate(sd.get('created_date'))
137 entries.append({
138 '_type': 'video',
139 'id': video_id,
140 'title': title,
141 'formats': formats,
142 'subtitles': subtitles,
143 'description': sd.get('description'),
144 'duration': int_or_none(sd.get('duration_in_seconds')),
145 'upload_date': upload_date,
146 'thumbnail': sd.get('image_full_url'),
147 })
148
149 return {
150 '_type': 'playlist',
151 'entries': entries,
152 'id': playlist_id,
153 }
154
155
156 class ORFRadioIE(InfoExtractor):
157 def _real_extract(self, url):
158 mobj = re.match(self._VALID_URL, url)
159 station = mobj.group('station')
160 show_date = mobj.group('date')
161 show_id = mobj.group('show')
162
163 if station == 'fm4':
164 show_id = '4%s' % show_id
165
166 data = self._download_json(
167 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date),
168 show_id
169 )
170
171 def extract_entry_dict(info, title, subtitle):
172 return {
173 'id': info['loopStreamId'].replace('.mp3', ''),
174 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']),
175 'title': title,
176 'description': subtitle,
177 'duration': (info['end'] - info['start']) / 1000,
178 'timestamp': info['start'] / 1000,
179 'ext': 'mp3'
180 }
181
182 entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]
183
184 return {
185 '_type': 'playlist',
186 'id': show_id,
187 'title': data['title'],
188 'description': data['subtitle'],
189 'entries': entries
190 }
191
192
193 class ORFFM4IE(ORFRadioIE):
194 IE_NAME = 'orf:fm4'
195 IE_DESC = 'radio FM4'
196 _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
197
198 _TEST = {
199 'url': 'http://fm4.orf.at/player/20170107/CC',
200 'md5': '2b0be47375432a7ef104453432a19212',
201 'info_dict': {
202 'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
203 'ext': 'mp3',
204 'title': 'Solid Steel Radioshow',
205 'description': 'Die Mixshow von Coldcut und Ninja Tune.',
206 'duration': 3599,
207 'timestamp': 1483819257,
208 'upload_date': '20170107',
209 },
210 'skip': 'Shows from ORF radios are only available for 7 days.'
211 }
212
213
214 class ORFOE1IE(ORFRadioIE):
215 IE_NAME = 'orf:oe1'
216 IE_DESC = 'Radio Österreich 1'
217 _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
218
219 _TEST = {
220 'url': 'http://oe1.orf.at/player/20170108/456544',
221 'md5': '34d8a6e67ea888293741c86a099b745b',
222 'info_dict': {
223 'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
224 'ext': 'mp3',
225 'title': 'Morgenjournal',
226 'duration': 609,
227 'timestamp': 1483858796,
228 'upload_date': '20170108',
229 },
230 'skip': 'Shows from ORF radios are only available for 7 days.'
231 }
232
233
234 class ORFIPTVIE(InfoExtractor):
235 IE_NAME = 'orf:iptv'
236 IE_DESC = 'iptv.ORF.at'
237 _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
238
239 _TEST = {
240 'url': 'http://iptv.orf.at/stories/2275236/',
241 'md5': 'c8b22af4718a4b4af58342529453e3e5',
242 'info_dict': {
243 'id': '350612',
244 'ext': 'flv',
245 'title': 'Weitere Evakuierungen um Vulkan Calbuco',
246 'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
247 'duration': 68.197,
248 'thumbnail': r're:^https?://.*\.jpg$',
249 'upload_date': '20150425',
250 },
251 }
252
253 def _real_extract(self, url):
254 story_id = self._match_id(url)
255
256 webpage = self._download_webpage(
257 'http://iptv.orf.at/stories/%s' % story_id, story_id)
258
259 video_id = self._search_regex(
260 r'data-video(?:id)?="(\d+)"', webpage, 'video id')
261
262 data = self._download_json(
263 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
264 video_id)[0]
265
266 duration = float_or_none(data['duration'], 1000)
267
268 video = data['sources']['default']
269 load_balancer_url = video['loadBalancerUrl']
270 abr = int_or_none(video.get('audioBitrate'))
271 vbr = int_or_none(video.get('bitrate'))
272 fps = int_or_none(video.get('videoFps'))
273 width = int_or_none(video.get('videoWidth'))
274 height = int_or_none(video.get('videoHeight'))
275 thumbnail = video.get('preview')
276
277 rendition = self._download_json(
278 load_balancer_url, video_id, transform_source=strip_jsonp)
279
280 f = {
281 'abr': abr,
282 'vbr': vbr,
283 'fps': fps,
284 'width': width,
285 'height': height,
286 }
287
288 formats = []
289 for format_id, format_url in rendition['redirect'].items():
290 if format_id == 'rtmp':
291 ff = f.copy()
292 ff.update({
293 'url': format_url,
294 'format_id': format_id,
295 })
296 formats.append(ff)
297 elif determine_ext(format_url) == 'f4m':
298 formats.extend(self._extract_f4m_formats(
299 format_url, video_id, f4m_id=format_id))
300 elif determine_ext(format_url) == 'm3u8':
301 formats.extend(self._extract_m3u8_formats(
302 format_url, video_id, 'mp4', m3u8_id=format_id))
303 else:
304 continue
305 self._sort_formats(formats)
306
307 title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
308 description = self._og_search_description(webpage)
309 upload_date = unified_strdate(self._html_search_meta(
310 'dc.date', webpage, 'upload date'))
311
312 return {
313 'id': video_id,
314 'title': title,
315 'description': description,
316 'duration': duration,
317 'thumbnail': thumbnail,
318 'upload_date': upload_date,
319 'formats': formats,
320 }
321
322
323 class ORFFM4StoryIE(InfoExtractor):
324 IE_NAME = 'orf:fm4:story'
325 IE_DESC = 'fm4.orf.at stories'
326 _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
327
328 _TEST = {
329 'url': 'http://fm4.orf.at/stories/2865738/',
330 'playlist': [{
331 'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
332 'info_dict': {
333 'id': '547792',
334 'ext': 'flv',
335 'title': 'Manu Delago und Inner Tongue live',
336 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
337 'duration': 1748.52,
338 'thumbnail': r're:^https?://.*\.jpg$',
339 'upload_date': '20170913',
340 },
341 }, {
342 'md5': 'c6dd2179731f86f4f55a7b49899d515f',
343 'info_dict': {
344 'id': '547798',
345 'ext': 'flv',
346 'title': 'Manu Delago und Inner Tongue live (2)',
347 'duration': 1504.08,
348 'thumbnail': r're:^https?://.*\.jpg$',
349 'upload_date': '20170913',
350 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
351 },
352 }],
353 }
354
355 def _real_extract(self, url):
356 story_id = self._match_id(url)
357 webpage = self._download_webpage(url, story_id)
358
359 entries = []
360 all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
361 for idx, video_id in enumerate(all_ids):
362 data = self._download_json(
363 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
364 video_id)[0]
365
366 duration = float_or_none(data['duration'], 1000)
367
368 video = data['sources']['q8c']
369 load_balancer_url = video['loadBalancerUrl']
370 abr = int_or_none(video.get('audioBitrate'))
371 vbr = int_or_none(video.get('bitrate'))
372 fps = int_or_none(video.get('videoFps'))
373 width = int_or_none(video.get('videoWidth'))
374 height = int_or_none(video.get('videoHeight'))
375 thumbnail = video.get('preview')
376
377 rendition = self._download_json(
378 load_balancer_url, video_id, transform_source=strip_jsonp)
379
380 f = {
381 'abr': abr,
382 'vbr': vbr,
383 'fps': fps,
384 'width': width,
385 'height': height,
386 }
387
388 formats = []
389 for format_id, format_url in rendition['redirect'].items():
390 if format_id == 'rtmp':
391 ff = f.copy()
392 ff.update({
393 'url': format_url,
394 'format_id': format_id,
395 })
396 formats.append(ff)
397 elif determine_ext(format_url) == 'f4m':
398 formats.extend(self._extract_f4m_formats(
399 format_url, video_id, f4m_id=format_id))
400 elif determine_ext(format_url) == 'm3u8':
401 formats.extend(self._extract_m3u8_formats(
402 format_url, video_id, 'mp4', m3u8_id=format_id))
403 else:
404 continue
405 self._sort_formats(formats)
406
407 title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
408 if idx >= 1:
409 # Titles are duplicates, make them unique
410 title += ' (' + str(idx + 1) + ')'
411 description = self._og_search_description(webpage)
412 upload_date = unified_strdate(self._html_search_meta(
413 'dc.date', webpage, 'upload date'))
414
415 entries.append({
416 'id': video_id,
417 'title': title,
418 'description': description,
419 'duration': duration,
420 'thumbnail': thumbnail,
421 'upload_date': upload_date,
422 'formats': formats,
423 })
424
425 return self.playlist_result(entries)