]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/orf.py
d/p/remove-autoupdate-mechanism.patch: Update patch with metadata at the top.
[youtubedl] / youtube_dl / extractor / orf.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9 determine_ext,
10 float_or_none,
11 HEADRequest,
12 int_or_none,
13 orderedSet,
14 remove_end,
15 strip_jsonp,
16 unescapeHTML,
17 unified_strdate,
18 )
19
20
21 class ORFTVthekIE(InfoExtractor):
22 IE_NAME = 'orf:tvthek'
23 IE_DESC = 'ORF TVthek'
24 _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
25
26 _TESTS = [{
27 'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
28 'playlist': [{
29 'md5': '2942210346ed779588f428a92db88712',
30 'info_dict': {
31 'id': '8896777',
32 'ext': 'mp4',
33 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
34 'description': 'md5:c1272f0245537812d4e36419c207b67d',
35 'duration': 2668,
36 'upload_date': '20141208',
37 },
38 }],
39 'skip': 'Blocked outside of Austria / Germany',
40 }, {
41 'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
42 'info_dict': {
43 'id': '7982259',
44 'ext': 'mp4',
45 'title': 'Best of Ingrid Thurnher',
46 'upload_date': '20140527',
47 'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
48 },
49 'params': {
50 'skip_download': True, # rtsp downloads
51 },
52 'skip': 'Blocked outside of Austria / Germany',
53 }, {
54 'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
55 'only_matching': True,
56 }, {
57 'url': 'http://tvthek.orf.at/profile/Universum/35429',
58 'only_matching': True,
59 }]
60
61 def _real_extract(self, url):
62 playlist_id = self._match_id(url)
63 webpage = self._download_webpage(url, playlist_id)
64
65 data_jsb = self._parse_json(
66 self._search_regex(
67 r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
68 webpage, 'playlist', group='json'),
69 playlist_id, transform_source=unescapeHTML)['playlist']['videos']
70
71 def quality_to_int(s):
72 m = re.search('([0-9]+)', s)
73 if m is None:
74 return -1
75 return int(m.group(1))
76
77 entries = []
78 for sd in data_jsb:
79 video_id, title = sd.get('id'), sd.get('title')
80 if not video_id or not title:
81 continue
82 video_id = compat_str(video_id)
83 formats = [{
84 'preference': -10 if fd['delivery'] == 'hls' else None,
85 'format_id': '%s-%s-%s' % (
86 fd['delivery'], fd['quality'], fd['quality_string']),
87 'url': fd['src'],
88 'protocol': fd['protocol'],
89 'quality': quality_to_int(fd['quality']),
90 } for fd in sd['sources']]
91
92 # Check for geoblocking.
93 # There is a property is_geoprotection, but that's always false
94 geo_str = sd.get('geoprotection_string')
95 if geo_str:
96 try:
97 http_url = next(
98 f['url']
99 for f in formats
100 if re.match(r'^https?://.*\.mp4$', f['url']))
101 except StopIteration:
102 pass
103 else:
104 req = HEADRequest(http_url)
105 self._request_webpage(
106 req, video_id,
107 note='Testing for geoblocking',
108 errnote=((
109 'This video seems to be blocked outside of %s. '
110 'You may want to try the streaming-* formats.')
111 % geo_str),
112 fatal=False)
113
114 self._check_formats(formats, video_id)
115 self._sort_formats(formats)
116
117 subtitles = {}
118 for sub in sd.get('subtitles', []):
119 sub_src = sub.get('src')
120 if not sub_src:
121 continue
122 subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
123 'url': sub_src,
124 })
125
126 upload_date = unified_strdate(sd.get('created_date'))
127 entries.append({
128 '_type': 'video',
129 'id': video_id,
130 'title': title,
131 'formats': formats,
132 'subtitles': subtitles,
133 'description': sd.get('description'),
134 'duration': int_or_none(sd.get('duration_in_seconds')),
135 'upload_date': upload_date,
136 'thumbnail': sd.get('image_full_url'),
137 })
138
139 return {
140 '_type': 'playlist',
141 'entries': entries,
142 'id': playlist_id,
143 }
144
145
146 class ORFRadioIE(InfoExtractor):
147 def _real_extract(self, url):
148 mobj = re.match(self._VALID_URL, url)
149 station = mobj.group('station')
150 show_date = mobj.group('date')
151 show_id = mobj.group('show')
152
153 if station == 'fm4':
154 show_id = '4%s' % show_id
155
156 data = self._download_json(
157 'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date),
158 show_id
159 )
160
161 def extract_entry_dict(info, title, subtitle):
162 return {
163 'id': info['loopStreamId'].replace('.mp3', ''),
164 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']),
165 'title': title,
166 'description': subtitle,
167 'duration': (info['end'] - info['start']) / 1000,
168 'timestamp': info['start'] / 1000,
169 'ext': 'mp3'
170 }
171
172 entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]
173
174 return {
175 '_type': 'playlist',
176 'id': show_id,
177 'title': data['title'],
178 'description': data['subtitle'],
179 'entries': entries
180 }
181
182
183 class ORFFM4IE(ORFRadioIE):
184 IE_NAME = 'orf:fm4'
185 IE_DESC = 'radio FM4'
186 _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
187
188 _TEST = {
189 'url': 'http://fm4.orf.at/player/20170107/CC',
190 'md5': '2b0be47375432a7ef104453432a19212',
191 'info_dict': {
192 'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
193 'ext': 'mp3',
194 'title': 'Solid Steel Radioshow',
195 'description': 'Die Mixshow von Coldcut und Ninja Tune.',
196 'duration': 3599,
197 'timestamp': 1483819257,
198 'upload_date': '20170107',
199 },
200 'skip': 'Shows from ORF radios are only available for 7 days.'
201 }
202
203
204 class ORFOE1IE(ORFRadioIE):
205 IE_NAME = 'orf:oe1'
206 IE_DESC = 'Radio Österreich 1'
207 _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
208
209 _TEST = {
210 'url': 'http://oe1.orf.at/player/20170108/456544',
211 'md5': '34d8a6e67ea888293741c86a099b745b',
212 'info_dict': {
213 'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
214 'ext': 'mp3',
215 'title': 'Morgenjournal',
216 'duration': 609,
217 'timestamp': 1483858796,
218 'upload_date': '20170108',
219 },
220 'skip': 'Shows from ORF radios are only available for 7 days.'
221 }
222
223
224 class ORFIPTVIE(InfoExtractor):
225 IE_NAME = 'orf:iptv'
226 IE_DESC = 'iptv.ORF.at'
227 _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
228
229 _TEST = {
230 'url': 'http://iptv.orf.at/stories/2275236/',
231 'md5': 'c8b22af4718a4b4af58342529453e3e5',
232 'info_dict': {
233 'id': '350612',
234 'ext': 'flv',
235 'title': 'Weitere Evakuierungen um Vulkan Calbuco',
236 'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
237 'duration': 68.197,
238 'thumbnail': r're:^https?://.*\.jpg$',
239 'upload_date': '20150425',
240 },
241 }
242
243 def _real_extract(self, url):
244 story_id = self._match_id(url)
245
246 webpage = self._download_webpage(
247 'http://iptv.orf.at/stories/%s' % story_id, story_id)
248
249 video_id = self._search_regex(
250 r'data-video(?:id)?="(\d+)"', webpage, 'video id')
251
252 data = self._download_json(
253 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
254 video_id)[0]
255
256 duration = float_or_none(data['duration'], 1000)
257
258 video = data['sources']['default']
259 load_balancer_url = video['loadBalancerUrl']
260 abr = int_or_none(video.get('audioBitrate'))
261 vbr = int_or_none(video.get('bitrate'))
262 fps = int_or_none(video.get('videoFps'))
263 width = int_or_none(video.get('videoWidth'))
264 height = int_or_none(video.get('videoHeight'))
265 thumbnail = video.get('preview')
266
267 rendition = self._download_json(
268 load_balancer_url, video_id, transform_source=strip_jsonp)
269
270 f = {
271 'abr': abr,
272 'vbr': vbr,
273 'fps': fps,
274 'width': width,
275 'height': height,
276 }
277
278 formats = []
279 for format_id, format_url in rendition['redirect'].items():
280 if format_id == 'rtmp':
281 ff = f.copy()
282 ff.update({
283 'url': format_url,
284 'format_id': format_id,
285 })
286 formats.append(ff)
287 elif determine_ext(format_url) == 'f4m':
288 formats.extend(self._extract_f4m_formats(
289 format_url, video_id, f4m_id=format_id))
290 elif determine_ext(format_url) == 'm3u8':
291 formats.extend(self._extract_m3u8_formats(
292 format_url, video_id, 'mp4', m3u8_id=format_id))
293 else:
294 continue
295 self._sort_formats(formats)
296
297 title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
298 description = self._og_search_description(webpage)
299 upload_date = unified_strdate(self._html_search_meta(
300 'dc.date', webpage, 'upload date'))
301
302 return {
303 'id': video_id,
304 'title': title,
305 'description': description,
306 'duration': duration,
307 'thumbnail': thumbnail,
308 'upload_date': upload_date,
309 'formats': formats,
310 }
311
312
313 class ORFFM4StoryIE(InfoExtractor):
314 IE_NAME = 'orf:fm4:story'
315 IE_DESC = 'fm4.orf.at stories'
316 _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
317
318 _TEST = {
319 'url': 'http://fm4.orf.at/stories/2865738/',
320 'playlist': [{
321 'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
322 'info_dict': {
323 'id': '547792',
324 'ext': 'flv',
325 'title': 'Manu Delago und Inner Tongue live',
326 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
327 'duration': 1748.52,
328 'thumbnail': r're:^https?://.*\.jpg$',
329 'upload_date': '20170913',
330 },
331 }, {
332 'md5': 'c6dd2179731f86f4f55a7b49899d515f',
333 'info_dict': {
334 'id': '547798',
335 'ext': 'flv',
336 'title': 'Manu Delago und Inner Tongue live (2)',
337 'duration': 1504.08,
338 'thumbnail': r're:^https?://.*\.jpg$',
339 'upload_date': '20170913',
340 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
341 },
342 }],
343 }
344
345 def _real_extract(self, url):
346 story_id = self._match_id(url)
347 webpage = self._download_webpage(url, story_id)
348
349 entries = []
350 all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
351 for idx, video_id in enumerate(all_ids):
352 data = self._download_json(
353 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
354 video_id)[0]
355
356 duration = float_or_none(data['duration'], 1000)
357
358 video = data['sources']['q8c']
359 load_balancer_url = video['loadBalancerUrl']
360 abr = int_or_none(video.get('audioBitrate'))
361 vbr = int_or_none(video.get('bitrate'))
362 fps = int_or_none(video.get('videoFps'))
363 width = int_or_none(video.get('videoWidth'))
364 height = int_or_none(video.get('videoHeight'))
365 thumbnail = video.get('preview')
366
367 rendition = self._download_json(
368 load_balancer_url, video_id, transform_source=strip_jsonp)
369
370 f = {
371 'abr': abr,
372 'vbr': vbr,
373 'fps': fps,
374 'width': width,
375 'height': height,
376 }
377
378 formats = []
379 for format_id, format_url in rendition['redirect'].items():
380 if format_id == 'rtmp':
381 ff = f.copy()
382 ff.update({
383 'url': format_url,
384 'format_id': format_id,
385 })
386 formats.append(ff)
387 elif determine_ext(format_url) == 'f4m':
388 formats.extend(self._extract_f4m_formats(
389 format_url, video_id, f4m_id=format_id))
390 elif determine_ext(format_url) == 'm3u8':
391 formats.extend(self._extract_m3u8_formats(
392 format_url, video_id, 'mp4', m3u8_id=format_id))
393 else:
394 continue
395 self._sort_formats(formats)
396
397 title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
398 if idx >= 1:
399 # Titles are duplicates, make them unique
400 title += ' (' + str(idx + 1) + ')'
401 description = self._og_search_description(webpage)
402 upload_date = unified_strdate(self._html_search_meta(
403 'dc.date', webpage, 'upload date'))
404
405 entries.append({
406 'id': video_id,
407 'title': title,
408 'description': description,
409 'duration': duration,
410 'thumbnail': thumbnail,
411 'upload_date': upload_date,
412 'formats': formats,
413 })
414
415 return self.playlist_result(entries)