Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/orf.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import compat_str
   8 from ..utils import (
   9     determine_ext,
  10     float_or_none,
  11     HEADRequest,
  12     int_or_none,
  13     orderedSet,
  14     remove_end,
  15     strip_jsonp,
  16     unescapeHTML,
  17     unified_strdate,
  18 )
  19
  20
  21 class ORFTVthekIE(InfoExtractor):
  22     IE_NAME = 'orf:tvthek'
  23     IE_DESC = 'ORF TVthek'
  24     _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
  25
  26     _TESTS = [{
  27         'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
  28         'playlist': [{
  29             'md5': '2942210346ed779588f428a92db88712',
  30             'info_dict': {
  31                 'id': '8896777',
  32                 'ext': 'mp4',
  33                 'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
  34                 'description': 'md5:c1272f0245537812d4e36419c207b67d',
  35                 'duration': 2668,
  36                 'upload_date': '20141208',
  37             },
  38         }],
  39         'skip': 'Blocked outside of Austria / Germany',
  40     }, {
  41         'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
  42         'info_dict': {
  43             'id': '7982259',
  44             'ext': 'mp4',
  45             'title': 'Best of Ingrid Thurnher',
  46             'upload_date': '20140527',
  47             'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
  48         },
  49         'params': {
  50             'skip_download': True,  # rtsp downloads
  51         },
  52         'skip': 'Blocked outside of Austria / Germany',
  53     }, {
  54         'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
  55         'only_matching': True,
  56     }, {
  57         'url': 'http://tvthek.orf.at/profile/Universum/35429',
  58         'only_matching': True,
  59     }]
  60
  61     def _real_extract(self, url):
  62         playlist_id = self._match_id(url)
  63         webpage = self._download_webpage(url, playlist_id)
  64
  65         data_jsb = self._parse_json(
  66             self._search_regex(
  67                 r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
  68                 webpage, 'playlist', group='json'),
  69             playlist_id, transform_source=unescapeHTML)['playlist']['videos']
  70
  71         def quality_to_int(s):
  72             m = re.search('([0-9]+)', s)
  73             if m is None:
  74                 return -1
  75             return int(m.group(1))
  76
  77         entries = []
  78         for sd in data_jsb:
  79             video_id, title = sd.get('id'), sd.get('title')
  80             if not video_id or not title:
  81                 continue
  82             video_id = compat_str(video_id)
  83             formats = [{
  84                 'preference': -10 if fd['delivery'] == 'hls' else None,
  85                 'format_id': '%s-%s-%s' % (
  86                     fd['delivery'], fd['quality'], fd['quality_string']),
  87                 'url': fd['src'],
  88                 'protocol': fd['protocol'],
  89                 'quality': quality_to_int(fd['quality']),
  90             } for fd in sd['sources']]
  91
  92             # Check for geoblocking.
  93             # There is a property is_geoprotection, but that's always false
  94             geo_str = sd.get('geoprotection_string')
  95             if geo_str:
  96                 try:
  97                     http_url = next(
  98                         f['url']
  99                         for f in formats
 100                         if re.match(r'^https?://.*\.mp4$', f['url']))
 101                 except StopIteration:
 102                     pass
 103                 else:
 104                     req = HEADRequest(http_url)
 105                     self._request_webpage(
 106                         req, video_id,
 107                         note='Testing for geoblocking',
 108                         errnote=((
 109                             'This video seems to be blocked outside of %s. '
 110                             'You may want to try the streaming-* formats.')
 111                             % geo_str),
 112                         fatal=False)
 113
 114             self._check_formats(formats, video_id)
 115             self._sort_formats(formats)
 116
 117             subtitles = {}
 118             for sub in sd.get('subtitles', []):
 119                 sub_src = sub.get('src')
 120                 if not sub_src:
 121                     continue
 122                 subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
 123                     'url': sub_src,
 124                 })
 125
 126             upload_date = unified_strdate(sd.get('created_date'))
 127             entries.append({
 128                 '_type': 'video',
 129                 'id': video_id,
 130                 'title': title,
 131                 'formats': formats,
 132                 'subtitles': subtitles,
 133                 'description': sd.get('description'),
 134                 'duration': int_or_none(sd.get('duration_in_seconds')),
 135                 'upload_date': upload_date,
 136                 'thumbnail': sd.get('image_full_url'),
 137             })
 138
 139         return {
 140             '_type': 'playlist',
 141             'entries': entries,
 142             'id': playlist_id,
 143         }
 144
 145
 146 class ORFRadioIE(InfoExtractor):
 147     def _real_extract(self, url):
 148         mobj = re.match(self._VALID_URL, url)
 149         station = mobj.group('station')
 150         show_date = mobj.group('date')
 151         show_id = mobj.group('show')
 152
 153         if station == 'fm4':
 154             show_id = '4%s' % show_id
 155
 156         data = self._download_json(
 157             'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s' % (station, show_id, show_date),
 158             show_id
 159         )
 160
 161         def extract_entry_dict(info, title, subtitle):
 162             return {
 163                 'id': info['loopStreamId'].replace('.mp3', ''),
 164                 'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (station, info['loopStreamId']),
 165                 'title': title,
 166                 'description': subtitle,
 167                 'duration': (info['end'] - info['start']) / 1000,
 168                 'timestamp': info['start'] / 1000,
 169                 'ext': 'mp3'
 170             }
 171
 172         entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']]
 173
 174         return {
 175             '_type': 'playlist',
 176             'id': show_id,
 177             'title': data['title'],
 178             'description': data['subtitle'],
 179             'entries': entries
 180         }
 181
 182
 183 class ORFFM4IE(ORFRadioIE):
 184     IE_NAME = 'orf:fm4'
 185     IE_DESC = 'radio FM4'
 186     _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
 187
 188     _TEST = {
 189         'url': 'http://fm4.orf.at/player/20170107/CC',
 190         'md5': '2b0be47375432a7ef104453432a19212',
 191         'info_dict': {
 192             'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
 193             'ext': 'mp3',
 194             'title': 'Solid Steel Radioshow',
 195             'description': 'Die Mixshow von Coldcut und Ninja Tune.',
 196             'duration': 3599,
 197             'timestamp': 1483819257,
 198             'upload_date': '20170107',
 199         },
 200         'skip': 'Shows from ORF radios are only available for 7 days.'
 201     }
 202
 203
 204 class ORFOE1IE(ORFRadioIE):
 205     IE_NAME = 'orf:oe1'
 206     IE_DESC = 'Radio Österreich 1'
 207     _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
 208
 209     _TEST = {
 210         'url': 'http://oe1.orf.at/player/20170108/456544',
 211         'md5': '34d8a6e67ea888293741c86a099b745b',
 212         'info_dict': {
 213             'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
 214             'ext': 'mp3',
 215             'title': 'Morgenjournal',
 216             'duration': 609,
 217             'timestamp': 1483858796,
 218             'upload_date': '20170108',
 219         },
 220         'skip': 'Shows from ORF radios are only available for 7 days.'
 221     }
 222
 223
 224 class ORFIPTVIE(InfoExtractor):
 225     IE_NAME = 'orf:iptv'
 226     IE_DESC = 'iptv.ORF.at'
 227     _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
 228
 229     _TEST = {
 230         'url': 'http://iptv.orf.at/stories/2275236/',
 231         'md5': 'c8b22af4718a4b4af58342529453e3e5',
 232         'info_dict': {
 233             'id': '350612',
 234             'ext': 'flv',
 235             'title': 'Weitere Evakuierungen um Vulkan Calbuco',
 236             'description': 'md5:d689c959bdbcf04efeddedbf2299d633',
 237             'duration': 68.197,
 238             'thumbnail': r're:^https?://.*\.jpg$',
 239             'upload_date': '20150425',
 240         },
 241     }
 242
 243     def _real_extract(self, url):
 244         story_id = self._match_id(url)
 245
 246         webpage = self._download_webpage(
 247             'http://iptv.orf.at/stories/%s' % story_id, story_id)
 248
 249         video_id = self._search_regex(
 250             r'data-video(?:id)?="(\d+)"', webpage, 'video id')
 251
 252         data = self._download_json(
 253             'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
 254             video_id)[0]
 255
 256         duration = float_or_none(data['duration'], 1000)
 257
 258         video = data['sources']['default']
 259         load_balancer_url = video['loadBalancerUrl']
 260         abr = int_or_none(video.get('audioBitrate'))
 261         vbr = int_or_none(video.get('bitrate'))
 262         fps = int_or_none(video.get('videoFps'))
 263         width = int_or_none(video.get('videoWidth'))
 264         height = int_or_none(video.get('videoHeight'))
 265         thumbnail = video.get('preview')
 266
 267         rendition = self._download_json(
 268             load_balancer_url, video_id, transform_source=strip_jsonp)
 269
 270         f = {
 271             'abr': abr,
 272             'vbr': vbr,
 273             'fps': fps,
 274             'width': width,
 275             'height': height,
 276         }
 277
 278         formats = []
 279         for format_id, format_url in rendition['redirect'].items():
 280             if format_id == 'rtmp':
 281                 ff = f.copy()
 282                 ff.update({
 283                     'url': format_url,
 284                     'format_id': format_id,
 285                 })
 286                 formats.append(ff)
 287             elif determine_ext(format_url) == 'f4m':
 288                 formats.extend(self._extract_f4m_formats(
 289                     format_url, video_id, f4m_id=format_id))
 290             elif determine_ext(format_url) == 'm3u8':
 291                 formats.extend(self._extract_m3u8_formats(
 292                     format_url, video_id, 'mp4', m3u8_id=format_id))
 293             else:
 294                 continue
 295         self._sort_formats(formats)
 296
 297         title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
 298         description = self._og_search_description(webpage)
 299         upload_date = unified_strdate(self._html_search_meta(
 300             'dc.date', webpage, 'upload date'))
 301
 302         return {
 303             'id': video_id,
 304             'title': title,
 305             'description': description,
 306             'duration': duration,
 307             'thumbnail': thumbnail,
 308             'upload_date': upload_date,
 309             'formats': formats,
 310         }
 311
 312
 313 class ORFFM4StoryIE(InfoExtractor):
 314     IE_NAME = 'orf:fm4:story'
 315     IE_DESC = 'fm4.orf.at stories'
 316     _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
 317
 318     _TEST = {
 319         'url': 'http://fm4.orf.at/stories/2865738/',
 320         'playlist': [{
 321             'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
 322             'info_dict': {
 323                 'id': '547792',
 324                 'ext': 'flv',
 325                 'title': 'Manu Delago und Inner Tongue live',
 326                 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
 327                 'duration': 1748.52,
 328                 'thumbnail': r're:^https?://.*\.jpg$',
 329                 'upload_date': '20170913',
 330             },
 331         }, {
 332             'md5': 'c6dd2179731f86f4f55a7b49899d515f',
 333             'info_dict': {
 334                 'id': '547798',
 335                 'ext': 'flv',
 336                 'title': 'Manu Delago und Inner Tongue live (2)',
 337                 'duration': 1504.08,
 338                 'thumbnail': r're:^https?://.*\.jpg$',
 339                 'upload_date': '20170913',
 340                 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
 341             },
 342         }],
 343     }
 344
 345     def _real_extract(self, url):
 346         story_id = self._match_id(url)
 347         webpage = self._download_webpage(url, story_id)
 348
 349         entries = []
 350         all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
 351         for idx, video_id in enumerate(all_ids):
 352             data = self._download_json(
 353                 'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
 354                 video_id)[0]
 355
 356             duration = float_or_none(data['duration'], 1000)
 357
 358             video = data['sources']['q8c']
 359             load_balancer_url = video['loadBalancerUrl']
 360             abr = int_or_none(video.get('audioBitrate'))
 361             vbr = int_or_none(video.get('bitrate'))
 362             fps = int_or_none(video.get('videoFps'))
 363             width = int_or_none(video.get('videoWidth'))
 364             height = int_or_none(video.get('videoHeight'))
 365             thumbnail = video.get('preview')
 366
 367             rendition = self._download_json(
 368                 load_balancer_url, video_id, transform_source=strip_jsonp)
 369
 370             f = {
 371                 'abr': abr,
 372                 'vbr': vbr,
 373                 'fps': fps,
 374                 'width': width,
 375                 'height': height,
 376             }
 377
 378             formats = []
 379             for format_id, format_url in rendition['redirect'].items():
 380                 if format_id == 'rtmp':
 381                     ff = f.copy()
 382                     ff.update({
 383                         'url': format_url,
 384                         'format_id': format_id,
 385                     })
 386                     formats.append(ff)
 387                 elif determine_ext(format_url) == 'f4m':
 388                     formats.extend(self._extract_f4m_formats(
 389                         format_url, video_id, f4m_id=format_id))
 390                 elif determine_ext(format_url) == 'm3u8':
 391                     formats.extend(self._extract_m3u8_formats(
 392                         format_url, video_id, 'mp4', m3u8_id=format_id))
 393                 else:
 394                     continue
 395             self._sort_formats(formats)
 396
 397             title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
 398             if idx >= 1:
 399                 # Titles are duplicates, make them unique
 400                 title += ' (' + str(idx + 1) + ')'
 401             description = self._og_search_description(webpage)
 402             upload_date = unified_strdate(self._html_search_meta(
 403                 'dc.date', webpage, 'upload date'))
 404
 405             entries.append({
 406                 'id': video_id,
 407                 'title': title,
 408                 'description': description,
 409                 'duration': duration,
 410                 'thumbnail': thumbnail,
 411                 'upload_date': upload_date,
 412                 'formats': formats,
 413             })
 414
 415         return self.playlist_result(entries)