Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/ceskatelevize.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_urllib_parse_unquote,
   9     compat_urllib_parse_urlparse,
  10 )
  11 from ..utils import (
  12     ExtractorError,
  13     float_or_none,
  14     sanitized_Request,
  15     urlencode_postdata,
  16     USER_AGENTS,
  17 )
  18
  19
  20 class CeskaTelevizeIE(InfoExtractor):
  21     _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(porady|ivysilani)/(?:[^/]+/)*(?P<id>[^/#?]+)/*(?:[#?].*)?$'
  22     _TESTS = [{
  23         'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
  24         'info_dict': {
  25             'id': '61924494877246241',
  26             'ext': 'mp4',
  27             'title': 'Hyde Park Civilizace: Život v Grónsku',
  28             'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
  29             'thumbnail': r're:^https?://.*\.jpg',
  30             'duration': 3350,
  31         },
  32         'params': {
  33             # m3u8 download
  34             'skip_download': True,
  35         },
  36     }, {
  37         'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
  38         'info_dict': {
  39             'id': '61924494877028507',
  40             'ext': 'mp4',
  41             'title': 'Hyde Park Civilizace: Bonus 01 - En',
  42             'description': 'English Subtittles',
  43             'thumbnail': r're:^https?://.*\.jpg',
  44             'duration': 81.3,
  45         },
  46         'params': {
  47             # m3u8 download
  48             'skip_download': True,
  49         },
  50     }, {
  51         # live stream
  52         'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
  53         'info_dict': {
  54             'id': 402,
  55             'ext': 'mp4',
  56             'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
  57             'is_live': True,
  58         },
  59         'params': {
  60             # m3u8 download
  61             'skip_download': True,
  62         },
  63         'skip': 'Georestricted to Czech Republic',
  64     }, {
  65         # video with 18+ caution trailer
  66         'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
  67         'info_dict': {
  68             'id': '215562210900007-bogotart',
  69             'title': 'Queer: Bogotart',
  70             'description': 'Alternativní průvodce současným queer světem',
  71         },
  72         'playlist': [{
  73             'info_dict': {
  74                 'id': '61924494876844842',
  75                 'ext': 'mp4',
  76                 'title': 'Queer: Bogotart (Varování 18+)',
  77                 'duration': 10.2,
  78             },
  79         }, {
  80             'info_dict': {
  81                 'id': '61924494877068022',
  82                 'ext': 'mp4',
  83                 'title': 'Queer: Bogotart (Queer)',
  84                 'thumbnail': r're:^https?://.*\.jpg',
  85                 'duration': 1558.3,
  86             },
  87         }],
  88         'params': {
  89             # m3u8 download
  90             'skip_download': True,
  91         },
  92     }]
  93
  94     def _real_extract(self, url):
  95         url = url.replace('/porady/', '/ivysilani/').replace('/video/', '')
  96
  97         mobj = re.match(self._VALID_URL, url)
  98         playlist_id = mobj.group('id')
  99
 100         webpage = self._download_webpage(url, playlist_id)
 101
 102         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
 103         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
 104             raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 105
 106         typ = self._html_search_regex(
 107             r'getPlaylistUrl\(\[\{"type":"(.+?)","id":".+?"\}\],', webpage, 'type')
 108         episode_id = self._html_search_regex(
 109             r'getPlaylistUrl\(\[\{"type":".+?","id":"(.+?)"\}\],', webpage, 'episode_id')
 110
 111         data = {
 112             'playlist[0][type]': typ,
 113             'playlist[0][id]': episode_id,
 114             'requestUrl': compat_urllib_parse_urlparse(url).path,
 115             'requestSource': 'iVysilani',
 116         }
 117
 118         entries = []
 119
 120         for user_agent in (None, USER_AGENTS['Safari']):
 121             req = sanitized_Request(
 122                 'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
 123                 data=urlencode_postdata(data))
 124
 125             req.add_header('Content-type', 'application/x-www-form-urlencoded')
 126             req.add_header('x-addr', '127.0.0.1')
 127             req.add_header('X-Requested-With', 'XMLHttpRequest')
 128             if user_agent:
 129                 req.add_header('User-Agent', user_agent)
 130             req.add_header('Referer', url)
 131
 132             playlistpage = self._download_json(req, playlist_id, fatal=False)
 133
 134             if not playlistpage:
 135                 continue
 136
 137             playlist_url = playlistpage['url']
 138             if playlist_url == 'error_region':
 139                 raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
 140
 141             req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
 142             req.add_header('Referer', url)
 143
 144             playlist_title = self._og_search_title(webpage, default=None)
 145             playlist_description = self._og_search_description(webpage, default=None)
 146
 147             playlist = self._download_json(req, playlist_id, fatal=False)
 148             if not playlist:
 149                 continue
 150
 151             playlist = playlist.get('playlist')
 152             if not isinstance(playlist, list):
 153                 continue
 154
 155             playlist_len = len(playlist)
 156
 157             for num, item in enumerate(playlist):
 158                 is_live = item.get('type') == 'LIVE'
 159                 formats = []
 160                 for format_id, stream_url in item.get('streamUrls', {}).items():
 161                     if 'playerType=flash' in stream_url:
 162                         stream_formats = self._extract_m3u8_formats(
 163                             stream_url, playlist_id, 'mp4', 'm3u8_native',
 164                             m3u8_id='hls-%s' % format_id, fatal=False)
 165                     else:
 166                         stream_formats = self._extract_mpd_formats(
 167                             stream_url, playlist_id,
 168                             mpd_id='dash-%s' % format_id, fatal=False)
 169                     # See https://github.com/rg3/youtube-dl/issues/12119#issuecomment-280037031
 170                     if format_id == 'audioDescription':
 171                         for f in stream_formats:
 172                             f['source_preference'] = -10
 173                     formats.extend(stream_formats)
 174
 175                 if user_agent and len(entries) == playlist_len:
 176                     entries[num]['formats'].extend(formats)
 177                     continue
 178
 179                 item_id = item.get('id') or item['assetId']
 180                 title = item['title']
 181
 182                 duration = float_or_none(item.get('duration'))
 183                 thumbnail = item.get('previewImageUrl')
 184
 185                 subtitles = {}
 186                 if item.get('type') == 'VOD':
 187                     subs = item.get('subtitles')
 188                     if subs:
 189                         subtitles = self.extract_subtitles(episode_id, subs)
 190
 191                 if playlist_len == 1:
 192                     final_title = playlist_title or title
 193                     if is_live:
 194                         final_title = self._live_title(final_title)
 195                 else:
 196                     final_title = '%s (%s)' % (playlist_title, title)
 197
 198                 entries.append({
 199                     'id': item_id,
 200                     'title': final_title,
 201                     'description': playlist_description if playlist_len == 1 else None,
 202                     'thumbnail': thumbnail,
 203                     'duration': duration,
 204                     'formats': formats,
 205                     'subtitles': subtitles,
 206                     'is_live': is_live,
 207                 })
 208
 209         for e in entries:
 210             self._sort_formats(e['formats'])
 211
 212         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 213
 214     def _get_subtitles(self, episode_id, subs):
 215         original_subtitles = self._download_webpage(
 216             subs[0]['url'], episode_id, 'Downloading subtitles')
 217         srt_subs = self._fix_subtitles(original_subtitles)
 218         return {
 219             'cs': [{
 220                 'ext': 'srt',
 221                 'data': srt_subs,
 222             }]
 223         }
 224
 225     @staticmethod
 226     def _fix_subtitles(subtitles):
 227         """ Convert millisecond-based subtitles to SRT """
 228
 229         def _msectotimecode(msec):
 230             """ Helper utility to convert milliseconds to timecode """
 231             components = []
 232             for divider in [1000, 60, 60, 100]:
 233                 components.append(msec % divider)
 234                 msec //= divider
 235             return '{3:02}:{2:02}:{1:02},{0:03}'.format(*components)
 236
 237         def _fix_subtitle(subtitle):
 238             for line in subtitle.splitlines():
 239                 m = re.match(r'^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$', line)
 240                 if m:
 241                     yield m.group(1)
 242                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
 243                     yield '{0} --> {1}'.format(start, stop)
 244                 else:
 245                     yield line
 246
 247         return '\r\n'.join(_fix_subtitle(subtitles))