Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/canvas.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4 import json
   5
   6 from .common import InfoExtractor
   7 from .gigya import GigyaBaseIE
   8 from ..compat import compat_HTTPError
   9 from ..utils import (
  10     ExtractorError,
  11     strip_or_none,
  12     float_or_none,
  13     int_or_none,
  14     merge_dicts,
  15     parse_iso8601,
  16 )
  17
  18
  19 class CanvasIE(InfoExtractor):
  20     _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrtvideo)/assets/(?P<id>[^/?#&]+)'
  21     _TESTS = [{
  22         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  23         'md5': '90139b746a0a9bd7bb631283f6e2a64e',
  24         'info_dict': {
  25             'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  26             'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  27             'ext': 'flv',
  28             'title': 'Nachtwacht: De Greystook',
  29             'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
  30             'thumbnail': r're:^https?://.*\.jpg$',
  31             'duration': 1468.03,
  32         },
  33         'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
  34     }, {
  35         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
  36         'only_matching': True,
  37     }]
  38
  39     def _real_extract(self, url):
  40         mobj = re.match(self._VALID_URL, url)
  41         site_id, video_id = mobj.group('site_id'), mobj.group('id')
  42
  43         data = self._download_json(
  44             'https://mediazone.vrt.be/api/v1/%s/assets/%s'
  45             % (site_id, video_id), video_id)
  46
  47         title = data['title']
  48         description = data.get('description')
  49
  50         formats = []
  51         for target in data['targetUrls']:
  52             format_url, format_type = target.get('url'), target.get('type')
  53             if not format_url or not format_type:
  54                 continue
  55             if format_type == 'HLS':
  56                 formats.extend(self._extract_m3u8_formats(
  57                     format_url, video_id, 'mp4', entry_protocol='m3u8_native',
  58                     m3u8_id=format_type, fatal=False))
  59             elif format_type == 'HDS':
  60                 formats.extend(self._extract_f4m_formats(
  61                     format_url, video_id, f4m_id=format_type, fatal=False))
  62             elif format_type == 'MPEG_DASH':
  63                 formats.extend(self._extract_mpd_formats(
  64                     format_url, video_id, mpd_id=format_type, fatal=False))
  65             elif format_type == 'HSS':
  66                 formats.extend(self._extract_ism_formats(
  67                     format_url, video_id, ism_id='mss', fatal=False))
  68             else:
  69                 formats.append({
  70                     'format_id': format_type,
  71                     'url': format_url,
  72                 })
  73         self._sort_formats(formats)
  74
  75         subtitles = {}
  76         subtitle_urls = data.get('subtitleUrls')
  77         if isinstance(subtitle_urls, list):
  78             for subtitle in subtitle_urls:
  79                 subtitle_url = subtitle.get('url')
  80                 if subtitle_url and subtitle.get('type') == 'CLOSED':
  81                     subtitles.setdefault('nl', []).append({'url': subtitle_url})
  82
  83         return {
  84             'id': video_id,
  85             'display_id': video_id,
  86             'title': title,
  87             'description': description,
  88             'formats': formats,
  89             'duration': float_or_none(data.get('duration'), 1000),
  90             'thumbnail': data.get('posterImageUrl'),
  91             'subtitles': subtitles,
  92         }
  93
  94
  95 class CanvasEenIE(InfoExtractor):
  96     IE_DESC = 'canvas.be and een.be'
  97     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
  98     _TESTS = [{
  99         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
 100         'md5': 'ed66976748d12350b118455979cca293',
 101         'info_dict': {
 102             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
 103             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
 104             'ext': 'flv',
 105             'title': 'De afspraak veilt voor de Warmste Week',
 106             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
 107             'thumbnail': r're:^https?://.*\.jpg$',
 108             'duration': 49.02,
 109         },
 110         'expected_warnings': ['is not a supported codec'],
 111     }, {
 112         # with subtitles
 113         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
 114         'info_dict': {
 115             'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
 116             'display_id': 'pieter-0167',
 117             'ext': 'mp4',
 118             'title': 'Pieter 0167',
 119             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
 120             'thumbnail': r're:^https?://.*\.jpg$',
 121             'duration': 2553.08,
 122             'subtitles': {
 123                 'nl': [{
 124                     'ext': 'vtt',
 125                 }],
 126             },
 127         },
 128         'params': {
 129             'skip_download': True,
 130         },
 131         'skip': 'Pagina niet gevonden',
 132     }, {
 133         'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles',
 134         'info_dict': {
 135             'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f',
 136             'display_id': 'herbekijk-sorry-voor-alles',
 137             'ext': 'mp4',
 138             'title': 'Herbekijk Sorry voor alles',
 139             'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3',
 140             'thumbnail': r're:^https?://.*\.jpg$',
 141             'duration': 3788.06,
 142         },
 143         'params': {
 144             'skip_download': True,
 145         },
 146         'skip': 'Episode no longer available',
 147     }, {
 148         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
 149         'only_matching': True,
 150     }]
 151
 152     def _real_extract(self, url):
 153         mobj = re.match(self._VALID_URL, url)
 154         site_id, display_id = mobj.group('site_id'), mobj.group('id')
 155
 156         webpage = self._download_webpage(url, display_id)
 157
 158         title = strip_or_none(self._search_regex(
 159             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
 160             webpage, 'title', default=None) or self._og_search_title(
 161             webpage, default=None))
 162
 163         video_id = self._html_search_regex(
 164             r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 165             group='id')
 166
 167         return {
 168             '_type': 'url_transparent',
 169             'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
 170             'ie_key': CanvasIE.ie_key(),
 171             'id': video_id,
 172             'display_id': display_id,
 173             'title': title,
 174             'description': self._og_search_description(webpage),
 175         }
 176
 177
 178 class VrtNUIE(GigyaBaseIE):
 179     IE_DESC = 'VrtNU.be'
 180     _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 181     _TESTS = [{
 182         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
 183         'info_dict': {
 184             'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
 185             'ext': 'flv',
 186             'title': 'De zwarte weduwe',
 187             'description': 'md5:d90c21dced7db869a85db89a623998d4',
 188             'duration': 1457.04,
 189             'thumbnail': r're:^https?://.*\.jpg$',
 190             'season': '1',
 191             'season_number': 1,
 192             'episode_number': 1,
 193         },
 194         'skip': 'This video is only available for registered users'
 195     }]
 196     _NETRC_MACHINE = 'vrtnu'
 197     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
 198     _CONTEXT_ID = 'R3595707040'
 199
 200     def _real_initialize(self):
 201         self._login()
 202
 203     def _login(self):
 204         username, password = self._get_login_info()
 205         if username is None:
 206             return
 207
 208         auth_data = {
 209             'APIKey': self._APIKEY,
 210             'targetEnv': 'jssdk',
 211             'loginID': username,
 212             'password': password,
 213             'authMode': 'cookie',
 214         }
 215
 216         auth_info = self._gigya_login(auth_data)
 217
 218         # Sometimes authentication fails for no good reason, retry
 219         login_attempt = 1
 220         while login_attempt <= 3:
 221             try:
 222                 # When requesting a token, no actual token is returned, but the
 223                 # necessary cookies are set.
 224                 self._request_webpage(
 225                     'https://token.vrt.be',
 226                     None, note='Requesting a token', errnote='Could not get a token',
 227                     headers={
 228                         'Content-Type': 'application/json',
 229                         'Referer': 'https://www.vrt.be/vrtnu/',
 230                     },
 231                     data=json.dumps({
 232                         'uid': auth_info['UID'],
 233                         'uidsig': auth_info['UIDSignature'],
 234                         'ts': auth_info['signatureTimestamp'],
 235                         'email': auth_info['profile']['email'],
 236                     }).encode('utf-8'))
 237             except ExtractorError as e:
 238                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 239                     login_attempt += 1
 240                     self.report_warning('Authentication failed')
 241                     self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
 242                 else:
 243                     raise e
 244             else:
 245                 break
 246
 247     def _real_extract(self, url):
 248         display_id = self._match_id(url)
 249
 250         webpage, urlh = self._download_webpage_handle(url, display_id)
 251
 252         info = self._search_json_ld(webpage, display_id, default={})
 253
 254         # title is optional here since it may be extracted by extractor
 255         # that is delegated from here
 256         title = strip_or_none(self._html_search_regex(
 257             r'(?ms)<h1 class="content__heading">(.+?)</h1>',
 258             webpage, 'title', default=None))
 259
 260         description = self._html_search_regex(
 261             r'(?ms)<div class="content__description">(.+?)</div>',
 262             webpage, 'description', default=None)
 263
 264         season = self._html_search_regex(
 265             [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s*
 266                     <span>seizoen\ (.+?)</span>\s*
 267                 </div>''',
 268              r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
 269             webpage, 'season', default=None)
 270
 271         season_number = int_or_none(season)
 272
 273         episode_number = int_or_none(self._html_search_regex(
 274             r'''(?xms)<div\ class="content__episode">\s*
 275                     <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span>
 276                 </div>''',
 277             webpage, 'episode_number', default=None))
 278
 279         release_date = parse_iso8601(self._html_search_regex(
 280             r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"',
 281             webpage, 'release_date', default=None))
 282
 283         # If there's a ? or a # in the URL, remove them and everything after
 284         clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/')
 285         securevideo_url = clean_url + '.mssecurevideo.json'
 286
 287         try:
 288             video = self._download_json(securevideo_url, display_id)
 289         except ExtractorError as e:
 290             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 291                 self.raise_login_required()
 292             raise
 293
 294         # We are dealing with a '../<show>.relevant' URL
 295         redirect_url = video.get('url')
 296         if redirect_url:
 297             return self.url_result(self._proto_relative_url(redirect_url, 'https:'))
 298
 299         # There is only one entry, but with an unknown key, so just get
 300         # the first one
 301         video_id = list(video.values())[0].get('videoid')
 302
 303         return merge_dicts(info, {
 304             '_type': 'url_transparent',
 305             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
 306             'ie_key': CanvasIE.ie_key(),
 307             'id': video_id,
 308             'display_id': display_id,
 309             'title': title,
 310             'description': description,
 311             'season': season,
 312             'season_number': season_number,
 313             'episode_number': episode_number,
 314             'release_date': release_date,
 315         })