Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/canvas.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4 import json
   5
   6 from .common import InfoExtractor
   7 from .gigya import GigyaBaseIE
   8 from ..compat import compat_HTTPError
   9 from ..utils import (
  10     ExtractorError,
  11     strip_or_none,
  12     float_or_none,
  13     int_or_none,
  14     parse_iso8601,
  15 )
  16
  17
  18 class CanvasIE(InfoExtractor):
  19     _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrtvideo)/assets/(?P<id>[^/?#&]+)'
  20     _TESTS = [{
  21         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  22         'md5': '90139b746a0a9bd7bb631283f6e2a64e',
  23         'info_dict': {
  24             'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  25             'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
  26             'ext': 'flv',
  27             'title': 'Nachtwacht: De Greystook',
  28             'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
  29             'thumbnail': r're:^https?://.*\.jpg$',
  30             'duration': 1468.03,
  31         },
  32         'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
  33     }, {
  34         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
  35         'only_matching': True,
  36     }]
  37
  38     def _real_extract(self, url):
  39         mobj = re.match(self._VALID_URL, url)
  40         site_id, video_id = mobj.group('site_id'), mobj.group('id')
  41
  42         data = self._download_json(
  43             'https://mediazone.vrt.be/api/v1/%s/assets/%s'
  44             % (site_id, video_id), video_id)
  45
  46         title = data['title']
  47         description = data.get('description')
  48
  49         formats = []
  50         for target in data['targetUrls']:
  51             format_url, format_type = target.get('url'), target.get('type')
  52             if not format_url or not format_type:
  53                 continue
  54             if format_type == 'HLS':
  55                 formats.extend(self._extract_m3u8_formats(
  56                     format_url, video_id, 'mp4', entry_protocol='m3u8_native',
  57                     m3u8_id=format_type, fatal=False))
  58             elif format_type == 'HDS':
  59                 formats.extend(self._extract_f4m_formats(
  60                     format_url, video_id, f4m_id=format_type, fatal=False))
  61             elif format_type == 'MPEG_DASH':
  62                 formats.extend(self._extract_mpd_formats(
  63                     format_url, video_id, mpd_id=format_type, fatal=False))
  64             elif format_type == 'HSS':
  65                 formats.extend(self._extract_ism_formats(
  66                     format_url, video_id, ism_id='mss', fatal=False))
  67             else:
  68                 formats.append({
  69                     'format_id': format_type,
  70                     'url': format_url,
  71                 })
  72         self._sort_formats(formats)
  73
  74         subtitles = {}
  75         subtitle_urls = data.get('subtitleUrls')
  76         if isinstance(subtitle_urls, list):
  77             for subtitle in subtitle_urls:
  78                 subtitle_url = subtitle.get('url')
  79                 if subtitle_url and subtitle.get('type') == 'CLOSED':
  80                     subtitles.setdefault('nl', []).append({'url': subtitle_url})
  81
  82         return {
  83             'id': video_id,
  84             'display_id': video_id,
  85             'title': title,
  86             'description': description,
  87             'formats': formats,
  88             'duration': float_or_none(data.get('duration'), 1000),
  89             'thumbnail': data.get('posterImageUrl'),
  90             'subtitles': subtitles,
  91         }
  92
  93
  94 class CanvasEenIE(InfoExtractor):
  95     IE_DESC = 'canvas.be and een.be'
  96     _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
  97     _TESTS = [{
  98         'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
  99         'md5': 'ed66976748d12350b118455979cca293',
 100         'info_dict': {
 101             'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
 102             'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
 103             'ext': 'flv',
 104             'title': 'De afspraak veilt voor de Warmste Week',
 105             'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
 106             'thumbnail': r're:^https?://.*\.jpg$',
 107             'duration': 49.02,
 108         },
 109         'expected_warnings': ['is not a supported codec'],
 110     }, {
 111         # with subtitles
 112         'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
 113         'info_dict': {
 114             'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
 115             'display_id': 'pieter-0167',
 116             'ext': 'mp4',
 117             'title': 'Pieter 0167',
 118             'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
 119             'thumbnail': r're:^https?://.*\.jpg$',
 120             'duration': 2553.08,
 121             'subtitles': {
 122                 'nl': [{
 123                     'ext': 'vtt',
 124                 }],
 125             },
 126         },
 127         'params': {
 128             'skip_download': True,
 129         },
 130         'skip': 'Pagina niet gevonden',
 131     }, {
 132         'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles',
 133         'info_dict': {
 134             'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f',
 135             'display_id': 'herbekijk-sorry-voor-alles',
 136             'ext': 'mp4',
 137             'title': 'Herbekijk Sorry voor alles',
 138             'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3',
 139             'thumbnail': r're:^https?://.*\.jpg$',
 140             'duration': 3788.06,
 141         },
 142         'params': {
 143             'skip_download': True,
 144         },
 145         'skip': 'Episode no longer available',
 146     }, {
 147         'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
 148         'only_matching': True,
 149     }]
 150
 151     def _real_extract(self, url):
 152         mobj = re.match(self._VALID_URL, url)
 153         site_id, display_id = mobj.group('site_id'), mobj.group('id')
 154
 155         webpage = self._download_webpage(url, display_id)
 156
 157         title = strip_or_none(self._search_regex(
 158             r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
 159             webpage, 'title', default=None) or self._og_search_title(
 160             webpage, default=None))
 161
 162         video_id = self._html_search_regex(
 163             r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
 164             group='id')
 165
 166         return {
 167             '_type': 'url_transparent',
 168             'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
 169             'ie_key': CanvasIE.ie_key(),
 170             'id': video_id,
 171             'display_id': display_id,
 172             'title': title,
 173             'description': self._og_search_description(webpage),
 174         }
 175
 176
 177 class VrtNUIE(GigyaBaseIE):
 178     IE_DESC = 'VrtNU.be'
 179     _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 180     _TESTS = [{
 181         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
 182         'info_dict': {
 183             'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
 184             'ext': 'flv',
 185             'title': 'De zwarte weduwe',
 186             'description': 'md5:d90c21dced7db869a85db89a623998d4',
 187             'duration': 1457.04,
 188             'thumbnail': r're:^https?://.*\.jpg$',
 189             'season': '1',
 190             'season_number': 1,
 191             'episode_number': 1,
 192         },
 193         'skip': 'This video is only available for registered users'
 194     }]
 195     _NETRC_MACHINE = 'vrtnu'
 196     _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
 197     _CONTEXT_ID = 'R3595707040'
 198
 199     def _real_initialize(self):
 200         self._login()
 201
 202     def _login(self):
 203         username, password = self._get_login_info()
 204         if username is None:
 205             return
 206
 207         auth_data = {
 208             'APIKey': self._APIKEY,
 209             'targetEnv': 'jssdk',
 210             'loginID': username,
 211             'password': password,
 212             'authMode': 'cookie',
 213         }
 214
 215         auth_info = self._gigya_login(auth_data)
 216
 217         # Sometimes authentication fails for no good reason, retry
 218         login_attempt = 1
 219         while login_attempt <= 3:
 220             try:
 221                 # When requesting a token, no actual token is returned, but the
 222                 # necessary cookies are set.
 223                 self._request_webpage(
 224                     'https://token.vrt.be',
 225                     None, note='Requesting a token', errnote='Could not get a token',
 226                     headers={
 227                         'Content-Type': 'application/json',
 228                         'Referer': 'https://www.vrt.be/vrtnu/',
 229                     },
 230                     data=json.dumps({
 231                         'uid': auth_info['UID'],
 232                         'uidsig': auth_info['UIDSignature'],
 233                         'ts': auth_info['signatureTimestamp'],
 234                         'email': auth_info['profile']['email'],
 235                     }).encode('utf-8'))
 236             except ExtractorError as e:
 237                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 238                     login_attempt += 1
 239                     self.report_warning('Authentication failed')
 240                     self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
 241                 else:
 242                     raise e
 243             else:
 244                 break
 245
 246     def _real_extract(self, url):
 247         display_id = self._match_id(url)
 248
 249         webpage, urlh = self._download_webpage_handle(url, display_id)
 250
 251         title = self._html_search_regex(
 252             r'(?ms)<h1 class="content__heading">(.+?)</h1>',
 253             webpage, 'title').strip()
 254
 255         description = self._html_search_regex(
 256             r'(?ms)<div class="content__description">(.+?)</div>',
 257             webpage, 'description', default=None)
 258
 259         season = self._html_search_regex(
 260             [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s*
 261                     <span>seizoen\ (.+?)</span>\s*
 262                 </div>''',
 263              r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
 264             webpage, 'season', default=None)
 265
 266         season_number = int_or_none(season)
 267
 268         episode_number = int_or_none(self._html_search_regex(
 269             r'''(?xms)<div\ class="content__episode">\s*
 270                     <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span>
 271                 </div>''',
 272             webpage, 'episode_number', default=None))
 273
 274         release_date = parse_iso8601(self._html_search_regex(
 275             r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"',
 276             webpage, 'release_date', default=None))
 277
 278         # If there's a ? or a # in the URL, remove them and everything after
 279         clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/')
 280         securevideo_url = clean_url + '.mssecurevideo.json'
 281
 282         try:
 283             video = self._download_json(securevideo_url, display_id)
 284         except ExtractorError as e:
 285             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 286                 self.raise_login_required()
 287             raise
 288
 289         # We are dealing with a '../<show>.relevant' URL
 290         redirect_url = video.get('url')
 291         if redirect_url:
 292             return self.url_result(self._proto_relative_url(redirect_url, 'https:'))
 293
 294         # There is only one entry, but with an unknown key, so just get
 295         # the first one
 296         video_id = list(video.values())[0].get('videoid')
 297
 298         return {
 299             '_type': 'url_transparent',
 300             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
 301             'ie_key': CanvasIE.ie_key(),
 302             'id': video_id,
 303             'display_id': display_id,
 304             'title': title,
 305             'description': description,
 306             'season': season,
 307             'season_number': season_number,
 308             'episode_number': episode_number,
 309             'release_date': release_date,
 310         }