]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/canvas.py
New upstream version 2018.06.18
[youtubedl] / youtube_dl / extractor / canvas.py
1 from __future__ import unicode_literals
2
3 import re
4 import json
5
6 from .common import InfoExtractor
7 from .gigya import GigyaBaseIE
8 from ..compat import compat_HTTPError
9 from ..utils import (
10 ExtractorError,
11 strip_or_none,
12 float_or_none,
13 int_or_none,
14 parse_iso8601,
15 )
16
17
18 class CanvasIE(InfoExtractor):
19 _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrtvideo)/assets/(?P<id>[^/?#&]+)'
20 _TESTS = [{
21 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
22 'md5': '90139b746a0a9bd7bb631283f6e2a64e',
23 'info_dict': {
24 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
25 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
26 'ext': 'flv',
27 'title': 'Nachtwacht: De Greystook',
28 'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
29 'thumbnail': r're:^https?://.*\.jpg$',
30 'duration': 1468.03,
31 },
32 'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
33 }, {
34 'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
35 'only_matching': True,
36 }]
37
38 def _real_extract(self, url):
39 mobj = re.match(self._VALID_URL, url)
40 site_id, video_id = mobj.group('site_id'), mobj.group('id')
41
42 data = self._download_json(
43 'https://mediazone.vrt.be/api/v1/%s/assets/%s'
44 % (site_id, video_id), video_id)
45
46 title = data['title']
47 description = data.get('description')
48
49 formats = []
50 for target in data['targetUrls']:
51 format_url, format_type = target.get('url'), target.get('type')
52 if not format_url or not format_type:
53 continue
54 if format_type == 'HLS':
55 formats.extend(self._extract_m3u8_formats(
56 format_url, video_id, 'mp4', entry_protocol='m3u8_native',
57 m3u8_id=format_type, fatal=False))
58 elif format_type == 'HDS':
59 formats.extend(self._extract_f4m_formats(
60 format_url, video_id, f4m_id=format_type, fatal=False))
61 elif format_type == 'MPEG_DASH':
62 formats.extend(self._extract_mpd_formats(
63 format_url, video_id, mpd_id=format_type, fatal=False))
64 elif format_type == 'HSS':
65 formats.extend(self._extract_ism_formats(
66 format_url, video_id, ism_id='mss', fatal=False))
67 else:
68 formats.append({
69 'format_id': format_type,
70 'url': format_url,
71 })
72 self._sort_formats(formats)
73
74 subtitles = {}
75 subtitle_urls = data.get('subtitleUrls')
76 if isinstance(subtitle_urls, list):
77 for subtitle in subtitle_urls:
78 subtitle_url = subtitle.get('url')
79 if subtitle_url and subtitle.get('type') == 'CLOSED':
80 subtitles.setdefault('nl', []).append({'url': subtitle_url})
81
82 return {
83 'id': video_id,
84 'display_id': video_id,
85 'title': title,
86 'description': description,
87 'formats': formats,
88 'duration': float_or_none(data.get('duration'), 1000),
89 'thumbnail': data.get('posterImageUrl'),
90 'subtitles': subtitles,
91 }
92
93
94 class CanvasEenIE(InfoExtractor):
95 IE_DESC = 'canvas.be and een.be'
96 _VALID_URL = r'https?://(?:www\.)?(?P<site_id>canvas|een)\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
97 _TESTS = [{
98 'url': 'http://www.canvas.be/video/de-afspraak/najaar-2015/de-afspraak-veilt-voor-de-warmste-week',
99 'md5': 'ed66976748d12350b118455979cca293',
100 'info_dict': {
101 'id': 'mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
102 'display_id': 'de-afspraak-veilt-voor-de-warmste-week',
103 'ext': 'flv',
104 'title': 'De afspraak veilt voor de Warmste Week',
105 'description': 'md5:24cb860c320dc2be7358e0e5aa317ba6',
106 'thumbnail': r're:^https?://.*\.jpg$',
107 'duration': 49.02,
108 },
109 'expected_warnings': ['is not a supported codec'],
110 }, {
111 # with subtitles
112 'url': 'http://www.canvas.be/video/panorama/2016/pieter-0167',
113 'info_dict': {
114 'id': 'mz-ast-5240ff21-2d30-4101-bba6-92b5ec67c625',
115 'display_id': 'pieter-0167',
116 'ext': 'mp4',
117 'title': 'Pieter 0167',
118 'description': 'md5:943cd30f48a5d29ba02c3a104dc4ec4e',
119 'thumbnail': r're:^https?://.*\.jpg$',
120 'duration': 2553.08,
121 'subtitles': {
122 'nl': [{
123 'ext': 'vtt',
124 }],
125 },
126 },
127 'params': {
128 'skip_download': True,
129 },
130 'skip': 'Pagina niet gevonden',
131 }, {
132 'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles',
133 'info_dict': {
134 'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f',
135 'display_id': 'herbekijk-sorry-voor-alles',
136 'ext': 'mp4',
137 'title': 'Herbekijk Sorry voor alles',
138 'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3',
139 'thumbnail': r're:^https?://.*\.jpg$',
140 'duration': 3788.06,
141 },
142 'params': {
143 'skip_download': True,
144 },
145 'skip': 'Episode no longer available',
146 }, {
147 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend',
148 'only_matching': True,
149 }]
150
151 def _real_extract(self, url):
152 mobj = re.match(self._VALID_URL, url)
153 site_id, display_id = mobj.group('site_id'), mobj.group('id')
154
155 webpage = self._download_webpage(url, display_id)
156
157 title = strip_or_none(self._search_regex(
158 r'<h1[^>]+class="video__body__header__title"[^>]*>(.+?)</h1>',
159 webpage, 'title', default=None) or self._og_search_title(
160 webpage, default=None))
161
162 video_id = self._html_search_regex(
163 r'data-video=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
164 group='id')
165
166 return {
167 '_type': 'url_transparent',
168 'url': 'https://mediazone.vrt.be/api/v1/%s/assets/%s' % (site_id, video_id),
169 'ie_key': CanvasIE.ie_key(),
170 'id': video_id,
171 'display_id': display_id,
172 'title': title,
173 'description': self._og_search_description(webpage),
174 }
175
176
177 class VrtNUIE(GigyaBaseIE):
178 IE_DESC = 'VrtNU.be'
179 _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
180 _TESTS = [{
181 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
182 'info_dict': {
183 'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
184 'ext': 'flv',
185 'title': 'De zwarte weduwe',
186 'description': 'md5:d90c21dced7db869a85db89a623998d4',
187 'duration': 1457.04,
188 'thumbnail': r're:^https?://.*\.jpg$',
189 'season': '1',
190 'season_number': 1,
191 'episode_number': 1,
192 },
193 'skip': 'This video is only available for registered users'
194 }]
195 _NETRC_MACHINE = 'vrtnu'
196 _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
197 _CONTEXT_ID = 'R3595707040'
198
199 def _real_initialize(self):
200 self._login()
201
202 def _login(self):
203 username, password = self._get_login_info()
204 if username is None:
205 return
206
207 auth_data = {
208 'APIKey': self._APIKEY,
209 'targetEnv': 'jssdk',
210 'loginID': username,
211 'password': password,
212 'authMode': 'cookie',
213 }
214
215 auth_info = self._gigya_login(auth_data)
216
217 # Sometimes authentication fails for no good reason, retry
218 login_attempt = 1
219 while login_attempt <= 3:
220 try:
221 # When requesting a token, no actual token is returned, but the
222 # necessary cookies are set.
223 self._request_webpage(
224 'https://token.vrt.be',
225 None, note='Requesting a token', errnote='Could not get a token',
226 headers={
227 'Content-Type': 'application/json',
228 'Referer': 'https://www.vrt.be/vrtnu/',
229 },
230 data=json.dumps({
231 'uid': auth_info['UID'],
232 'uidsig': auth_info['UIDSignature'],
233 'ts': auth_info['signatureTimestamp'],
234 'email': auth_info['profile']['email'],
235 }).encode('utf-8'))
236 except ExtractorError as e:
237 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
238 login_attempt += 1
239 self.report_warning('Authentication failed')
240 self._sleep(1, None, msg_template='Waiting for %(timeout)s seconds before trying again')
241 else:
242 raise e
243 else:
244 break
245
246 def _real_extract(self, url):
247 display_id = self._match_id(url)
248
249 webpage, urlh = self._download_webpage_handle(url, display_id)
250
251 title = self._html_search_regex(
252 r'(?ms)<h1 class="content__heading">(.+?)</h1>',
253 webpage, 'title').strip()
254
255 description = self._html_search_regex(
256 r'(?ms)<div class="content__description">(.+?)</div>',
257 webpage, 'description', default=None)
258
259 season = self._html_search_regex(
260 [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s*
261 <span>seizoen\ (.+?)</span>\s*
262 </div>''',
263 r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
264 webpage, 'season', default=None)
265
266 season_number = int_or_none(season)
267
268 episode_number = int_or_none(self._html_search_regex(
269 r'''(?xms)<div\ class="content__episode">\s*
270 <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span>
271 </div>''',
272 webpage, 'episode_number', default=None))
273
274 release_date = parse_iso8601(self._html_search_regex(
275 r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"',
276 webpage, 'release_date', default=None))
277
278 # If there's a ? or a # in the URL, remove them and everything after
279 clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/')
280 securevideo_url = clean_url + '.mssecurevideo.json'
281
282 try:
283 video = self._download_json(securevideo_url, display_id)
284 except ExtractorError as e:
285 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
286 self.raise_login_required()
287 raise
288
289 # We are dealing with a '../<show>.relevant' URL
290 redirect_url = video.get('url')
291 if redirect_url:
292 return self.url_result(self._proto_relative_url(redirect_url, 'https:'))
293
294 # There is only one entry, but with an unknown key, so just get
295 # the first one
296 video_id = list(video.values())[0].get('videoid')
297
298 return {
299 '_type': 'url_transparent',
300 'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
301 'ie_key': CanvasIE.ie_key(),
302 'id': video_id,
303 'display_id': display_id,
304 'title': title,
305 'description': description,
306 'season': season,
307 'season_number': season_number,
308 'episode_number': episode_number,
309 'release_date': release_date,
310 }