]> Raphaƫl G. Git Repositories - youtubedl/blob - youtube_dl/extractor/twitch.py
890f551800d011c423636530dfd8d70e4e926290
[youtubedl] / youtube_dl / extractor / twitch.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import itertools
5 import re
6 import random
7
8 from .common import InfoExtractor
9 from ..compat import (
10 compat_parse_qs,
11 compat_str,
12 compat_urllib_parse_urlencode,
13 compat_urllib_parse_urlparse,
14 compat_urlparse,
15 )
16 from ..utils import (
17 ExtractorError,
18 int_or_none,
19 js_to_json,
20 orderedSet,
21 parse_duration,
22 parse_iso8601,
23 sanitized_Request,
24 urlencode_postdata,
25 )
26
27
28 class TwitchBaseIE(InfoExtractor):
29 _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
30
31 _API_BASE = 'https://api.twitch.tv'
32 _USHER_BASE = 'https://usher.ttvnw.net'
33 _LOGIN_URL = 'http://www.twitch.tv/login'
34 _NETRC_MACHINE = 'twitch'
35
36 def _handle_error(self, response):
37 if not isinstance(response, dict):
38 return
39 error = response.get('error')
40 if error:
41 raise ExtractorError(
42 '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
43 expected=True)
44
45 def _download_json(self, url, video_id, note='Downloading JSON metadata'):
46 headers = {
47 'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
48 'X-Requested-With': 'XMLHttpRequest',
49 }
50 for cookie in self._downloader.cookiejar:
51 if cookie.name == 'api_token':
52 headers['Twitch-Api-Token'] = cookie.value
53 request = sanitized_Request(url, headers=headers)
54 response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
55 self._handle_error(response)
56 return response
57
58 def _real_initialize(self):
59 self._login()
60
61 def _login(self):
62 (username, password) = self._get_login_info()
63 if username is None:
64 return
65
66 login_page, handle = self._download_webpage_handle(
67 self._LOGIN_URL, None, 'Downloading login page')
68
69 login_form = self._hidden_inputs(login_page)
70
71 login_form.update({
72 'username': username,
73 'password': password,
74 })
75
76 redirect_url = handle.geturl()
77
78 post_url = self._search_regex(
79 r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
80 'post url', default=redirect_url, group='url')
81
82 if not post_url.startswith('http'):
83 post_url = compat_urlparse.urljoin(redirect_url, post_url)
84
85 request = sanitized_Request(
86 post_url, urlencode_postdata(login_form))
87 request.add_header('Referer', redirect_url)
88 response = self._download_webpage(
89 request, None, 'Logging in as %s' % username)
90
91 error_message = self._search_regex(
92 r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
93 response, 'error message', default=None)
94 if error_message:
95 raise ExtractorError(
96 'Unable to login. Twitch said: %s' % error_message, expected=True)
97
98 if '>Reset your password<' in response:
99 self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
100
101 def _prefer_source(self, formats):
102 try:
103 source = next(f for f in formats if f['format_id'] == 'Source')
104 source['preference'] = 10
105 except StopIteration:
106 pass # No Source stream present
107 self._sort_formats(formats)
108
109
110 class TwitchItemBaseIE(TwitchBaseIE):
111 def _download_info(self, item, item_id):
112 return self._extract_info(self._download_json(
113 '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
114 'Downloading %s info JSON' % self._ITEM_TYPE))
115
116 def _extract_media(self, item_id):
117 info = self._download_info(self._ITEM_SHORTCUT, item_id)
118 response = self._download_json(
119 '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
120 'Downloading %s playlist JSON' % self._ITEM_TYPE)
121 entries = []
122 chunks = response['chunks']
123 qualities = list(chunks.keys())
124 for num, fragment in enumerate(zip(*chunks.values()), start=1):
125 formats = []
126 for fmt_num, fragment_fmt in enumerate(fragment):
127 format_id = qualities[fmt_num]
128 fmt = {
129 'url': fragment_fmt['url'],
130 'format_id': format_id,
131 'quality': 1 if format_id == 'live' else 0,
132 }
133 m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
134 if m:
135 fmt['height'] = int(m.group('height'))
136 formats.append(fmt)
137 self._sort_formats(formats)
138 entry = dict(info)
139 entry['id'] = '%s_%d' % (entry['id'], num)
140 entry['title'] = '%s part %d' % (entry['title'], num)
141 entry['formats'] = formats
142 entries.append(entry)
143 return self.playlist_result(entries, info['id'], info['title'])
144
145 def _extract_info(self, info):
146 return {
147 'id': info['_id'],
148 'title': info.get('title') or 'Untitled Broadcast',
149 'description': info.get('description'),
150 'duration': int_or_none(info.get('length')),
151 'thumbnail': info.get('preview'),
152 'uploader': info.get('channel', {}).get('display_name'),
153 'uploader_id': info.get('channel', {}).get('name'),
154 'timestamp': parse_iso8601(info.get('recorded_at')),
155 'view_count': int_or_none(info.get('views')),
156 }
157
158 def _real_extract(self, url):
159 return self._extract_media(self._match_id(url))
160
161
162 class TwitchVideoIE(TwitchItemBaseIE):
163 IE_NAME = 'twitch:video'
164 _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
165 _ITEM_TYPE = 'video'
166 _ITEM_SHORTCUT = 'a'
167
168 _TEST = {
169 'url': 'http://www.twitch.tv/riotgames/b/577357806',
170 'info_dict': {
171 'id': 'a577357806',
172 'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
173 },
174 'playlist_mincount': 12,
175 'skip': 'HTTP Error 404: Not Found',
176 }
177
178
179 class TwitchChapterIE(TwitchItemBaseIE):
180 IE_NAME = 'twitch:chapter'
181 _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
182 _ITEM_TYPE = 'chapter'
183 _ITEM_SHORTCUT = 'c'
184
185 _TESTS = [{
186 'url': 'http://www.twitch.tv/acracingleague/c/5285812',
187 'info_dict': {
188 'id': 'c5285812',
189 'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
190 },
191 'playlist_mincount': 3,
192 'skip': 'HTTP Error 404: Not Found',
193 }, {
194 'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
195 'only_matching': True,
196 }]
197
198
199 class TwitchVodIE(TwitchItemBaseIE):
200 IE_NAME = 'twitch:vod'
201 _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
202 _ITEM_TYPE = 'vod'
203 _ITEM_SHORTCUT = 'v'
204
205 _TESTS = [{
206 'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
207 'info_dict': {
208 'id': 'v6528877',
209 'ext': 'mp4',
210 'title': 'LCK Summer Split - Week 6 Day 1',
211 'thumbnail': 're:^https?://.*\.jpg$',
212 'duration': 17208,
213 'timestamp': 1435131709,
214 'upload_date': '20150624',
215 'uploader': 'Riot Games',
216 'uploader_id': 'riotgames',
217 'view_count': int,
218 'start_time': 310,
219 },
220 'params': {
221 # m3u8 download
222 'skip_download': True,
223 },
224 }, {
225 # Untitled broadcast (title is None)
226 'url': 'http://www.twitch.tv/belkao_o/v/11230755',
227 'info_dict': {
228 'id': 'v11230755',
229 'ext': 'mp4',
230 'title': 'Untitled Broadcast',
231 'thumbnail': 're:^https?://.*\.jpg$',
232 'duration': 1638,
233 'timestamp': 1439746708,
234 'upload_date': '20150816',
235 'uploader': 'BelkAO_o',
236 'uploader_id': 'belkao_o',
237 'view_count': int,
238 },
239 'params': {
240 # m3u8 download
241 'skip_download': True,
242 },
243 }]
244
245 def _real_extract(self, url):
246 item_id = self._match_id(url)
247
248 info = self._download_info(self._ITEM_SHORTCUT, item_id)
249 access_token = self._download_json(
250 '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
251 'Downloading %s access token' % self._ITEM_TYPE)
252
253 formats = self._extract_m3u8_formats(
254 '%s/vod/%s?%s' % (
255 self._USHER_BASE, item_id,
256 compat_urllib_parse_urlencode({
257 'allow_source': 'true',
258 'allow_audio_only': 'true',
259 'allow_spectre': 'true',
260 'player': 'twitchweb',
261 'nauth': access_token['token'],
262 'nauthsig': access_token['sig'],
263 })),
264 item_id, 'mp4', entry_protocol='m3u8_native')
265
266 self._prefer_source(formats)
267 info['formats'] = formats
268
269 parsed_url = compat_urllib_parse_urlparse(url)
270 query = compat_parse_qs(parsed_url.query)
271 if 't' in query:
272 info['start_time'] = parse_duration(query['t'][0])
273
274 return info
275
276
277 class TwitchPlaylistBaseIE(TwitchBaseIE):
278 _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
279 _PAGE_LIMIT = 100
280
281 def _extract_playlist(self, channel_id):
282 info = self._download_json(
283 '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
284 channel_id, 'Downloading channel info JSON')
285 channel_name = info.get('display_name') or info.get('name')
286 entries = []
287 offset = 0
288 limit = self._PAGE_LIMIT
289 broken_paging_detected = False
290 counter_override = None
291 for counter in itertools.count(1):
292 response = self._download_json(
293 self._PLAYLIST_URL % (channel_id, offset, limit),
294 channel_id,
295 'Downloading %s videos JSON page %s'
296 % (self._PLAYLIST_TYPE, counter_override or counter))
297 page_entries = self._extract_playlist_page(response)
298 if not page_entries:
299 break
300 total = int_or_none(response.get('_total'))
301 # Since the beginning of March 2016 twitch's paging mechanism
302 # is completely broken on the twitch side. It simply ignores
303 # a limit and returns the whole offset number of videos.
304 # Working around by just requesting all videos at once.
305 # Upd: pagination bug was fixed by twitch on 15.03.2016.
306 if not broken_paging_detected and total and len(page_entries) > limit:
307 self.report_warning(
308 'Twitch pagination is broken on twitch side, requesting all videos at once',
309 channel_id)
310 broken_paging_detected = True
311 offset = total
312 counter_override = '(all at once)'
313 continue
314 entries.extend(page_entries)
315 if broken_paging_detected or total and len(page_entries) >= total:
316 break
317 offset += limit
318 return self.playlist_result(
319 [self.url_result(entry) for entry in orderedSet(entries)],
320 channel_id, channel_name)
321
322 def _extract_playlist_page(self, response):
323 videos = response.get('videos')
324 return [video['url'] for video in videos] if videos else []
325
326 def _real_extract(self, url):
327 return self._extract_playlist(self._match_id(url))
328
329
330 class TwitchProfileIE(TwitchPlaylistBaseIE):
331 IE_NAME = 'twitch:profile'
332 _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
333 _PLAYLIST_TYPE = 'profile'
334
335 _TEST = {
336 'url': 'http://www.twitch.tv/vanillatv/profile',
337 'info_dict': {
338 'id': 'vanillatv',
339 'title': 'VanillaTV',
340 },
341 'playlist_mincount': 412,
342 }
343
344
345 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
346 IE_NAME = 'twitch:past_broadcasts'
347 _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
348 _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
349 _PLAYLIST_TYPE = 'past broadcasts'
350
351 _TEST = {
352 'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
353 'info_dict': {
354 'id': 'spamfish',
355 'title': 'Spamfish',
356 },
357 'playlist_mincount': 54,
358 }
359
360
361 class TwitchStreamIE(TwitchBaseIE):
362 IE_NAME = 'twitch:stream'
363 _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
364
365 _TESTS = [{
366 'url': 'http://www.twitch.tv/shroomztv',
367 'info_dict': {
368 'id': '12772022048',
369 'display_id': 'shroomztv',
370 'ext': 'mp4',
371 'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
372 'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
373 'is_live': True,
374 'timestamp': 1421928037,
375 'upload_date': '20150122',
376 'uploader': 'ShroomzTV',
377 'uploader_id': 'shroomztv',
378 'view_count': int,
379 },
380 'params': {
381 # m3u8 download
382 'skip_download': True,
383 },
384 }, {
385 'url': 'http://www.twitch.tv/miracle_doto#profile-0',
386 'only_matching': True,
387 }]
388
389 def _real_extract(self, url):
390 channel_id = self._match_id(url)
391
392 stream = self._download_json(
393 '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
394 'Downloading stream JSON').get('stream')
395
396 # Fallback on profile extraction if stream is offline
397 if not stream:
398 return self.url_result(
399 'http://www.twitch.tv/%s/profile' % channel_id,
400 'TwitchProfile', channel_id)
401
402 # Channel name may be typed if different case than the original channel name
403 # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
404 # an invalid m3u8 URL. Working around by use of original channel name from stream
405 # JSON and fallback to lowercase if it's not available.
406 channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
407
408 access_token = self._download_json(
409 '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
410 'Downloading channel access token')
411
412 query = {
413 'allow_source': 'true',
414 'allow_audio_only': 'true',
415 'p': random.randint(1000000, 10000000),
416 'player': 'twitchweb',
417 'segment_preference': '4',
418 'sig': access_token['sig'].encode('utf-8'),
419 'token': access_token['token'].encode('utf-8'),
420 }
421 formats = self._extract_m3u8_formats(
422 '%s/api/channel/hls/%s.m3u8?%s'
423 % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),
424 channel_id, 'mp4')
425 self._prefer_source(formats)
426
427 view_count = stream.get('viewers')
428 timestamp = parse_iso8601(stream.get('created_at'))
429
430 channel = stream['channel']
431 title = self._live_title(channel.get('display_name') or channel.get('name'))
432 description = channel.get('status')
433
434 thumbnails = []
435 for thumbnail_key, thumbnail_url in stream['preview'].items():
436 m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
437 if not m:
438 continue
439 thumbnails.append({
440 'url': thumbnail_url,
441 'width': int(m.group('width')),
442 'height': int(m.group('height')),
443 })
444
445 return {
446 'id': compat_str(stream['_id']),
447 'display_id': channel_id,
448 'title': title,
449 'description': description,
450 'thumbnails': thumbnails,
451 'uploader': channel.get('display_name'),
452 'uploader_id': channel.get('name'),
453 'timestamp': timestamp,
454 'view_count': view_count,
455 'formats': formats,
456 'is_live': True,
457 }
458
459
460 class TwitchClipsIE(InfoExtractor):
461 IE_NAME = 'twitch:clips'
462 _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
463
464 _TESTS = [{
465 'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
466 'md5': '761769e1eafce0ffebfb4089cb3847cd',
467 'info_dict': {
468 'id': 'AggressiveCobraPoooound',
469 'ext': 'mp4',
470 'title': 'EA Play 2016 Live from the Novo Theatre',
471 'thumbnail': 're:^https?://.*\.jpg',
472 'creator': 'EA',
473 'uploader': 'stereotype_',
474 'uploader_id': 'stereotype_',
475 },
476 }, {
477 # multiple formats
478 'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
479 'only_matching': True,
480 }]
481
482 def _real_extract(self, url):
483 video_id = self._match_id(url)
484
485 webpage = self._download_webpage(url, video_id)
486
487 clip = self._parse_json(
488 self._search_regex(
489 r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
490 video_id, transform_source=js_to_json)
491
492 title = clip.get('channel_title') or self._og_search_title(webpage)
493
494 formats = [{
495 'url': option['source'],
496 'format_id': option.get('quality'),
497 'height': int_or_none(option.get('quality')),
498 } for option in clip.get('quality_options', []) if option.get('source')]
499
500 if not formats:
501 formats = [{
502 'url': clip['clip_video_url'],
503 }]
504
505 self._sort_formats(formats)
506
507 return {
508 'id': video_id,
509 'title': title,
510 'thumbnail': self._og_search_thumbnail(webpage),
511 'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
512 'uploader': clip.get('curator_login'),
513 'uploader_id': clip.get('curator_display_name'),
514 'formats': formats,
515 }