2 from __future__
import unicode_literals
8 from .common
import InfoExtractor
12 compat_urllib_parse_urlencode
,
13 compat_urllib_parse_urlparse
,
28 class TwitchBaseIE(InfoExtractor
):
29 _VALID_URL_BASE
= r
'https?://(?:www\.)?twitch\.tv'
31 _API_BASE
= 'https://api.twitch.tv'
32 _USHER_BASE
= 'https://usher.ttvnw.net'
33 _LOGIN_URL
= 'http://www.twitch.tv/login'
34 _NETRC_MACHINE
= 'twitch'
36 def _handle_error(self
, response
):
37 if not isinstance(response
, dict):
39 error
= response
.get('error')
42 '%s returned error: %s - %s' % (self
.IE_NAME
, error
, response
.get('message')),
45 def _download_json(self
, url
, video_id
, note
='Downloading JSON metadata'):
47 'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
48 'X-Requested-With': 'XMLHttpRequest',
50 for cookie
in self
._downloader
.cookiejar
:
51 if cookie
.name
== 'api_token':
52 headers
['Twitch-Api-Token'] = cookie
.value
53 request
= sanitized_Request(url
, headers
=headers
)
54 response
= super(TwitchBaseIE
, self
)._download
_json
(request
, video_id
, note
)
55 self
._handle
_error
(response
)
58 def _real_initialize(self
):
62 (username
, password
) = self
._get
_login
_info
()
66 login_page
, handle
= self
._download
_webpage
_handle
(
67 self
._LOGIN
_URL
, None, 'Downloading login page')
69 login_form
= self
._hidden
_inputs
(login_page
)
76 redirect_url
= handle
.geturl()
78 post_url
= self
._search
_regex
(
79 r
'<form[^>]+action=(["\'])(?P
<url
>.+?
)\
1', login_page,
80 'post url
', default=redirect_url, group='url
')
82 if not post_url.startswith('http
'):
83 post_url = compat_urlparse.urljoin(redirect_url, post_url)
85 request = sanitized_Request(
86 post_url, urlencode_postdata(login_form))
87 request.add_header('Referer
', redirect_url)
88 response = self._download_webpage(
89 request, None, 'Logging
in as %s' % username)
91 error_message = self._search_regex(
92 r'<div
[^
>]+class="subwindow_notice"[^
>]*>([^
<]+)</div
>',
93 response, 'error message
', default=None)
96 'Unable to login
. Twitch said
: %s' % error_message, expected=True)
98 if '>Reset your password
<' in response:
99 self.report_warning('Twitch asks you to reset your password
, go to https
://secure
.twitch
.tv
/reset
/submit
')
101 def _prefer_source(self, formats):
103 source = next(f for f in formats if f['format_id
'] == 'Source
')
104 source['preference
'] = 10
105 except StopIteration:
106 pass # No Source stream present
107 self._sort_formats(formats)
110 class TwitchItemBaseIE(TwitchBaseIE):
111 def _download_info(self, item, item_id):
112 return self._extract_info(self._download_json(
113 '%s/kraken
/videos
/%s%s' % (self._API_BASE, item, item_id), item_id,
114 'Downloading
%s info JSON
' % self._ITEM_TYPE))
116 def _extract_media(self, item_id):
117 info = self._download_info(self._ITEM_SHORTCUT, item_id)
118 response = self._download_json(
119 '%s/api
/videos
/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
120 'Downloading
%s playlist JSON
' % self._ITEM_TYPE)
122 chunks = response['chunks
']
123 qualities = list(chunks.keys())
124 for num, fragment in enumerate(zip(*chunks.values()), start=1):
126 for fmt_num, fragment_fmt in enumerate(fragment):
127 format_id = qualities[fmt_num]
129 'url
': fragment_fmt['url
'],
130 'format_id
': format_id,
131 'quality
': 1 if format_id == 'live
' else 0,
133 m = re.search(r'^
(?P
<height
>\d
+)[Pp
]', format_id)
135 fmt['height
'] = int(m.group('height
'))
137 self._sort_formats(formats)
139 entry['id'] = '%s_%d' % (entry['id'], num)
140 entry['title
'] = '%s part
%d' % (entry['title
'], num)
141 entry['formats
'] = formats
142 entries.append(entry)
143 return self.playlist_result(entries, info['id'], info['title
'])
145 def _extract_info(self, info):
148 'title
': info.get('title
') or 'Untitled Broadcast
',
149 'description
': info.get('description
'),
150 'duration
': int_or_none(info.get('length
')),
151 'thumbnail
': info.get('preview
'),
152 'uploader
': info.get('channel
', {}).get('display_name
'),
153 'uploader_id
': info.get('channel
', {}).get('name
'),
154 'timestamp
': parse_iso8601(info.get('recorded_at
')),
155 'view_count
': int_or_none(info.get('views
')),
158 def _real_extract(self, url):
159 return self._extract_media(self._match_id(url))
162 class TwitchVideoIE(TwitchItemBaseIE):
163 IE_NAME = 'twitch
:video
'
164 _VALID_URL = r'%s/[^
/]+/b
/(?P
<id>\d
+)' % TwitchBaseIE._VALID_URL_BASE
169 'url
': 'http
://www
.twitch
.tv
/riotgames
/b
/577357806',
172 'title
': 'Worlds Semifinals
- Star Horn Royal Club vs
. OMG
',
174 'playlist_mincount
': 12,
175 'skip
': 'HTTP Error
404: Not Found
',
179 class TwitchChapterIE(TwitchItemBaseIE):
180 IE_NAME = 'twitch
:chapter
'
181 _VALID_URL = r'%s/[^
/]+/c
/(?P
<id>\d
+)' % TwitchBaseIE._VALID_URL_BASE
182 _ITEM_TYPE = 'chapter
'
186 'url
': 'http
://www
.twitch
.tv
/acracingleague
/c
/5285812',
189 'title
': 'ACRL Off Season
- Sports Cars
@ Nordschleife
',
191 'playlist_mincount
': 3,
192 'skip
': 'HTTP Error
404: Not Found
',
194 'url
': 'http
://www
.twitch
.tv
/tsm_theoddone
/c
/2349361',
195 'only_matching
': True,
199 class TwitchVodIE(TwitchItemBaseIE):
200 IE_NAME = 'twitch
:vod
'
201 _VALID_URL = r'%s/[^
/]+/v
/(?P
<id>\d
+)' % TwitchBaseIE._VALID_URL_BASE
206 'url
': 'http
://www
.twitch
.tv
/riotgames
/v
/6528877?t
=5m10s
',
210 'title
': 'LCK Summer Split
- Week
6 Day
1',
211 'thumbnail
': 're
:^https?
://.*\
.jpg$
',
213 'timestamp
': 1435131709,
214 'upload_date
': '20150624',
215 'uploader
': 'Riot Games
',
216 'uploader_id
': 'riotgames
',
222 'skip_download
': True,
225 # Untitled broadcast (title is None)
226 'url
': 'http
://www
.twitch
.tv
/belkao_o
/v
/11230755',
230 'title
': 'Untitled Broadcast
',
231 'thumbnail
': 're
:^https?
://.*\
.jpg$
',
233 'timestamp
': 1439746708,
234 'upload_date
': '20150816',
235 'uploader
': 'BelkAO_o
',
236 'uploader_id
': 'belkao_o
',
241 'skip_download
': True,
245 def _real_extract(self, url):
246 item_id = self._match_id(url)
248 info = self._download_info(self._ITEM_SHORTCUT, item_id)
249 access_token = self._download_json(
250 '%s/api
/vods
/%s/access_token
' % (self._API_BASE, item_id), item_id,
251 'Downloading
%s access token
' % self._ITEM_TYPE)
253 formats = self._extract_m3u8_formats(
255 self._USHER_BASE, item_id,
256 compat_urllib_parse_urlencode({
257 'allow_source
': 'true
',
258 'allow_audio_only
': 'true
',
259 'allow_spectre
': 'true
',
260 'player
': 'twitchweb
',
261 'nauth
': access_token['token
'],
262 'nauthsig
': access_token['sig
'],
264 item_id, 'mp4
', entry_protocol='m3u8_native
')
266 self._prefer_source(formats)
267 info['formats
'] = formats
269 parsed_url = compat_urllib_parse_urlparse(url)
270 query = compat_parse_qs(parsed_url.query)
272 info['start_time
'] = parse_duration(query['t
'][0])
277 class TwitchPlaylistBaseIE(TwitchBaseIE):
278 _PLAYLIST_URL = '%s/kraken
/channels
/%%s/videos
/?offset
=%%d&limit
=%%d' % TwitchBaseIE._API_BASE
281 def _extract_playlist(self, channel_id):
282 info = self._download_json(
283 '%s/kraken
/channels
/%s' % (self._API_BASE, channel_id),
284 channel_id, 'Downloading channel info JSON
')
285 channel_name = info.get('display_name
') or info.get('name
')
288 limit = self._PAGE_LIMIT
289 broken_paging_detected = False
290 counter_override = None
291 for counter in itertools.count(1):
292 response = self._download_json(
293 self._PLAYLIST_URL % (channel_id, offset, limit),
295 'Downloading
%s videos JSON page
%s'
296 % (self._PLAYLIST_TYPE, counter_override or counter))
297 page_entries = self._extract_playlist_page(response)
300 total = int_or_none(response.get('_total
'))
301 # Since the beginning of March 2016 twitch's paging mechanism
302 # is completely broken on the twitch side. It simply ignores
303 # a limit and returns the whole offset number of videos.
304 # Working around by just requesting all videos at once.
305 # Upd: pagination bug was fixed by twitch on 15.03.2016.
306 if not broken_paging_detected
and total
and len(page_entries
) > limit
:
308 'Twitch pagination is broken on twitch side, requesting all videos at once',
310 broken_paging_detected
= True
312 counter_override
= '(all at once)'
314 entries
.extend(page_entries
)
315 if broken_paging_detected
or total
and len(page_entries
) >= total
:
318 return self
.playlist_result(
319 [self
.url_result(entry
) for entry
in orderedSet(entries
)],
320 channel_id
, channel_name
)
322 def _extract_playlist_page(self
, response
):
323 videos
= response
.get('videos')
324 return [video
['url'] for video
in videos
] if videos
else []
326 def _real_extract(self
, url
):
327 return self
._extract
_playlist
(self
._match
_id
(url
))
330 class TwitchProfileIE(TwitchPlaylistBaseIE
):
331 IE_NAME
= 'twitch:profile'
332 _VALID_URL
= r
'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE
._VALID
_URL
_BASE
333 _PLAYLIST_TYPE
= 'profile'
336 'url': 'http://www.twitch.tv/vanillatv/profile',
339 'title': 'VanillaTV',
341 'playlist_mincount': 412,
345 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE
):
346 IE_NAME
= 'twitch:past_broadcasts'
347 _VALID_URL
= r
'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE
._VALID
_URL
_BASE
348 _PLAYLIST_URL
= TwitchPlaylistBaseIE
._PLAYLIST
_URL
+ '&broadcasts=true'
349 _PLAYLIST_TYPE
= 'past broadcasts'
352 'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
357 'playlist_mincount': 54,
361 class TwitchStreamIE(TwitchBaseIE
):
362 IE_NAME
= 'twitch:stream'
363 _VALID_URL
= r
'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE
._VALID
_URL
_BASE
366 'url': 'http://www.twitch.tv/shroomztv',
369 'display_id': 'shroomztv',
371 'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
372 'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
374 'timestamp': 1421928037,
375 'upload_date': '20150122',
376 'uploader': 'ShroomzTV',
377 'uploader_id': 'shroomztv',
382 'skip_download': True,
385 'url': 'http://www.twitch.tv/miracle_doto#profile-0',
386 'only_matching': True,
389 def _real_extract(self
, url
):
390 channel_id
= self
._match
_id
(url
)
392 stream
= self
._download
_json
(
393 '%s/kraken/streams/%s' % (self
._API
_BASE
, channel_id
), channel_id
,
394 'Downloading stream JSON').get('stream')
396 # Fallback on profile extraction if stream is offline
398 return self
.url_result(
399 'http://www.twitch.tv/%s/profile' % channel_id
,
400 'TwitchProfile', channel_id
)
402 # Channel name may be typed if different case than the original channel name
403 # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
404 # an invalid m3u8 URL. Working around by use of original channel name from stream
405 # JSON and fallback to lowercase if it's not available.
406 channel_id
= stream
.get('channel', {}).get('name') or channel_id
.lower()
408 access_token
= self
._download
_json
(
409 '%s/api/channels/%s/access_token' % (self
._API
_BASE
, channel_id
), channel_id
,
410 'Downloading channel access token')
413 'allow_source': 'true',
414 'allow_audio_only': 'true',
415 'p': random
.randint(1000000, 10000000),
416 'player': 'twitchweb',
417 'segment_preference': '4',
418 'sig': access_token
['sig'].encode('utf-8'),
419 'token': access_token
['token'].encode('utf-8'),
421 formats
= self
._extract
_m
3u8_formats
(
422 '%s/api/channel/hls/%s.m3u8?%s'
423 % (self
._USHER
_BASE
, channel_id
, compat_urllib_parse_urlencode(query
)),
425 self
._prefer
_source
(formats
)
427 view_count
= stream
.get('viewers')
428 timestamp
= parse_iso8601(stream
.get('created_at'))
430 channel
= stream
['channel']
431 title
= self
._live
_title
(channel
.get('display_name') or channel
.get('name'))
432 description
= channel
.get('status')
435 for thumbnail_key
, thumbnail_url
in stream
['preview'].items():
436 m
= re
.search(r
'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key
)
440 'url': thumbnail_url
,
441 'width': int(m
.group('width')),
442 'height': int(m
.group('height')),
446 'id': compat_str(stream
['_id']),
447 'display_id': channel_id
,
449 'description': description
,
450 'thumbnails': thumbnails
,
451 'uploader': channel
.get('display_name'),
452 'uploader_id': channel
.get('name'),
453 'timestamp': timestamp
,
454 'view_count': view_count
,
460 class TwitchClipsIE(InfoExtractor
):
461 IE_NAME
= 'twitch:clips'
462 _VALID_URL
= r
'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
465 'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
466 'md5': '761769e1eafce0ffebfb4089cb3847cd',
468 'id': 'AggressiveCobraPoooound',
470 'title': 'EA Play 2016 Live from the Novo Theatre',
471 'thumbnail': 're:^https?://.*\.jpg',
473 'uploader': 'stereotype_',
474 'uploader_id': 'stereotype_',
478 'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
479 'only_matching': True,
482 def _real_extract(self
, url
):
483 video_id
= self
._match
_id
(url
)
485 webpage
= self
._download
_webpage
(url
, video_id
)
487 clip
= self
._parse
_json
(
489 r
'(?s)clipInfo\s*=\s*({.+?});', webpage
, 'clip info'),
490 video_id
, transform_source
=js_to_json
)
492 title
= clip
.get('channel_title') or self
._og
_search
_title
(webpage
)
495 'url': option
['source'],
496 'format_id': option
.get('quality'),
497 'height': int_or_none(option
.get('quality')),
498 } for option
in clip
.get('quality_options', []) if option
.get('source')]
502 'url': clip
['clip_video_url'],
505 self
._sort
_formats
(formats
)
510 'thumbnail': self
._og
_search
_thumbnail
(webpage
),
511 'creator': clip
.get('broadcaster_display_name') or clip
.get('broadcaster_login'),
512 'uploader': clip
.get('curator_login'),
513 'uploader_id': clip
.get('curator_display_name'),