New upstream version 2018.01.27

[youtubedl] / youtube_dl / extractor / twitch.py

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 69882da6337bc6f20d0578f6d612e799429db3e8..1981b4d4a8064541c0fcf2faa04d863c72eebb14 100644 (file)
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -7,28 +7,33 @@ import random
 
 from .common import InfoExtractor
 from ..compat import (
 
 from .common import InfoExtractor
 from ..compat import (

+    compat_HTTPError,

     compat_parse_qs,
     compat_str,
     compat_parse_qs,
     compat_str,

-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,

     compat_urllib_parse_urlparse,
     compat_urllib_parse_urlparse,

-    compat_urlparse,

 )
 from ..utils import (
 )
 from ..utils import (

-    encode_dict,
+    clean_html,

     ExtractorError,
     int_or_none,
     ExtractorError,
     int_or_none,

+    js_to_json,
+    orderedSet,

     parse_duration,
     parse_iso8601,
     parse_duration,
     parse_iso8601,

-    sanitized_Request,
+    update_url_query,
+    urlencode_postdata,
+    urljoin,

 )
 
 
 class TwitchBaseIE(InfoExtractor):
 )
 
 
 class TwitchBaseIE(InfoExtractor):

-    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
+    _VALID_URL_BASE = r'https?://(?:(?:www|go)\.)?twitch\.tv'

 
     _API_BASE = 'https://api.twitch.tv'
 
     _API_BASE = 'https://api.twitch.tv'

-    _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'http://www.twitch.tv/login'
+    _USHER_BASE = 'https://usher.ttvnw.net'
+    _LOGIN_URL = 'https://www.twitch.tv/login'
+    _CLIENT_ID = 'jzkbprff40iqj646a697cyrvl0zt2m6'

     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):


@@ -40,16 +45,10 @@ class TwitchBaseIE(InfoExtractor):
                 '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                 expected=True)
 
                 '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
                 expected=True)
 

-    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        headers = {
-            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
-            'X-Requested-With': 'XMLHttpRequest',
-        }
-        for cookie in self._downloader.cookiejar:
-            if cookie.name == 'api_token':
-                headers['Twitch-Api-Token'] = cookie.value
-        request = sanitized_Request(url, headers=headers)
-        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
+    def _call_api(self, path, item_id, note):
+        response = self._download_json(
+            '%s/%s' % (self._API_BASE, path), item_id, note,
+            headers={'Client-ID': self._CLIENT_ID})

         self._handle_error(response)
         return response
 
         self._handle_error(response)
         return response
 


@@ -61,40 +60,68 @@ class TwitchBaseIE(InfoExtractor):
         if username is None:
             return
 
         if username is None:
             return
 

+        def fail(message):
+            raise ExtractorError(
+                'Unable to login. Twitch said: %s' % message, expected=True)
+
+        def login_step(page, urlh, note, data):
+            form = self._hidden_inputs(page)
+            form.update(data)
+
+            page_url = urlh.geturl()
+            post_url = self._search_regex(
+                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
+                'post url', default=page_url, group='url')
+            post_url = urljoin(page_url, post_url)
+
+            headers = {'Referer': page_url}
+
+            try:
+                response = self._download_json(
+                    post_url, None, note,
+                    data=urlencode_postdata(form),
+                    headers=headers)
+            except ExtractorError as e:
+                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                    response = self._parse_json(
+                        e.cause.read().decode('utf-8'), None)
+                    fail(response.get('message') or response['errors'][0])
+                raise
+
+            if 'Authenticated successfully' in response.get('message', ''):
+                return None, None
+
+            redirect_url = urljoin(
+                post_url,
+                response.get('redirect') or response['redirect_path'])
+            return self._download_webpage_handle(
+                redirect_url, None, 'Downloading login redirect page',
+                headers=headers)
+

         login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
 
         login_page, handle = self._download_webpage_handle(
             self._LOGIN_URL, None, 'Downloading login page')
 

-        login_form = self._hidden_inputs(login_page)
-
-        login_form.update({
-            'username': username,
-            'password': password,
-        })
+        # Some TOR nodes and public proxies are blocked completely
+        if 'blacklist_message' in login_page:
+            fail(clean_html(login_page))

 
 

-        redirect_url = handle.geturl()
-
-        post_url = self._search_regex(
-            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
-            'post url', default=redirect_url, group='url')
-
-        if not post_url.startswith('http'):
-            post_url = compat_urlparse.urljoin(redirect_url, post_url)
-
-        request = sanitized_Request(
-            post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
-        request.add_header('Referer', redirect_url)
-        response = self._download_webpage(
-            request, None, 'Logging in as %s' % username)
+        redirect_page, handle = login_step(
+            login_page, handle, 'Logging in', {
+                'username': username,
+                'password': password,
+            })

 
 

-        error_message = self._search_regex(
-            r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
-            response, 'error message', default=None)
-        if error_message:
-            raise ExtractorError(
-                'Unable to login. Twitch said: %s' % error_message, expected=True)
+        # Successful login
+        if not redirect_page:
+            return

 
 

-        if '>Reset your password<' in response:
-            self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
+        if re.search(r'(?i)<form[^>]+id="two-factor-submit"', redirect_page) is not None:
+            # TODO: Add mechanism to request an SMS or phone call
+            tfa_token = self._get_tfa_info('two-factor authentication token')
+            login_step(redirect_page, handle, 'Submitting TFA token', {
+                'authy_token': tfa_token,
+                'remember_2fa': 'true',
+            })

 
     def _prefer_source(self, formats):
         try:
 
     def _prefer_source(self, formats):
         try:


@@ -107,14 +134,14 @@ class TwitchBaseIE(InfoExtractor):
 
 class TwitchItemBaseIE(TwitchBaseIE):
     def _download_info(self, item, item_id):
 
 class TwitchItemBaseIE(TwitchBaseIE):
     def _download_info(self, item, item_id):

-        return self._extract_info(self._download_json(
-            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+        return self._extract_info(self._call_api(
+            'kraken/videos/%s%s' % (item, item_id), item_id,

             'Downloading %s info JSON' % self._ITEM_TYPE))
 
     def _extract_media(self, item_id):
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
             'Downloading %s info JSON' % self._ITEM_TYPE))
 
     def _extract_media(self, item_id):
         info = self._download_info(self._ITEM_SHORTCUT, item_id)

-        response = self._download_json(
-            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
+        response = self._call_api(
+            'api/videos/%s%s' % (self._ITEM_SHORTCUT, item_id), item_id,

             'Downloading %s playlist JSON' % self._ITEM_TYPE)
         entries = []
         chunks = response['chunks']
             'Downloading %s playlist JSON' % self._ITEM_TYPE)
         entries = []
         chunks = response['chunks']


@@ -170,6 +197,7 @@ class TwitchVideoIE(TwitchItemBaseIE):
             'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
         },
         'playlist_mincount': 12,
             'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
         },
         'playlist_mincount': 12,

+        'skip': 'HTTP Error 404: Not Found',

     }
 
 
     }
 
 


@@ -186,6 +214,7 @@ class TwitchChapterIE(TwitchItemBaseIE):
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,

+        'skip': 'HTTP Error 404: Not Found',

     }, {
         'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
         'only_matching': True,
     }, {
         'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
         'only_matching': True,


@@ -194,7 +223,14 @@ class TwitchChapterIE(TwitchItemBaseIE):
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'

-    _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:(?:www|go)\.)?twitch\.tv/(?:[^/]+/v|videos)/|
+                            player\.twitch\.tv/\?.*?\bvideo=v
+                        )
+                        (?P<id>\d+)
+                    '''

     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 


@@ -204,7 +240,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             'id': 'v6528877',
             'ext': 'mp4',
             'title': 'LCK Summer Split - Week 6 Day 1',
             'id': 'v6528877',
             'ext': 'mp4',
             'title': 'LCK Summer Split - Week 6 Day 1',

-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',

             'duration': 17208,
             'timestamp': 1435131709,
             'upload_date': '20150624',
             'duration': 17208,
             'timestamp': 1435131709,
             'upload_date': '20150624',


@@ -224,7 +260,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             'id': 'v11230755',
             'ext': 'mp4',
             'title': 'Untitled Broadcast',
             'id': 'v11230755',
             'ext': 'mp4',
             'title': 'Untitled Broadcast',

-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.jpg$',

             'duration': 1638,
             'timestamp': 1439746708,
             'upload_date': '20150816',
             'duration': 1638,
             'timestamp': 1439746708,
             'upload_date': '20150816',


@@ -236,27 +272,35 @@ class TwitchVodIE(TwitchItemBaseIE):
             # m3u8 download
             'skip_download': True,
         },
             # m3u8 download
             'skip_download': True,
         },

+        'skip': 'HTTP Error 404: Not Found',
+    }, {
+        'url': 'http://player.twitch.tv/?t=5m10s&video=v6528877',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.twitch.tv/videos/6528877',
+        'only_matching': True,

     }]
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
 
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
     }]
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
 
         info = self._download_info(self._ITEM_SHORTCUT, item_id)

-        access_token = self._download_json(
-            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+        access_token = self._call_api(
+            'api/vods/%s/access_token' % item_id, item_id,

             'Downloading %s access token' % self._ITEM_TYPE)
 
         formats = self._extract_m3u8_formats(
             '%s/vod/%s?%s' % (
                 self._USHER_BASE, item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
 
         formats = self._extract_m3u8_formats(
             '%s/vod/%s?%s' % (
                 self._USHER_BASE, item_id,

-                compat_urllib_parse.urlencode({
+                compat_urllib_parse_urlencode({

                     'allow_source': 'true',
                     'allow_source': 'true',

+                    'allow_audio_only': 'true',

                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'nauth': access_token['token'],
                     'nauthsig': access_token['sig'],
                 })),
                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'nauth': access_token['token'],
                     'nauthsig': access_token['sig'],
                 })),

-            item_id, 'mp4')
+            item_id, 'mp4', entry_protocol='m3u8_native')

 
         self._prefer_source(formats)
         info['formats'] = formats
 
         self._prefer_source(formats)
         info['formats'] = formats


@@ -266,34 +310,73 @@ class TwitchVodIE(TwitchItemBaseIE):
         if 't' in query:
             info['start_time'] = parse_duration(query['t'][0])
 
         if 't' in query:
             info['start_time'] = parse_duration(query['t'][0])
 

+        if info.get('timestamp') is not None:
+            info['subtitles'] = {
+                'rechat': [{
+                    'url': update_url_query(
+                        'https://rechat.twitch.tv/rechat-messages', {
+                            'video_id': 'v%s' % item_id,
+                            'start': info['timestamp'],
+                        }),
+                    'ext': 'json',
+                }],
+            }
+

         return info
 
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):
         return info
 
 
 class TwitchPlaylistBaseIE(TwitchBaseIE):

-    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PLAYLIST_PATH = 'kraken/channels/%s/videos/?offset=%d&limit=%d'

     _PAGE_LIMIT = 100
 
     def _extract_playlist(self, channel_id):
     _PAGE_LIMIT = 100
 
     def _extract_playlist(self, channel_id):

-        info = self._download_json(
-            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
+        info = self._call_api(
+            'kraken/channels/%s' % channel_id,

             channel_id, 'Downloading channel info JSON')
         channel_name = info.get('display_name') or info.get('name')
         entries = []
         offset = 0
         limit = self._PAGE_LIMIT
             channel_id, 'Downloading channel info JSON')
         channel_name = info.get('display_name') or info.get('name')
         entries = []
         offset = 0
         limit = self._PAGE_LIMIT

+        broken_paging_detected = False
+        counter_override = None

         for counter in itertools.count(1):
         for counter in itertools.count(1):

-            response = self._download_json(
-                self._PLAYLIST_URL % (channel_id, offset, limit),
-                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+            response = self._call_api(
+                self._PLAYLIST_PATH % (channel_id, offset, limit),
+                channel_id,
+                'Downloading %s JSON page %s'
+                % (self._PLAYLIST_TYPE, counter_override or counter))

             page_entries = self._extract_playlist_page(response)
             if not page_entries:
                 break
             page_entries = self._extract_playlist_page(response)
             if not page_entries:
                 break

+            total = int_or_none(response.get('_total'))
+            # Since the beginning of March 2016 twitch's paging mechanism
+            # is completely broken on the twitch side. It simply ignores
+            # a limit and returns the whole offset number of videos.
+            # Working around by just requesting all videos at once.
+            # Upd: pagination bug was fixed by twitch on 15.03.2016.
+            if not broken_paging_detected and total and len(page_entries) > limit:
+                self.report_warning(
+                    'Twitch pagination is broken on twitch side, requesting all videos at once',
+                    channel_id)
+                broken_paging_detected = True
+                offset = total
+                counter_override = '(all at once)'
+                continue

             entries.extend(page_entries)
             entries.extend(page_entries)

+            if broken_paging_detected or total and len(page_entries) >= total:
+                break

             offset += limit
         return self.playlist_result(
             offset += limit
         return self.playlist_result(

-            [self.url_result(entry) for entry in set(entries)],
+            [self._make_url_result(entry) for entry in orderedSet(entries)],

             channel_id, channel_name)
 
             channel_id, channel_name)
 

+    def _make_url_result(self, url):
+        try:
+            video_id = 'v%s' % TwitchVodIE._match_id(url)
+            return self.url_result(url, TwitchVodIE.ie_key(), video_id=video_id)
+        except AssertionError:
+            return self.url_result(url)
+

     def _extract_playlist_page(self, response):
         videos = response.get('videos')
         return [video['url'] for video in videos] if videos else []
     def _extract_playlist_page(self, response):
         videos = response.get('videos')
         return [video['url'] for video in videos] if videos else []


@@ -317,50 +400,85 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
     }
 
 
     }
 
 

-class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:past_broadcasts'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
-    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
-    _PLAYLIST_TYPE = 'past broadcasts'
+class TwitchVideosBaseIE(TwitchPlaylistBaseIE):
+    _VALID_URL_VIDEOS_BASE = r'%s/(?P<id>[^/]+)/videos' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_PATH = TwitchPlaylistBaseIE._PLAYLIST_PATH + '&broadcast_type='
+
+
+class TwitchAllVideosIE(TwitchVideosBaseIE):
+    IE_NAME = 'twitch:videos:all'
+    _VALID_URL = r'%s/all' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
+    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'archive,upload,highlight'
+    _PLAYLIST_TYPE = 'all videos'

 
     _TEST = {
 
     _TEST = {

-        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
+        'url': 'https://www.twitch.tv/spamfish/videos/all',

         'info_dict': {
             'id': 'spamfish',
             'title': 'Spamfish',
         },
         'info_dict': {
             'id': 'spamfish',
             'title': 'Spamfish',
         },

-        'playlist_mincount': 54,
+        'playlist_mincount': 869,

     }
 
 
     }
 
 

-class TwitchBookmarksIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:bookmarks'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
-    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PLAYLIST_TYPE = 'bookmarks'
+class TwitchUploadsIE(TwitchVideosBaseIE):
+    IE_NAME = 'twitch:videos:uploads'
+    _VALID_URL = r'%s/uploads' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
+    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'upload'
+    _PLAYLIST_TYPE = 'uploads'

 
     _TEST = {
 
     _TEST = {

-        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
+        'url': 'https://www.twitch.tv/spamfish/videos/uploads',

         'info_dict': {
         'info_dict': {

-            'id': 'ognos',
-            'title': 'Ognos',
+            'id': 'spamfish',
+            'title': 'Spamfish',

         },
         },

-        'playlist_mincount': 3,
+        'playlist_mincount': 0,

     }
 
     }
 

-    def _extract_playlist_page(self, response):
-        entries = []
-        for bookmark in response.get('bookmarks', []):
-            video = bookmark.get('video')
-            if not video:
-                continue
-            entries.append(video['url'])
-        return entries
+
+class TwitchPastBroadcastsIE(TwitchVideosBaseIE):
+    IE_NAME = 'twitch:videos:past-broadcasts'
+    _VALID_URL = r'%s/past-broadcasts' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
+    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'archive'
+    _PLAYLIST_TYPE = 'past broadcasts'
+
+    _TEST = {
+        'url': 'https://www.twitch.tv/spamfish/videos/past-broadcasts',
+        'info_dict': {
+            'id': 'spamfish',
+            'title': 'Spamfish',
+        },
+        'playlist_mincount': 0,
+    }
+
+
+class TwitchHighlightsIE(TwitchVideosBaseIE):
+    IE_NAME = 'twitch:videos:highlights'
+    _VALID_URL = r'%s/highlights' % TwitchVideosBaseIE._VALID_URL_VIDEOS_BASE
+    _PLAYLIST_PATH = TwitchVideosBaseIE._PLAYLIST_PATH + 'highlight'
+    _PLAYLIST_TYPE = 'highlights'
+
+    _TEST = {
+        'url': 'https://www.twitch.tv/spamfish/videos/highlights',
+        'info_dict': {
+            'id': 'spamfish',
+            'title': 'Spamfish',
+        },
+        'playlist_mincount': 805,
+    }

 
 
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
 
 
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'

-    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:(?:www|go)\.)?twitch\.tv/|
+                            player\.twitch\.tv/\?.*?\bchannel=
+                        )
+                        (?P<id>[^/#?]+)
+                    '''

 
     _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',
 
     _TESTS = [{
         'url': 'http://www.twitch.tv/shroomztv',


@@ -384,20 +502,37 @@ class TwitchStreamIE(TwitchBaseIE):
     }, {
         'url': 'http://www.twitch.tv/miracle_doto#profile-0',
         'only_matching': True,
     }, {
         'url': 'http://www.twitch.tv/miracle_doto#profile-0',
         'only_matching': True,

+    }, {
+        'url': 'https://player.twitch.tv/?channel=lotsofs',
+        'only_matching': True,
+    }, {
+        'url': 'https://go.twitch.tv/food',
+        'only_matching': True,

     }]
 
     }]
 

+    @classmethod
+    def suitable(cls, url):
+        return (False
+                if any(ie.suitable(url) for ie in (
+                    TwitchVideoIE,
+                    TwitchChapterIE,
+                    TwitchVodIE,
+                    TwitchProfileIE,
+                    TwitchAllVideosIE,
+                    TwitchUploadsIE,
+                    TwitchPastBroadcastsIE,
+                    TwitchHighlightsIE))
+                else super(TwitchStreamIE, cls).suitable(url))
+

     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 

-        stream = self._download_json(
-            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
+        stream = self._call_api(
+            'kraken/streams/%s?stream_type=all' % channel_id, channel_id,

             'Downloading stream JSON').get('stream')
 
             'Downloading stream JSON').get('stream')
 

-        # Fallback on profile extraction if stream is offline

         if not stream:
         if not stream:

-            return self.url_result(
-                'http://www.twitch.tv/%s/profile' % channel_id,
-                'TwitchProfile', channel_id)
+            raise ExtractorError('%s is offline' % channel_id, expected=True)

 
         # Channel name may be typed if different case than the original channel name
         # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
 
         # Channel name may be typed if different case than the original channel name
         # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing


@@ -405,12 +540,14 @@ class TwitchStreamIE(TwitchBaseIE):
         # JSON and fallback to lowercase if it's not available.
         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
 
         # JSON and fallback to lowercase if it's not available.
         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
 

-        access_token = self._download_json(
-            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
+        access_token = self._call_api(
+            'api/channels/%s/access_token' % channel_id, channel_id,

             'Downloading channel access token')
 
         query = {
             'allow_source': 'true',
             'Downloading channel access token')
 
         query = {
             'allow_source': 'true',

+            'allow_audio_only': 'true',
+            'allow_spectre': 'true',

             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',


@@ -419,7 +556,7 @@ class TwitchStreamIE(TwitchBaseIE):
         }
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'
         }
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'

-            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
+            % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),

             channel_id, 'mp4')
         self._prefer_source(formats)
 
             channel_id, 'mp4')
         self._prefer_source(formats)
 


@@ -454,3 +591,61 @@ class TwitchStreamIE(TwitchBaseIE):
             'formats': formats,
             'is_live': True,
         }
             'formats': formats,
             'is_live': True,
         }

+
+
+class TwitchClipsIE(InfoExtractor):
+    IE_NAME = 'twitch:clips'
+    _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
+        'md5': '761769e1eafce0ffebfb4089cb3847cd',
+        'info_dict': {
+            'id': 'AggressiveCobraPoooound',
+            'ext': 'mp4',
+            'title': 'EA Play 2016 Live from the Novo Theatre',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'creator': 'EA',
+            'uploader': 'stereotype_',
+            'uploader_id': 'stereotype_',
+        },
+    }, {
+        # multiple formats
+        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        clip = self._parse_json(
+            self._search_regex(
+                r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
+            video_id, transform_source=js_to_json)
+
+        title = clip.get('title') or clip.get('channel_title') or self._og_search_title(webpage)
+
+        formats = [{
+            'url': option['source'],
+            'format_id': option.get('quality'),
+            'height': int_or_none(option.get('quality')),
+        } for option in clip.get('quality_options', []) if option.get('source')]
+
+        if not formats:
+            formats = [{
+                'url': clip['clip_video_url'],
+            }]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
+            'uploader': clip.get('curator_login'),
+            'uploader_id': clip.get('curator_display_name'),
+            'formats': formats,
+        }