Tidy up cleanup rule and keep lintian happy.

[youtubedl] / youtube_dl / extractor / twitch.py

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 87290d002e44850e6b3584a97ff2a3e1be7c1a0f..890f551800d011c423636530dfd8d70e4e926290 100644 (file)
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -7,13 +7,21 @@ import random
 
 from .common import InfoExtractor
 from ..compat import (
 
 from .common import InfoExtractor
 from ..compat import (

+    compat_parse_qs,

     compat_str,
     compat_str,

-    compat_urllib_parse,
-    compat_urllib_request,
+    compat_urllib_parse_urlencode,
+    compat_urllib_parse_urlparse,
+    compat_urlparse,

 )
 from ..utils import (
     ExtractorError,
 )
 from ..utils import (
     ExtractorError,

+    int_or_none,
+    js_to_json,
+    orderedSet,
+    parse_duration,

     parse_iso8601,
     parse_iso8601,

+    sanitized_Request,
+    urlencode_postdata,

 )
 
 
 )
 
 


@@ -21,8 +29,9 @@ class TwitchBaseIE(InfoExtractor):
     _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
     _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'

-    _USHER_BASE = 'http://usher.twitch.tv'
-    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _USHER_BASE = 'https://usher.ttvnw.net'
+    _LOGIN_URL = 'http://www.twitch.tv/login'
+    _NETRC_MACHINE = 'twitch'

 
     def _handle_error(self, response):
         if not isinstance(response, dict):
 
     def _handle_error(self, response):
         if not isinstance(response, dict):


@@ -34,7 +43,15 @@ class TwitchBaseIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):

-        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
+        headers = {
+            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+        for cookie in self._downloader.cookiejar:
+            if cookie.name == 'api_token':
+                headers['Twitch-Api-Token'] = cookie.value
+        request = sanitized_Request(url, headers=headers)
+        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)

         self._handle_error(response)
         return response
 
         self._handle_error(response)
         return response
 


@@ -46,35 +63,48 @@ class TwitchBaseIE(InfoExtractor):
         if username is None:
             return
 
         if username is None:
             return
 

-        login_page = self._download_webpage(
+        login_page, handle = self._download_webpage_handle(

             self._LOGIN_URL, None, 'Downloading login page')
 
             self._LOGIN_URL, None, 'Downloading login page')
 

-        authenticity_token = self._search_regex(
-            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
-            login_page, 'authenticity token')
-
-        login_form = {
-            'utf8': '✓'.encode('utf-8'),
-            'authenticity_token': authenticity_token,
-            'redirect_on_login': '',
-            'embed_form': 'false',
-            'mp_source_action': '',
-            'follow': '',
-            'user[login]': username,
-            'user[password]': password,
-        }
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'username': username,
+            'password': password,
+        })
+
+        redirect_url = handle.geturl()
+
+        post_url = self._search_regex(
+            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
+            'post url', default=redirect_url, group='url')

 
 

-        request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
-        request.add_header('Referer', self._LOGIN_URL)
+        if not post_url.startswith('http'):
+            post_url = compat_urlparse.urljoin(redirect_url, post_url)
+
+        request = sanitized_Request(
+            post_url, urlencode_postdata(login_form))
+        request.add_header('Referer', redirect_url)

         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)
 

-        m = re.search(
-            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
-        if m:
+        error_message = self._search_regex(
+            r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
+            response, 'error message', default=None)
+        if error_message:

             raise ExtractorError(
             raise ExtractorError(

-                'Unable to login: %s' % m.group('msg').strip(), expected=True)
+                'Unable to login. Twitch said: %s' % error_message, expected=True)
+
+        if '>Reset your password<' in response:
+            self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
+
+    def _prefer_source(self, formats):
+        try:
+            source = next(f for f in formats if f['format_id'] == 'Source')
+            source['preference'] = 10
+        except StopIteration:
+            pass  # No Source stream present
+        self._sort_formats(formats)

 
 
 class TwitchItemBaseIE(TwitchBaseIE):
 
 
 class TwitchItemBaseIE(TwitchBaseIE):


@@ -115,14 +145,14 @@ class TwitchItemBaseIE(TwitchBaseIE):
     def _extract_info(self, info):
         return {
             'id': info['_id'],
     def _extract_info(self, info):
         return {
             'id': info['_id'],

-            'title': info['title'],
-            'description': info['description'],
-            'duration': info['length'],
-            'thumbnail': info['preview'],
-            'uploader': info['channel']['display_name'],
-            'uploader_id': info['channel']['name'],
-            'timestamp': parse_iso8601(info['recorded_at']),
-            'view_count': info['views'],
+            'title': info.get('title') or 'Untitled Broadcast',
+            'description': info.get('description'),
+            'duration': int_or_none(info.get('length')),
+            'thumbnail': info.get('preview'),
+            'uploader': info.get('channel', {}).get('display_name'),
+            'uploader_id': info.get('channel', {}).get('name'),
+            'timestamp': parse_iso8601(info.get('recorded_at')),
+            'view_count': int_or_none(info.get('views')),

         }
 
     def _real_extract(self, url):
         }
 
     def _real_extract(self, url):


@@ -131,7 +161,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
 
 class TwitchVideoIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:video'
 
 class TwitchVideoIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:video'

-    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE

     _ITEM_TYPE = 'video'
     _ITEM_SHORTCUT = 'a'
 
     _ITEM_TYPE = 'video'
     _ITEM_SHORTCUT = 'a'
 


@@ -142,12 +172,13 @@ class TwitchVideoIE(TwitchItemBaseIE):
             'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
         },
         'playlist_mincount': 12,
             'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
         },
         'playlist_mincount': 12,

+        'skip': 'HTTP Error 404: Not Found',

     }
 
 
 class TwitchChapterIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:chapter'
     }
 
 
 class TwitchChapterIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:chapter'

-    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE

     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 


@@ -158,6 +189,7 @@ class TwitchChapterIE(TwitchItemBaseIE):
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,

+        'skip': 'HTTP Error 404: Not Found',

     }, {
         'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
         'only_matching': True,
     }, {
         'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
         'only_matching': True,


@@ -166,41 +198,79 @@ class TwitchChapterIE(TwitchItemBaseIE):
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'

-    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE

     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 

-    _TEST = {
-        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+    _TESTS = [{
+        'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',

         'info_dict': {
         'info_dict': {

-            'id': 'v3622000',
+            'id': 'v6528877',

             'ext': 'mp4',
             'ext': 'mp4',

-            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'title': 'LCK Summer Split - Week 6 Day 1',

             'thumbnail': 're:^https?://.*\.jpg$',
             'thumbnail': 're:^https?://.*\.jpg$',

-            'duration': 6951,
-            'timestamp': 1419028564,
-            'upload_date': '20141219',
-            'uploader': 'KSPTV',
-            'uploader_id': 'ksptv',
+            'duration': 17208,
+            'timestamp': 1435131709,
+            'upload_date': '20150624',
+            'uploader': 'Riot Games',
+            'uploader_id': 'riotgames',

             'view_count': int,
             'view_count': int,

+            'start_time': 310,

         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },

-    }
+    }, {
+        # Untitled broadcast (title is None)
+        'url': 'http://www.twitch.tv/belkao_o/v/11230755',
+        'info_dict': {
+            'id': 'v11230755',
+            'ext': 'mp4',
+            'title': 'Untitled Broadcast',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1638,
+            'timestamp': 1439746708,
+            'upload_date': '20150816',
+            'uploader': 'BelkAO_o',
+            'uploader_id': 'belkao_o',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]

 
     def _real_extract(self, url):
         item_id = self._match_id(url)
 
     def _real_extract(self, url):
         item_id = self._match_id(url)

+

         info = self._download_info(self._ITEM_SHORTCUT, item_id)
         access_token = self._download_json(
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         info = self._download_info(self._ITEM_SHORTCUT, item_id)
         access_token = self._download_json(
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)

+

         formats = self._extract_m3u8_formats(
         formats = self._extract_m3u8_formats(

-            '%s/vod/%s?nauth=%s&nauthsig=%s'
-            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
-            item_id, 'mp4')
+            '%s/vod/%s?%s' % (
+                self._USHER_BASE, item_id,
+                compat_urllib_parse_urlencode({
+                    'allow_source': 'true',
+                    'allow_audio_only': 'true',
+                    'allow_spectre': 'true',
+                    'player': 'twitchweb',
+                    'nauth': access_token['token'],
+                    'nauthsig': access_token['sig'],
+                })),
+            item_id, 'mp4', entry_protocol='m3u8_native')
+
+        self._prefer_source(formats)

         info['formats'] = formats
         info['formats'] = formats

+
+        parsed_url = compat_urllib_parse_urlparse(url)
+        query = compat_parse_qs(parsed_url.query)
+        if 't' in query:
+            info['start_time'] = parse_duration(query['t'][0])
+

         return info
 
 
         return info
 
 


@@ -216,17 +286,37 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
         entries = []
         offset = 0
         limit = self._PAGE_LIMIT
         entries = []
         offset = 0
         limit = self._PAGE_LIMIT

+        broken_paging_detected = False
+        counter_override = None

         for counter in itertools.count(1):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
         for counter in itertools.count(1):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),

-                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+                channel_id,
+                'Downloading %s videos JSON page %s'
+                % (self._PLAYLIST_TYPE, counter_override or counter))

             page_entries = self._extract_playlist_page(response)
             if not page_entries:
                 break
             page_entries = self._extract_playlist_page(response)
             if not page_entries:
                 break

+            total = int_or_none(response.get('_total'))
+            # Since the beginning of March 2016 twitch's paging mechanism
+            # is completely broken on the twitch side. It simply ignores
+            # a limit and returns the whole offset number of videos.
+            # Working around by just requesting all videos at once.
+            # Upd: pagination bug was fixed by twitch on 15.03.2016.
+            if not broken_paging_detected and total and len(page_entries) > limit:
+                self.report_warning(
+                    'Twitch pagination is broken on twitch side, requesting all videos at once',
+                    channel_id)
+                broken_paging_detected = True
+                offset = total
+                counter_override = '(all at once)'
+                continue

             entries.extend(page_entries)
             entries.extend(page_entries)

+            if broken_paging_detected or total and len(page_entries) >= total:
+                break

             offset += limit
         return self.playlist_result(
             offset += limit
         return self.playlist_result(

-            [self.url_result(entry) for entry in set(entries)],
+            [self.url_result(entry) for entry in orderedSet(entries)],

             channel_id, channel_name)
 
     def _extract_playlist_page(self, response):
             channel_id, channel_name)
 
     def _extract_playlist_page(self, response):


@@ -268,36 +358,11 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     }
 
 
     }
 
 

-class TwitchBookmarksIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:bookmarks'
-    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
-    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
-    _PLAYLIST_TYPE = 'bookmarks'
-
-    _TEST = {
-        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
-        'info_dict': {
-            'id': 'ognos',
-            'title': 'Ognos',
-        },
-        'playlist_mincount': 3,
-    }
-
-    def _extract_playlist_page(self, response):
-        entries = []
-        for bookmark in response.get('bookmarks', []):
-            video = bookmark.get('video')
-            if not video:
-                continue
-            entries.append(video['url'])
-        return entries
-
-

 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'

-    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE

 
 

-    _TEST = {
+    _TESTS = [{

         'url': 'http://www.twitch.tv/shroomztv',
         'info_dict': {
             'id': '12772022048',
         'url': 'http://www.twitch.tv/shroomztv',
         'info_dict': {
             'id': '12772022048',


@@ -316,7 +381,10 @@ class TwitchStreamIE(TwitchBaseIE):
             # m3u8 download
             'skip_download': True,
         },
             # m3u8 download
             'skip_download': True,
         },

-    }
+    }, {
+        'url': 'http://www.twitch.tv/miracle_doto#profile-0',
+        'only_matching': True,
+    }]

 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)


@@ -331,23 +399,30 @@ class TwitchStreamIE(TwitchBaseIE):
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 
                 'http://www.twitch.tv/%s/profile' % channel_id,
                 'TwitchProfile', channel_id)
 

+        # Channel name may be typed if different case than the original channel name
+        # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
+        # an invalid m3u8 URL. Working around by use of original channel name from stream
+        # JSON and fallback to lowercase if it's not available.
+        channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
+

         access_token = self._download_json(
             '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
             'Downloading channel access token')
 
         query = {
             'allow_source': 'true',
         access_token = self._download_json(
             '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
             'Downloading channel access token')
 
         query = {
             'allow_source': 'true',

+            'allow_audio_only': 'true',

             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',

-            'sig': access_token['sig'],
-            'token': access_token['token'],
+            'sig': access_token['sig'].encode('utf-8'),
+            'token': access_token['token'].encode('utf-8'),

         }
         }

-

         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'

-            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),

             channel_id, 'mp4')
             channel_id, 'mp4')

+        self._prefer_source(formats)

 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))


@@ -380,3 +455,61 @@ class TwitchStreamIE(TwitchBaseIE):
             'formats': formats,
             'is_live': True,
         }
             'formats': formats,
             'is_live': True,
         }

+
+
+class TwitchClipsIE(InfoExtractor):
+    IE_NAME = 'twitch:clips'
+    _VALID_URL = r'https?://clips\.twitch\.tv/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+
+    _TESTS = [{
+        'url': 'https://clips.twitch.tv/ea/AggressiveCobraPoooound',
+        'md5': '761769e1eafce0ffebfb4089cb3847cd',
+        'info_dict': {
+            'id': 'AggressiveCobraPoooound',
+            'ext': 'mp4',
+            'title': 'EA Play 2016 Live from the Novo Theatre',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'creator': 'EA',
+            'uploader': 'stereotype_',
+            'uploader_id': 'stereotype_',
+        },
+    }, {
+        # multiple formats
+        'url': 'https://clips.twitch.tv/rflegendary/UninterestedBeeDAESuppy',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        clip = self._parse_json(
+            self._search_regex(
+                r'(?s)clipInfo\s*=\s*({.+?});', webpage, 'clip info'),
+            video_id, transform_source=js_to_json)
+
+        title = clip.get('channel_title') or self._og_search_title(webpage)
+
+        formats = [{
+            'url': option['source'],
+            'format_id': option.get('quality'),
+            'height': int_or_none(option.get('quality')),
+        } for option in clip.get('quality_options', []) if option.get('source')]
+
+        if not formats:
+            formats = [{
+                'url': clip['clip_video_url'],
+            }]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'creator': clip.get('broadcaster_display_name') or clip.get('broadcaster_login'),
+            'uploader': clip.get('curator_login'),
+            'uploader_id': clip.get('curator_display_name'),
+            'formats': formats,
+        }