debian/changelog: Annotate bugs to close.

[youtubedl] / youtube_dl / extractor / common.py

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df1a4417bb99fe6cf415b293ce068787d1568dc8..cecf917ffb67040739bbacf573297ba891ec9ea1 100644 (file)
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -23,11 +23,11 @@ from ..compat import (
 )
 from ..utils import (
     age_restricted,
 )
 from ..utils import (
     age_restricted,

+    bug_reports_message,

     clean_html,
     compiled_regex_type,
     ExtractorError,
     float_or_none,
     clean_html,
     compiled_regex_type,
     ExtractorError,
     float_or_none,

-    HEADRequest,

     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,


@@ -47,7 +47,7 @@ class InfoExtractor(object):
     information possibly downloading the video to the file system, among
     other possible outcomes.
 
     information possibly downloading the video to the file system, among
     other possible outcomes.
 

-    The type field determines the the type of the result.
+    The type field determines the type of the result.

     By far the most common value (and the default if _type is missing) is
     "video", which indicates a single video.
 
     By far the most common value (and the default if _type is missing) is
     "video", which indicates a single video.
 


@@ -111,11 +111,8 @@ class InfoExtractor(object):
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.

-                    * http_method  HTTP method to use for the download.

                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.
                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.

-                    * http_post_data  Additional data to send with a POST
-                                 request.

                     * stretched_ratio  If given and not 1, indicates that the
                                  video's pixels are not square.
                                  width : height ratio as float.
                     * stretched_ratio  If given and not 1, indicates that the
                                  video's pixels are not square.
                                  width : height ratio as float.


@@ -151,12 +148,19 @@ class InfoExtractor(object):
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
     location:       Physical location where the video was filmed.
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
     location:       Physical location where the video was filmed.

-    subtitles:      The subtitle file contents as a dictionary in the format
-                    {language: subtitles}.
+    subtitles:      The available subtitles as a dictionary in the format
+                    {language: subformats}. "subformats" is a list sorted from
+                    lower to higher preference, each element is a dictionary
+                    with the "ext" entry and one of:
+                        * "data": The subtitles file contents
+                        * "url": A url pointing to the subtitles file
+    automatic_captions: Like 'subtitles', used by the YoutubeIE for
+                    automatically generated captions

     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video

+    average_rating: Average rating give by users, the scale used depends on the webpage

     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following
                     properties (all but one of text or html optional):
     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following
                     properties (all but one of text or html optional):


@@ -264,8 +268,15 @@ class InfoExtractor(object):
 
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
 
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""

-        self.initialize()
-        return self._real_extract(url)
+        try:
+            self.initialize()
+            return self._real_extract(url)
+        except ExtractorError:
+            raise
+        except compat_http_client.IncompleteRead as e:
+            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+        except (KeyError, StopIteration) as e:
+            raise ExtractorError('An extractor error has occured.', cause=e)

 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""


@@ -311,7 +322,7 @@ class InfoExtractor(object):
                 self._downloader.report_warning(errmsg)
                 return False
 
                 self._downloader.report_warning(errmsg)
                 return False
 

-    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):

         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
         """ Returns a tuple (page content as string, URL handle) """
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):


@@ -321,14 +332,11 @@ class InfoExtractor(object):
         if urlh is False:
             assert not fatal
             return False
         if urlh is False:
             assert not fatal
             return False

-        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
+        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)

         return (content, urlh)
 
         return (content, urlh)
 

-    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
-        content_type = urlh.headers.get('Content-Type', '')
-        webpage_bytes = urlh.read()
-        if prefix is not None:
-            webpage_bytes = prefix + webpage_bytes
+    @staticmethod
+    def _guess_encoding_from_content(content_type, webpage_bytes):

         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
         if m:
             encoding = m.group(1)
         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
         if m:
             encoding = m.group(1)


@@ -341,6 +349,16 @@ class InfoExtractor(object):
                 encoding = 'utf-16'
             else:
                 encoding = 'utf-8'
                 encoding = 'utf-16'
             else:
                 encoding = 'utf-8'

+
+        return encoding
+
+    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
+        content_type = urlh.headers.get('Content-Type', '')
+        webpage_bytes = urlh.read()
+        if prefix is not None:
+            webpage_bytes = prefix + webpage_bytes
+        if not encoding:
+            encoding = self._guess_encoding_from_content(content_type, webpage_bytes)

         if self._downloader.params.get('dump_intermediate_pages', False):
             try:
                 url = url_or_request.get_full_url()
         if self._downloader.params.get('dump_intermediate_pages', False):
             try:
                 url = url_or_request.get_full_url()


@@ -384,16 +402,26 @@ class InfoExtractor(object):
             if blocked_iframe:
                 msg += ' Visit %s for more details' % blocked_iframe
             raise ExtractorError(msg, expected=True)
             if blocked_iframe:
                 msg += ' Visit %s for more details' % blocked_iframe
             raise ExtractorError(msg, expected=True)

+        if '<title>The URL you requested has been blocked</title>' in content[:512]:
+            msg = (
+                'Access to this webpage has been blocked by Indian censorship. '
+                'Use a VPN or proxy server (with --proxy) to route around it.')
+            block_msg = self._html_search_regex(
+                r'</h1><p>(.*?)</p>',
+                content, 'block message', default=None)
+            if block_msg:
+                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
+            raise ExtractorError(msg, expected=True)

 
         return content
 
 
         return content
 

-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):

         """ Returns the data of the page as a string """
         success = False
         try_count = 0
         while success is False:
             try:
         """ Returns the data of the page as a string """
         success = False
         try_count = 0
         while success is False:
             try:

-                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)

                 success = True
             except compat_http_client.IncompleteRead as e:
                 try_count += 1
                 success = True
             except compat_http_client.IncompleteRead as e:
                 try_count += 1


@@ -408,10 +436,10 @@ class InfoExtractor(object):
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',
 
     def _download_xml(self, url_or_request, video_id,
                       note='Downloading XML', errnote='Unable to download XML',

-                      transform_source=None, fatal=True):
+                      transform_source=None, fatal=True, encoding=None):

         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(
         """Return the xml as an xml.etree.ElementTree.Element"""
         xml_string = self._download_webpage(

-            url_or_request, video_id, note, errnote, fatal=fatal)
+            url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)

         if xml_string is False:
             return xml_string
         if transform_source:
         if xml_string is False:
             return xml_string
         if transform_source:


@@ -422,9 +450,10 @@ class InfoExtractor(object):
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,
                        note='Downloading JSON metadata',
                        errnote='Unable to download JSON metadata',
                        transform_source=None,

-                       fatal=True):
+                       fatal=True, encoding=None):

         json_string = self._download_webpage(
         json_string = self._download_webpage(

-            url_or_request, video_id, note, errnote, fatal=fatal)
+            url_or_request, video_id, note, errnote, fatal=fatal,
+            encoding=encoding)

         if (not fatal) and json_string is False:
             return None
         return self._parse_json(
         if (not fatal) and json_string is False:
             return None
         return self._parse_json(


@@ -469,7 +498,7 @@ class InfoExtractor(object):
 
     # Methods for following #608
     @staticmethod
 
     # Methods for following #608
     @staticmethod

-    def url_result(url, ie=None, video_id=None):
+    def url_result(url, ie=None, video_id=None, video_title=None):

         """Returns a url that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
         """Returns a url that points to a page that should be processed"""
         # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',


@@ -477,6 +506,8 @@ class InfoExtractor(object):
                       'ie_key': ie}
         if video_id is not None:
             video_info['id'] = video_id
                       'ie_key': ie}
         if video_id is not None:
             video_info['id'] = video_id

+        if video_title is not None:
+            video_info['title'] = video_title

         return video_info
 
     @staticmethod
         return video_info
 
     @staticmethod


@@ -507,7 +538,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
                 if mobj:
                     break
 

-        if os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():

             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name


@@ -523,8 +554,7 @@ class InfoExtractor(object):
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
         else:
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
         else:

-            self._downloader.report_warning('unable to extract %s; '
-                                            'please report this issue on http://yt-dl.org/bug' % _name)
+            self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())

             return None
 
     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
             return None
 
     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):


@@ -539,7 +569,7 @@ class InfoExtractor(object):
 
     def _get_login_info(self):
         """
 
     def _get_login_info(self):
         """

-        Get the the login info as (username, password)
+        Get the login info as (username, password)

         It will look in the netrc file using the _NETRC_MACHINE value
         If there's no info available, return (None, None)
         """
         It will look in the netrc file using the _NETRC_MACHINE value
         If there's no info available, return (None, None)
         """


@@ -656,11 +686,26 @@ class InfoExtractor(object):
         }
         return RATING_TABLE.get(rating.lower(), None)
 
         }
         return RATING_TABLE.get(rating.lower(), None)
 

+    def _family_friendly_search(self, html):
+        # See http://schema.org/VideoObject
+        family_friendly = self._html_search_meta('isFamilyFriendly', html)
+
+        if not family_friendly:
+            return None
+
+        RATING_TABLE = {
+            '1': 0,
+            'true': 0,
+            '0': 18,
+            'false': 18,
+        }
+        return RATING_TABLE.get(family_friendly.lower(), None)
+

     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')
 

-    def _sort_formats(self, formats):
+    def _sort_formats(self, formats, field_preference=None):

         if not formats:
             raise ExtractorError('No video formats found')
 
         if not formats:
             raise ExtractorError('No video formats found')
 


@@ -670,6 +715,9 @@ class InfoExtractor(object):
             if not f.get('ext') and 'url' in f:
                 f['ext'] = determine_ext(f['url'])
 
             if not f.get('ext') and 'url' in f:
                 f['ext'] = determine_ext(f['url'])
 

+            if isinstance(field_preference, (list, tuple)):
+                return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
+

             preference = f.get('preference')
             if preference is None:
                 proto = f.get('protocol')
             preference = f.get('preference')
             if preference is None:
                 proto = f.get('protocol')


@@ -706,17 +754,17 @@ class InfoExtractor(object):
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,

+                f.get('filesize') if f.get('filesize') is not None else -1,

                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,

-                ext_preference,

                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,

+                ext_preference,

                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

-                f.get('filesize') if f.get('filesize') is not None else -1,

                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,

-                f.get('format_id'),
+                f.get('format_id') if f.get('format_id') is not None else '',

             )
         formats.sort(key=_formats_key)
 
             )
         formats.sort(key=_formats_key)
 


@@ -729,15 +777,17 @@ class InfoExtractor(object):
                 formats)
 
     def _is_valid_url(self, url, video_id, item='video'):
                 formats)
 
     def _is_valid_url(self, url, video_id, item='video'):

+        url = self._proto_relative_url(url, scheme='http:')
+        # For now assume non HTTP(S) URLs always valid
+        if not (url.startswith('http://') or url.startswith('https://')):
+            return True

         try:
         try:

-            self._request_webpage(
-                HEADRequest(url), video_id,
-                'Checking %s URL' % item)
+            self._request_webpage(url, video_id, 'Checking %s URL' % item)

             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):

-                self.report_warning(
-                    '%s URL is invalid, skipping' % item, video_id)
+                self.to_screen(
+                    '%s: %s URL is invalid, skipping' % (video_id, item))

                 return False
             raise
 
                 return False
             raise
 


@@ -778,11 +828,11 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':

-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/'
-                                + (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
+                                (media_el.attrib.get('href') or media_el.attrib.get('url')))

             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({

-                'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
+                'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,
                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,


@@ -799,11 +849,11 @@ class InfoExtractor(object):
                               m3u8_id=None):
 
         formats = [{
                               m3u8_id=None):
 
         formats = [{

-            'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
+            'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),

             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',

-            'preference': -1,
+            'preference': preference - 1 if preference else -1,

             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
         }]
             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
         }]


@@ -818,6 +868,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None

+        last_media = None

         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():


@@ -828,6 +879,13 @@ class InfoExtractor(object):
                     if v.startswith('"'):
                         v = v[1:-1]
                     last_info[m.group('key')] = v
                     if v.startswith('"'):
                         v = v[1:-1]
                     last_info[m.group('key')] = v

+            elif line.startswith('#EXT-X-MEDIA:'):
+                last_media = {}
+                for m in kv_rex.finditer(line):
+                    v = m.group('val')
+                    if v.startswith('"'):
+                        v = v[1:-1]
+                    last_media[m.group('key')] = v

             elif line.startswith('#') or not line.strip():
                 continue
             else:
             elif line.startswith('#') or not line.strip():
                 continue
             else:


@@ -835,8 +893,13 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)

+                format_id = []
+                if m3u8_id:
+                    format_id.append(m3u8_id)
+                last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
+                format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))

                 f = {
                 f = {

-                    'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
+                    'format_id': '-'.join(format_id),

                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,


@@ -856,6 +919,9 @@ class InfoExtractor(object):
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)

+                if last_media is not None:
+                    f['m3u8_media'] = last_media
+                    last_media = None

                 formats.append(f)
                 last_info = {}
         self._sort_formats(formats)
                 formats.append(f)
                 last_info = {}
         self._sort_formats(formats)


@@ -874,39 +940,57 @@ class InfoExtractor(object):
 
         formats = []
         rtmp_count = 0
 
         formats = []
         rtmp_count = 0

-        for video in smil.findall('./body/switch/video'):
-            src = video.get('src')
-            if not src:
-                continue
-            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-            width = int_or_none(video.get('width'))
-            height = int_or_none(video.get('height'))
-            proto = video.get('proto')
-            if not proto:
-                if base:
-                    if base.startswith('rtmp'):
-                        proto = 'rtmp'
-                    elif base.startswith('http'):
-                        proto = 'http'
-            ext = video.get('ext')
-            if proto == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(src, video_id, ext))
-            elif proto == 'rtmp':
-                rtmp_count += 1
-                streamer = video.get('streamer') or base
-                formats.append({
-                    'url': streamer,
-                    'play_path': src,
-                    'ext': 'flv',
-                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
-                    'tbr': bitrate,
-                    'width': width,
-                    'height': height,
-                })
+        if smil.findall('./body/seq/video'):
+            video = smil.findall('./body/seq/video')[0]
+            fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
+            formats.extend(fmts)
+        else:
+            for video in smil.findall('./body/switch/video'):
+                fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
+                formats.extend(fmts)
+

         self._sort_formats(formats)
 
         return formats
 
         self._sort_formats(formats)
 
         return formats
 

+    def _parse_smil_video(self, video, video_id, base, rtmp_count):
+        src = video.get('src')
+        if not src:
+            return ([], rtmp_count)
+        bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+        width = int_or_none(video.get('width'))
+        height = int_or_none(video.get('height'))
+        proto = video.get('proto')
+        if not proto:
+            if base:
+                if base.startswith('rtmp'):
+                    proto = 'rtmp'
+                elif base.startswith('http'):
+                    proto = 'http'
+        ext = video.get('ext')
+        if proto == 'm3u8':
+            return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
+        elif proto == 'rtmp':
+            rtmp_count += 1
+            streamer = video.get('streamer') or base
+            return ([{
+                'url': streamer,
+                'play_path': src,
+                'ext': 'flv',
+                'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                'tbr': bitrate,
+                'width': width,
+                'height': height,
+            }], rtmp_count)
+        elif proto.startswith('http'):
+            return ([{
+                'url': base + src,
+                'ext': ext or 'flv',
+                'tbr': bitrate,
+                'width': width,
+                'height': height,
+            }], rtmp_count)
+

     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()


@@ -970,6 +1054,24 @@ class InfoExtractor(object):
             any_restricted = any_restricted or is_restricted
         return not any_restricted
 
             any_restricted = any_restricted or is_restricted
         return not any_restricted
 

+    def extract_subtitles(self, *args, **kwargs):
+        if (self._downloader.params.get('writesubtitles', False) or
+                self._downloader.params.get('listsubtitles')):
+            return self._get_subtitles(*args, **kwargs)
+        return {}
+
+    def _get_subtitles(self, *args, **kwargs):
+        raise NotImplementedError("This method must be implemented by subclasses")
+
+    def extract_automatic_captions(self, *args, **kwargs):
+        if (self._downloader.params.get('writeautomaticsub', False) or
+                self._downloader.params.get('listsubtitles')):
+            return self._get_automatic_captions(*args, **kwargs)
+        return {}
+
+    def _get_automatic_captions(self, *args, **kwargs):
+        raise NotImplementedError("This method must be implemented by subclasses")
+

 
 class SearchInfoExtractor(InfoExtractor):
     """
 
 class SearchInfoExtractor(InfoExtractor):
     """