Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/common.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import datetime
   5 import hashlib
   6 import json
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import sys
  12 import time
  13 import xml.etree.ElementTree
  14
  15 from ..compat import (
  16     compat_cookiejar,
  17     compat_http_client,
  18     compat_urllib_error,
  19     compat_urllib_parse_urlparse,
  20     compat_urlparse,
  21     compat_str,
  22 )
  23 from ..utils import (
  24     age_restricted,
  25     clean_html,
  26     compiled_regex_type,
  27     ExtractorError,
  28     float_or_none,
  29     int_or_none,
  30     RegexNotFoundError,
  31     sanitize_filename,
  32     unescapeHTML,
  33 )
  34 _NO_DEFAULT = object()
  35
  36
  37 class InfoExtractor(object):
  38     """Information Extractor class.
  39
  40     Information extractors are the classes that, given a URL, extract
  41     information about the video (or videos) the URL refers to. This
  42     information includes the real video URL, the video title, author and
  43     others. The information is stored in a dictionary which is then
  44     passed to the YoutubeDL. The YoutubeDL processes this
  45     information possibly downloading the video to the file system, among
  46     other possible outcomes.
  47
  48     The type field determines the the type of the result.
  49     By far the most common value (and the default if _type is missing) is
  50     "video", which indicates a single video.
  51
  52     For a video, the dictionaries must include the following fields:
  53
  54     id:             Video identifier.
  55     title:          Video title, unescaped.
  56
  57     Additionally, it must contain either a formats entry or a url one:
  58
  59     formats:        A list of dictionaries for each format available, ordered
  60                     from worst to best quality.
  61
  62                     Potential fields:
  63                     * url        Mandatory. The URL of the video file
  64                     * ext        Will be calculated from url if missing
  65                     * format     A human-readable description of the format
  66                                  ("mp4 container with h264/opus").
  67                                  Calculated from the format_id, width, height.
  68                                  and format_note fields if missing.
  69                     * format_id  A short description of the format
  70                                  ("mp4_h264_opus" or "19").
  71                                 Technically optional, but strongly recommended.
  72                     * format_note Additional info about the format
  73                                  ("3D" or "DASH video")
  74                     * width      Width of the video, if known
  75                     * height     Height of the video, if known
  76                     * resolution Textual description of width and height
  77                     * tbr        Average bitrate of audio and video in KBit/s
  78                     * abr        Average audio bitrate in KBit/s
  79                     * acodec     Name of the audio codec in use
  80                     * asr        Audio sampling rate in Hertz
  81                     * vbr        Average video bitrate in KBit/s
  82                     * fps        Frame rate
  83                     * vcodec     Name of the video codec in use
  84                     * container  Name of the container format
  85                     * filesize   The number of bytes, if known in advance
  86                     * filesize_approx  An estimate for the number of bytes
  87                     * player_url SWF Player URL (used for rtmpdump).
  88                     * protocol   The protocol that will be used for the actual
  89                                  download, lower-case.
  90                                  "http", "https", "rtsp", "rtmp", "m3u8" or so.
  91                     * preference Order number of this format. If this field is
  92                                  present and not None, the formats get sorted
  93                                  by this field, regardless of all other values.
  94                                  -1 for default (order by other properties),
  95                                  -2 or smaller for less than default.
  96                                  < -1000 to hide the format (if there is
  97                                     another one which is strictly better)
  98                     * language_preference  Is this in the correct requested
  99                                  language?
 100                                  10 if it's what the URL is about,
 101                                  -1 for default (don't know),
 102                                  -10 otherwise, other values reserved for now.
 103                     * quality    Order number of the video quality of this
 104                                  format, irrespective of the file format.
 105                                  -1 for default (order by other properties),
 106                                  -2 or smaller for less than default.
 107                     * source_preference  Order number for this video source
 108                                   (quality takes higher priority)
 109                                  -1 for default (order by other properties),
 110                                  -2 or smaller for less than default.
 111                     * http_referer  HTTP Referer header value to set.
 112                     * http_method  HTTP method to use for the download.
 113                     * http_headers  A dictionary of additional HTTP headers
 114                                  to add to the request.
 115                     * http_post_data  Additional data to send with a POST
 116                                  request.
 117                     * stretched_ratio  If given and not 1, indicates that the
 118                                        video's pixels are not square.
 119                                        width : height ratio as float.
 120     url:            Final video URL.
 121     ext:            Video filename extension.
 122     format:         The video format, defaults to ext (used for --get-format)
 123     player_url:     SWF Player URL (used for rtmpdump).
 124
 125     The following fields are optional:
 126
 127     alt_title:      A secondary title of the video.
 128     display_id      An alternative identifier for the video, not necessarily
 129                     unique, but available before title. Typically, id is
 130                     something like "4234987", title "Dancing naked mole rats",
 131                     and display_id "dancing-naked-mole-rats"
 132     thumbnails:     A list of dictionaries, with the following entries:
 133                         * "url"
 134                         * "width" (optional, int)
 135                         * "height" (optional, int)
 136                         * "resolution" (optional, string "{width}x{height"},
 137                                         deprecated)
 138     thumbnail:      Full URL to a video thumbnail image.
 139     description:    Full video description.
 140     uploader:       Full name of the video uploader.
 141     timestamp:      UNIX timestamp of the moment the video became available.
 142     upload_date:    Video upload date (YYYYMMDD).
 143                     If not explicitly set, calculated from timestamp.
 144     uploader_id:    Nickname or id of the video uploader.
 145     location:       Physical location where the video was filmed.
 146     subtitles:      The subtitle file contents as a dictionary in the format
 147                     {language: subtitles}.
 148     duration:       Length of the video in seconds, as an integer.
 149     view_count:     How many users have watched the video on the platform.
 150     like_count:     Number of positive ratings of the video
 151     dislike_count:  Number of negative ratings of the video
 152     comment_count:  Number of comments on the video
 153     comments:       A list of comments, each with one or more of the following
 154                     properties (all but one of text or html optional):
 155                         * "author" - human-readable name of the comment author
 156                         * "author_id" - user ID of the comment author
 157                         * "id" - Comment ID
 158                         * "html" - Comment as HTML
 159                         * "text" - Plain text of the comment
 160                         * "timestamp" - UNIX timestamp of comment
 161                         * "parent" - ID of the comment this one is replying to.
 162                                      Set to "root" to indicate that this is a
 163                                      comment to the original video.
 164     age_limit:      Age restriction for the video, as an integer (years)
 165     webpage_url:    The url to the video webpage, if given to youtube-dl it
 166                     should allow to get the same result again. (It will be set
 167                     by YoutubeDL if it's missing)
 168     categories:     A list of categories that the video falls in, for example
 169                     ["Sports", "Berlin"]
 170     is_live:        True, False, or None (=unknown). Whether this video is a
 171                     live stream that goes on instead of a fixed-length video.
 172
 173     Unless mentioned otherwise, the fields should be Unicode strings.
 174
 175     Unless mentioned otherwise, None is equivalent to absence of information.
 176
 177
 178     _type "playlist" indicates multiple videos.
 179     There must be a key "entries", which is a list, an iterable, or a PagedList
 180     object, each element of which is a valid dictionary by this specification.
 181
 182     Additionally, playlists can have "title" and "id" attributes with the same
 183     semantics as videos (see above).
 184
 185
 186     _type "multi_video" indicates that there are multiple videos that
 187     form a single show, for examples multiple acts of an opera or TV episode.
 188     It must have an entries key like a playlist and contain all the keys
 189     required for a video at the same time.
 190
 191
 192     _type "url" indicates that the video must be extracted from another
 193     location, possibly by a different extractor. Its only required key is:
 194     "url" - the next URL to extract.
 195     The key "ie_key" can be set to the class name (minus the trailing "IE",
 196     e.g. "Youtube") if the extractor class is known in advance.
 197     Additionally, the dictionary may have any properties of the resolved entity
 198     known in advance, for example "title" if the title of the referred video is
 199     known ahead of time.
 200
 201
 202     _type "url_transparent" entities have the same specification as "url", but
 203     indicate that the given additional information is more precise than the one
 204     associated with the resolved URL.
 205     This is useful when a site employs a video service that hosts the video and
 206     its technical metadata, but that video service does not embed a useful
 207     title, description etc.
 208
 209
 210     Subclasses of this one should re-define the _real_initialize() and
 211     _real_extract() methods and define a _VALID_URL regexp.
 212     Probably, they should also be added to the list of extractors.
 213
 214     Finally, the _WORKING attribute should be set to False for broken IEs
 215     in order to warn the users and skip the tests.
 216     """
 217
 218     _ready = False
 219     _downloader = None
 220     _WORKING = True
 221
 222     def __init__(self, downloader=None):
 223         """Constructor. Receives an optional downloader."""
 224         self._ready = False
 225         self.set_downloader(downloader)
 226
 227     @classmethod
 228     def suitable(cls, url):
 229         """Receives a URL and returns True if suitable for this IE."""
 230
 231         # This does not use has/getattr intentionally - we want to know whether
 232         # we have cached the regexp for *this* class, whereas getattr would also
 233         # match the superclass
 234         if '_VALID_URL_RE' not in cls.__dict__:
 235             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 236         return cls._VALID_URL_RE.match(url) is not None
 237
 238     @classmethod
 239     def _match_id(cls, url):
 240         if '_VALID_URL_RE' not in cls.__dict__:
 241             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 242         m = cls._VALID_URL_RE.match(url)
 243         assert m
 244         return m.group('id')
 245
 246     @classmethod
 247     def working(cls):
 248         """Getter method for _WORKING."""
 249         return cls._WORKING
 250
 251     def initialize(self):
 252         """Initializes an instance (authentication, etc)."""
 253         if not self._ready:
 254             self._real_initialize()
 255             self._ready = True
 256
 257     def extract(self, url):
 258         """Extracts URL information and returns it in list of dicts."""
 259         self.initialize()
 260         return self._real_extract(url)
 261
 262     def set_downloader(self, downloader):
 263         """Sets the downloader for this IE."""
 264         self._downloader = downloader
 265
 266     def _real_initialize(self):
 267         """Real initialization process. Redefine in subclasses."""
 268         pass
 269
 270     def _real_extract(self, url):
 271         """Real extraction process. Redefine in subclasses."""
 272         pass
 273
 274     @classmethod
 275     def ie_key(cls):
 276         """A string for getting the InfoExtractor with get_info_extractor"""
 277         return cls.__name__[:-2]
 278
 279     @property
 280     def IE_NAME(self):
 281         return type(self).__name__[:-2]
 282
 283     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 284         """ Returns the response handle """
 285         if note is None:
 286             self.report_download_webpage(video_id)
 287         elif note is not False:
 288             if video_id is None:
 289                 self.to_screen('%s' % (note,))
 290             else:
 291                 self.to_screen('%s: %s' % (video_id, note))
 292         try:
 293             return self._downloader.urlopen(url_or_request)
 294         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 295             if errnote is False:
 296                 return False
 297             if errnote is None:
 298                 errnote = 'Unable to download webpage'
 299             errmsg = '%s: %s' % (errnote, compat_str(err))
 300             if fatal:
 301                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 302             else:
 303                 self._downloader.report_warning(errmsg)
 304                 return False
 305
 306     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 307         """ Returns a tuple (page content as string, URL handle) """
 308         # Strip hashes from the URL (#1038)
 309         if isinstance(url_or_request, (compat_str, str)):
 310             url_or_request = url_or_request.partition('#')[0]
 311
 312         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 313         if urlh is False:
 314             assert not fatal
 315             return False
 316         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
 317         return (content, urlh)
 318
 319     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
 320         content_type = urlh.headers.get('Content-Type', '')
 321         webpage_bytes = urlh.read()
 322         if prefix is not None:
 323             webpage_bytes = prefix + webpage_bytes
 324         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 325         if m:
 326             encoding = m.group(1)
 327         else:
 328             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 329                           webpage_bytes[:1024])
 330             if m:
 331                 encoding = m.group(1).decode('ascii')
 332             elif webpage_bytes.startswith(b'\xff\xfe'):
 333                 encoding = 'utf-16'
 334             else:
 335                 encoding = 'utf-8'
 336         if self._downloader.params.get('dump_intermediate_pages', False):
 337             try:
 338                 url = url_or_request.get_full_url()
 339             except AttributeError:
 340                 url = url_or_request
 341             self.to_screen('Dumping request to ' + url)
 342             dump = base64.b64encode(webpage_bytes).decode('ascii')
 343             self._downloader.to_screen(dump)
 344         if self._downloader.params.get('write_pages', False):
 345             try:
 346                 url = url_or_request.get_full_url()
 347             except AttributeError:
 348                 url = url_or_request
 349             basen = '%s_%s' % (video_id, url)
 350             if len(basen) > 240:
 351                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 352                 basen = basen[:240 - len(h)] + h
 353             raw_filename = basen + '.dump'
 354             filename = sanitize_filename(raw_filename, restricted=True)
 355             self.to_screen('Saving request to ' + filename)
 356             # Working around MAX_PATH limitation on Windows (see
 357             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 358             if os.name == 'nt':
 359                 absfilepath = os.path.abspath(filename)
 360                 if len(absfilepath) > 259:
 361                     filename = '\\\\?\\' + absfilepath
 362             with open(filename, 'wb') as outf:
 363                 outf.write(webpage_bytes)
 364
 365         try:
 366             content = webpage_bytes.decode(encoding, 'replace')
 367         except LookupError:
 368             content = webpage_bytes.decode('utf-8', 'replace')
 369
 370         if ('<title>Access to this site is blocked</title>' in content and
 371                 'Websense' in content[:512]):
 372             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 373             blocked_iframe = self._html_search_regex(
 374                 r'<iframe src="([^"]+)"', content,
 375                 'Websense information URL', default=None)
 376             if blocked_iframe:
 377                 msg += ' Visit %s for more details' % blocked_iframe
 378             raise ExtractorError(msg, expected=True)
 379
 380         return content
 381
 382     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
 383         """ Returns the data of the page as a string """
 384         success = False
 385         try_count = 0
 386         while success is False:
 387             try:
 388                 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
 389                 success = True
 390             except compat_http_client.IncompleteRead as e:
 391                 try_count += 1
 392                 if try_count >= tries:
 393                     raise e
 394                 self._sleep(timeout, video_id)
 395         if res is False:
 396             return res
 397         else:
 398             content, _ = res
 399             return content
 400
 401     def _download_xml(self, url_or_request, video_id,
 402                       note='Downloading XML', errnote='Unable to download XML',
 403                       transform_source=None, fatal=True):
 404         """Return the xml as an xml.etree.ElementTree.Element"""
 405         xml_string = self._download_webpage(
 406             url_or_request, video_id, note, errnote, fatal=fatal)
 407         if xml_string is False:
 408             return xml_string
 409         if transform_source:
 410             xml_string = transform_source(xml_string)
 411         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 412
 413     def _download_json(self, url_or_request, video_id,
 414                        note='Downloading JSON metadata',
 415                        errnote='Unable to download JSON metadata',
 416                        transform_source=None,
 417                        fatal=True):
 418         json_string = self._download_webpage(
 419             url_or_request, video_id, note, errnote, fatal=fatal)
 420         if (not fatal) and json_string is False:
 421             return None
 422         return self._parse_json(
 423             json_string, video_id, transform_source=transform_source, fatal=fatal)
 424
 425     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 426         if transform_source:
 427             json_string = transform_source(json_string)
 428         try:
 429             return json.loads(json_string)
 430         except ValueError as ve:
 431             errmsg = '%s: Failed to parse JSON ' % video_id
 432             if fatal:
 433                 raise ExtractorError(errmsg, cause=ve)
 434             else:
 435                 self.report_warning(errmsg + str(ve))
 436
 437     def report_warning(self, msg, video_id=None):
 438         idstr = '' if video_id is None else '%s: ' % video_id
 439         self._downloader.report_warning(
 440             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 441
 442     def to_screen(self, msg):
 443         """Print msg to screen, prefixing it with '[ie_name]'"""
 444         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 445
 446     def report_extraction(self, id_or_name):
 447         """Report information extraction."""
 448         self.to_screen('%s: Extracting information' % id_or_name)
 449
 450     def report_download_webpage(self, video_id):
 451         """Report webpage download."""
 452         self.to_screen('%s: Downloading webpage' % video_id)
 453
 454     def report_age_confirmation(self):
 455         """Report attempt to confirm age."""
 456         self.to_screen('Confirming age')
 457
 458     def report_login(self):
 459         """Report attempt to log in."""
 460         self.to_screen('Logging in')
 461
 462     # Methods for following #608
 463     @staticmethod
 464     def url_result(url, ie=None, video_id=None):
 465         """Returns a url that points to a page that should be processed"""
 466         # TODO: ie should be the class used for getting the info
 467         video_info = {'_type': 'url',
 468                       'url': url,
 469                       'ie_key': ie}
 470         if video_id is not None:
 471             video_info['id'] = video_id
 472         return video_info
 473
 474     @staticmethod
 475     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 476         """Returns a playlist"""
 477         video_info = {'_type': 'playlist',
 478                       'entries': entries}
 479         if playlist_id:
 480             video_info['id'] = playlist_id
 481         if playlist_title:
 482             video_info['title'] = playlist_title
 483         if playlist_description:
 484             video_info['description'] = playlist_description
 485         return video_info
 486
 487     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
 488         """
 489         Perform a regex search on the given string, using a single or a list of
 490         patterns returning the first matching group.
 491         In case of failure return a default value or raise a WARNING or a
 492         RegexNotFoundError, depending on fatal, specifying the field name.
 493         """
 494         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 495             mobj = re.search(pattern, string, flags)
 496         else:
 497             for p in pattern:
 498                 mobj = re.search(p, string, flags)
 499                 if mobj:
 500                     break
 501
 502         if os.name != 'nt' and sys.stderr.isatty():
 503             _name = '\033[0;34m%s\033[0m' % name
 504         else:
 505             _name = name
 506
 507         if mobj:
 508             if group is None:
 509                 # return the first matching group
 510                 return next(g for g in mobj.groups() if g is not None)
 511             else:
 512                 return mobj.group(group)
 513         elif default is not _NO_DEFAULT:
 514             return default
 515         elif fatal:
 516             raise RegexNotFoundError('Unable to extract %s' % _name)
 517         else:
 518             self._downloader.report_warning('unable to extract %s; '
 519                                             'please report this issue on http://yt-dl.org/bug' % _name)
 520             return None
 521
 522     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
 523         """
 524         Like _search_regex, but strips HTML tags and unescapes entities.
 525         """
 526         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
 527         if res:
 528             return clean_html(res).strip()
 529         else:
 530             return res
 531
 532     def _get_login_info(self):
 533         """
 534         Get the the login info as (username, password)
 535         It will look in the netrc file using the _NETRC_MACHINE value
 536         If there's no info available, return (None, None)
 537         """
 538         if self._downloader is None:
 539             return (None, None)
 540
 541         username = None
 542         password = None
 543         downloader_params = self._downloader.params
 544
 545         # Attempt to use provided username and password or .netrc data
 546         if downloader_params.get('username', None) is not None:
 547             username = downloader_params['username']
 548             password = downloader_params['password']
 549         elif downloader_params.get('usenetrc', False):
 550             try:
 551                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 552                 if info is not None:
 553                     username = info[0]
 554                     password = info[2]
 555                 else:
 556                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 557             except (IOError, netrc.NetrcParseError) as err:
 558                 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
 559
 560         return (username, password)
 561
 562     def _get_tfa_info(self):
 563         """
 564         Get the two-factor authentication info
 565         TODO - asking the user will be required for sms/phone verify
 566         currently just uses the command line option
 567         If there's no info available, return None
 568         """
 569         if self._downloader is None:
 570             return None
 571         downloader_params = self._downloader.params
 572
 573         if downloader_params.get('twofactor', None) is not None:
 574             return downloader_params['twofactor']
 575
 576         return None
 577
 578     # Helper functions for extracting OpenGraph info
 579     @staticmethod
 580     def _og_regexes(prop):
 581         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 582         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 583         template = r'<meta[^>]+?%s[^>]+?%s'
 584         return [
 585             template % (property_re, content_re),
 586             template % (content_re, property_re),
 587         ]
 588
 589     def _og_search_property(self, prop, html, name=None, **kargs):
 590         if name is None:
 591             name = 'OpenGraph %s' % prop
 592         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 593         if escaped is None:
 594             return None
 595         return unescapeHTML(escaped)
 596
 597     def _og_search_thumbnail(self, html, **kargs):
 598         return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
 599
 600     def _og_search_description(self, html, **kargs):
 601         return self._og_search_property('description', html, fatal=False, **kargs)
 602
 603     def _og_search_title(self, html, **kargs):
 604         return self._og_search_property('title', html, **kargs)
 605
 606     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 607         regexes = self._og_regexes('video') + self._og_regexes('video:url')
 608         if secure:
 609             regexes = self._og_regexes('video:secure_url') + regexes
 610         return self._html_search_regex(regexes, html, name, **kargs)
 611
 612     def _og_search_url(self, html, **kargs):
 613         return self._og_search_property('url', html, **kargs)
 614
 615     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
 616         if display_name is None:
 617             display_name = name
 618         return self._html_search_regex(
 619             r'''(?isx)<meta
 620                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
 621                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
 622             html, display_name, fatal=fatal, group='content', **kwargs)
 623
 624     def _dc_search_uploader(self, html):
 625         return self._html_search_meta('dc.creator', html, 'uploader')
 626
 627     def _rta_search(self, html):
 628         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 629         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 630                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 631                      html):
 632             return 18
 633         return 0
 634
 635     def _media_rating_search(self, html):
 636         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 637         rating = self._html_search_meta('rating', html)
 638
 639         if not rating:
 640             return None
 641
 642         RATING_TABLE = {
 643             'safe for kids': 0,
 644             'general': 8,
 645             '14 years': 14,
 646             'mature': 17,
 647             'restricted': 19,
 648         }
 649         return RATING_TABLE.get(rating.lower(), None)
 650
 651     def _twitter_search_player(self, html):
 652         return self._html_search_meta('twitter:player', html,
 653                                       'twitter card player')
 654
 655     def _sort_formats(self, formats):
 656         if not formats:
 657             raise ExtractorError('No video formats found')
 658
 659         def _formats_key(f):
 660             # TODO remove the following workaround
 661             from ..utils import determine_ext
 662             if not f.get('ext') and 'url' in f:
 663                 f['ext'] = determine_ext(f['url'])
 664
 665             preference = f.get('preference')
 666             if preference is None:
 667                 proto = f.get('protocol')
 668                 if proto is None:
 669                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 670
 671                 preference = 0 if proto in ['http', 'https'] else -0.1
 672                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 673                     preference -= 0.5
 674
 675             if f.get('vcodec') == 'none':  # audio only
 676                 if self._downloader.params.get('prefer_free_formats'):
 677                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
 678                 else:
 679                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
 680                 ext_preference = 0
 681                 try:
 682                     audio_ext_preference = ORDER.index(f['ext'])
 683                 except ValueError:
 684                     audio_ext_preference = -1
 685             else:
 686                 if self._downloader.params.get('prefer_free_formats'):
 687                     ORDER = ['flv', 'mp4', 'webm']
 688                 else:
 689                     ORDER = ['webm', 'flv', 'mp4']
 690                 try:
 691                     ext_preference = ORDER.index(f['ext'])
 692                 except ValueError:
 693                     ext_preference = -1
 694                 audio_ext_preference = 0
 695
 696             return (
 697                 preference,
 698                 f.get('language_preference') if f.get('language_preference') is not None else -1,
 699                 f.get('quality') if f.get('quality') is not None else -1,
 700                 f.get('height') if f.get('height') is not None else -1,
 701                 f.get('width') if f.get('width') is not None else -1,
 702                 ext_preference,
 703                 f.get('tbr') if f.get('tbr') is not None else -1,
 704                 f.get('vbr') if f.get('vbr') is not None else -1,
 705                 f.get('abr') if f.get('abr') is not None else -1,
 706                 audio_ext_preference,
 707                 f.get('fps') if f.get('fps') is not None else -1,
 708                 f.get('filesize') if f.get('filesize') is not None else -1,
 709                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
 710                 f.get('source_preference') if f.get('source_preference') is not None else -1,
 711                 f.get('format_id'),
 712             )
 713         formats.sort(key=_formats_key)
 714
 715     def http_scheme(self):
 716         """ Either "http:" or "https:", depending on the user's preferences """
 717         return (
 718             'http:'
 719             if self._downloader.params.get('prefer_insecure', False)
 720             else 'https:')
 721
 722     def _proto_relative_url(self, url, scheme=None):
 723         if url is None:
 724             return url
 725         if url.startswith('//'):
 726             if scheme is None:
 727                 scheme = self.http_scheme()
 728             return scheme + url
 729         else:
 730             return url
 731
 732     def _sleep(self, timeout, video_id, msg_template=None):
 733         if msg_template is None:
 734             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
 735         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
 736         self.to_screen(msg)
 737         time.sleep(timeout)
 738
 739     def _extract_f4m_formats(self, manifest_url, video_id):
 740         manifest = self._download_xml(
 741             manifest_url, video_id, 'Downloading f4m manifest',
 742             'Unable to download f4m manifest')
 743
 744         formats = []
 745         manifest_version = '1.0'
 746         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
 747         if not media_nodes:
 748             manifest_version = '2.0'
 749             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
 750         for i, media_el in enumerate(media_nodes):
 751             if manifest_version == '2.0':
 752                 manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
 753             tbr = int_or_none(media_el.attrib.get('bitrate'))
 754             format_id = 'f4m-%d' % (i if tbr is None else tbr)
 755             formats.append({
 756                 'format_id': format_id,
 757                 'url': manifest_url,
 758                 'ext': 'flv',
 759                 'tbr': tbr,
 760                 'width': int_or_none(media_el.attrib.get('width')),
 761                 'height': int_or_none(media_el.attrib.get('height')),
 762             })
 763         self._sort_formats(formats)
 764
 765         return formats
 766
 767     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
 768                               entry_protocol='m3u8', preference=None):
 769
 770         formats = [{
 771             'format_id': 'm3u8-meta',
 772             'url': m3u8_url,
 773             'ext': ext,
 774             'protocol': 'm3u8',
 775             'preference': -1,
 776             'resolution': 'multiple',
 777             'format_note': 'Quality selection URL',
 778         }]
 779
 780         format_url = lambda u: (
 781             u
 782             if re.match(r'^https?://', u)
 783             else compat_urlparse.urljoin(m3u8_url, u))
 784
 785         m3u8_doc = self._download_webpage(
 786             m3u8_url, video_id,
 787             note='Downloading m3u8 information',
 788             errnote='Failed to download m3u8 information')
 789         last_info = None
 790         kv_rex = re.compile(
 791             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
 792         for line in m3u8_doc.splitlines():
 793             if line.startswith('#EXT-X-STREAM-INF:'):
 794                 last_info = {}
 795                 for m in kv_rex.finditer(line):
 796                     v = m.group('val')
 797                     if v.startswith('"'):
 798                         v = v[1:-1]
 799                     last_info[m.group('key')] = v
 800             elif line.startswith('#') or not line.strip():
 801                 continue
 802             else:
 803                 if last_info is None:
 804                     formats.append({'url': format_url(line)})
 805                     continue
 806                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 807
 808                 f = {
 809                     'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
 810                     'url': format_url(line.strip()),
 811                     'tbr': tbr,
 812                     'ext': ext,
 813                     'protocol': entry_protocol,
 814                     'preference': preference,
 815                 }
 816                 codecs = last_info.get('CODECS')
 817                 if codecs:
 818                     # TODO: looks like video codec is not always necessarily goes first
 819                     va_codecs = codecs.split(',')
 820                     if va_codecs[0]:
 821                         f['vcodec'] = va_codecs[0].partition('.')[0]
 822                     if len(va_codecs) > 1 and va_codecs[1]:
 823                         f['acodec'] = va_codecs[1].partition('.')[0]
 824                 resolution = last_info.get('RESOLUTION')
 825                 if resolution:
 826                     width_str, height_str = resolution.split('x')
 827                     f['width'] = int(width_str)
 828                     f['height'] = int(height_str)
 829                 formats.append(f)
 830                 last_info = {}
 831         self._sort_formats(formats)
 832         return formats
 833
 834     # TODO: improve extraction
 835     def _extract_smil_formats(self, smil_url, video_id):
 836         smil = self._download_xml(
 837             smil_url, video_id, 'Downloading SMIL file',
 838             'Unable to download SMIL file')
 839
 840         base = smil.find('./head/meta').get('base')
 841
 842         formats = []
 843         rtmp_count = 0
 844         for video in smil.findall('./body/switch/video'):
 845             src = video.get('src')
 846             if not src:
 847                 continue
 848             bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
 849             width = int_or_none(video.get('width'))
 850             height = int_or_none(video.get('height'))
 851             proto = video.get('proto')
 852             if not proto:
 853                 if base:
 854                     if base.startswith('rtmp'):
 855                         proto = 'rtmp'
 856                     elif base.startswith('http'):
 857                         proto = 'http'
 858             ext = video.get('ext')
 859             if proto == 'm3u8':
 860                 formats.extend(self._extract_m3u8_formats(src, video_id, ext))
 861             elif proto == 'rtmp':
 862                 rtmp_count += 1
 863                 streamer = video.get('streamer') or base
 864                 formats.append({
 865                     'url': streamer,
 866                     'play_path': src,
 867                     'ext': 'flv',
 868                     'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
 869                     'tbr': bitrate,
 870                     'width': width,
 871                     'height': height,
 872                 })
 873         self._sort_formats(formats)
 874
 875         return formats
 876
 877     def _live_title(self, name):
 878         """ Generate the title for a live video """
 879         now = datetime.datetime.now()
 880         now_str = now.strftime("%Y-%m-%d %H:%M")
 881         return name + ' ' + now_str
 882
 883     def _int(self, v, name, fatal=False, **kwargs):
 884         res = int_or_none(v, **kwargs)
 885         if 'get_attr' in kwargs:
 886             print(getattr(v, kwargs['get_attr']))
 887         if res is None:
 888             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
 889             if fatal:
 890                 raise ExtractorError(msg)
 891             else:
 892                 self._downloader.report_warning(msg)
 893         return res
 894
 895     def _float(self, v, name, fatal=False, **kwargs):
 896         res = float_or_none(v, **kwargs)
 897         if res is None:
 898             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
 899             if fatal:
 900                 raise ExtractorError(msg)
 901             else:
 902                 self._downloader.report_warning(msg)
 903         return res
 904
 905     def _set_cookie(self, domain, name, value, expire_time=None):
 906         cookie = compat_cookiejar.Cookie(
 907             0, name, value, None, None, domain, None,
 908             None, '/', True, False, expire_time, '', None, None, None)
 909         self._downloader.cookiejar.set_cookie(cookie)
 910
 911     def get_testcases(self, include_onlymatching=False):
 912         t = getattr(self, '_TEST', None)
 913         if t:
 914             assert not hasattr(self, '_TESTS'), \
 915                 '%s has _TEST and _TESTS' % type(self).__name__
 916             tests = [t]
 917         else:
 918             tests = getattr(self, '_TESTS', [])
 919         for t in tests:
 920             if not include_onlymatching and t.get('only_matching', False):
 921                 continue
 922             t['name'] = type(self).__name__[:-len('IE')]
 923             yield t
 924
 925     def is_suitable(self, age_limit):
 926         """ Test whether the extractor is generally suitable for the given
 927         age limit (i.e. pornographic sites are not, all others usually are) """
 928
 929         any_restricted = False
 930         for tc in self.get_testcases(include_onlymatching=False):
 931             if 'playlist' in tc:
 932                 tc = tc['playlist'][0]
 933             is_restricted = age_restricted(
 934                 tc.get('info_dict', {}).get('age_limit'), age_limit)
 935             if not is_restricted:
 936                 return True
 937             any_restricted = any_restricted or is_restricted
 938         return not any_restricted
 939
 940
 941 class SearchInfoExtractor(InfoExtractor):
 942     """
 943     Base class for paged search queries extractors.
 944     They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
 945     Instances should define _SEARCH_KEY and _MAX_RESULTS.
 946     """
 947
 948     @classmethod
 949     def _make_valid_url(cls):
 950         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
 951
 952     @classmethod
 953     def suitable(cls, url):
 954         return re.match(cls._make_valid_url(), url) is not None
 955
 956     def _real_extract(self, query):
 957         mobj = re.match(self._make_valid_url(), query)
 958         if mobj is None:
 959             raise ExtractorError('Invalid search query "%s"' % query)
 960
 961         prefix = mobj.group('prefix')
 962         query = mobj.group('query')
 963         if prefix == '':
 964             return self._get_n_results(query, 1)
 965         elif prefix == 'all':
 966             return self._get_n_results(query, self._MAX_RESULTS)
 967         else:
 968             n = int(prefix)
 969             if n <= 0:
 970                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
 971             elif n > self._MAX_RESULTS:
 972                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
 973                 n = self._MAX_RESULTS
 974             return self._get_n_results(query, n)
 975
 976     def _get_n_results(self, query, n):
 977         """Get a specified number of results for a query"""
 978         raise NotImplementedError("This method must be implemented by subclasses")
 979
 980     @property
 981     def SEARCH_KEY(self):
 982         return self._SEARCH_KEY