Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/common.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import datetime
   5 import hashlib
   6 import json
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import sys
  12 import time
  13 import xml.etree.ElementTree
  14
  15 from ..compat import (
  16     compat_cookiejar,
  17     compat_HTTPError,
  18     compat_http_client,
  19     compat_urllib_error,
  20     compat_urllib_parse_urlparse,
  21     compat_urlparse,
  22     compat_str,
  23 )
  24 from ..utils import (
  25     age_restricted,
  26     clean_html,
  27     compiled_regex_type,
  28     ExtractorError,
  29     float_or_none,
  30     int_or_none,
  31     RegexNotFoundError,
  32     sanitize_filename,
  33     unescapeHTML,
  34 )
  35 _NO_DEFAULT = object()
  36
  37
  38 class InfoExtractor(object):
  39     """Information Extractor class.
  40
  41     Information extractors are the classes that, given a URL, extract
  42     information about the video (or videos) the URL refers to. This
  43     information includes the real video URL, the video title, author and
  44     others. The information is stored in a dictionary which is then
  45     passed to the YoutubeDL. The YoutubeDL processes this
  46     information possibly downloading the video to the file system, among
  47     other possible outcomes.
  48
  49     The type field determines the the type of the result.
  50     By far the most common value (and the default if _type is missing) is
  51     "video", which indicates a single video.
  52
  53     For a video, the dictionaries must include the following fields:
  54
  55     id:             Video identifier.
  56     title:          Video title, unescaped.
  57
  58     Additionally, it must contain either a formats entry or a url one:
  59
  60     formats:        A list of dictionaries for each format available, ordered
  61                     from worst to best quality.
  62
  63                     Potential fields:
  64                     * url        Mandatory. The URL of the video file
  65                     * ext        Will be calculated from url if missing
  66                     * format     A human-readable description of the format
  67                                  ("mp4 container with h264/opus").
  68                                  Calculated from the format_id, width, height.
  69                                  and format_note fields if missing.
  70                     * format_id  A short description of the format
  71                                  ("mp4_h264_opus" or "19").
  72                                 Technically optional, but strongly recommended.
  73                     * format_note Additional info about the format
  74                                  ("3D" or "DASH video")
  75                     * width      Width of the video, if known
  76                     * height     Height of the video, if known
  77                     * resolution Textual description of width and height
  78                     * tbr        Average bitrate of audio and video in KBit/s
  79                     * abr        Average audio bitrate in KBit/s
  80                     * acodec     Name of the audio codec in use
  81                     * asr        Audio sampling rate in Hertz
  82                     * vbr        Average video bitrate in KBit/s
  83                     * fps        Frame rate
  84                     * vcodec     Name of the video codec in use
  85                     * container  Name of the container format
  86                     * filesize   The number of bytes, if known in advance
  87                     * filesize_approx  An estimate for the number of bytes
  88                     * player_url SWF Player URL (used for rtmpdump).
  89                     * protocol   The protocol that will be used for the actual
  90                                  download, lower-case.
  91                                  "http", "https", "rtsp", "rtmp", "rtmpe",
  92                                  "m3u8", or "m3u8_native".
  93                     * preference Order number of this format. If this field is
  94                                  present and not None, the formats get sorted
  95                                  by this field, regardless of all other values.
  96                                  -1 for default (order by other properties),
  97                                  -2 or smaller for less than default.
  98                                  < -1000 to hide the format (if there is
  99                                     another one which is strictly better)
 100                     * language_preference  Is this in the correct requested
 101                                  language?
 102                                  10 if it's what the URL is about,
 103                                  -1 for default (don't know),
 104                                  -10 otherwise, other values reserved for now.
 105                     * quality    Order number of the video quality of this
 106                                  format, irrespective of the file format.
 107                                  -1 for default (order by other properties),
 108                                  -2 or smaller for less than default.
 109                     * source_preference  Order number for this video source
 110                                   (quality takes higher priority)
 111                                  -1 for default (order by other properties),
 112                                  -2 or smaller for less than default.
 113                     * http_method  HTTP method to use for the download.
 114                     * http_headers  A dictionary of additional HTTP headers
 115                                  to add to the request.
 116                     * http_post_data  Additional data to send with a POST
 117                                  request.
 118                     * stretched_ratio  If given and not 1, indicates that the
 119                                  video's pixels are not square.
 120                                  width : height ratio as float.
 121                     * no_resume  The server does not support resuming the
 122                                  (HTTP or RTMP) download. Boolean.
 123
 124     url:            Final video URL.
 125     ext:            Video filename extension.
 126     format:         The video format, defaults to ext (used for --get-format)
 127     player_url:     SWF Player URL (used for rtmpdump).
 128
 129     The following fields are optional:
 130
 131     alt_title:      A secondary title of the video.
 132     display_id      An alternative identifier for the video, not necessarily
 133                     unique, but available before title. Typically, id is
 134                     something like "4234987", title "Dancing naked mole rats",
 135                     and display_id "dancing-naked-mole-rats"
 136     thumbnails:     A list of dictionaries, with the following entries:
 137                         * "id" (optional, string) - Thumbnail format ID
 138                         * "url"
 139                         * "preference" (optional, int) - quality of the image
 140                         * "width" (optional, int)
 141                         * "height" (optional, int)
 142                         * "resolution" (optional, string "{width}x{height"},
 143                                         deprecated)
 144     thumbnail:      Full URL to a video thumbnail image.
 145     description:    Full video description.
 146     uploader:       Full name of the video uploader.
 147     creator:        The main artist who created the video.
 148     timestamp:      UNIX timestamp of the moment the video became available.
 149     upload_date:    Video upload date (YYYYMMDD).
 150                     If not explicitly set, calculated from timestamp.
 151     uploader_id:    Nickname or id of the video uploader.
 152     location:       Physical location where the video was filmed.
 153     subtitles:      The available subtitles as a dictionary in the format
 154                     {language: subformats}. "subformats" is a list sorted from
 155                     lower to higher preference, each element is a dictionary
 156                     with the "ext" entry and one of:
 157                         * "data": The subtitles file contents
 158                         * "url": A url pointing to the subtitles file
 159     automatic_captions: Like 'subtitles', used by the YoutubeIE for
 160                     automatically generated captions
 161     duration:       Length of the video in seconds, as an integer.
 162     view_count:     How many users have watched the video on the platform.
 163     like_count:     Number of positive ratings of the video
 164     dislike_count:  Number of negative ratings of the video
 165     average_rating: Average rating give by users, the scale used depends on the webpage
 166     comment_count:  Number of comments on the video
 167     comments:       A list of comments, each with one or more of the following
 168                     properties (all but one of text or html optional):
 169                         * "author" - human-readable name of the comment author
 170                         * "author_id" - user ID of the comment author
 171                         * "id" - Comment ID
 172                         * "html" - Comment as HTML
 173                         * "text" - Plain text of the comment
 174                         * "timestamp" - UNIX timestamp of comment
 175                         * "parent" - ID of the comment this one is replying to.
 176                                      Set to "root" to indicate that this is a
 177                                      comment to the original video.
 178     age_limit:      Age restriction for the video, as an integer (years)
 179     webpage_url:    The url to the video webpage, if given to youtube-dl it
 180                     should allow to get the same result again. (It will be set
 181                     by YoutubeDL if it's missing)
 182     categories:     A list of categories that the video falls in, for example
 183                     ["Sports", "Berlin"]
 184     is_live:        True, False, or None (=unknown). Whether this video is a
 185                     live stream that goes on instead of a fixed-length video.
 186
 187     Unless mentioned otherwise, the fields should be Unicode strings.
 188
 189     Unless mentioned otherwise, None is equivalent to absence of information.
 190
 191
 192     _type "playlist" indicates multiple videos.
 193     There must be a key "entries", which is a list, an iterable, or a PagedList
 194     object, each element of which is a valid dictionary by this specification.
 195
 196     Additionally, playlists can have "title" and "id" attributes with the same
 197     semantics as videos (see above).
 198
 199
 200     _type "multi_video" indicates that there are multiple videos that
 201     form a single show, for examples multiple acts of an opera or TV episode.
 202     It must have an entries key like a playlist and contain all the keys
 203     required for a video at the same time.
 204
 205
 206     _type "url" indicates that the video must be extracted from another
 207     location, possibly by a different extractor. Its only required key is:
 208     "url" - the next URL to extract.
 209     The key "ie_key" can be set to the class name (minus the trailing "IE",
 210     e.g. "Youtube") if the extractor class is known in advance.
 211     Additionally, the dictionary may have any properties of the resolved entity
 212     known in advance, for example "title" if the title of the referred video is
 213     known ahead of time.
 214
 215
 216     _type "url_transparent" entities have the same specification as "url", but
 217     indicate that the given additional information is more precise than the one
 218     associated with the resolved URL.
 219     This is useful when a site employs a video service that hosts the video and
 220     its technical metadata, but that video service does not embed a useful
 221     title, description etc.
 222
 223
 224     Subclasses of this one should re-define the _real_initialize() and
 225     _real_extract() methods and define a _VALID_URL regexp.
 226     Probably, they should also be added to the list of extractors.
 227
 228     Finally, the _WORKING attribute should be set to False for broken IEs
 229     in order to warn the users and skip the tests.
 230     """
 231
 232     _ready = False
 233     _downloader = None
 234     _WORKING = True
 235
 236     def __init__(self, downloader=None):
 237         """Constructor. Receives an optional downloader."""
 238         self._ready = False
 239         self.set_downloader(downloader)
 240
 241     @classmethod
 242     def suitable(cls, url):
 243         """Receives a URL and returns True if suitable for this IE."""
 244
 245         # This does not use has/getattr intentionally - we want to know whether
 246         # we have cached the regexp for *this* class, whereas getattr would also
 247         # match the superclass
 248         if '_VALID_URL_RE' not in cls.__dict__:
 249             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 250         return cls._VALID_URL_RE.match(url) is not None
 251
 252     @classmethod
 253     def _match_id(cls, url):
 254         if '_VALID_URL_RE' not in cls.__dict__:
 255             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 256         m = cls._VALID_URL_RE.match(url)
 257         assert m
 258         return m.group('id')
 259
 260     @classmethod
 261     def working(cls):
 262         """Getter method for _WORKING."""
 263         return cls._WORKING
 264
 265     def initialize(self):
 266         """Initializes an instance (authentication, etc)."""
 267         if not self._ready:
 268             self._real_initialize()
 269             self._ready = True
 270
 271     def extract(self, url):
 272         """Extracts URL information and returns it in list of dicts."""
 273         try:
 274             self.initialize()
 275             return self._real_extract(url)
 276         except ExtractorError:
 277             raise
 278         except compat_http_client.IncompleteRead as e:
 279             raise ExtractorError('A network error has occured.', cause=e, expected=True)
 280         except (KeyError, StopIteration) as e:
 281             raise ExtractorError('An extractor error has occured.', cause=e)
 282
 283     def set_downloader(self, downloader):
 284         """Sets the downloader for this IE."""
 285         self._downloader = downloader
 286
 287     def _real_initialize(self):
 288         """Real initialization process. Redefine in subclasses."""
 289         pass
 290
 291     def _real_extract(self, url):
 292         """Real extraction process. Redefine in subclasses."""
 293         pass
 294
 295     @classmethod
 296     def ie_key(cls):
 297         """A string for getting the InfoExtractor with get_info_extractor"""
 298         return cls.__name__[:-2]
 299
 300     @property
 301     def IE_NAME(self):
 302         return type(self).__name__[:-2]
 303
 304     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 305         """ Returns the response handle """
 306         if note is None:
 307             self.report_download_webpage(video_id)
 308         elif note is not False:
 309             if video_id is None:
 310                 self.to_screen('%s' % (note,))
 311             else:
 312                 self.to_screen('%s: %s' % (video_id, note))
 313         try:
 314             return self._downloader.urlopen(url_or_request)
 315         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 316             if errnote is False:
 317                 return False
 318             if errnote is None:
 319                 errnote = 'Unable to download webpage'
 320             errmsg = '%s: %s' % (errnote, compat_str(err))
 321             if fatal:
 322                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 323             else:
 324                 self._downloader.report_warning(errmsg)
 325                 return False
 326
 327     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 328         """ Returns a tuple (page content as string, URL handle) """
 329         # Strip hashes from the URL (#1038)
 330         if isinstance(url_or_request, (compat_str, str)):
 331             url_or_request = url_or_request.partition('#')[0]
 332
 333         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 334         if urlh is False:
 335             assert not fatal
 336             return False
 337         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
 338         return (content, urlh)
 339
 340     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
 341         content_type = urlh.headers.get('Content-Type', '')
 342         webpage_bytes = urlh.read()
 343         if prefix is not None:
 344             webpage_bytes = prefix + webpage_bytes
 345         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 346         if m:
 347             encoding = m.group(1)
 348         else:
 349             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 350                           webpage_bytes[:1024])
 351             if m:
 352                 encoding = m.group(1).decode('ascii')
 353             elif webpage_bytes.startswith(b'\xff\xfe'):
 354                 encoding = 'utf-16'
 355             else:
 356                 encoding = 'utf-8'
 357         if self._downloader.params.get('dump_intermediate_pages', False):
 358             try:
 359                 url = url_or_request.get_full_url()
 360             except AttributeError:
 361                 url = url_or_request
 362             self.to_screen('Dumping request to ' + url)
 363             dump = base64.b64encode(webpage_bytes).decode('ascii')
 364             self._downloader.to_screen(dump)
 365         if self._downloader.params.get('write_pages', False):
 366             try:
 367                 url = url_or_request.get_full_url()
 368             except AttributeError:
 369                 url = url_or_request
 370             basen = '%s_%s' % (video_id, url)
 371             if len(basen) > 240:
 372                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 373                 basen = basen[:240 - len(h)] + h
 374             raw_filename = basen + '.dump'
 375             filename = sanitize_filename(raw_filename, restricted=True)
 376             self.to_screen('Saving request to ' + filename)
 377             # Working around MAX_PATH limitation on Windows (see
 378             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 379             if os.name == 'nt':
 380                 absfilepath = os.path.abspath(filename)
 381                 if len(absfilepath) > 259:
 382                     filename = '\\\\?\\' + absfilepath
 383             with open(filename, 'wb') as outf:
 384                 outf.write(webpage_bytes)
 385
 386         try:
 387             content = webpage_bytes.decode(encoding, 'replace')
 388         except LookupError:
 389             content = webpage_bytes.decode('utf-8', 'replace')
 390
 391         if ('<title>Access to this site is blocked</title>' in content and
 392                 'Websense' in content[:512]):
 393             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 394             blocked_iframe = self._html_search_regex(
 395                 r'<iframe src="([^"]+)"', content,
 396                 'Websense information URL', default=None)
 397             if blocked_iframe:
 398                 msg += ' Visit %s for more details' % blocked_iframe
 399             raise ExtractorError(msg, expected=True)
 400         if '<title>The URL you requested has been blocked</title>' in content[:512]:
 401             msg = (
 402                 'Access to this webpage has been blocked by Indian censorship. '
 403                 'Use a VPN or proxy server (with --proxy) to route around it.')
 404             block_msg = self._html_search_regex(
 405                 r'</h1><p>(.*?)</p>',
 406                 content, 'block message', default=None)
 407             if block_msg:
 408                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
 409             raise ExtractorError(msg, expected=True)
 410
 411         return content
 412
 413     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
 414         """ Returns the data of the page as a string """
 415         success = False
 416         try_count = 0
 417         while success is False:
 418             try:
 419                 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
 420                 success = True
 421             except compat_http_client.IncompleteRead as e:
 422                 try_count += 1
 423                 if try_count >= tries:
 424                     raise e
 425                 self._sleep(timeout, video_id)
 426         if res is False:
 427             return res
 428         else:
 429             content, _ = res
 430             return content
 431
 432     def _download_xml(self, url_or_request, video_id,
 433                       note='Downloading XML', errnote='Unable to download XML',
 434                       transform_source=None, fatal=True):
 435         """Return the xml as an xml.etree.ElementTree.Element"""
 436         xml_string = self._download_webpage(
 437             url_or_request, video_id, note, errnote, fatal=fatal)
 438         if xml_string is False:
 439             return xml_string
 440         if transform_source:
 441             xml_string = transform_source(xml_string)
 442         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 443
 444     def _download_json(self, url_or_request, video_id,
 445                        note='Downloading JSON metadata',
 446                        errnote='Unable to download JSON metadata',
 447                        transform_source=None,
 448                        fatal=True):
 449         json_string = self._download_webpage(
 450             url_or_request, video_id, note, errnote, fatal=fatal)
 451         if (not fatal) and json_string is False:
 452             return None
 453         return self._parse_json(
 454             json_string, video_id, transform_source=transform_source, fatal=fatal)
 455
 456     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 457         if transform_source:
 458             json_string = transform_source(json_string)
 459         try:
 460             return json.loads(json_string)
 461         except ValueError as ve:
 462             errmsg = '%s: Failed to parse JSON ' % video_id
 463             if fatal:
 464                 raise ExtractorError(errmsg, cause=ve)
 465             else:
 466                 self.report_warning(errmsg + str(ve))
 467
 468     def report_warning(self, msg, video_id=None):
 469         idstr = '' if video_id is None else '%s: ' % video_id
 470         self._downloader.report_warning(
 471             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 472
 473     def to_screen(self, msg):
 474         """Print msg to screen, prefixing it with '[ie_name]'"""
 475         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 476
 477     def report_extraction(self, id_or_name):
 478         """Report information extraction."""
 479         self.to_screen('%s: Extracting information' % id_or_name)
 480
 481     def report_download_webpage(self, video_id):
 482         """Report webpage download."""
 483         self.to_screen('%s: Downloading webpage' % video_id)
 484
 485     def report_age_confirmation(self):
 486         """Report attempt to confirm age."""
 487         self.to_screen('Confirming age')
 488
 489     def report_login(self):
 490         """Report attempt to log in."""
 491         self.to_screen('Logging in')
 492
 493     # Methods for following #608
 494     @staticmethod
 495     def url_result(url, ie=None, video_id=None):
 496         """Returns a url that points to a page that should be processed"""
 497         # TODO: ie should be the class used for getting the info
 498         video_info = {'_type': 'url',
 499                       'url': url,
 500                       'ie_key': ie}
 501         if video_id is not None:
 502             video_info['id'] = video_id
 503         return video_info
 504
 505     @staticmethod
 506     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 507         """Returns a playlist"""
 508         video_info = {'_type': 'playlist',
 509                       'entries': entries}
 510         if playlist_id:
 511             video_info['id'] = playlist_id
 512         if playlist_title:
 513             video_info['title'] = playlist_title
 514         if playlist_description:
 515             video_info['description'] = playlist_description
 516         return video_info
 517
 518     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
 519         """
 520         Perform a regex search on the given string, using a single or a list of
 521         patterns returning the first matching group.
 522         In case of failure return a default value or raise a WARNING or a
 523         RegexNotFoundError, depending on fatal, specifying the field name.
 524         """
 525         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 526             mobj = re.search(pattern, string, flags)
 527         else:
 528             for p in pattern:
 529                 mobj = re.search(p, string, flags)
 530                 if mobj:
 531                     break
 532
 533         if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
 534             _name = '\033[0;34m%s\033[0m' % name
 535         else:
 536             _name = name
 537
 538         if mobj:
 539             if group is None:
 540                 # return the first matching group
 541                 return next(g for g in mobj.groups() if g is not None)
 542             else:
 543                 return mobj.group(group)
 544         elif default is not _NO_DEFAULT:
 545             return default
 546         elif fatal:
 547             raise RegexNotFoundError('Unable to extract %s' % _name)
 548         else:
 549             self._downloader.report_warning('unable to extract %s; '
 550                                             'please report this issue on http://yt-dl.org/bug' % _name)
 551             return None
 552
 553     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
 554         """
 555         Like _search_regex, but strips HTML tags and unescapes entities.
 556         """
 557         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
 558         if res:
 559             return clean_html(res).strip()
 560         else:
 561             return res
 562
 563     def _get_login_info(self):
 564         """
 565         Get the the login info as (username, password)
 566         It will look in the netrc file using the _NETRC_MACHINE value
 567         If there's no info available, return (None, None)
 568         """
 569         if self._downloader is None:
 570             return (None, None)
 571
 572         username = None
 573         password = None
 574         downloader_params = self._downloader.params
 575
 576         # Attempt to use provided username and password or .netrc data
 577         if downloader_params.get('username', None) is not None:
 578             username = downloader_params['username']
 579             password = downloader_params['password']
 580         elif downloader_params.get('usenetrc', False):
 581             try:
 582                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 583                 if info is not None:
 584                     username = info[0]
 585                     password = info[2]
 586                 else:
 587                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 588             except (IOError, netrc.NetrcParseError) as err:
 589                 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
 590
 591         return (username, password)
 592
 593     def _get_tfa_info(self):
 594         """
 595         Get the two-factor authentication info
 596         TODO - asking the user will be required for sms/phone verify
 597         currently just uses the command line option
 598         If there's no info available, return None
 599         """
 600         if self._downloader is None:
 601             return None
 602         downloader_params = self._downloader.params
 603
 604         if downloader_params.get('twofactor', None) is not None:
 605             return downloader_params['twofactor']
 606
 607         return None
 608
 609     # Helper functions for extracting OpenGraph info
 610     @staticmethod
 611     def _og_regexes(prop):
 612         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 613         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 614         template = r'<meta[^>]+?%s[^>]+?%s'
 615         return [
 616             template % (property_re, content_re),
 617             template % (content_re, property_re),
 618         ]
 619
 620     def _og_search_property(self, prop, html, name=None, **kargs):
 621         if name is None:
 622             name = 'OpenGraph %s' % prop
 623         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 624         if escaped is None:
 625             return None
 626         return unescapeHTML(escaped)
 627
 628     def _og_search_thumbnail(self, html, **kargs):
 629         return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
 630
 631     def _og_search_description(self, html, **kargs):
 632         return self._og_search_property('description', html, fatal=False, **kargs)
 633
 634     def _og_search_title(self, html, **kargs):
 635         return self._og_search_property('title', html, **kargs)
 636
 637     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 638         regexes = self._og_regexes('video') + self._og_regexes('video:url')
 639         if secure:
 640             regexes = self._og_regexes('video:secure_url') + regexes
 641         return self._html_search_regex(regexes, html, name, **kargs)
 642
 643     def _og_search_url(self, html, **kargs):
 644         return self._og_search_property('url', html, **kargs)
 645
 646     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
 647         if display_name is None:
 648             display_name = name
 649         return self._html_search_regex(
 650             r'''(?isx)<meta
 651                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
 652                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
 653             html, display_name, fatal=fatal, group='content', **kwargs)
 654
 655     def _dc_search_uploader(self, html):
 656         return self._html_search_meta('dc.creator', html, 'uploader')
 657
 658     def _rta_search(self, html):
 659         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 660         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 661                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 662                      html):
 663             return 18
 664         return 0
 665
 666     def _media_rating_search(self, html):
 667         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 668         rating = self._html_search_meta('rating', html)
 669
 670         if not rating:
 671             return None
 672
 673         RATING_TABLE = {
 674             'safe for kids': 0,
 675             'general': 8,
 676             '14 years': 14,
 677             'mature': 17,
 678             'restricted': 19,
 679         }
 680         return RATING_TABLE.get(rating.lower(), None)
 681
 682     def _family_friendly_search(self, html):
 683         # See http://schema.org/VideoObject
 684         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 685
 686         if not family_friendly:
 687             return None
 688
 689         RATING_TABLE = {
 690             '1': 0,
 691             'true': 0,
 692             '0': 18,
 693             'false': 18,
 694         }
 695         return RATING_TABLE.get(family_friendly.lower(), None)
 696
 697     def _twitter_search_player(self, html):
 698         return self._html_search_meta('twitter:player', html,
 699                                       'twitter card player')
 700
 701     def _sort_formats(self, formats):
 702         if not formats:
 703             raise ExtractorError('No video formats found')
 704
 705         def _formats_key(f):
 706             # TODO remove the following workaround
 707             from ..utils import determine_ext
 708             if not f.get('ext') and 'url' in f:
 709                 f['ext'] = determine_ext(f['url'])
 710
 711             preference = f.get('preference')
 712             if preference is None:
 713                 proto = f.get('protocol')
 714                 if proto is None:
 715                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 716
 717                 preference = 0 if proto in ['http', 'https'] else -0.1
 718                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 719                     preference -= 0.5
 720
 721             if f.get('vcodec') == 'none':  # audio only
 722                 if self._downloader.params.get('prefer_free_formats'):
 723                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
 724                 else:
 725                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
 726                 ext_preference = 0
 727                 try:
 728                     audio_ext_preference = ORDER.index(f['ext'])
 729                 except ValueError:
 730                     audio_ext_preference = -1
 731             else:
 732                 if self._downloader.params.get('prefer_free_formats'):
 733                     ORDER = ['flv', 'mp4', 'webm']
 734                 else:
 735                     ORDER = ['webm', 'flv', 'mp4']
 736                 try:
 737                     ext_preference = ORDER.index(f['ext'])
 738                 except ValueError:
 739                     ext_preference = -1
 740                 audio_ext_preference = 0
 741
 742             return (
 743                 preference,
 744                 f.get('language_preference') if f.get('language_preference') is not None else -1,
 745                 f.get('quality') if f.get('quality') is not None else -1,
 746                 f.get('tbr') if f.get('tbr') is not None else -1,
 747                 f.get('filesize') if f.get('filesize') is not None else -1,
 748                 f.get('vbr') if f.get('vbr') is not None else -1,
 749                 f.get('height') if f.get('height') is not None else -1,
 750                 f.get('width') if f.get('width') is not None else -1,
 751                 ext_preference,
 752                 f.get('abr') if f.get('abr') is not None else -1,
 753                 audio_ext_preference,
 754                 f.get('fps') if f.get('fps') is not None else -1,
 755                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
 756                 f.get('source_preference') if f.get('source_preference') is not None else -1,
 757                 f.get('format_id'),
 758             )
 759         formats.sort(key=_formats_key)
 760
 761     def _check_formats(self, formats, video_id):
 762         if formats:
 763             formats[:] = filter(
 764                 lambda f: self._is_valid_url(
 765                     f['url'], video_id,
 766                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
 767                 formats)
 768
 769     def _is_valid_url(self, url, video_id, item='video'):
 770         try:
 771             self._request_webpage(url, video_id, 'Checking %s URL' % item)
 772             return True
 773         except ExtractorError as e:
 774             if isinstance(e.cause, compat_HTTPError):
 775                 self.report_warning(
 776                     '%s URL is invalid, skipping' % item, video_id)
 777                 return False
 778             raise
 779
 780     def http_scheme(self):
 781         """ Either "http:" or "https:", depending on the user's preferences """
 782         return (
 783             'http:'
 784             if self._downloader.params.get('prefer_insecure', False)
 785             else 'https:')
 786
 787     def _proto_relative_url(self, url, scheme=None):
 788         if url is None:
 789             return url
 790         if url.startswith('//'):
 791             if scheme is None:
 792                 scheme = self.http_scheme()
 793             return scheme + url
 794         else:
 795             return url
 796
 797     def _sleep(self, timeout, video_id, msg_template=None):
 798         if msg_template is None:
 799             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
 800         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
 801         self.to_screen(msg)
 802         time.sleep(timeout)
 803
 804     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
 805         manifest = self._download_xml(
 806             manifest_url, video_id, 'Downloading f4m manifest',
 807             'Unable to download f4m manifest')
 808
 809         formats = []
 810         manifest_version = '1.0'
 811         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
 812         if not media_nodes:
 813             manifest_version = '2.0'
 814             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
 815         for i, media_el in enumerate(media_nodes):
 816             if manifest_version == '2.0':
 817                 manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
 818                                 (media_el.attrib.get('href') or media_el.attrib.get('url')))
 819             tbr = int_or_none(media_el.attrib.get('bitrate'))
 820             formats.append({
 821                 'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
 822                 'url': manifest_url,
 823                 'ext': 'flv',
 824                 'tbr': tbr,
 825                 'width': int_or_none(media_el.attrib.get('width')),
 826                 'height': int_or_none(media_el.attrib.get('height')),
 827                 'preference': preference,
 828             })
 829         self._sort_formats(formats)
 830
 831         return formats
 832
 833     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
 834                               entry_protocol='m3u8', preference=None,
 835                               m3u8_id=None):
 836
 837         formats = [{
 838             'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
 839             'url': m3u8_url,
 840             'ext': ext,
 841             'protocol': 'm3u8',
 842             'preference': preference - 1 if preference else -1,
 843             'resolution': 'multiple',
 844             'format_note': 'Quality selection URL',
 845         }]
 846
 847         format_url = lambda u: (
 848             u
 849             if re.match(r'^https?://', u)
 850             else compat_urlparse.urljoin(m3u8_url, u))
 851
 852         m3u8_doc = self._download_webpage(
 853             m3u8_url, video_id,
 854             note='Downloading m3u8 information',
 855             errnote='Failed to download m3u8 information')
 856         last_info = None
 857         last_media = None
 858         kv_rex = re.compile(
 859             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
 860         for line in m3u8_doc.splitlines():
 861             if line.startswith('#EXT-X-STREAM-INF:'):
 862                 last_info = {}
 863                 for m in kv_rex.finditer(line):
 864                     v = m.group('val')
 865                     if v.startswith('"'):
 866                         v = v[1:-1]
 867                     last_info[m.group('key')] = v
 868             elif line.startswith('#EXT-X-MEDIA:'):
 869                 last_media = {}
 870                 for m in kv_rex.finditer(line):
 871                     v = m.group('val')
 872                     if v.startswith('"'):
 873                         v = v[1:-1]
 874                     last_media[m.group('key')] = v
 875             elif line.startswith('#') or not line.strip():
 876                 continue
 877             else:
 878                 if last_info is None:
 879                     formats.append({'url': format_url(line)})
 880                     continue
 881                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 882                 f = {
 883                     'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
 884                     'url': format_url(line.strip()),
 885                     'tbr': tbr,
 886                     'ext': ext,
 887                     'protocol': entry_protocol,
 888                     'preference': preference,
 889                 }
 890                 codecs = last_info.get('CODECS')
 891                 if codecs:
 892                     # TODO: looks like video codec is not always necessarily goes first
 893                     va_codecs = codecs.split(',')
 894                     if va_codecs[0]:
 895                         f['vcodec'] = va_codecs[0].partition('.')[0]
 896                     if len(va_codecs) > 1 and va_codecs[1]:
 897                         f['acodec'] = va_codecs[1].partition('.')[0]
 898                 resolution = last_info.get('RESOLUTION')
 899                 if resolution:
 900                     width_str, height_str = resolution.split('x')
 901                     f['width'] = int(width_str)
 902                     f['height'] = int(height_str)
 903                 if last_media is not None:
 904                     f['m3u8_media'] = last_media
 905                     last_media = None
 906                 formats.append(f)
 907                 last_info = {}
 908         self._sort_formats(formats)
 909         return formats
 910
 911     # TODO: improve extraction
 912     def _extract_smil_formats(self, smil_url, video_id, fatal=True):
 913         smil = self._download_xml(
 914             smil_url, video_id, 'Downloading SMIL file',
 915             'Unable to download SMIL file', fatal=fatal)
 916         if smil is False:
 917             assert not fatal
 918             return []
 919
 920         base = smil.find('./head/meta').get('base')
 921
 922         formats = []
 923         rtmp_count = 0
 924         if smil.findall('./body/seq/video'):
 925             video = smil.findall('./body/seq/video')[0]
 926             fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
 927             formats.extend(fmts)
 928         else:
 929             for video in smil.findall('./body/switch/video'):
 930                 fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
 931                 formats.extend(fmts)
 932
 933         self._sort_formats(formats)
 934
 935         return formats
 936
 937     def _parse_smil_video(self, video, video_id, base, rtmp_count):
 938         src = video.get('src')
 939         if not src:
 940             return ([], rtmp_count)
 941         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
 942         width = int_or_none(video.get('width'))
 943         height = int_or_none(video.get('height'))
 944         proto = video.get('proto')
 945         if not proto:
 946             if base:
 947                 if base.startswith('rtmp'):
 948                     proto = 'rtmp'
 949                 elif base.startswith('http'):
 950                     proto = 'http'
 951         ext = video.get('ext')
 952         if proto == 'm3u8':
 953             return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
 954         elif proto == 'rtmp':
 955             rtmp_count += 1
 956             streamer = video.get('streamer') or base
 957             return ([{
 958                 'url': streamer,
 959                 'play_path': src,
 960                 'ext': 'flv',
 961                 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
 962                 'tbr': bitrate,
 963                 'width': width,
 964                 'height': height,
 965             }], rtmp_count)
 966         elif proto.startswith('http'):
 967             return ([{
 968                 'url': base + src,
 969                 'ext': ext or 'flv',
 970                 'tbr': bitrate,
 971                 'width': width,
 972                 'height': height,
 973             }], rtmp_count)
 974
 975     def _live_title(self, name):
 976         """ Generate the title for a live video """
 977         now = datetime.datetime.now()
 978         now_str = now.strftime("%Y-%m-%d %H:%M")
 979         return name + ' ' + now_str
 980
 981     def _int(self, v, name, fatal=False, **kwargs):
 982         res = int_or_none(v, **kwargs)
 983         if 'get_attr' in kwargs:
 984             print(getattr(v, kwargs['get_attr']))
 985         if res is None:
 986             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
 987             if fatal:
 988                 raise ExtractorError(msg)
 989             else:
 990                 self._downloader.report_warning(msg)
 991         return res
 992
 993     def _float(self, v, name, fatal=False, **kwargs):
 994         res = float_or_none(v, **kwargs)
 995         if res is None:
 996             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
 997             if fatal:
 998                 raise ExtractorError(msg)
 999             else:
1000                 self._downloader.report_warning(msg)
1001         return res
1002
1003     def _set_cookie(self, domain, name, value, expire_time=None):
1004         cookie = compat_cookiejar.Cookie(
1005             0, name, value, None, None, domain, None,
1006             None, '/', True, False, expire_time, '', None, None, None)
1007         self._downloader.cookiejar.set_cookie(cookie)
1008
1009     def get_testcases(self, include_onlymatching=False):
1010         t = getattr(self, '_TEST', None)
1011         if t:
1012             assert not hasattr(self, '_TESTS'), \
1013                 '%s has _TEST and _TESTS' % type(self).__name__
1014             tests = [t]
1015         else:
1016             tests = getattr(self, '_TESTS', [])
1017         for t in tests:
1018             if not include_onlymatching and t.get('only_matching', False):
1019                 continue
1020             t['name'] = type(self).__name__[:-len('IE')]
1021             yield t
1022
1023     def is_suitable(self, age_limit):
1024         """ Test whether the extractor is generally suitable for the given
1025         age limit (i.e. pornographic sites are not, all others usually are) """
1026
1027         any_restricted = False
1028         for tc in self.get_testcases(include_onlymatching=False):
1029             if 'playlist' in tc:
1030                 tc = tc['playlist'][0]
1031             is_restricted = age_restricted(
1032                 tc.get('info_dict', {}).get('age_limit'), age_limit)
1033             if not is_restricted:
1034                 return True
1035             any_restricted = any_restricted or is_restricted
1036         return not any_restricted
1037
1038     def extract_subtitles(self, *args, **kwargs):
1039         if (self._downloader.params.get('writesubtitles', False) or
1040                 self._downloader.params.get('listsubtitles')):
1041             return self._get_subtitles(*args, **kwargs)
1042         return {}
1043
1044     def _get_subtitles(self, *args, **kwargs):
1045         raise NotImplementedError("This method must be implemented by subclasses")
1046
1047     def extract_automatic_captions(self, *args, **kwargs):
1048         if (self._downloader.params.get('writeautomaticsub', False) or
1049                 self._downloader.params.get('listsubtitles')):
1050             return self._get_automatic_captions(*args, **kwargs)
1051         return {}
1052
1053     def _get_automatic_captions(self, *args, **kwargs):
1054         raise NotImplementedError("This method must be implemented by subclasses")
1055
1056
1057 class SearchInfoExtractor(InfoExtractor):
1058     """
1059     Base class for paged search queries extractors.
1060     They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
1061     Instances should define _SEARCH_KEY and _MAX_RESULTS.
1062     """
1063
1064     @classmethod
1065     def _make_valid_url(cls):
1066         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
1067
1068     @classmethod
1069     def suitable(cls, url):
1070         return re.match(cls._make_valid_url(), url) is not None
1071
1072     def _real_extract(self, query):
1073         mobj = re.match(self._make_valid_url(), query)
1074         if mobj is None:
1075             raise ExtractorError('Invalid search query "%s"' % query)
1076
1077         prefix = mobj.group('prefix')
1078         query = mobj.group('query')
1079         if prefix == '':
1080             return self._get_n_results(query, 1)
1081         elif prefix == 'all':
1082             return self._get_n_results(query, self._MAX_RESULTS)
1083         else:
1084             n = int(prefix)
1085             if n <= 0:
1086                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
1087             elif n > self._MAX_RESULTS:
1088                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
1089                 n = self._MAX_RESULTS
1090             return self._get_n_results(query, n)
1091
1092     def _get_n_results(self, query, n):
1093         """Get a specified number of results for a query"""
1094         raise NotImplementedError("This method must be implemented by subclasses")
1095
1096     @property
1097     def SEARCH_KEY(self):
1098         return self._SEARCH_KEY