Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/extractor/common.py

   1 from __future__ import unicode_literals
   2
   3 import base64
   4 import datetime
   5 import hashlib
   6 import json
   7 import netrc
   8 import os
   9 import re
  10 import socket
  11 import sys
  12 import time
  13 import xml.etree.ElementTree
  14
  15 from ..compat import (
  16     compat_cookiejar,
  17     compat_HTTPError,
  18     compat_http_client,
  19     compat_urllib_error,
  20     compat_urllib_parse_urlparse,
  21     compat_urlparse,
  22     compat_str,
  23 )
  24 from ..utils import (
  25     age_restricted,
  26     bug_reports_message,
  27     clean_html,
  28     compiled_regex_type,
  29     ExtractorError,
  30     float_or_none,
  31     int_or_none,
  32     RegexNotFoundError,
  33     sanitize_filename,
  34     unescapeHTML,
  35 )
  36 _NO_DEFAULT = object()
  37
  38
  39 class InfoExtractor(object):
  40     """Information Extractor class.
  41
  42     Information extractors are the classes that, given a URL, extract
  43     information about the video (or videos) the URL refers to. This
  44     information includes the real video URL, the video title, author and
  45     others. The information is stored in a dictionary which is then
  46     passed to the YoutubeDL. The YoutubeDL processes this
  47     information possibly downloading the video to the file system, among
  48     other possible outcomes.
  49
  50     The type field determines the type of the result.
  51     By far the most common value (and the default if _type is missing) is
  52     "video", which indicates a single video.
  53
  54     For a video, the dictionaries must include the following fields:
  55
  56     id:             Video identifier.
  57     title:          Video title, unescaped.
  58
  59     Additionally, it must contain either a formats entry or a url one:
  60
  61     formats:        A list of dictionaries for each format available, ordered
  62                     from worst to best quality.
  63
  64                     Potential fields:
  65                     * url        Mandatory. The URL of the video file
  66                     * ext        Will be calculated from url if missing
  67                     * format     A human-readable description of the format
  68                                  ("mp4 container with h264/opus").
  69                                  Calculated from the format_id, width, height.
  70                                  and format_note fields if missing.
  71                     * format_id  A short description of the format
  72                                  ("mp4_h264_opus" or "19").
  73                                 Technically optional, but strongly recommended.
  74                     * format_note Additional info about the format
  75                                  ("3D" or "DASH video")
  76                     * width      Width of the video, if known
  77                     * height     Height of the video, if known
  78                     * resolution Textual description of width and height
  79                     * tbr        Average bitrate of audio and video in KBit/s
  80                     * abr        Average audio bitrate in KBit/s
  81                     * acodec     Name of the audio codec in use
  82                     * asr        Audio sampling rate in Hertz
  83                     * vbr        Average video bitrate in KBit/s
  84                     * fps        Frame rate
  85                     * vcodec     Name of the video codec in use
  86                     * container  Name of the container format
  87                     * filesize   The number of bytes, if known in advance
  88                     * filesize_approx  An estimate for the number of bytes
  89                     * player_url SWF Player URL (used for rtmpdump).
  90                     * protocol   The protocol that will be used for the actual
  91                                  download, lower-case.
  92                                  "http", "https", "rtsp", "rtmp", "rtmpe",
  93                                  "m3u8", or "m3u8_native".
  94                     * preference Order number of this format. If this field is
  95                                  present and not None, the formats get sorted
  96                                  by this field, regardless of all other values.
  97                                  -1 for default (order by other properties),
  98                                  -2 or smaller for less than default.
  99                                  < -1000 to hide the format (if there is
 100                                     another one which is strictly better)
 101                     * language_preference  Is this in the correct requested
 102                                  language?
 103                                  10 if it's what the URL is about,
 104                                  -1 for default (don't know),
 105                                  -10 otherwise, other values reserved for now.
 106                     * quality    Order number of the video quality of this
 107                                  format, irrespective of the file format.
 108                                  -1 for default (order by other properties),
 109                                  -2 or smaller for less than default.
 110                     * source_preference  Order number for this video source
 111                                   (quality takes higher priority)
 112                                  -1 for default (order by other properties),
 113                                  -2 or smaller for less than default.
 114                     * http_headers  A dictionary of additional HTTP headers
 115                                  to add to the request.
 116                     * stretched_ratio  If given and not 1, indicates that the
 117                                  video's pixels are not square.
 118                                  width : height ratio as float.
 119                     * no_resume  The server does not support resuming the
 120                                  (HTTP or RTMP) download. Boolean.
 121
 122     url:            Final video URL.
 123     ext:            Video filename extension.
 124     format:         The video format, defaults to ext (used for --get-format)
 125     player_url:     SWF Player URL (used for rtmpdump).
 126
 127     The following fields are optional:
 128
 129     alt_title:      A secondary title of the video.
 130     display_id      An alternative identifier for the video, not necessarily
 131                     unique, but available before title. Typically, id is
 132                     something like "4234987", title "Dancing naked mole rats",
 133                     and display_id "dancing-naked-mole-rats"
 134     thumbnails:     A list of dictionaries, with the following entries:
 135                         * "id" (optional, string) - Thumbnail format ID
 136                         * "url"
 137                         * "preference" (optional, int) - quality of the image
 138                         * "width" (optional, int)
 139                         * "height" (optional, int)
 140                         * "resolution" (optional, string "{width}x{height"},
 141                                         deprecated)
 142     thumbnail:      Full URL to a video thumbnail image.
 143     description:    Full video description.
 144     uploader:       Full name of the video uploader.
 145     creator:        The main artist who created the video.
 146     timestamp:      UNIX timestamp of the moment the video became available.
 147     upload_date:    Video upload date (YYYYMMDD).
 148                     If not explicitly set, calculated from timestamp.
 149     uploader_id:    Nickname or id of the video uploader.
 150     location:       Physical location where the video was filmed.
 151     subtitles:      The available subtitles as a dictionary in the format
 152                     {language: subformats}. "subformats" is a list sorted from
 153                     lower to higher preference, each element is a dictionary
 154                     with the "ext" entry and one of:
 155                         * "data": The subtitles file contents
 156                         * "url": A url pointing to the subtitles file
 157     automatic_captions: Like 'subtitles', used by the YoutubeIE for
 158                     automatically generated captions
 159     duration:       Length of the video in seconds, as an integer.
 160     view_count:     How many users have watched the video on the platform.
 161     like_count:     Number of positive ratings of the video
 162     dislike_count:  Number of negative ratings of the video
 163     average_rating: Average rating give by users, the scale used depends on the webpage
 164     comment_count:  Number of comments on the video
 165     comments:       A list of comments, each with one or more of the following
 166                     properties (all but one of text or html optional):
 167                         * "author" - human-readable name of the comment author
 168                         * "author_id" - user ID of the comment author
 169                         * "id" - Comment ID
 170                         * "html" - Comment as HTML
 171                         * "text" - Plain text of the comment
 172                         * "timestamp" - UNIX timestamp of comment
 173                         * "parent" - ID of the comment this one is replying to.
 174                                      Set to "root" to indicate that this is a
 175                                      comment to the original video.
 176     age_limit:      Age restriction for the video, as an integer (years)
 177     webpage_url:    The url to the video webpage, if given to youtube-dl it
 178                     should allow to get the same result again. (It will be set
 179                     by YoutubeDL if it's missing)
 180     categories:     A list of categories that the video falls in, for example
 181                     ["Sports", "Berlin"]
 182     is_live:        True, False, or None (=unknown). Whether this video is a
 183                     live stream that goes on instead of a fixed-length video.
 184
 185     Unless mentioned otherwise, the fields should be Unicode strings.
 186
 187     Unless mentioned otherwise, None is equivalent to absence of information.
 188
 189
 190     _type "playlist" indicates multiple videos.
 191     There must be a key "entries", which is a list, an iterable, or a PagedList
 192     object, each element of which is a valid dictionary by this specification.
 193
 194     Additionally, playlists can have "title" and "id" attributes with the same
 195     semantics as videos (see above).
 196
 197
 198     _type "multi_video" indicates that there are multiple videos that
 199     form a single show, for examples multiple acts of an opera or TV episode.
 200     It must have an entries key like a playlist and contain all the keys
 201     required for a video at the same time.
 202
 203
 204     _type "url" indicates that the video must be extracted from another
 205     location, possibly by a different extractor. Its only required key is:
 206     "url" - the next URL to extract.
 207     The key "ie_key" can be set to the class name (minus the trailing "IE",
 208     e.g. "Youtube") if the extractor class is known in advance.
 209     Additionally, the dictionary may have any properties of the resolved entity
 210     known in advance, for example "title" if the title of the referred video is
 211     known ahead of time.
 212
 213
 214     _type "url_transparent" entities have the same specification as "url", but
 215     indicate that the given additional information is more precise than the one
 216     associated with the resolved URL.
 217     This is useful when a site employs a video service that hosts the video and
 218     its technical metadata, but that video service does not embed a useful
 219     title, description etc.
 220
 221
 222     Subclasses of this one should re-define the _real_initialize() and
 223     _real_extract() methods and define a _VALID_URL regexp.
 224     Probably, they should also be added to the list of extractors.
 225
 226     Finally, the _WORKING attribute should be set to False for broken IEs
 227     in order to warn the users and skip the tests.
 228     """
 229
 230     _ready = False
 231     _downloader = None
 232     _WORKING = True
 233
 234     def __init__(self, downloader=None):
 235         """Constructor. Receives an optional downloader."""
 236         self._ready = False
 237         self.set_downloader(downloader)
 238
 239     @classmethod
 240     def suitable(cls, url):
 241         """Receives a URL and returns True if suitable for this IE."""
 242
 243         # This does not use has/getattr intentionally - we want to know whether
 244         # we have cached the regexp for *this* class, whereas getattr would also
 245         # match the superclass
 246         if '_VALID_URL_RE' not in cls.__dict__:
 247             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 248         return cls._VALID_URL_RE.match(url) is not None
 249
 250     @classmethod
 251     def _match_id(cls, url):
 252         if '_VALID_URL_RE' not in cls.__dict__:
 253             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 254         m = cls._VALID_URL_RE.match(url)
 255         assert m
 256         return m.group('id')
 257
 258     @classmethod
 259     def working(cls):
 260         """Getter method for _WORKING."""
 261         return cls._WORKING
 262
 263     def initialize(self):
 264         """Initializes an instance (authentication, etc)."""
 265         if not self._ready:
 266             self._real_initialize()
 267             self._ready = True
 268
 269     def extract(self, url):
 270         """Extracts URL information and returns it in list of dicts."""
 271         try:
 272             self.initialize()
 273             return self._real_extract(url)
 274         except ExtractorError:
 275             raise
 276         except compat_http_client.IncompleteRead as e:
 277             raise ExtractorError('A network error has occured.', cause=e, expected=True)
 278         except (KeyError, StopIteration) as e:
 279             raise ExtractorError('An extractor error has occured.', cause=e)
 280
 281     def set_downloader(self, downloader):
 282         """Sets the downloader for this IE."""
 283         self._downloader = downloader
 284
 285     def _real_initialize(self):
 286         """Real initialization process. Redefine in subclasses."""
 287         pass
 288
 289     def _real_extract(self, url):
 290         """Real extraction process. Redefine in subclasses."""
 291         pass
 292
 293     @classmethod
 294     def ie_key(cls):
 295         """A string for getting the InfoExtractor with get_info_extractor"""
 296         return cls.__name__[:-2]
 297
 298     @property
 299     def IE_NAME(self):
 300         return type(self).__name__[:-2]
 301
 302     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
 303         """ Returns the response handle """
 304         if note is None:
 305             self.report_download_webpage(video_id)
 306         elif note is not False:
 307             if video_id is None:
 308                 self.to_screen('%s' % (note,))
 309             else:
 310                 self.to_screen('%s: %s' % (video_id, note))
 311         try:
 312             return self._downloader.urlopen(url_or_request)
 313         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 314             if errnote is False:
 315                 return False
 316             if errnote is None:
 317                 errnote = 'Unable to download webpage'
 318             errmsg = '%s: %s' % (errnote, compat_str(err))
 319             if fatal:
 320                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 321             else:
 322                 self._downloader.report_warning(errmsg)
 323                 return False
 324
 325     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None):
 326         """ Returns a tuple (page content as string, URL handle) """
 327         # Strip hashes from the URL (#1038)
 328         if isinstance(url_or_request, (compat_str, str)):
 329             url_or_request = url_or_request.partition('#')[0]
 330
 331         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
 332         if urlh is False:
 333             assert not fatal
 334             return False
 335         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 336         return (content, urlh)
 337
 338     @staticmethod
 339     def _guess_encoding_from_content(content_type, webpage_bytes):
 340         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 341         if m:
 342             encoding = m.group(1)
 343         else:
 344             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 345                           webpage_bytes[:1024])
 346             if m:
 347                 encoding = m.group(1).decode('ascii')
 348             elif webpage_bytes.startswith(b'\xff\xfe'):
 349                 encoding = 'utf-16'
 350             else:
 351                 encoding = 'utf-8'
 352
 353         return encoding
 354
 355     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
 356         content_type = urlh.headers.get('Content-Type', '')
 357         webpage_bytes = urlh.read()
 358         if prefix is not None:
 359             webpage_bytes = prefix + webpage_bytes
 360         if not encoding:
 361             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
 362         if self._downloader.params.get('dump_intermediate_pages', False):
 363             try:
 364                 url = url_or_request.get_full_url()
 365             except AttributeError:
 366                 url = url_or_request
 367             self.to_screen('Dumping request to ' + url)
 368             dump = base64.b64encode(webpage_bytes).decode('ascii')
 369             self._downloader.to_screen(dump)
 370         if self._downloader.params.get('write_pages', False):
 371             try:
 372                 url = url_or_request.get_full_url()
 373             except AttributeError:
 374                 url = url_or_request
 375             basen = '%s_%s' % (video_id, url)
 376             if len(basen) > 240:
 377                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 378                 basen = basen[:240 - len(h)] + h
 379             raw_filename = basen + '.dump'
 380             filename = sanitize_filename(raw_filename, restricted=True)
 381             self.to_screen('Saving request to ' + filename)
 382             # Working around MAX_PATH limitation on Windows (see
 383             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 384             if os.name == 'nt':
 385                 absfilepath = os.path.abspath(filename)
 386                 if len(absfilepath) > 259:
 387                     filename = '\\\\?\\' + absfilepath
 388             with open(filename, 'wb') as outf:
 389                 outf.write(webpage_bytes)
 390
 391         try:
 392             content = webpage_bytes.decode(encoding, 'replace')
 393         except LookupError:
 394             content = webpage_bytes.decode('utf-8', 'replace')
 395
 396         if ('<title>Access to this site is blocked</title>' in content and
 397                 'Websense' in content[:512]):
 398             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 399             blocked_iframe = self._html_search_regex(
 400                 r'<iframe src="([^"]+)"', content,
 401                 'Websense information URL', default=None)
 402             if blocked_iframe:
 403                 msg += ' Visit %s for more details' % blocked_iframe
 404             raise ExtractorError(msg, expected=True)
 405         if '<title>The URL you requested has been blocked</title>' in content[:512]:
 406             msg = (
 407                 'Access to this webpage has been blocked by Indian censorship. '
 408                 'Use a VPN or proxy server (with --proxy) to route around it.')
 409             block_msg = self._html_search_regex(
 410                 r'</h1><p>(.*?)</p>',
 411                 content, 'block message', default=None)
 412             if block_msg:
 413                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
 414             raise ExtractorError(msg, expected=True)
 415
 416         return content
 417
 418     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
 419         """ Returns the data of the page as a string """
 420         success = False
 421         try_count = 0
 422         while success is False:
 423             try:
 424                 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 425                 success = True
 426             except compat_http_client.IncompleteRead as e:
 427                 try_count += 1
 428                 if try_count >= tries:
 429                     raise e
 430                 self._sleep(timeout, video_id)
 431         if res is False:
 432             return res
 433         else:
 434             content, _ = res
 435             return content
 436
 437     def _download_xml(self, url_or_request, video_id,
 438                       note='Downloading XML', errnote='Unable to download XML',
 439                       transform_source=None, fatal=True, encoding=None):
 440         """Return the xml as an xml.etree.ElementTree.Element"""
 441         xml_string = self._download_webpage(
 442             url_or_request, video_id, note, errnote, fatal=fatal, encoding=encoding)
 443         if xml_string is False:
 444             return xml_string
 445         if transform_source:
 446             xml_string = transform_source(xml_string)
 447         return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
 448
 449     def _download_json(self, url_or_request, video_id,
 450                        note='Downloading JSON metadata',
 451                        errnote='Unable to download JSON metadata',
 452                        transform_source=None,
 453                        fatal=True, encoding=None):
 454         json_string = self._download_webpage(
 455             url_or_request, video_id, note, errnote, fatal=fatal,
 456             encoding=encoding)
 457         if (not fatal) and json_string is False:
 458             return None
 459         return self._parse_json(
 460             json_string, video_id, transform_source=transform_source, fatal=fatal)
 461
 462     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 463         if transform_source:
 464             json_string = transform_source(json_string)
 465         try:
 466             return json.loads(json_string)
 467         except ValueError as ve:
 468             errmsg = '%s: Failed to parse JSON ' % video_id
 469             if fatal:
 470                 raise ExtractorError(errmsg, cause=ve)
 471             else:
 472                 self.report_warning(errmsg + str(ve))
 473
 474     def report_warning(self, msg, video_id=None):
 475         idstr = '' if video_id is None else '%s: ' % video_id
 476         self._downloader.report_warning(
 477             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 478
 479     def to_screen(self, msg):
 480         """Print msg to screen, prefixing it with '[ie_name]'"""
 481         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 482
 483     def report_extraction(self, id_or_name):
 484         """Report information extraction."""
 485         self.to_screen('%s: Extracting information' % id_or_name)
 486
 487     def report_download_webpage(self, video_id):
 488         """Report webpage download."""
 489         self.to_screen('%s: Downloading webpage' % video_id)
 490
 491     def report_age_confirmation(self):
 492         """Report attempt to confirm age."""
 493         self.to_screen('Confirming age')
 494
 495     def report_login(self):
 496         """Report attempt to log in."""
 497         self.to_screen('Logging in')
 498
 499     # Methods for following #608
 500     @staticmethod
 501     def url_result(url, ie=None, video_id=None, video_title=None):
 502         """Returns a url that points to a page that should be processed"""
 503         # TODO: ie should be the class used for getting the info
 504         video_info = {'_type': 'url',
 505                       'url': url,
 506                       'ie_key': ie}
 507         if video_id is not None:
 508             video_info['id'] = video_id
 509         if video_title is not None:
 510             video_info['title'] = video_title
 511         return video_info
 512
 513     @staticmethod
 514     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 515         """Returns a playlist"""
 516         video_info = {'_type': 'playlist',
 517                       'entries': entries}
 518         if playlist_id:
 519             video_info['id'] = playlist_id
 520         if playlist_title:
 521             video_info['title'] = playlist_title
 522         if playlist_description:
 523             video_info['description'] = playlist_description
 524         return video_info
 525
 526     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
 527         """
 528         Perform a regex search on the given string, using a single or a list of
 529         patterns returning the first matching group.
 530         In case of failure return a default value or raise a WARNING or a
 531         RegexNotFoundError, depending on fatal, specifying the field name.
 532         """
 533         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 534             mobj = re.search(pattern, string, flags)
 535         else:
 536             for p in pattern:
 537                 mobj = re.search(p, string, flags)
 538                 if mobj:
 539                     break
 540
 541         if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
 542             _name = '\033[0;34m%s\033[0m' % name
 543         else:
 544             _name = name
 545
 546         if mobj:
 547             if group is None:
 548                 # return the first matching group
 549                 return next(g for g in mobj.groups() if g is not None)
 550             else:
 551                 return mobj.group(group)
 552         elif default is not _NO_DEFAULT:
 553             return default
 554         elif fatal:
 555             raise RegexNotFoundError('Unable to extract %s' % _name)
 556         else:
 557             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
 558             return None
 559
 560     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
 561         """
 562         Like _search_regex, but strips HTML tags and unescapes entities.
 563         """
 564         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
 565         if res:
 566             return clean_html(res).strip()
 567         else:
 568             return res
 569
 570     def _get_login_info(self):
 571         """
 572         Get the login info as (username, password)
 573         It will look in the netrc file using the _NETRC_MACHINE value
 574         If there's no info available, return (None, None)
 575         """
 576         if self._downloader is None:
 577             return (None, None)
 578
 579         username = None
 580         password = None
 581         downloader_params = self._downloader.params
 582
 583         # Attempt to use provided username and password or .netrc data
 584         if downloader_params.get('username', None) is not None:
 585             username = downloader_params['username']
 586             password = downloader_params['password']
 587         elif downloader_params.get('usenetrc', False):
 588             try:
 589                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 590                 if info is not None:
 591                     username = info[0]
 592                     password = info[2]
 593                 else:
 594                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 595             except (IOError, netrc.NetrcParseError) as err:
 596                 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
 597
 598         return (username, password)
 599
 600     def _get_tfa_info(self):
 601         """
 602         Get the two-factor authentication info
 603         TODO - asking the user will be required for sms/phone verify
 604         currently just uses the command line option
 605         If there's no info available, return None
 606         """
 607         if self._downloader is None:
 608             return None
 609         downloader_params = self._downloader.params
 610
 611         if downloader_params.get('twofactor', None) is not None:
 612             return downloader_params['twofactor']
 613
 614         return None
 615
 616     # Helper functions for extracting OpenGraph info
 617     @staticmethod
 618     def _og_regexes(prop):
 619         content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
 620         property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
 621         template = r'<meta[^>]+?%s[^>]+?%s'
 622         return [
 623             template % (property_re, content_re),
 624             template % (content_re, property_re),
 625         ]
 626
 627     def _og_search_property(self, prop, html, name=None, **kargs):
 628         if name is None:
 629             name = 'OpenGraph %s' % prop
 630         escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
 631         if escaped is None:
 632             return None
 633         return unescapeHTML(escaped)
 634
 635     def _og_search_thumbnail(self, html, **kargs):
 636         return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
 637
 638     def _og_search_description(self, html, **kargs):
 639         return self._og_search_property('description', html, fatal=False, **kargs)
 640
 641     def _og_search_title(self, html, **kargs):
 642         return self._og_search_property('title', html, **kargs)
 643
 644     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
 645         regexes = self._og_regexes('video') + self._og_regexes('video:url')
 646         if secure:
 647             regexes = self._og_regexes('video:secure_url') + regexes
 648         return self._html_search_regex(regexes, html, name, **kargs)
 649
 650     def _og_search_url(self, html, **kargs):
 651         return self._og_search_property('url', html, **kargs)
 652
 653     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
 654         if display_name is None:
 655             display_name = name
 656         return self._html_search_regex(
 657             r'''(?isx)<meta
 658                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
 659                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
 660             html, display_name, fatal=fatal, group='content', **kwargs)
 661
 662     def _dc_search_uploader(self, html):
 663         return self._html_search_meta('dc.creator', html, 'uploader')
 664
 665     def _rta_search(self, html):
 666         # See http://www.rtalabel.org/index.php?content=howtofaq#single
 667         if re.search(r'(?ix)<meta\s+name="rating"\s+'
 668                      r'     content="RTA-5042-1996-1400-1577-RTA"',
 669                      html):
 670             return 18
 671         return 0
 672
 673     def _media_rating_search(self, html):
 674         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
 675         rating = self._html_search_meta('rating', html)
 676
 677         if not rating:
 678             return None
 679
 680         RATING_TABLE = {
 681             'safe for kids': 0,
 682             'general': 8,
 683             '14 years': 14,
 684             'mature': 17,
 685             'restricted': 19,
 686         }
 687         return RATING_TABLE.get(rating.lower(), None)
 688
 689     def _family_friendly_search(self, html):
 690         # See http://schema.org/VideoObject
 691         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 692
 693         if not family_friendly:
 694             return None
 695
 696         RATING_TABLE = {
 697             '1': 0,
 698             'true': 0,
 699             '0': 18,
 700             'false': 18,
 701         }
 702         return RATING_TABLE.get(family_friendly.lower(), None)
 703
 704     def _twitter_search_player(self, html):
 705         return self._html_search_meta('twitter:player', html,
 706                                       'twitter card player')
 707
 708     def _sort_formats(self, formats, field_preference=None):
 709         if not formats:
 710             raise ExtractorError('No video formats found')
 711
 712         def _formats_key(f):
 713             # TODO remove the following workaround
 714             from ..utils import determine_ext
 715             if not f.get('ext') and 'url' in f:
 716                 f['ext'] = determine_ext(f['url'])
 717
 718             if isinstance(field_preference, (list, tuple)):
 719                 return tuple(f.get(field) if f.get(field) is not None else -1 for field in field_preference)
 720
 721             preference = f.get('preference')
 722             if preference is None:
 723                 proto = f.get('protocol')
 724                 if proto is None:
 725                     proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
 726
 727                 preference = 0 if proto in ['http', 'https'] else -0.1
 728                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
 729                     preference -= 0.5
 730
 731             if f.get('vcodec') == 'none':  # audio only
 732                 if self._downloader.params.get('prefer_free_formats'):
 733                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
 734                 else:
 735                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
 736                 ext_preference = 0
 737                 try:
 738                     audio_ext_preference = ORDER.index(f['ext'])
 739                 except ValueError:
 740                     audio_ext_preference = -1
 741             else:
 742                 if self._downloader.params.get('prefer_free_formats'):
 743                     ORDER = ['flv', 'mp4', 'webm']
 744                 else:
 745                     ORDER = ['webm', 'flv', 'mp4']
 746                 try:
 747                     ext_preference = ORDER.index(f['ext'])
 748                 except ValueError:
 749                     ext_preference = -1
 750                 audio_ext_preference = 0
 751
 752             return (
 753                 preference,
 754                 f.get('language_preference') if f.get('language_preference') is not None else -1,
 755                 f.get('quality') if f.get('quality') is not None else -1,
 756                 f.get('tbr') if f.get('tbr') is not None else -1,
 757                 f.get('filesize') if f.get('filesize') is not None else -1,
 758                 f.get('vbr') if f.get('vbr') is not None else -1,
 759                 f.get('height') if f.get('height') is not None else -1,
 760                 f.get('width') if f.get('width') is not None else -1,
 761                 ext_preference,
 762                 f.get('abr') if f.get('abr') is not None else -1,
 763                 audio_ext_preference,
 764                 f.get('fps') if f.get('fps') is not None else -1,
 765                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
 766                 f.get('source_preference') if f.get('source_preference') is not None else -1,
 767                 f.get('format_id') if f.get('format_id') is not None else '',
 768             )
 769         formats.sort(key=_formats_key)
 770
 771     def _check_formats(self, formats, video_id):
 772         if formats:
 773             formats[:] = filter(
 774                 lambda f: self._is_valid_url(
 775                     f['url'], video_id,
 776                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
 777                 formats)
 778
 779     def _is_valid_url(self, url, video_id, item='video'):
 780         url = self._proto_relative_url(url, scheme='http:')
 781         # For now assume non HTTP(S) URLs always valid
 782         if not (url.startswith('http://') or url.startswith('https://')):
 783             return True
 784         try:
 785             self._request_webpage(url, video_id, 'Checking %s URL' % item)
 786             return True
 787         except ExtractorError as e:
 788             if isinstance(e.cause, compat_HTTPError):
 789                 self.to_screen(
 790                     '%s: %s URL is invalid, skipping' % (video_id, item))
 791                 return False
 792             raise
 793
 794     def http_scheme(self):
 795         """ Either "http:" or "https:", depending on the user's preferences """
 796         return (
 797             'http:'
 798             if self._downloader.params.get('prefer_insecure', False)
 799             else 'https:')
 800
 801     def _proto_relative_url(self, url, scheme=None):
 802         if url is None:
 803             return url
 804         if url.startswith('//'):
 805             if scheme is None:
 806                 scheme = self.http_scheme()
 807             return scheme + url
 808         else:
 809             return url
 810
 811     def _sleep(self, timeout, video_id, msg_template=None):
 812         if msg_template is None:
 813             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
 814         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
 815         self.to_screen(msg)
 816         time.sleep(timeout)
 817
 818     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
 819         manifest = self._download_xml(
 820             manifest_url, video_id, 'Downloading f4m manifest',
 821             'Unable to download f4m manifest')
 822
 823         formats = []
 824         manifest_version = '1.0'
 825         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
 826         if not media_nodes:
 827             manifest_version = '2.0'
 828             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
 829         for i, media_el in enumerate(media_nodes):
 830             if manifest_version == '2.0':
 831                 manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
 832                                 (media_el.attrib.get('href') or media_el.attrib.get('url')))
 833             tbr = int_or_none(media_el.attrib.get('bitrate'))
 834             formats.append({
 835                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),
 836                 'url': manifest_url,
 837                 'ext': 'flv',
 838                 'tbr': tbr,
 839                 'width': int_or_none(media_el.attrib.get('width')),
 840                 'height': int_or_none(media_el.attrib.get('height')),
 841                 'preference': preference,
 842             })
 843         self._sort_formats(formats)
 844
 845         return formats
 846
 847     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
 848                               entry_protocol='m3u8', preference=None,
 849                               m3u8_id=None):
 850
 851         formats = [{
 852             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
 853             'url': m3u8_url,
 854             'ext': ext,
 855             'protocol': 'm3u8',
 856             'preference': preference - 1 if preference else -1,
 857             'resolution': 'multiple',
 858             'format_note': 'Quality selection URL',
 859         }]
 860
 861         format_url = lambda u: (
 862             u
 863             if re.match(r'^https?://', u)
 864             else compat_urlparse.urljoin(m3u8_url, u))
 865
 866         m3u8_doc = self._download_webpage(
 867             m3u8_url, video_id,
 868             note='Downloading m3u8 information',
 869             errnote='Failed to download m3u8 information')
 870         last_info = None
 871         last_media = None
 872         kv_rex = re.compile(
 873             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
 874         for line in m3u8_doc.splitlines():
 875             if line.startswith('#EXT-X-STREAM-INF:'):
 876                 last_info = {}
 877                 for m in kv_rex.finditer(line):
 878                     v = m.group('val')
 879                     if v.startswith('"'):
 880                         v = v[1:-1]
 881                     last_info[m.group('key')] = v
 882             elif line.startswith('#EXT-X-MEDIA:'):
 883                 last_media = {}
 884                 for m in kv_rex.finditer(line):
 885                     v = m.group('val')
 886                     if v.startswith('"'):
 887                         v = v[1:-1]
 888                     last_media[m.group('key')] = v
 889             elif line.startswith('#') or not line.strip():
 890                 continue
 891             else:
 892                 if last_info is None:
 893                     formats.append({'url': format_url(line)})
 894                     continue
 895                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 896                 format_id = []
 897                 if m3u8_id:
 898                     format_id.append(m3u8_id)
 899                 last_media_name = last_media.get('NAME') if last_media and last_media.get('TYPE') != 'SUBTITLES' else None
 900                 format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
 901                 f = {
 902                     'format_id': '-'.join(format_id),
 903                     'url': format_url(line.strip()),
 904                     'tbr': tbr,
 905                     'ext': ext,
 906                     'protocol': entry_protocol,
 907                     'preference': preference,
 908                 }
 909                 codecs = last_info.get('CODECS')
 910                 if codecs:
 911                     # TODO: looks like video codec is not always necessarily goes first
 912                     va_codecs = codecs.split(',')
 913                     if va_codecs[0]:
 914                         f['vcodec'] = va_codecs[0].partition('.')[0]
 915                     if len(va_codecs) > 1 and va_codecs[1]:
 916                         f['acodec'] = va_codecs[1].partition('.')[0]
 917                 resolution = last_info.get('RESOLUTION')
 918                 if resolution:
 919                     width_str, height_str = resolution.split('x')
 920                     f['width'] = int(width_str)
 921                     f['height'] = int(height_str)
 922                 if last_media is not None:
 923                     f['m3u8_media'] = last_media
 924                     last_media = None
 925                 formats.append(f)
 926                 last_info = {}
 927         self._sort_formats(formats)
 928         return formats
 929
 930     # TODO: improve extraction
 931     def _extract_smil_formats(self, smil_url, video_id, fatal=True):
 932         smil = self._download_xml(
 933             smil_url, video_id, 'Downloading SMIL file',
 934             'Unable to download SMIL file', fatal=fatal)
 935         if smil is False:
 936             assert not fatal
 937             return []
 938
 939         base = smil.find('./head/meta').get('base')
 940
 941         formats = []
 942         rtmp_count = 0
 943         if smil.findall('./body/seq/video'):
 944             video = smil.findall('./body/seq/video')[0]
 945             fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
 946             formats.extend(fmts)
 947         else:
 948             for video in smil.findall('./body/switch/video'):
 949                 fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
 950                 formats.extend(fmts)
 951
 952         self._sort_formats(formats)
 953
 954         return formats
 955
 956     def _parse_smil_video(self, video, video_id, base, rtmp_count):
 957         src = video.get('src')
 958         if not src:
 959             return ([], rtmp_count)
 960         bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
 961         width = int_or_none(video.get('width'))
 962         height = int_or_none(video.get('height'))
 963         proto = video.get('proto')
 964         if not proto:
 965             if base:
 966                 if base.startswith('rtmp'):
 967                     proto = 'rtmp'
 968                 elif base.startswith('http'):
 969                     proto = 'http'
 970         ext = video.get('ext')
 971         if proto == 'm3u8':
 972             return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
 973         elif proto == 'rtmp':
 974             rtmp_count += 1
 975             streamer = video.get('streamer') or base
 976             return ([{
 977                 'url': streamer,
 978                 'play_path': src,
 979                 'ext': 'flv',
 980                 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
 981                 'tbr': bitrate,
 982                 'width': width,
 983                 'height': height,
 984             }], rtmp_count)
 985         elif proto.startswith('http'):
 986             return ([{
 987                 'url': base + src,
 988                 'ext': ext or 'flv',
 989                 'tbr': bitrate,
 990                 'width': width,
 991                 'height': height,
 992             }], rtmp_count)
 993
 994     def _live_title(self, name):
 995         """ Generate the title for a live video """
 996         now = datetime.datetime.now()
 997         now_str = now.strftime("%Y-%m-%d %H:%M")
 998         return name + ' ' + now_str
 999
1000     def _int(self, v, name, fatal=False, **kwargs):
1001         res = int_or_none(v, **kwargs)
1002         if 'get_attr' in kwargs:
1003             print(getattr(v, kwargs['get_attr']))
1004         if res is None:
1005             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1006             if fatal:
1007                 raise ExtractorError(msg)
1008             else:
1009                 self._downloader.report_warning(msg)
1010         return res
1011
1012     def _float(self, v, name, fatal=False, **kwargs):
1013         res = float_or_none(v, **kwargs)
1014         if res is None:
1015             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
1016             if fatal:
1017                 raise ExtractorError(msg)
1018             else:
1019                 self._downloader.report_warning(msg)
1020         return res
1021
1022     def _set_cookie(self, domain, name, value, expire_time=None):
1023         cookie = compat_cookiejar.Cookie(
1024             0, name, value, None, None, domain, None,
1025             None, '/', True, False, expire_time, '', None, None, None)
1026         self._downloader.cookiejar.set_cookie(cookie)
1027
1028     def get_testcases(self, include_onlymatching=False):
1029         t = getattr(self, '_TEST', None)
1030         if t:
1031             assert not hasattr(self, '_TESTS'), \
1032                 '%s has _TEST and _TESTS' % type(self).__name__
1033             tests = [t]
1034         else:
1035             tests = getattr(self, '_TESTS', [])
1036         for t in tests:
1037             if not include_onlymatching and t.get('only_matching', False):
1038                 continue
1039             t['name'] = type(self).__name__[:-len('IE')]
1040             yield t
1041
1042     def is_suitable(self, age_limit):
1043         """ Test whether the extractor is generally suitable for the given
1044         age limit (i.e. pornographic sites are not, all others usually are) """
1045
1046         any_restricted = False
1047         for tc in self.get_testcases(include_onlymatching=False):
1048             if 'playlist' in tc:
1049                 tc = tc['playlist'][0]
1050             is_restricted = age_restricted(
1051                 tc.get('info_dict', {}).get('age_limit'), age_limit)
1052             if not is_restricted:
1053                 return True
1054             any_restricted = any_restricted or is_restricted
1055         return not any_restricted
1056
1057     def extract_subtitles(self, *args, **kwargs):
1058         if (self._downloader.params.get('writesubtitles', False) or
1059                 self._downloader.params.get('listsubtitles')):
1060             return self._get_subtitles(*args, **kwargs)
1061         return {}
1062
1063     def _get_subtitles(self, *args, **kwargs):
1064         raise NotImplementedError("This method must be implemented by subclasses")
1065
1066     def extract_automatic_captions(self, *args, **kwargs):
1067         if (self._downloader.params.get('writeautomaticsub', False) or
1068                 self._downloader.params.get('listsubtitles')):
1069             return self._get_automatic_captions(*args, **kwargs)
1070         return {}
1071
1072     def _get_automatic_captions(self, *args, **kwargs):
1073         raise NotImplementedError("This method must be implemented by subclasses")
1074
1075
1076 class SearchInfoExtractor(InfoExtractor):
1077     """
1078     Base class for paged search queries extractors.
1079     They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
1080     Instances should define _SEARCH_KEY and _MAX_RESULTS.
1081     """
1082
1083     @classmethod
1084     def _make_valid_url(cls):
1085         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
1086
1087     @classmethod
1088     def suitable(cls, url):
1089         return re.match(cls._make_valid_url(), url) is not None
1090
1091     def _real_extract(self, query):
1092         mobj = re.match(self._make_valid_url(), query)
1093         if mobj is None:
1094             raise ExtractorError('Invalid search query "%s"' % query)
1095
1096         prefix = mobj.group('prefix')
1097         query = mobj.group('query')
1098         if prefix == '':
1099             return self._get_n_results(query, 1)
1100         elif prefix == 'all':
1101             return self._get_n_results(query, self._MAX_RESULTS)
1102         else:
1103             n = int(prefix)
1104             if n <= 0:
1105                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
1106             elif n > self._MAX_RESULTS:
1107                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
1108                 n = self._MAX_RESULTS
1109             return self._get_n_results(query, n)
1110
1111     def _get_n_results(self, query, n):
1112         """Get a specified number of results for a query"""
1113         raise NotImplementedError("This method must be implemented by subclasses")
1114
1115     @property
1116     def SEARCH_KEY(self):
1117         return self._SEARCH_KEY