Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import datetime
   8 import errno
   9 import io
  10 import itertools
  11 import json
  12 import locale
  13 import operator
  14 import os
  15 import platform
  16 import re
  17 import shutil
  18 import subprocess
  19 import socket
  20 import sys
  21 import time
  22 import traceback
  23
  24 if os.name == 'nt':
  25     import ctypes
  26
  27 from .compat import (
  28     compat_basestring,
  29     compat_cookiejar,
  30     compat_expanduser,
  31     compat_http_client,
  32     compat_kwargs,
  33     compat_str,
  34     compat_urllib_error,
  35     compat_urllib_request,
  36 )
  37 from .utils import (
  38     escape_url,
  39     ContentTooShortError,
  40     date_from_str,
  41     DateRange,
  42     DEFAULT_OUTTMPL,
  43     determine_ext,
  44     DownloadError,
  45     encodeFilename,
  46     ExtractorError,
  47     format_bytes,
  48     formatSeconds,
  49     get_term_width,
  50     locked_file,
  51     make_HTTPS_handler,
  52     MaxDownloadsReached,
  53     PagedList,
  54     parse_filesize,
  55     PostProcessingError,
  56     platform_name,
  57     preferredencoding,
  58     render_table,
  59     SameFileError,
  60     sanitize_filename,
  61     std_headers,
  62     subtitles_filename,
  63     takewhile_inclusive,
  64     UnavailableVideoError,
  65     url_basename,
  66     version_tuple,
  67     write_json_file,
  68     write_string,
  69     YoutubeDLHandler,
  70     prepend_extension,
  71     args_to_str,
  72     age_restricted,
  73 )
  74 from .cache import Cache
  75 from .extractor import get_info_extractor, gen_extractors
  76 from .downloader import get_suitable_downloader
  77 from .downloader.rtmp import rtmpdump_version
  78 from .postprocessor import (
  79     FFmpegFixupM4aPP,
  80     FFmpegFixupStretchedPP,
  81     FFmpegMergerPP,
  82     FFmpegPostProcessor,
  83     get_postprocessor,
  84 )
  85 from .version import __version__
  86
  87
  88 class YoutubeDL(object):
  89     """YoutubeDL class.
  90
  91     YoutubeDL objects are the ones responsible of downloading the
  92     actual video file and writing it to disk if the user has requested
  93     it, among some other tasks. In most cases there should be one per
  94     program. As, given a video URL, the downloader doesn't know how to
  95     extract all the needed information, task that InfoExtractors do, it
  96     has to pass the URL to one of them.
  97
  98     For this, YoutubeDL objects have a method that allows
  99     InfoExtractors to be registered in a given order. When it is passed
 100     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 101     finds that reports being able to handle it. The InfoExtractor extracts
 102     all the information about the video or videos the URL refers to, and
 103     YoutubeDL process the extracted information, possibly using a File
 104     Downloader to download the video.
 105
 106     YoutubeDL objects accept a lot of parameters. In order not to saturate
 107     the object constructor with arguments, it receives a dictionary of
 108     options instead. These options are available through the params
 109     attribute for the InfoExtractors to use. The YoutubeDL also
 110     registers itself as the downloader in charge for the InfoExtractors
 111     that are added to it, so this is a "mutual registration".
 112
 113     Available options:
 114
 115     username:          Username for authentication purposes.
 116     password:          Password for authentication purposes.
 117     videopassword:     Password for acces a video.
 118     usenetrc:          Use netrc for authentication instead.
 119     verbose:           Print additional info to stdout.
 120     quiet:             Do not print messages to stdout.
 121     no_warnings:       Do not print out anything for warnings.
 122     forceurl:          Force printing final URL.
 123     forcetitle:        Force printing title.
 124     forceid:           Force printing ID.
 125     forcethumbnail:    Force printing thumbnail URL.
 126     forcedescription:  Force printing description.
 127     forcefilename:     Force printing final filename.
 128     forceduration:     Force printing duration.
 129     forcejson:         Force printing info_dict as JSON.
 130     dump_single_json:  Force printing the info_dict of the whole playlist
 131                        (or video) as a single JSON line.
 132     simulate:          Do not download the video files.
 133     format:            Video format code. See options.py for more information.
 134     format_limit:      Highest quality format to try.
 135     outtmpl:           Template for output names.
 136     restrictfilenames: Do not allow "&" and spaces in file names
 137     ignoreerrors:      Do not stop on download errors.
 138     nooverwrites:      Prevent overwriting files.
 139     playliststart:     Playlist item to start at.
 140     playlistend:       Playlist item to end at.
 141     playlist_items:    Specific indices of playlist to download.
 142     playlistreverse:   Download playlist items in reverse order.
 143     matchtitle:        Download only matching titles.
 144     rejecttitle:       Reject downloads for matching titles.
 145     logger:            Log messages to a logging.Logger instance.
 146     logtostderr:       Log messages to stderr instead of stdout.
 147     writedescription:  Write the video description to a .description file
 148     writeinfojson:     Write the video description to a .info.json file
 149     writeannotations:  Write the video annotations to a .annotations.xml file
 150     writethumbnail:    Write the thumbnail image to a file
 151     write_all_thumbnails:  Write all thumbnail formats to files
 152     writesubtitles:    Write the video subtitles to a file
 153     writeautomaticsub: Write the automatic subtitles to a file
 154     allsubtitles:      Downloads all the subtitles of the video
 155                        (requires writesubtitles or writeautomaticsub)
 156     listsubtitles:     Lists all available subtitles for the video
 157     subtitlesformat:   Subtitle format [srt/sbv/vtt] (default=srt)
 158     subtitleslangs:    List of languages of the subtitles to download
 159     keepvideo:         Keep the video file after post-processing
 160     daterange:         A DateRange object, download only if the upload_date is in the range.
 161     skip_download:     Skip the actual download of the video file
 162     cachedir:          Location of the cache files in the filesystem.
 163                        False to disable filesystem cache.
 164     noplaylist:        Download single video instead of a playlist if in doubt.
 165     age_limit:         An integer representing the user's age in years.
 166                        Unsuitable videos for the given age are skipped.
 167     min_views:         An integer representing the minimum view count the video
 168                        must have in order to not be skipped.
 169                        Videos without view count information are always
 170                        downloaded. None for no limit.
 171     max_views:         An integer representing the maximum view count.
 172                        Videos that are more popular than that are not
 173                        downloaded.
 174                        Videos without view count information are always
 175                        downloaded. None for no limit.
 176     download_archive:  File name of a file where all downloads are recorded.
 177                        Videos already present in the file are not downloaded
 178                        again.
 179     cookiefile:        File name where cookies should be read from and dumped to.
 180     nocheckcertificate:Do not verify SSL certificates
 181     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 182                        At the moment, this is only supported by YouTube.
 183     proxy:             URL of the proxy server to use
 184     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 185     bidi_workaround:   Work around buggy terminals without bidirectional text
 186                        support, using fridibi
 187     debug_printtraffic:Print out sent and received HTTP traffic
 188     include_ads:       Download ads as well
 189     default_search:    Prepend this string if an input url is not valid.
 190                        'auto' for elaborate guessing
 191     encoding:          Use this encoding instead of the system-specified.
 192     extract_flat:      Do not resolve URLs, return the immediate result.
 193                        Pass in 'in_playlist' to only show this behavior for
 194                        playlist items.
 195     postprocessors:    A list of dictionaries, each with an entry
 196                        * key:  The name of the postprocessor. See
 197                                youtube_dl/postprocessor/__init__.py for a list.
 198                        as well as any further keyword arguments for the
 199                        postprocessor.
 200     progress_hooks:    A list of functions that get called on download
 201                        progress, with a dictionary with the entries
 202                        * status: One of "downloading" and "finished".
 203                                  Check this first and ignore unknown values.
 204
 205                        If status is one of "downloading" or "finished", the
 206                        following properties may also be present:
 207                        * filename: The final filename (always present)
 208                        * downloaded_bytes: Bytes on disk
 209                        * total_bytes: Size of the whole file, None if unknown
 210                        * tmpfilename: The filename we're currently writing to
 211                        * eta: The estimated time in seconds, None if unknown
 212                        * speed: The download speed in bytes/second, None if
 213                                 unknown
 214
 215                        Progress hooks are guaranteed to be called at least once
 216                        (with status "finished") if the download is successful.
 217     merge_output_format: Extension to use when merging formats.
 218     fixup:             Automatically correct known faults of the file.
 219                        One of:
 220                        - "never": do nothing
 221                        - "warn": only emit a warning
 222                        - "detect_or_warn": check whether we can do anything
 223                                            about it, warn otherwise (default)
 224     source_address:    (Experimental) Client-side IP address to bind to.
 225     call_home:         Boolean, true iff we are allowed to contact the
 226                        youtube-dl servers for debugging.
 227     sleep_interval:    Number of seconds to sleep before each download.
 228     external_downloader:  Executable of the external downloader to call.
 229     listformats:       Print an overview of available video formats and exit.
 230     list_thumbnails:   Print a table of all thumbnails and exit.
 231
 232
 233     The following parameters are not used by YoutubeDL itself, they are used by
 234     the FileDownloader:
 235     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 236     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 237     xattr_set_filesize.
 238
 239     The following options are used by the post processors:
 240     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 241                        otherwise prefer avconv.
 242     exec_cmd:          Arbitrary command to run after downloading
 243     """
 244
 245     params = None
 246     _ies = []
 247     _pps = []
 248     _download_retcode = None
 249     _num_downloads = None
 250     _screen_file = None
 251
 252     def __init__(self, params=None, auto_init=True):
 253         """Create a FileDownloader object with the given options."""
 254         if params is None:
 255             params = {}
 256         self._ies = []
 257         self._ies_instances = {}
 258         self._pps = []
 259         self._progress_hooks = []
 260         self._download_retcode = 0
 261         self._num_downloads = 0
 262         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 263         self._err_file = sys.stderr
 264         self.params = params
 265         self.cache = Cache(self)
 266
 267         if params.get('bidi_workaround', False):
 268             try:
 269                 import pty
 270                 master, slave = pty.openpty()
 271                 width = get_term_width()
 272                 if width is None:
 273                     width_args = []
 274                 else:
 275                     width_args = ['-w', str(width)]
 276                 sp_kwargs = dict(
 277                     stdin=subprocess.PIPE,
 278                     stdout=slave,
 279                     stderr=self._err_file)
 280                 try:
 281                     self._output_process = subprocess.Popen(
 282                         ['bidiv'] + width_args, **sp_kwargs
 283                     )
 284                 except OSError:
 285                     self._output_process = subprocess.Popen(
 286                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 287                 self._output_channel = os.fdopen(master, 'rb')
 288             except OSError as ose:
 289                 if ose.errno == 2:
 290                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 291                 else:
 292                     raise
 293
 294         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 295                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
 296                 and not params.get('restrictfilenames', False)):
 297             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 298             self.report_warning(
 299                 'Assuming --restrict-filenames since file system encoding '
 300                 'cannot encode all characters. '
 301                 'Set the LC_ALL environment variable to fix this.')
 302             self.params['restrictfilenames'] = True
 303
 304         if '%(stitle)s' in self.params.get('outtmpl', ''):
 305             self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 306
 307         self._setup_opener()
 308
 309         if auto_init:
 310             self.print_debug_header()
 311             self.add_default_info_extractors()
 312
 313         for pp_def_raw in self.params.get('postprocessors', []):
 314             pp_class = get_postprocessor(pp_def_raw['key'])
 315             pp_def = dict(pp_def_raw)
 316             del pp_def['key']
 317             pp = pp_class(self, **compat_kwargs(pp_def))
 318             self.add_post_processor(pp)
 319
 320         for ph in self.params.get('progress_hooks', []):
 321             self.add_progress_hook(ph)
 322
 323     def warn_if_short_id(self, argv):
 324         # short YouTube ID starting with dash?
 325         idxs = [
 326             i for i, a in enumerate(argv)
 327             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 328         if idxs:
 329             correct_argv = (
 330                 ['youtube-dl'] +
 331                 [a for i, a in enumerate(argv) if i not in idxs] +
 332                 ['--'] + [argv[i] for i in idxs]
 333             )
 334             self.report_warning(
 335                 'Long argument string detected. '
 336                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 337                 args_to_str(correct_argv))
 338
 339     def add_info_extractor(self, ie):
 340         """Add an InfoExtractor object to the end of the list."""
 341         self._ies.append(ie)
 342         self._ies_instances[ie.ie_key()] = ie
 343         ie.set_downloader(self)
 344
 345     def get_info_extractor(self, ie_key):
 346         """
 347         Get an instance of an IE with name ie_key, it will try to get one from
 348         the _ies list, if there's no instance it will create a new one and add
 349         it to the extractor list.
 350         """
 351         ie = self._ies_instances.get(ie_key)
 352         if ie is None:
 353             ie = get_info_extractor(ie_key)()
 354             self.add_info_extractor(ie)
 355         return ie
 356
 357     def add_default_info_extractors(self):
 358         """
 359         Add the InfoExtractors returned by gen_extractors to the end of the list
 360         """
 361         for ie in gen_extractors():
 362             self.add_info_extractor(ie)
 363
 364     def add_post_processor(self, pp):
 365         """Add a PostProcessor object to the end of the chain."""
 366         self._pps.append(pp)
 367         pp.set_downloader(self)
 368
 369     def add_progress_hook(self, ph):
 370         """Add the progress hook (currently only for the file downloader)"""
 371         self._progress_hooks.append(ph)
 372
 373     def _bidi_workaround(self, message):
 374         if not hasattr(self, '_output_channel'):
 375             return message
 376
 377         assert hasattr(self, '_output_process')
 378         assert isinstance(message, compat_str)
 379         line_count = message.count('\n') + 1
 380         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 381         self._output_process.stdin.flush()
 382         res = ''.join(self._output_channel.readline().decode('utf-8')
 383                       for _ in range(line_count))
 384         return res[:-len('\n')]
 385
 386     def to_screen(self, message, skip_eol=False):
 387         """Print message to stdout if not in quiet mode."""
 388         return self.to_stdout(message, skip_eol, check_quiet=True)
 389
 390     def _write_string(self, s, out=None):
 391         write_string(s, out=out, encoding=self.params.get('encoding'))
 392
 393     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 394         """Print message to stdout if not in quiet mode."""
 395         if self.params.get('logger'):
 396             self.params['logger'].debug(message)
 397         elif not check_quiet or not self.params.get('quiet', False):
 398             message = self._bidi_workaround(message)
 399             terminator = ['\n', ''][skip_eol]
 400             output = message + terminator
 401
 402             self._write_string(output, self._screen_file)
 403
 404     def to_stderr(self, message):
 405         """Print message to stderr."""
 406         assert isinstance(message, compat_str)
 407         if self.params.get('logger'):
 408             self.params['logger'].error(message)
 409         else:
 410             message = self._bidi_workaround(message)
 411             output = message + '\n'
 412             self._write_string(output, self._err_file)
 413
 414     def to_console_title(self, message):
 415         if not self.params.get('consoletitle', False):
 416             return
 417         if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 418             # c_wchar_p() might not be necessary if `message` is
 419             # already of type unicode()
 420             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 421         elif 'TERM' in os.environ:
 422             self._write_string('\033]0;%s\007' % message, self._screen_file)
 423
 424     def save_console_title(self):
 425         if not self.params.get('consoletitle', False):
 426             return
 427         if 'TERM' in os.environ:
 428             # Save the title on stack
 429             self._write_string('\033[22;0t', self._screen_file)
 430
 431     def restore_console_title(self):
 432         if not self.params.get('consoletitle', False):
 433             return
 434         if 'TERM' in os.environ:
 435             # Restore the title from stack
 436             self._write_string('\033[23;0t', self._screen_file)
 437
 438     def __enter__(self):
 439         self.save_console_title()
 440         return self
 441
 442     def __exit__(self, *args):
 443         self.restore_console_title()
 444
 445         if self.params.get('cookiefile') is not None:
 446             self.cookiejar.save()
 447
 448     def trouble(self, message=None, tb=None):
 449         """Determine action to take when a download problem appears.
 450
 451         Depending on if the downloader has been configured to ignore
 452         download errors or not, this method may throw an exception or
 453         not when errors are found, after printing the message.
 454
 455         tb, if given, is additional traceback information.
 456         """
 457         if message is not None:
 458             self.to_stderr(message)
 459         if self.params.get('verbose'):
 460             if tb is None:
 461                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 462                     tb = ''
 463                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 464                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 465                     tb += compat_str(traceback.format_exc())
 466                 else:
 467                     tb_data = traceback.format_list(traceback.extract_stack())
 468                     tb = ''.join(tb_data)
 469             self.to_stderr(tb)
 470         if not self.params.get('ignoreerrors', False):
 471             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 472                 exc_info = sys.exc_info()[1].exc_info
 473             else:
 474                 exc_info = sys.exc_info()
 475             raise DownloadError(message, exc_info)
 476         self._download_retcode = 1
 477
 478     def report_warning(self, message):
 479         '''
 480         Print the message to stderr, it will be prefixed with 'WARNING:'
 481         If stderr is a tty file the 'WARNING:' will be colored
 482         '''
 483         if self.params.get('logger') is not None:
 484             self.params['logger'].warning(message)
 485         else:
 486             if self.params.get('no_warnings'):
 487                 return
 488             if self._err_file.isatty() and os.name != 'nt':
 489                 _msg_header = '\033[0;33mWARNING:\033[0m'
 490             else:
 491                 _msg_header = 'WARNING:'
 492             warning_message = '%s %s' % (_msg_header, message)
 493             self.to_stderr(warning_message)
 494
 495     def report_error(self, message, tb=None):
 496         '''
 497         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 498         in red if stderr is a tty file.
 499         '''
 500         if self._err_file.isatty() and os.name != 'nt':
 501             _msg_header = '\033[0;31mERROR:\033[0m'
 502         else:
 503             _msg_header = 'ERROR:'
 504         error_message = '%s %s' % (_msg_header, message)
 505         self.trouble(error_message, tb)
 506
 507     def report_file_already_downloaded(self, file_name):
 508         """Report file has already been fully downloaded."""
 509         try:
 510             self.to_screen('[download] %s has already been downloaded' % file_name)
 511         except UnicodeEncodeError:
 512             self.to_screen('[download] The file has already been downloaded')
 513
 514     def prepare_filename(self, info_dict):
 515         """Generate the output filename."""
 516         try:
 517             template_dict = dict(info_dict)
 518
 519             template_dict['epoch'] = int(time.time())
 520             autonumber_size = self.params.get('autonumber_size')
 521             if autonumber_size is None:
 522                 autonumber_size = 5
 523             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 524             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 525             if template_dict.get('playlist_index') is not None:
 526                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 527             if template_dict.get('resolution') is None:
 528                 if template_dict.get('width') and template_dict.get('height'):
 529                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 530                 elif template_dict.get('height'):
 531                     template_dict['resolution'] = '%sp' % template_dict['height']
 532                 elif template_dict.get('width'):
 533                     template_dict['resolution'] = '?x%d' % template_dict['width']
 534
 535             sanitize = lambda k, v: sanitize_filename(
 536                 compat_str(v),
 537                 restricted=self.params.get('restrictfilenames'),
 538                 is_id=(k == 'id'))
 539             template_dict = dict((k, sanitize(k, v))
 540                                  for k, v in template_dict.items()
 541                                  if v is not None)
 542             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 543
 544             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 545             tmpl = compat_expanduser(outtmpl)
 546             filename = tmpl % template_dict
 547             # Temporary fix for #4787
 548             # 'Treat' all problem characters by passing filename through preferredencoding
 549             # to workaround encoding issues with subprocess on python2 @ Windows
 550             if sys.version_info < (3, 0) and sys.platform == 'win32':
 551                 filename = encodeFilename(filename, True).decode(preferredencoding())
 552             return filename
 553         except ValueError as err:
 554             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 555             return None
 556
 557     def _match_entry(self, info_dict):
 558         """ Returns None iff the file should be downloaded """
 559
 560         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 561         if 'title' in info_dict:
 562             # This can happen when we're just evaluating the playlist
 563             title = info_dict['title']
 564             matchtitle = self.params.get('matchtitle', False)
 565             if matchtitle:
 566                 if not re.search(matchtitle, title, re.IGNORECASE):
 567                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 568             rejecttitle = self.params.get('rejecttitle', False)
 569             if rejecttitle:
 570                 if re.search(rejecttitle, title, re.IGNORECASE):
 571                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 572         date = info_dict.get('upload_date', None)
 573         if date is not None:
 574             dateRange = self.params.get('daterange', DateRange())
 575             if date not in dateRange:
 576                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 577         view_count = info_dict.get('view_count', None)
 578         if view_count is not None:
 579             min_views = self.params.get('min_views')
 580             if min_views is not None and view_count < min_views:
 581                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 582             max_views = self.params.get('max_views')
 583             if max_views is not None and view_count > max_views:
 584                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 585         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 586             return 'Skipping "%s" because it is age restricted' % title
 587         if self.in_download_archive(info_dict):
 588             return '%s has already been recorded in archive' % video_title
 589         return None
 590
 591     @staticmethod
 592     def add_extra_info(info_dict, extra_info):
 593         '''Set the keys from extra_info in info dict if they are missing'''
 594         for key, value in extra_info.items():
 595             info_dict.setdefault(key, value)
 596
 597     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 598                      process=True):
 599         '''
 600         Returns a list with a dictionary for each video we find.
 601         If 'download', also downloads the videos.
 602         extra_info is a dict containing the extra values to add to each result
 603          '''
 604
 605         if ie_key:
 606             ies = [self.get_info_extractor(ie_key)]
 607         else:
 608             ies = self._ies
 609
 610         for ie in ies:
 611             if not ie.suitable(url):
 612                 continue
 613
 614             if not ie.working():
 615                 self.report_warning('The program functionality for this site has been marked as broken, '
 616                                     'and will probably not work.')
 617
 618             try:
 619                 ie_result = ie.extract(url)
 620                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 621                     break
 622                 if isinstance(ie_result, list):
 623                     # Backwards compatibility: old IE result format
 624                     ie_result = {
 625                         '_type': 'compat_list',
 626                         'entries': ie_result,
 627                     }
 628                 self.add_default_extra_info(ie_result, ie, url)
 629                 if process:
 630                     return self.process_ie_result(ie_result, download, extra_info)
 631                 else:
 632                     return ie_result
 633             except ExtractorError as de:  # An error we somewhat expected
 634                 self.report_error(compat_str(de), de.format_traceback())
 635                 break
 636             except MaxDownloadsReached:
 637                 raise
 638             except Exception as e:
 639                 if self.params.get('ignoreerrors', False):
 640                     self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
 641                     break
 642                 else:
 643                     raise
 644         else:
 645             self.report_error('no suitable InfoExtractor for URL %s' % url)
 646
 647     def add_default_extra_info(self, ie_result, ie, url):
 648         self.add_extra_info(ie_result, {
 649             'extractor': ie.IE_NAME,
 650             'webpage_url': url,
 651             'webpage_url_basename': url_basename(url),
 652             'extractor_key': ie.ie_key(),
 653         })
 654
 655     def process_ie_result(self, ie_result, download=True, extra_info={}):
 656         """
 657         Take the result of the ie(may be modified) and resolve all unresolved
 658         references (URLs, playlist items).
 659
 660         It will also download the videos if 'download'.
 661         Returns the resolved ie_result.
 662         """
 663
 664         result_type = ie_result.get('_type', 'video')
 665
 666         if result_type in ('url', 'url_transparent'):
 667             extract_flat = self.params.get('extract_flat', False)
 668             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 669                     extract_flat is True):
 670                 if self.params.get('forcejson', False):
 671                     self.to_stdout(json.dumps(ie_result))
 672                 return ie_result
 673
 674         if result_type == 'video':
 675             self.add_extra_info(ie_result, extra_info)
 676             return self.process_video_result(ie_result, download=download)
 677         elif result_type == 'url':
 678             # We have to add extra_info to the results because it may be
 679             # contained in a playlist
 680             return self.extract_info(ie_result['url'],
 681                                      download,
 682                                      ie_key=ie_result.get('ie_key'),
 683                                      extra_info=extra_info)
 684         elif result_type == 'url_transparent':
 685             # Use the information from the embedding page
 686             info = self.extract_info(
 687                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 688                 extra_info=extra_info, download=False, process=False)
 689
 690             force_properties = dict(
 691                 (k, v) for k, v in ie_result.items() if v is not None)
 692             for f in ('_type', 'url'):
 693                 if f in force_properties:
 694                     del force_properties[f]
 695             new_result = info.copy()
 696             new_result.update(force_properties)
 697
 698             assert new_result.get('_type') != 'url_transparent'
 699
 700             return self.process_ie_result(
 701                 new_result, download=download, extra_info=extra_info)
 702         elif result_type == 'playlist' or result_type == 'multi_video':
 703             # We process each entry in the playlist
 704             playlist = ie_result.get('title', None) or ie_result.get('id', None)
 705             self.to_screen('[download] Downloading playlist: %s' % playlist)
 706
 707             playlist_results = []
 708
 709             playliststart = self.params.get('playliststart', 1) - 1
 710             playlistend = self.params.get('playlistend', None)
 711             # For backwards compatibility, interpret -1 as whole list
 712             if playlistend == -1:
 713                 playlistend = None
 714
 715             playlistitems_str = self.params.get('playlist_items', None)
 716             playlistitems = None
 717             if playlistitems_str is not None:
 718                 def iter_playlistitems(format):
 719                     for string_segment in format.split(','):
 720                         if '-' in string_segment:
 721                             start, end = string_segment.split('-')
 722                             for item in range(int(start), int(end) + 1):
 723                                 yield int(item)
 724                         else:
 725                             yield int(string_segment)
 726                 playlistitems = iter_playlistitems(playlistitems_str)
 727
 728             ie_entries = ie_result['entries']
 729             if isinstance(ie_entries, list):
 730                 n_all_entries = len(ie_entries)
 731                 if playlistitems:
 732                     entries = [ie_entries[i - 1] for i in playlistitems]
 733                 else:
 734                     entries = ie_entries[playliststart:playlistend]
 735                 n_entries = len(entries)
 736                 self.to_screen(
 737                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
 738                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 739             elif isinstance(ie_entries, PagedList):
 740                 if playlistitems:
 741                     entries = []
 742                     for item in playlistitems:
 743                         entries.extend(ie_entries.getslice(
 744                             item - 1, item
 745                         ))
 746                 else:
 747                     entries = ie_entries.getslice(
 748                         playliststart, playlistend)
 749                 n_entries = len(entries)
 750                 self.to_screen(
 751                     "[%s] playlist %s: Downloading %d videos" %
 752                     (ie_result['extractor'], playlist, n_entries))
 753             else:  # iterable
 754                 if playlistitems:
 755                     entry_list = list(ie_entries)
 756                     entries = [entry_list[i - 1] for i in playlistitems]
 757                 else:
 758                     entries = list(itertools.islice(
 759                         ie_entries, playliststart, playlistend))
 760                 n_entries = len(entries)
 761                 self.to_screen(
 762                     "[%s] playlist %s: Downloading %d videos" %
 763                     (ie_result['extractor'], playlist, n_entries))
 764
 765             if self.params.get('playlistreverse', False):
 766                 entries = entries[::-1]
 767
 768             for i, entry in enumerate(entries, 1):
 769                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 770                 extra = {
 771                     'n_entries': n_entries,
 772                     'playlist': playlist,
 773                     'playlist_id': ie_result.get('id'),
 774                     'playlist_title': ie_result.get('title'),
 775                     'playlist_index': i + playliststart,
 776                     'extractor': ie_result['extractor'],
 777                     'webpage_url': ie_result['webpage_url'],
 778                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 779                     'extractor_key': ie_result['extractor_key'],
 780                 }
 781
 782                 reason = self._match_entry(entry)
 783                 if reason is not None:
 784                     self.to_screen('[download] ' + reason)
 785                     continue
 786
 787                 entry_result = self.process_ie_result(entry,
 788                                                       download=download,
 789                                                       extra_info=extra)
 790                 playlist_results.append(entry_result)
 791             ie_result['entries'] = playlist_results
 792             return ie_result
 793         elif result_type == 'compat_list':
 794             self.report_warning(
 795                 'Extractor %s returned a compat_list result. '
 796                 'It needs to be updated.' % ie_result.get('extractor'))
 797
 798             def _fixup(r):
 799                 self.add_extra_info(
 800                     r,
 801                     {
 802                         'extractor': ie_result['extractor'],
 803                         'webpage_url': ie_result['webpage_url'],
 804                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 805                         'extractor_key': ie_result['extractor_key'],
 806                     }
 807                 )
 808                 return r
 809             ie_result['entries'] = [
 810                 self.process_ie_result(_fixup(r), download, extra_info)
 811                 for r in ie_result['entries']
 812             ]
 813             return ie_result
 814         else:
 815             raise Exception('Invalid result type: %s' % result_type)
 816
 817     def _apply_format_filter(self, format_spec, available_formats):
 818         " Returns a tuple of the remaining format_spec and filtered formats "
 819
 820         OPERATORS = {
 821             '<': operator.lt,
 822             '<=': operator.le,
 823             '>': operator.gt,
 824             '>=': operator.ge,
 825             '=': operator.eq,
 826             '!=': operator.ne,
 827         }
 828         operator_rex = re.compile(r'''(?x)\s*\[
 829             (?P<key>width|height|tbr|abr|vbr|filesize|fps)
 830             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 831             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 832             \]$
 833             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 834         m = operator_rex.search(format_spec)
 835         if not m:
 836             raise ValueError('Invalid format specification %r' % format_spec)
 837
 838         try:
 839             comparison_value = int(m.group('value'))
 840         except ValueError:
 841             comparison_value = parse_filesize(m.group('value'))
 842             if comparison_value is None:
 843                 comparison_value = parse_filesize(m.group('value') + 'B')
 844             if comparison_value is None:
 845                 raise ValueError(
 846                     'Invalid value %r in format specification %r' % (
 847                         m.group('value'), format_spec))
 848         op = OPERATORS[m.group('op')]
 849
 850         def _filter(f):
 851             actual_value = f.get(m.group('key'))
 852             if actual_value is None:
 853                 return m.group('none_inclusive')
 854             return op(actual_value, comparison_value)
 855         new_formats = [f for f in available_formats if _filter(f)]
 856
 857         new_format_spec = format_spec[:-len(m.group(0))]
 858         if not new_format_spec:
 859             new_format_spec = 'best'
 860
 861         return (new_format_spec, new_formats)
 862
 863     def select_format(self, format_spec, available_formats):
 864         while format_spec.endswith(']'):
 865             format_spec, available_formats = self._apply_format_filter(
 866                 format_spec, available_formats)
 867         if not available_formats:
 868             return None
 869
 870         if format_spec == 'best' or format_spec is None:
 871             return available_formats[-1]
 872         elif format_spec == 'worst':
 873             return available_formats[0]
 874         elif format_spec == 'bestaudio':
 875             audio_formats = [
 876                 f for f in available_formats
 877                 if f.get('vcodec') == 'none']
 878             if audio_formats:
 879                 return audio_formats[-1]
 880         elif format_spec == 'worstaudio':
 881             audio_formats = [
 882                 f for f in available_formats
 883                 if f.get('vcodec') == 'none']
 884             if audio_formats:
 885                 return audio_formats[0]
 886         elif format_spec == 'bestvideo':
 887             video_formats = [
 888                 f for f in available_formats
 889                 if f.get('acodec') == 'none']
 890             if video_formats:
 891                 return video_formats[-1]
 892         elif format_spec == 'worstvideo':
 893             video_formats = [
 894                 f for f in available_formats
 895                 if f.get('acodec') == 'none']
 896             if video_formats:
 897                 return video_formats[0]
 898         else:
 899             extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
 900             if format_spec in extensions:
 901                 filter_f = lambda f: f['ext'] == format_spec
 902             else:
 903                 filter_f = lambda f: f['format_id'] == format_spec
 904             matches = list(filter(filter_f, available_formats))
 905             if matches:
 906                 return matches[-1]
 907         return None
 908
 909     def _calc_headers(self, info_dict):
 910         res = std_headers.copy()
 911
 912         add_headers = info_dict.get('http_headers')
 913         if add_headers:
 914             res.update(add_headers)
 915
 916         cookies = self._calc_cookies(info_dict)
 917         if cookies:
 918             res['Cookie'] = cookies
 919
 920         return res
 921
 922     def _calc_cookies(self, info_dict):
 923         class _PseudoRequest(object):
 924             def __init__(self, url):
 925                 self.url = url
 926                 self.headers = {}
 927                 self.unverifiable = False
 928
 929             def add_unredirected_header(self, k, v):
 930                 self.headers[k] = v
 931
 932             def get_full_url(self):
 933                 return self.url
 934
 935             def is_unverifiable(self):
 936                 return self.unverifiable
 937
 938             def has_header(self, h):
 939                 return h in self.headers
 940
 941         pr = _PseudoRequest(info_dict['url'])
 942         self.cookiejar.add_cookie_header(pr)
 943         return pr.headers.get('Cookie')
 944
 945     def process_video_result(self, info_dict, download=True):
 946         assert info_dict.get('_type', 'video') == 'video'
 947
 948         if 'id' not in info_dict:
 949             raise ExtractorError('Missing "id" field in extractor result')
 950         if 'title' not in info_dict:
 951             raise ExtractorError('Missing "title" field in extractor result')
 952
 953         if 'playlist' not in info_dict:
 954             # It isn't part of a playlist
 955             info_dict['playlist'] = None
 956             info_dict['playlist_index'] = None
 957
 958         thumbnails = info_dict.get('thumbnails')
 959         if thumbnails is None:
 960             thumbnail = info_dict.get('thumbnail')
 961             if thumbnail:
 962                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
 963         if thumbnails:
 964             thumbnails.sort(key=lambda t: (
 965                 t.get('preference'), t.get('width'), t.get('height'),
 966                 t.get('id'), t.get('url')))
 967             for i, t in enumerate(thumbnails):
 968                 if 'width' in t and 'height' in t:
 969                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
 970                 if t.get('id') is None:
 971                     t['id'] = '%d' % i
 972
 973         if thumbnails and 'thumbnail' not in info_dict:
 974             info_dict['thumbnail'] = thumbnails[-1]['url']
 975
 976         if 'display_id' not in info_dict and 'id' in info_dict:
 977             info_dict['display_id'] = info_dict['id']
 978
 979         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
 980             # Working around negative timestamps in Windows
 981             # (see http://bugs.python.org/issue1646728)
 982             if info_dict['timestamp'] < 0 and os.name == 'nt':
 983                 info_dict['timestamp'] = 0
 984             upload_date = datetime.datetime.utcfromtimestamp(
 985                 info_dict['timestamp'])
 986             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
 987
 988         # This extractors handle format selection themselves
 989         if info_dict['extractor'] in ['Youku']:
 990             if download:
 991                 self.process_info(info_dict)
 992             return info_dict
 993
 994         # We now pick which formats have to be downloaded
 995         if info_dict.get('formats') is None:
 996             # There's only one format available
 997             formats = [info_dict]
 998         else:
 999             formats = info_dict['formats']
1000
1001         if not formats:
1002             raise ExtractorError('No video formats found!')
1003
1004         # We check that all the formats have the format and format_id fields
1005         for i, format in enumerate(formats):
1006             if 'url' not in format:
1007                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1008
1009             if format.get('format_id') is None:
1010                 format['format_id'] = compat_str(i)
1011             if format.get('format') is None:
1012                 format['format'] = '{id} - {res}{note}'.format(
1013                     id=format['format_id'],
1014                     res=self.format_resolution(format),
1015                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1016                 )
1017             # Automatically determine file extension if missing
1018             if 'ext' not in format:
1019                 format['ext'] = determine_ext(format['url']).lower()
1020             # Add HTTP headers, so that external programs can use them from the
1021             # json output
1022             full_format_info = info_dict.copy()
1023             full_format_info.update(format)
1024             format['http_headers'] = self._calc_headers(full_format_info)
1025
1026         format_limit = self.params.get('format_limit', None)
1027         if format_limit:
1028             formats = list(takewhile_inclusive(
1029                 lambda f: f['format_id'] != format_limit, formats
1030             ))
1031
1032         # TODO Central sorting goes here
1033
1034         if formats[0] is not info_dict:
1035             # only set the 'formats' fields if the original info_dict list them
1036             # otherwise we end up with a circular reference, the first (and unique)
1037             # element in the 'formats' field in info_dict is info_dict itself,
1038             # wich can't be exported to json
1039             info_dict['formats'] = formats
1040         if self.params.get('listformats'):
1041             self.list_formats(info_dict)
1042             return
1043         if self.params.get('list_thumbnails'):
1044             self.list_thumbnails(info_dict)
1045             return
1046
1047         req_format = self.params.get('format')
1048         if req_format is None:
1049             req_format = 'best'
1050         formats_to_download = []
1051         # The -1 is for supporting YoutubeIE
1052         if req_format in ('-1', 'all'):
1053             formats_to_download = formats
1054         else:
1055             for rfstr in req_format.split(','):
1056                 # We can accept formats requested in the format: 34/5/best, we pick
1057                 # the first that is available, starting from left
1058                 req_formats = rfstr.split('/')
1059                 for rf in req_formats:
1060                     if re.match(r'.+?\+.+?', rf) is not None:
1061                         # Two formats have been requested like '137+139'
1062                         format_1, format_2 = rf.split('+')
1063                         formats_info = (self.select_format(format_1, formats),
1064                                         self.select_format(format_2, formats))
1065                         if all(formats_info):
1066                             # The first format must contain the video and the
1067                             # second the audio
1068                             if formats_info[0].get('vcodec') == 'none':
1069                                 self.report_error('The first format must '
1070                                                   'contain the video, try using '
1071                                                   '"-f %s+%s"' % (format_2, format_1))
1072                                 return
1073                             output_ext = (
1074                                 formats_info[0]['ext']
1075                                 if self.params.get('merge_output_format') is None
1076                                 else self.params['merge_output_format'])
1077                             selected_format = {
1078                                 'requested_formats': formats_info,
1079                                 'format': '%s+%s' % (formats_info[0].get('format'),
1080                                                      formats_info[1].get('format')),
1081                                 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1082                                                         formats_info[1].get('format_id')),
1083                                 'width': formats_info[0].get('width'),
1084                                 'height': formats_info[0].get('height'),
1085                                 'resolution': formats_info[0].get('resolution'),
1086                                 'fps': formats_info[0].get('fps'),
1087                                 'vcodec': formats_info[0].get('vcodec'),
1088                                 'vbr': formats_info[0].get('vbr'),
1089                                 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1090                                 'acodec': formats_info[1].get('acodec'),
1091                                 'abr': formats_info[1].get('abr'),
1092                                 'ext': output_ext,
1093                             }
1094                         else:
1095                             selected_format = None
1096                     else:
1097                         selected_format = self.select_format(rf, formats)
1098                     if selected_format is not None:
1099                         formats_to_download.append(selected_format)
1100                         break
1101         if not formats_to_download:
1102             raise ExtractorError('requested format not available',
1103                                  expected=True)
1104
1105         if download:
1106             if len(formats_to_download) > 1:
1107                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1108             for format in formats_to_download:
1109                 new_info = dict(info_dict)
1110                 new_info.update(format)
1111                 self.process_info(new_info)
1112         # We update the info dict with the best quality format (backwards compatibility)
1113         info_dict.update(formats_to_download[-1])
1114         return info_dict
1115
1116     def process_info(self, info_dict):
1117         """Process a single resolved IE result."""
1118
1119         assert info_dict.get('_type', 'video') == 'video'
1120
1121         max_downloads = self.params.get('max_downloads')
1122         if max_downloads is not None:
1123             if self._num_downloads >= int(max_downloads):
1124                 raise MaxDownloadsReached()
1125
1126         info_dict['fulltitle'] = info_dict['title']
1127         if len(info_dict['title']) > 200:
1128             info_dict['title'] = info_dict['title'][:197] + '...'
1129
1130         # Keep for backwards compatibility
1131         info_dict['stitle'] = info_dict['title']
1132
1133         if 'format' not in info_dict:
1134             info_dict['format'] = info_dict['ext']
1135
1136         reason = self._match_entry(info_dict)
1137         if reason is not None:
1138             self.to_screen('[download] ' + reason)
1139             return
1140
1141         self._num_downloads += 1
1142
1143         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1144
1145         # Forced printings
1146         if self.params.get('forcetitle', False):
1147             self.to_stdout(info_dict['fulltitle'])
1148         if self.params.get('forceid', False):
1149             self.to_stdout(info_dict['id'])
1150         if self.params.get('forceurl', False):
1151             if info_dict.get('requested_formats') is not None:
1152                 for f in info_dict['requested_formats']:
1153                     self.to_stdout(f['url'] + f.get('play_path', ''))
1154             else:
1155                 # For RTMP URLs, also include the playpath
1156                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1157         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1158             self.to_stdout(info_dict['thumbnail'])
1159         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1160             self.to_stdout(info_dict['description'])
1161         if self.params.get('forcefilename', False) and filename is not None:
1162             self.to_stdout(filename)
1163         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1164             self.to_stdout(formatSeconds(info_dict['duration']))
1165         if self.params.get('forceformat', False):
1166             self.to_stdout(info_dict['format'])
1167         if self.params.get('forcejson', False):
1168             self.to_stdout(json.dumps(info_dict))
1169
1170         # Do nothing else if in simulate mode
1171         if self.params.get('simulate', False):
1172             return
1173
1174         if filename is None:
1175             return
1176
1177         try:
1178             dn = os.path.dirname(encodeFilename(filename))
1179             if dn and not os.path.exists(dn):
1180                 os.makedirs(dn)
1181         except (OSError, IOError) as err:
1182             self.report_error('unable to create directory ' + compat_str(err))
1183             return
1184
1185         if self.params.get('writedescription', False):
1186             descfn = filename + '.description'
1187             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1188                 self.to_screen('[info] Video description is already present')
1189             elif info_dict.get('description') is None:
1190                 self.report_warning('There\'s no description to write.')
1191             else:
1192                 try:
1193                     self.to_screen('[info] Writing video description to: ' + descfn)
1194                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1195                         descfile.write(info_dict['description'])
1196                 except (OSError, IOError):
1197                     self.report_error('Cannot write description file ' + descfn)
1198                     return
1199
1200         if self.params.get('writeannotations', False):
1201             annofn = filename + '.annotations.xml'
1202             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1203                 self.to_screen('[info] Video annotations are already present')
1204             else:
1205                 try:
1206                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1207                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1208                         annofile.write(info_dict['annotations'])
1209                 except (KeyError, TypeError):
1210                     self.report_warning('There are no annotations to write.')
1211                 except (OSError, IOError):
1212                     self.report_error('Cannot write annotations file: ' + annofn)
1213                     return
1214
1215         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1216                                        self.params.get('writeautomaticsub')])
1217
1218         if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
1219             # subtitles download errors are already managed as troubles in relevant IE
1220             # that way it will silently go on when used with unsupporting IE
1221             subtitles = info_dict['subtitles']
1222             sub_format = self.params.get('subtitlesformat', 'srt')
1223             for sub_lang in subtitles.keys():
1224                 sub = subtitles[sub_lang]
1225                 if sub is None:
1226                     continue
1227                 try:
1228                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1229                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1230                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1231                     else:
1232                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1233                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1234                             subfile.write(sub)
1235                 except (OSError, IOError):
1236                     self.report_error('Cannot write subtitles file ' + sub_filename)
1237                     return
1238
1239         if self.params.get('writeinfojson', False):
1240             infofn = os.path.splitext(filename)[0] + '.info.json'
1241             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1242                 self.to_screen('[info] Video description metadata is already present')
1243             else:
1244                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1245                 try:
1246                     write_json_file(info_dict, infofn)
1247                 except (OSError, IOError):
1248                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1249                     return
1250
1251         self._write_thumbnails(info_dict, filename)
1252
1253         if not self.params.get('skip_download', False):
1254             try:
1255                 def dl(name, info):
1256                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1257                     for ph in self._progress_hooks:
1258                         fd.add_progress_hook(ph)
1259                     if self.params.get('verbose'):
1260                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1261                     return fd.download(name, info)
1262
1263                 if info_dict.get('requested_formats') is not None:
1264                     downloaded = []
1265                     success = True
1266                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1267                     if not merger._executable:
1268                         postprocessors = []
1269                         self.report_warning('You have requested multiple '
1270                                             'formats but ffmpeg or avconv are not installed.'
1271                                             ' The formats won\'t be merged')
1272                     else:
1273                         postprocessors = [merger]
1274                     for f in info_dict['requested_formats']:
1275                         new_info = dict(info_dict)
1276                         new_info.update(f)
1277                         fname = self.prepare_filename(new_info)
1278                         fname = prepend_extension(fname, 'f%s' % f['format_id'])
1279                         downloaded.append(fname)
1280                         partial_success = dl(fname, new_info)
1281                         success = success and partial_success
1282                     info_dict['__postprocessors'] = postprocessors
1283                     info_dict['__files_to_merge'] = downloaded
1284                 else:
1285                     # Just a single file
1286                     success = dl(filename, info_dict)
1287             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1288                 self.report_error('unable to download video data: %s' % str(err))
1289                 return
1290             except (OSError, IOError) as err:
1291                 raise UnavailableVideoError(err)
1292             except (ContentTooShortError, ) as err:
1293                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1294                 return
1295
1296             if success:
1297                 # Fixup content
1298                 fixup_policy = self.params.get('fixup')
1299                 if fixup_policy is None:
1300                     fixup_policy = 'detect_or_warn'
1301
1302                 stretched_ratio = info_dict.get('stretched_ratio')
1303                 if stretched_ratio is not None and stretched_ratio != 1:
1304                     if fixup_policy == 'warn':
1305                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1306                             info_dict['id'], stretched_ratio))
1307                     elif fixup_policy == 'detect_or_warn':
1308                         stretched_pp = FFmpegFixupStretchedPP(self)
1309                         if stretched_pp.available:
1310                             info_dict.setdefault('__postprocessors', [])
1311                             info_dict['__postprocessors'].append(stretched_pp)
1312                         else:
1313                             self.report_warning(
1314                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1315                                     info_dict['id'], stretched_ratio))
1316                     else:
1317                         assert fixup_policy in ('ignore', 'never')
1318
1319                 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1320                     if fixup_policy == 'warn':
1321                         self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1322                             info_dict['id']))
1323                     elif fixup_policy == 'detect_or_warn':
1324                         fixup_pp = FFmpegFixupM4aPP(self)
1325                         if fixup_pp.available:
1326                             info_dict.setdefault('__postprocessors', [])
1327                             info_dict['__postprocessors'].append(fixup_pp)
1328                         else:
1329                             self.report_warning(
1330                                 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1331                                     info_dict['id']))
1332                     else:
1333                         assert fixup_policy in ('ignore', 'never')
1334
1335                 try:
1336                     self.post_process(filename, info_dict)
1337                 except (PostProcessingError) as err:
1338                     self.report_error('postprocessing: %s' % str(err))
1339                     return
1340                 self.record_download_archive(info_dict)
1341
1342     def download(self, url_list):
1343         """Download a given list of URLs."""
1344         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1345         if (len(url_list) > 1 and
1346                 '%' not in outtmpl
1347                 and self.params.get('max_downloads') != 1):
1348             raise SameFileError(outtmpl)
1349
1350         for url in url_list:
1351             try:
1352                 # It also downloads the videos
1353                 res = self.extract_info(url)
1354             except UnavailableVideoError:
1355                 self.report_error('unable to download video')
1356             except MaxDownloadsReached:
1357                 self.to_screen('[info] Maximum number of downloaded files reached.')
1358                 raise
1359             else:
1360                 if self.params.get('dump_single_json', False):
1361                     self.to_stdout(json.dumps(res))
1362
1363         return self._download_retcode
1364
1365     def download_with_info_file(self, info_filename):
1366         with io.open(info_filename, 'r', encoding='utf-8') as f:
1367             info = json.load(f)
1368         try:
1369             self.process_ie_result(info, download=True)
1370         except DownloadError:
1371             webpage_url = info.get('webpage_url')
1372             if webpage_url is not None:
1373                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1374                 return self.download([webpage_url])
1375             else:
1376                 raise
1377         return self._download_retcode
1378
1379     def post_process(self, filename, ie_info):
1380         """Run all the postprocessors on the given file."""
1381         info = dict(ie_info)
1382         info['filepath'] = filename
1383         pps_chain = []
1384         if ie_info.get('__postprocessors') is not None:
1385             pps_chain.extend(ie_info['__postprocessors'])
1386         pps_chain.extend(self._pps)
1387         for pp in pps_chain:
1388             keep_video = None
1389             old_filename = info['filepath']
1390             try:
1391                 keep_video_wish, info = pp.run(info)
1392                 if keep_video_wish is not None:
1393                     if keep_video_wish:
1394                         keep_video = keep_video_wish
1395                     elif keep_video is None:
1396                         # No clear decision yet, let IE decide
1397                         keep_video = keep_video_wish
1398             except PostProcessingError as e:
1399                 self.report_error(e.msg)
1400             if keep_video is False and not self.params.get('keepvideo', False):
1401                 try:
1402                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1403                     os.remove(encodeFilename(old_filename))
1404                 except (IOError, OSError):
1405                     self.report_warning('Unable to remove downloaded video file')
1406
1407     def _make_archive_id(self, info_dict):
1408         # Future-proof against any change in case
1409         # and backwards compatibility with prior versions
1410         extractor = info_dict.get('extractor_key')
1411         if extractor is None:
1412             if 'id' in info_dict:
1413                 extractor = info_dict.get('ie_key')  # key in a playlist
1414         if extractor is None:
1415             return None  # Incomplete video information
1416         return extractor.lower() + ' ' + info_dict['id']
1417
1418     def in_download_archive(self, info_dict):
1419         fn = self.params.get('download_archive')
1420         if fn is None:
1421             return False
1422
1423         vid_id = self._make_archive_id(info_dict)
1424         if vid_id is None:
1425             return False  # Incomplete video information
1426
1427         try:
1428             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1429                 for line in archive_file:
1430                     if line.strip() == vid_id:
1431                         return True
1432         except IOError as ioe:
1433             if ioe.errno != errno.ENOENT:
1434                 raise
1435         return False
1436
1437     def record_download_archive(self, info_dict):
1438         fn = self.params.get('download_archive')
1439         if fn is None:
1440             return
1441         vid_id = self._make_archive_id(info_dict)
1442         assert vid_id
1443         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1444             archive_file.write(vid_id + '\n')
1445
1446     @staticmethod
1447     def format_resolution(format, default='unknown'):
1448         if format.get('vcodec') == 'none':
1449             return 'audio only'
1450         if format.get('resolution') is not None:
1451             return format['resolution']
1452         if format.get('height') is not None:
1453             if format.get('width') is not None:
1454                 res = '%sx%s' % (format['width'], format['height'])
1455             else:
1456                 res = '%sp' % format['height']
1457         elif format.get('width') is not None:
1458             res = '?x%d' % format['width']
1459         else:
1460             res = default
1461         return res
1462
1463     def _format_note(self, fdict):
1464         res = ''
1465         if fdict.get('ext') in ['f4f', 'f4m']:
1466             res += '(unsupported) '
1467         if fdict.get('format_note') is not None:
1468             res += fdict['format_note'] + ' '
1469         if fdict.get('tbr') is not None:
1470             res += '%4dk ' % fdict['tbr']
1471         if fdict.get('container') is not None:
1472             if res:
1473                 res += ', '
1474             res += '%s container' % fdict['container']
1475         if (fdict.get('vcodec') is not None and
1476                 fdict.get('vcodec') != 'none'):
1477             if res:
1478                 res += ', '
1479             res += fdict['vcodec']
1480             if fdict.get('vbr') is not None:
1481                 res += '@'
1482         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1483             res += 'video@'
1484         if fdict.get('vbr') is not None:
1485             res += '%4dk' % fdict['vbr']
1486         if fdict.get('fps') is not None:
1487             res += ', %sfps' % fdict['fps']
1488         if fdict.get('acodec') is not None:
1489             if res:
1490                 res += ', '
1491             if fdict['acodec'] == 'none':
1492                 res += 'video only'
1493             else:
1494                 res += '%-5s' % fdict['acodec']
1495         elif fdict.get('abr') is not None:
1496             if res:
1497                 res += ', '
1498             res += 'audio'
1499         if fdict.get('abr') is not None:
1500             res += '@%3dk' % fdict['abr']
1501         if fdict.get('asr') is not None:
1502             res += ' (%5dHz)' % fdict['asr']
1503         if fdict.get('filesize') is not None:
1504             if res:
1505                 res += ', '
1506             res += format_bytes(fdict['filesize'])
1507         elif fdict.get('filesize_approx') is not None:
1508             if res:
1509                 res += ', '
1510             res += '~' + format_bytes(fdict['filesize_approx'])
1511         return res
1512
1513     def list_formats(self, info_dict):
1514         def line(format, idlen=20):
1515             return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1516                 format['format_id'],
1517                 format['ext'],
1518                 self.format_resolution(format),
1519                 self._format_note(format),
1520             ))
1521
1522         formats = info_dict.get('formats', [info_dict])
1523         idlen = max(len('format code'),
1524                     max(len(f['format_id']) for f in formats))
1525         formats_s = [
1526             line(f, idlen) for f in formats
1527             if f.get('preference') is None or f['preference'] >= -1000]
1528         if len(formats) > 1:
1529             formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
1530             formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
1531
1532         header_line = line({
1533             'format_id': 'format code', 'ext': 'extension',
1534             'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1535         self.to_screen(
1536             '[info] Available formats for %s:\n%s\n%s' %
1537             (info_dict['id'], header_line, '\n'.join(formats_s)))
1538
1539     def list_thumbnails(self, info_dict):
1540         thumbnails = info_dict.get('thumbnails')
1541         if not thumbnails:
1542             tn_url = info_dict.get('thumbnail')
1543             if tn_url:
1544                 thumbnails = [{'id': '0', 'url': tn_url}]
1545             else:
1546                 self.to_screen(
1547                     '[info] No thumbnails present for %s' % info_dict['id'])
1548                 return
1549
1550         self.to_screen(
1551             '[info] Thumbnails for %s:' % info_dict['id'])
1552         self.to_screen(render_table(
1553             ['ID', 'width', 'height', 'URL'],
1554             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1555
1556     def urlopen(self, req):
1557         """ Start an HTTP download """
1558
1559         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1560         # always respected by websites, some tend to give out URLs with non percent-encoded
1561         # non-ASCII characters (see telemb.py, ard.py [#3412])
1562         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1563         # To work around aforementioned issue we will replace request's original URL with
1564         # percent-encoded one
1565         req_is_string = isinstance(req, compat_basestring)
1566         url = req if req_is_string else req.get_full_url()
1567         url_escaped = escape_url(url)
1568
1569         # Substitute URL if any change after escaping
1570         if url != url_escaped:
1571             if req_is_string:
1572                 req = url_escaped
1573             else:
1574                 req = compat_urllib_request.Request(
1575                     url_escaped, data=req.data, headers=req.headers,
1576                     origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1577
1578         return self._opener.open(req, timeout=self._socket_timeout)
1579
1580     def print_debug_header(self):
1581         if not self.params.get('verbose'):
1582             return
1583
1584         if type('') is not compat_str:
1585             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1586             self.report_warning(
1587                 'Your Python is broken! Update to a newer and supported version')
1588
1589         stdout_encoding = getattr(
1590             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1591         encoding_str = (
1592             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1593                 locale.getpreferredencoding(),
1594                 sys.getfilesystemencoding(),
1595                 stdout_encoding,
1596                 self.get_encoding()))
1597         write_string(encoding_str, encoding=None)
1598
1599         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1600         try:
1601             sp = subprocess.Popen(
1602                 ['git', 'rev-parse', '--short', 'HEAD'],
1603                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1604                 cwd=os.path.dirname(os.path.abspath(__file__)))
1605             out, err = sp.communicate()
1606             out = out.decode().strip()
1607             if re.match('[0-9a-f]+', out):
1608                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1609         except:
1610             try:
1611                 sys.exc_clear()
1612             except:
1613                 pass
1614         self._write_string('[debug] Python version %s - %s\n' % (
1615             platform.python_version(), platform_name()))
1616
1617         exe_versions = FFmpegPostProcessor.get_versions()
1618         exe_versions['rtmpdump'] = rtmpdump_version()
1619         exe_str = ', '.join(
1620             '%s %s' % (exe, v)
1621             for exe, v in sorted(exe_versions.items())
1622             if v
1623         )
1624         if not exe_str:
1625             exe_str = 'none'
1626         self._write_string('[debug] exe versions: %s\n' % exe_str)
1627
1628         proxy_map = {}
1629         for handler in self._opener.handlers:
1630             if hasattr(handler, 'proxies'):
1631                 proxy_map.update(handler.proxies)
1632         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1633
1634         if self.params.get('call_home', False):
1635             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1636             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1637             latest_version = self.urlopen(
1638                 'https://yt-dl.org/latest/version').read().decode('utf-8')
1639             if version_tuple(latest_version) > version_tuple(__version__):
1640                 self.report_warning(
1641                     'You are using an outdated version (newest version: %s)! '
1642                     'See https://yt-dl.org/update if you need help updating.' %
1643                     latest_version)
1644
1645     def _setup_opener(self):
1646         timeout_val = self.params.get('socket_timeout')
1647         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1648
1649         opts_cookiefile = self.params.get('cookiefile')
1650         opts_proxy = self.params.get('proxy')
1651
1652         if opts_cookiefile is None:
1653             self.cookiejar = compat_cookiejar.CookieJar()
1654         else:
1655             self.cookiejar = compat_cookiejar.MozillaCookieJar(
1656                 opts_cookiefile)
1657             if os.access(opts_cookiefile, os.R_OK):
1658                 self.cookiejar.load()
1659
1660         cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1661             self.cookiejar)
1662         if opts_proxy is not None:
1663             if opts_proxy == '':
1664                 proxies = {}
1665             else:
1666                 proxies = {'http': opts_proxy, 'https': opts_proxy}
1667         else:
1668             proxies = compat_urllib_request.getproxies()
1669             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1670             if 'http' in proxies and 'https' not in proxies:
1671                 proxies['https'] = proxies['http']
1672         proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1673
1674         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1675         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1676         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1677         opener = compat_urllib_request.build_opener(
1678             https_handler, proxy_handler, cookie_processor, ydlh)
1679         # Delete the default user-agent header, which would otherwise apply in
1680         # cases where our custom HTTP handler doesn't come into play
1681         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1682         opener.addheaders = []
1683         self._opener = opener
1684
1685     def encode(self, s):
1686         if isinstance(s, bytes):
1687             return s  # Already encoded
1688
1689         try:
1690             return s.encode(self.get_encoding())
1691         except UnicodeEncodeError as err:
1692             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1693             raise
1694
1695     def get_encoding(self):
1696         encoding = self.params.get('encoding')
1697         if encoding is None:
1698             encoding = preferredencoding()
1699         return encoding
1700
1701     def _write_thumbnails(self, info_dict, filename):
1702         if self.params.get('writethumbnail', False):
1703             thumbnails = info_dict.get('thumbnails')
1704             if thumbnails:
1705                 thumbnails = [thumbnails[-1]]
1706         elif self.params.get('write_all_thumbnails', False):
1707             thumbnails = info_dict.get('thumbnails')
1708         else:
1709             return
1710
1711         if not thumbnails:
1712             # No thumbnails present, so return immediately
1713             return
1714
1715         for t in thumbnails:
1716             thumb_ext = determine_ext(t['url'], 'jpg')
1717             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1718             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1719             thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1720
1721             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1722                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1723                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
1724             else:
1725                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1726                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
1727                 try:
1728                     uf = self.urlopen(t['url'])
1729                     with open(thumb_filename, 'wb') as thumbf:
1730                         shutil.copyfileobj(uf, thumbf)
1731                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1732                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1733                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1734                     self.report_warning('Unable to download thumbnail "%s": %s' %
1735                                         (t['url'], compat_str(err)))