Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import datetime
   9 import errno
  10 import fileinput
  11 import io
  12 import itertools
  13 import json
  14 import locale
  15 import operator
  16 import os
  17 import platform
  18 import re
  19 import shutil
  20 import subprocess
  21 import socket
  22 import sys
  23 import time
  24 import traceback
  25
  26 if os.name == 'nt':
  27     import ctypes
  28
  29 from .compat import (
  30     compat_basestring,
  31     compat_cookiejar,
  32     compat_expanduser,
  33     compat_get_terminal_size,
  34     compat_http_client,
  35     compat_kwargs,
  36     compat_str,
  37     compat_urllib_error,
  38     compat_urllib_request,
  39 )
  40 from .utils import (
  41     escape_url,
  42     ContentTooShortError,
  43     date_from_str,
  44     DateRange,
  45     DEFAULT_OUTTMPL,
  46     determine_ext,
  47     DownloadError,
  48     encodeFilename,
  49     ExtractorError,
  50     format_bytes,
  51     formatSeconds,
  52     HEADRequest,
  53     locked_file,
  54     make_HTTPS_handler,
  55     MaxDownloadsReached,
  56     PagedList,
  57     parse_filesize,
  58     PerRequestProxyHandler,
  59     PostProcessingError,
  60     platform_name,
  61     preferredencoding,
  62     render_table,
  63     SameFileError,
  64     sanitize_filename,
  65     sanitize_path,
  66     std_headers,
  67     subtitles_filename,
  68     UnavailableVideoError,
  69     url_basename,
  70     version_tuple,
  71     write_json_file,
  72     write_string,
  73     YoutubeDLHandler,
  74     prepend_extension,
  75     replace_extension,
  76     args_to_str,
  77     age_restricted,
  78 )
  79 from .cache import Cache
  80 from .extractor import get_info_extractor, gen_extractors
  81 from .downloader import get_suitable_downloader
  82 from .downloader.rtmp import rtmpdump_version
  83 from .postprocessor import (
  84     FFmpegFixupM4aPP,
  85     FFmpegFixupStretchedPP,
  86     FFmpegMergerPP,
  87     FFmpegPostProcessor,
  88     get_postprocessor,
  89 )
  90 from .version import __version__
  91
  92
  93 class YoutubeDL(object):
  94     """YoutubeDL class.
  95
  96     YoutubeDL objects are the ones responsible of downloading the
  97     actual video file and writing it to disk if the user has requested
  98     it, among some other tasks. In most cases there should be one per
  99     program. As, given a video URL, the downloader doesn't know how to
 100     extract all the needed information, task that InfoExtractors do, it
 101     has to pass the URL to one of them.
 102
 103     For this, YoutubeDL objects have a method that allows
 104     InfoExtractors to be registered in a given order. When it is passed
 105     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 106     finds that reports being able to handle it. The InfoExtractor extracts
 107     all the information about the video or videos the URL refers to, and
 108     YoutubeDL process the extracted information, possibly using a File
 109     Downloader to download the video.
 110
 111     YoutubeDL objects accept a lot of parameters. In order not to saturate
 112     the object constructor with arguments, it receives a dictionary of
 113     options instead. These options are available through the params
 114     attribute for the InfoExtractors to use. The YoutubeDL also
 115     registers itself as the downloader in charge for the InfoExtractors
 116     that are added to it, so this is a "mutual registration".
 117
 118     Available options:
 119
 120     username:          Username for authentication purposes.
 121     password:          Password for authentication purposes.
 122     videopassword:     Password for acces a video.
 123     usenetrc:          Use netrc for authentication instead.
 124     verbose:           Print additional info to stdout.
 125     quiet:             Do not print messages to stdout.
 126     no_warnings:       Do not print out anything for warnings.
 127     forceurl:          Force printing final URL.
 128     forcetitle:        Force printing title.
 129     forceid:           Force printing ID.
 130     forcethumbnail:    Force printing thumbnail URL.
 131     forcedescription:  Force printing description.
 132     forcefilename:     Force printing final filename.
 133     forceduration:     Force printing duration.
 134     forcejson:         Force printing info_dict as JSON.
 135     dump_single_json:  Force printing the info_dict of the whole playlist
 136                        (or video) as a single JSON line.
 137     simulate:          Do not download the video files.
 138     format:            Video format code. See options.py for more information.
 139     outtmpl:           Template for output names.
 140     restrictfilenames: Do not allow "&" and spaces in file names
 141     ignoreerrors:      Do not stop on download errors.
 142     nooverwrites:      Prevent overwriting files.
 143     playliststart:     Playlist item to start at.
 144     playlistend:       Playlist item to end at.
 145     playlist_items:    Specific indices of playlist to download.
 146     playlistreverse:   Download playlist items in reverse order.
 147     matchtitle:        Download only matching titles.
 148     rejecttitle:       Reject downloads for matching titles.
 149     logger:            Log messages to a logging.Logger instance.
 150     logtostderr:       Log messages to stderr instead of stdout.
 151     writedescription:  Write the video description to a .description file
 152     writeinfojson:     Write the video description to a .info.json file
 153     writeannotations:  Write the video annotations to a .annotations.xml file
 154     writethumbnail:    Write the thumbnail image to a file
 155     write_all_thumbnails:  Write all thumbnail formats to files
 156     writesubtitles:    Write the video subtitles to a file
 157     writeautomaticsub: Write the automatic subtitles to a file
 158     allsubtitles:      Downloads all the subtitles of the video
 159                        (requires writesubtitles or writeautomaticsub)
 160     listsubtitles:     Lists all available subtitles for the video
 161     subtitlesformat:   The format code for subtitles
 162     subtitleslangs:    List of languages of the subtitles to download
 163     keepvideo:         Keep the video file after post-processing
 164     daterange:         A DateRange object, download only if the upload_date is in the range.
 165     skip_download:     Skip the actual download of the video file
 166     cachedir:          Location of the cache files in the filesystem.
 167                        False to disable filesystem cache.
 168     noplaylist:        Download single video instead of a playlist if in doubt.
 169     age_limit:         An integer representing the user's age in years.
 170                        Unsuitable videos for the given age are skipped.
 171     min_views:         An integer representing the minimum view count the video
 172                        must have in order to not be skipped.
 173                        Videos without view count information are always
 174                        downloaded. None for no limit.
 175     max_views:         An integer representing the maximum view count.
 176                        Videos that are more popular than that are not
 177                        downloaded.
 178                        Videos without view count information are always
 179                        downloaded. None for no limit.
 180     download_archive:  File name of a file where all downloads are recorded.
 181                        Videos already present in the file are not downloaded
 182                        again.
 183     cookiefile:        File name where cookies should be read from and dumped to.
 184     nocheckcertificate:Do not verify SSL certificates
 185     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 186                        At the moment, this is only supported by YouTube.
 187     proxy:             URL of the proxy server to use
 188     cn_verification_proxy:  URL of the proxy to use for IP address verification
 189                        on Chinese sites. (Experimental)
 190     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 191     bidi_workaround:   Work around buggy terminals without bidirectional text
 192                        support, using fridibi
 193     debug_printtraffic:Print out sent and received HTTP traffic
 194     include_ads:       Download ads as well
 195     default_search:    Prepend this string if an input url is not valid.
 196                        'auto' for elaborate guessing
 197     encoding:          Use this encoding instead of the system-specified.
 198     extract_flat:      Do not resolve URLs, return the immediate result.
 199                        Pass in 'in_playlist' to only show this behavior for
 200                        playlist items.
 201     postprocessors:    A list of dictionaries, each with an entry
 202                        * key:  The name of the postprocessor. See
 203                                youtube_dl/postprocessor/__init__.py for a list.
 204                        as well as any further keyword arguments for the
 205                        postprocessor.
 206     progress_hooks:    A list of functions that get called on download
 207                        progress, with a dictionary with the entries
 208                        * status: One of "downloading", "error", or "finished".
 209                                  Check this first and ignore unknown values.
 210
 211                        If status is one of "downloading", or "finished", the
 212                        following properties may also be present:
 213                        * filename: The final filename (always present)
 214                        * tmpfilename: The filename we're currently writing to
 215                        * downloaded_bytes: Bytes on disk
 216                        * total_bytes: Size of the whole file, None if unknown
 217                        * total_bytes_estimate: Guess of the eventual file size,
 218                                                None if unavailable.
 219                        * elapsed: The number of seconds since download started.
 220                        * eta: The estimated time in seconds, None if unknown
 221                        * speed: The download speed in bytes/second, None if
 222                                 unknown
 223                        * fragment_index: The counter of the currently
 224                                          downloaded video fragment.
 225                        * fragment_count: The number of fragments (= individual
 226                                          files that will be merged)
 227
 228                        Progress hooks are guaranteed to be called at least once
 229                        (with status "finished") if the download is successful.
 230     merge_output_format: Extension to use when merging formats.
 231     fixup:             Automatically correct known faults of the file.
 232                        One of:
 233                        - "never": do nothing
 234                        - "warn": only emit a warning
 235                        - "detect_or_warn": check whether we can do anything
 236                                            about it, warn otherwise (default)
 237     source_address:    (Experimental) Client-side IP address to bind to.
 238     call_home:         Boolean, true iff we are allowed to contact the
 239                        youtube-dl servers for debugging.
 240     sleep_interval:    Number of seconds to sleep before each download.
 241     listformats:       Print an overview of available video formats and exit.
 242     list_thumbnails:   Print a table of all thumbnails and exit.
 243     match_filter:      A function that gets called with the info_dict of
 244                        every video.
 245                        If it returns a message, the video is ignored.
 246                        If it returns None, the video is downloaded.
 247                        match_filter_func in utils.py is one example for this.
 248     no_color:          Do not emit color codes in output.
 249
 250     The following options determine which downloader is picked:
 251     external_downloader: Executable of the external downloader to call.
 252                        None or unset for standard (built-in) downloader.
 253     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 254
 255     The following parameters are not used by YoutubeDL itself, they are used by
 256     the downloader (see youtube_dl/downloader/common.py):
 257     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 258     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 259     xattr_set_filesize, external_downloader_args.
 260
 261     The following options are used by the post processors:
 262     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 263                        otherwise prefer avconv.
 264     """
 265
 266     params = None
 267     _ies = []
 268     _pps = []
 269     _download_retcode = None
 270     _num_downloads = None
 271     _screen_file = None
 272
 273     def __init__(self, params=None, auto_init=True):
 274         """Create a FileDownloader object with the given options."""
 275         if params is None:
 276             params = {}
 277         self._ies = []
 278         self._ies_instances = {}
 279         self._pps = []
 280         self._progress_hooks = []
 281         self._download_retcode = 0
 282         self._num_downloads = 0
 283         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 284         self._err_file = sys.stderr
 285         self.params = params
 286         self.cache = Cache(self)
 287
 288         if params.get('bidi_workaround', False):
 289             try:
 290                 import pty
 291                 master, slave = pty.openpty()
 292                 width = compat_get_terminal_size().columns
 293                 if width is None:
 294                     width_args = []
 295                 else:
 296                     width_args = ['-w', str(width)]
 297                 sp_kwargs = dict(
 298                     stdin=subprocess.PIPE,
 299                     stdout=slave,
 300                     stderr=self._err_file)
 301                 try:
 302                     self._output_process = subprocess.Popen(
 303                         ['bidiv'] + width_args, **sp_kwargs
 304                     )
 305                 except OSError:
 306                     self._output_process = subprocess.Popen(
 307                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 308                 self._output_channel = os.fdopen(master, 'rb')
 309             except OSError as ose:
 310                 if ose.errno == 2:
 311                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 312                 else:
 313                     raise
 314
 315         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 316                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 317                 not params.get('restrictfilenames', False)):
 318             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 319             self.report_warning(
 320                 'Assuming --restrict-filenames since file system encoding '
 321                 'cannot encode all characters. '
 322                 'Set the LC_ALL environment variable to fix this.')
 323             self.params['restrictfilenames'] = True
 324
 325         if isinstance(params.get('outtmpl'), bytes):
 326             self.report_warning(
 327                 'Parameter outtmpl is bytes, but should be a unicode string. '
 328                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 329
 330         self._setup_opener()
 331
 332         if auto_init:
 333             self.print_debug_header()
 334             self.add_default_info_extractors()
 335
 336         for pp_def_raw in self.params.get('postprocessors', []):
 337             pp_class = get_postprocessor(pp_def_raw['key'])
 338             pp_def = dict(pp_def_raw)
 339             del pp_def['key']
 340             pp = pp_class(self, **compat_kwargs(pp_def))
 341             self.add_post_processor(pp)
 342
 343         for ph in self.params.get('progress_hooks', []):
 344             self.add_progress_hook(ph)
 345
 346     def warn_if_short_id(self, argv):
 347         # short YouTube ID starting with dash?
 348         idxs = [
 349             i for i, a in enumerate(argv)
 350             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 351         if idxs:
 352             correct_argv = (
 353                 ['youtube-dl'] +
 354                 [a for i, a in enumerate(argv) if i not in idxs] +
 355                 ['--'] + [argv[i] for i in idxs]
 356             )
 357             self.report_warning(
 358                 'Long argument string detected. '
 359                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 360                 args_to_str(correct_argv))
 361
 362     def add_info_extractor(self, ie):
 363         """Add an InfoExtractor object to the end of the list."""
 364         self._ies.append(ie)
 365         self._ies_instances[ie.ie_key()] = ie
 366         ie.set_downloader(self)
 367
 368     def get_info_extractor(self, ie_key):
 369         """
 370         Get an instance of an IE with name ie_key, it will try to get one from
 371         the _ies list, if there's no instance it will create a new one and add
 372         it to the extractor list.
 373         """
 374         ie = self._ies_instances.get(ie_key)
 375         if ie is None:
 376             ie = get_info_extractor(ie_key)()
 377             self.add_info_extractor(ie)
 378         return ie
 379
 380     def add_default_info_extractors(self):
 381         """
 382         Add the InfoExtractors returned by gen_extractors to the end of the list
 383         """
 384         for ie in gen_extractors():
 385             self.add_info_extractor(ie)
 386
 387     def add_post_processor(self, pp):
 388         """Add a PostProcessor object to the end of the chain."""
 389         self._pps.append(pp)
 390         pp.set_downloader(self)
 391
 392     def add_progress_hook(self, ph):
 393         """Add the progress hook (currently only for the file downloader)"""
 394         self._progress_hooks.append(ph)
 395
 396     def _bidi_workaround(self, message):
 397         if not hasattr(self, '_output_channel'):
 398             return message
 399
 400         assert hasattr(self, '_output_process')
 401         assert isinstance(message, compat_str)
 402         line_count = message.count('\n') + 1
 403         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 404         self._output_process.stdin.flush()
 405         res = ''.join(self._output_channel.readline().decode('utf-8')
 406                       for _ in range(line_count))
 407         return res[:-len('\n')]
 408
 409     def to_screen(self, message, skip_eol=False):
 410         """Print message to stdout if not in quiet mode."""
 411         return self.to_stdout(message, skip_eol, check_quiet=True)
 412
 413     def _write_string(self, s, out=None):
 414         write_string(s, out=out, encoding=self.params.get('encoding'))
 415
 416     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 417         """Print message to stdout if not in quiet mode."""
 418         if self.params.get('logger'):
 419             self.params['logger'].debug(message)
 420         elif not check_quiet or not self.params.get('quiet', False):
 421             message = self._bidi_workaround(message)
 422             terminator = ['\n', ''][skip_eol]
 423             output = message + terminator
 424
 425             self._write_string(output, self._screen_file)
 426
 427     def to_stderr(self, message):
 428         """Print message to stderr."""
 429         assert isinstance(message, compat_str)
 430         if self.params.get('logger'):
 431             self.params['logger'].error(message)
 432         else:
 433             message = self._bidi_workaround(message)
 434             output = message + '\n'
 435             self._write_string(output, self._err_file)
 436
 437     def to_console_title(self, message):
 438         if not self.params.get('consoletitle', False):
 439             return
 440         if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 441             # c_wchar_p() might not be necessary if `message` is
 442             # already of type unicode()
 443             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 444         elif 'TERM' in os.environ:
 445             self._write_string('\033]0;%s\007' % message, self._screen_file)
 446
 447     def save_console_title(self):
 448         if not self.params.get('consoletitle', False):
 449             return
 450         if 'TERM' in os.environ:
 451             # Save the title on stack
 452             self._write_string('\033[22;0t', self._screen_file)
 453
 454     def restore_console_title(self):
 455         if not self.params.get('consoletitle', False):
 456             return
 457         if 'TERM' in os.environ:
 458             # Restore the title from stack
 459             self._write_string('\033[23;0t', self._screen_file)
 460
 461     def __enter__(self):
 462         self.save_console_title()
 463         return self
 464
 465     def __exit__(self, *args):
 466         self.restore_console_title()
 467
 468         if self.params.get('cookiefile') is not None:
 469             self.cookiejar.save()
 470
 471     def trouble(self, message=None, tb=None):
 472         """Determine action to take when a download problem appears.
 473
 474         Depending on if the downloader has been configured to ignore
 475         download errors or not, this method may throw an exception or
 476         not when errors are found, after printing the message.
 477
 478         tb, if given, is additional traceback information.
 479         """
 480         if message is not None:
 481             self.to_stderr(message)
 482         if self.params.get('verbose'):
 483             if tb is None:
 484                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 485                     tb = ''
 486                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 487                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 488                     tb += compat_str(traceback.format_exc())
 489                 else:
 490                     tb_data = traceback.format_list(traceback.extract_stack())
 491                     tb = ''.join(tb_data)
 492             self.to_stderr(tb)
 493         if not self.params.get('ignoreerrors', False):
 494             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 495                 exc_info = sys.exc_info()[1].exc_info
 496             else:
 497                 exc_info = sys.exc_info()
 498             raise DownloadError(message, exc_info)
 499         self._download_retcode = 1
 500
 501     def report_warning(self, message):
 502         '''
 503         Print the message to stderr, it will be prefixed with 'WARNING:'
 504         If stderr is a tty file the 'WARNING:' will be colored
 505         '''
 506         if self.params.get('logger') is not None:
 507             self.params['logger'].warning(message)
 508         else:
 509             if self.params.get('no_warnings'):
 510                 return
 511             if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 512                 _msg_header = '\033[0;33mWARNING:\033[0m'
 513             else:
 514                 _msg_header = 'WARNING:'
 515             warning_message = '%s %s' % (_msg_header, message)
 516             self.to_stderr(warning_message)
 517
 518     def report_error(self, message, tb=None):
 519         '''
 520         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 521         in red if stderr is a tty file.
 522         '''
 523         if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 524             _msg_header = '\033[0;31mERROR:\033[0m'
 525         else:
 526             _msg_header = 'ERROR:'
 527         error_message = '%s %s' % (_msg_header, message)
 528         self.trouble(error_message, tb)
 529
 530     def report_file_already_downloaded(self, file_name):
 531         """Report file has already been fully downloaded."""
 532         try:
 533             self.to_screen('[download] %s has already been downloaded' % file_name)
 534         except UnicodeEncodeError:
 535             self.to_screen('[download] The file has already been downloaded')
 536
 537     def prepare_filename(self, info_dict):
 538         """Generate the output filename."""
 539         try:
 540             template_dict = dict(info_dict)
 541
 542             template_dict['epoch'] = int(time.time())
 543             autonumber_size = self.params.get('autonumber_size')
 544             if autonumber_size is None:
 545                 autonumber_size = 5
 546             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 547             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 548             if template_dict.get('playlist_index') is not None:
 549                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 550             if template_dict.get('resolution') is None:
 551                 if template_dict.get('width') and template_dict.get('height'):
 552                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 553                 elif template_dict.get('height'):
 554                     template_dict['resolution'] = '%sp' % template_dict['height']
 555                 elif template_dict.get('width'):
 556                     template_dict['resolution'] = '?x%d' % template_dict['width']
 557
 558             sanitize = lambda k, v: sanitize_filename(
 559                 compat_str(v),
 560                 restricted=self.params.get('restrictfilenames'),
 561                 is_id=(k == 'id'))
 562             template_dict = dict((k, sanitize(k, v))
 563                                  for k, v in template_dict.items()
 564                                  if v is not None)
 565             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 566
 567             outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
 568             tmpl = compat_expanduser(outtmpl)
 569             filename = tmpl % template_dict
 570             # Temporary fix for #4787
 571             # 'Treat' all problem characters by passing filename through preferredencoding
 572             # to workaround encoding issues with subprocess on python2 @ Windows
 573             if sys.version_info < (3, 0) and sys.platform == 'win32':
 574                 filename = encodeFilename(filename, True).decode(preferredencoding())
 575             return filename
 576         except ValueError as err:
 577             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 578             return None
 579
 580     def _match_entry(self, info_dict, incomplete):
 581         """ Returns None iff the file should be downloaded """
 582
 583         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 584         if 'title' in info_dict:
 585             # This can happen when we're just evaluating the playlist
 586             title = info_dict['title']
 587             matchtitle = self.params.get('matchtitle', False)
 588             if matchtitle:
 589                 if not re.search(matchtitle, title, re.IGNORECASE):
 590                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 591             rejecttitle = self.params.get('rejecttitle', False)
 592             if rejecttitle:
 593                 if re.search(rejecttitle, title, re.IGNORECASE):
 594                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 595         date = info_dict.get('upload_date', None)
 596         if date is not None:
 597             dateRange = self.params.get('daterange', DateRange())
 598             if date not in dateRange:
 599                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 600         view_count = info_dict.get('view_count', None)
 601         if view_count is not None:
 602             min_views = self.params.get('min_views')
 603             if min_views is not None and view_count < min_views:
 604                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 605             max_views = self.params.get('max_views')
 606             if max_views is not None and view_count > max_views:
 607                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 608         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 609             return 'Skipping "%s" because it is age restricted' % video_title
 610         if self.in_download_archive(info_dict):
 611             return '%s has already been recorded in archive' % video_title
 612
 613         if not incomplete:
 614             match_filter = self.params.get('match_filter')
 615             if match_filter is not None:
 616                 ret = match_filter(info_dict)
 617                 if ret is not None:
 618                     return ret
 619
 620         return None
 621
 622     @staticmethod
 623     def add_extra_info(info_dict, extra_info):
 624         '''Set the keys from extra_info in info dict if they are missing'''
 625         for key, value in extra_info.items():
 626             info_dict.setdefault(key, value)
 627
 628     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 629                      process=True):
 630         '''
 631         Returns a list with a dictionary for each video we find.
 632         If 'download', also downloads the videos.
 633         extra_info is a dict containing the extra values to add to each result
 634         '''
 635
 636         if ie_key:
 637             ies = [self.get_info_extractor(ie_key)]
 638         else:
 639             ies = self._ies
 640
 641         for ie in ies:
 642             if not ie.suitable(url):
 643                 continue
 644
 645             if not ie.working():
 646                 self.report_warning('The program functionality for this site has been marked as broken, '
 647                                     'and will probably not work.')
 648
 649             try:
 650                 ie_result = ie.extract(url)
 651                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 652                     break
 653                 if isinstance(ie_result, list):
 654                     # Backwards compatibility: old IE result format
 655                     ie_result = {
 656                         '_type': 'compat_list',
 657                         'entries': ie_result,
 658                     }
 659                 self.add_default_extra_info(ie_result, ie, url)
 660                 if process:
 661                     return self.process_ie_result(ie_result, download, extra_info)
 662                 else:
 663                     return ie_result
 664             except ExtractorError as de:  # An error we somewhat expected
 665                 self.report_error(compat_str(de), de.format_traceback())
 666                 break
 667             except MaxDownloadsReached:
 668                 raise
 669             except Exception as e:
 670                 if self.params.get('ignoreerrors', False):
 671                     self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
 672                     break
 673                 else:
 674                     raise
 675         else:
 676             self.report_error('no suitable InfoExtractor for URL %s' % url)
 677
 678     def add_default_extra_info(self, ie_result, ie, url):
 679         self.add_extra_info(ie_result, {
 680             'extractor': ie.IE_NAME,
 681             'webpage_url': url,
 682             'webpage_url_basename': url_basename(url),
 683             'extractor_key': ie.ie_key(),
 684         })
 685
 686     def process_ie_result(self, ie_result, download=True, extra_info={}):
 687         """
 688         Take the result of the ie(may be modified) and resolve all unresolved
 689         references (URLs, playlist items).
 690
 691         It will also download the videos if 'download'.
 692         Returns the resolved ie_result.
 693         """
 694
 695         result_type = ie_result.get('_type', 'video')
 696
 697         if result_type in ('url', 'url_transparent'):
 698             extract_flat = self.params.get('extract_flat', False)
 699             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 700                     extract_flat is True):
 701                 if self.params.get('forcejson', False):
 702                     self.to_stdout(json.dumps(ie_result))
 703                 return ie_result
 704
 705         if result_type == 'video':
 706             self.add_extra_info(ie_result, extra_info)
 707             return self.process_video_result(ie_result, download=download)
 708         elif result_type == 'url':
 709             # We have to add extra_info to the results because it may be
 710             # contained in a playlist
 711             return self.extract_info(ie_result['url'],
 712                                      download,
 713                                      ie_key=ie_result.get('ie_key'),
 714                                      extra_info=extra_info)
 715         elif result_type == 'url_transparent':
 716             # Use the information from the embedding page
 717             info = self.extract_info(
 718                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 719                 extra_info=extra_info, download=False, process=False)
 720
 721             force_properties = dict(
 722                 (k, v) for k, v in ie_result.items() if v is not None)
 723             for f in ('_type', 'url'):
 724                 if f in force_properties:
 725                     del force_properties[f]
 726             new_result = info.copy()
 727             new_result.update(force_properties)
 728
 729             assert new_result.get('_type') != 'url_transparent'
 730
 731             return self.process_ie_result(
 732                 new_result, download=download, extra_info=extra_info)
 733         elif result_type == 'playlist' or result_type == 'multi_video':
 734             # We process each entry in the playlist
 735             playlist = ie_result.get('title', None) or ie_result.get('id', None)
 736             self.to_screen('[download] Downloading playlist: %s' % playlist)
 737
 738             playlist_results = []
 739
 740             playliststart = self.params.get('playliststart', 1) - 1
 741             playlistend = self.params.get('playlistend', None)
 742             # For backwards compatibility, interpret -1 as whole list
 743             if playlistend == -1:
 744                 playlistend = None
 745
 746             playlistitems_str = self.params.get('playlist_items', None)
 747             playlistitems = None
 748             if playlistitems_str is not None:
 749                 def iter_playlistitems(format):
 750                     for string_segment in format.split(','):
 751                         if '-' in string_segment:
 752                             start, end = string_segment.split('-')
 753                             for item in range(int(start), int(end) + 1):
 754                                 yield int(item)
 755                         else:
 756                             yield int(string_segment)
 757                 playlistitems = iter_playlistitems(playlistitems_str)
 758
 759             ie_entries = ie_result['entries']
 760             if isinstance(ie_entries, list):
 761                 n_all_entries = len(ie_entries)
 762                 if playlistitems:
 763                     entries = [
 764                         ie_entries[i - 1] for i in playlistitems
 765                         if -n_all_entries <= i - 1 < n_all_entries]
 766                 else:
 767                     entries = ie_entries[playliststart:playlistend]
 768                 n_entries = len(entries)
 769                 self.to_screen(
 770                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
 771                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 772             elif isinstance(ie_entries, PagedList):
 773                 if playlistitems:
 774                     entries = []
 775                     for item in playlistitems:
 776                         entries.extend(ie_entries.getslice(
 777                             item - 1, item
 778                         ))
 779                 else:
 780                     entries = ie_entries.getslice(
 781                         playliststart, playlistend)
 782                 n_entries = len(entries)
 783                 self.to_screen(
 784                     "[%s] playlist %s: Downloading %d videos" %
 785                     (ie_result['extractor'], playlist, n_entries))
 786             else:  # iterable
 787                 if playlistitems:
 788                     entry_list = list(ie_entries)
 789                     entries = [entry_list[i - 1] for i in playlistitems]
 790                 else:
 791                     entries = list(itertools.islice(
 792                         ie_entries, playliststart, playlistend))
 793                 n_entries = len(entries)
 794                 self.to_screen(
 795                     "[%s] playlist %s: Downloading %d videos" %
 796                     (ie_result['extractor'], playlist, n_entries))
 797
 798             if self.params.get('playlistreverse', False):
 799                 entries = entries[::-1]
 800
 801             for i, entry in enumerate(entries, 1):
 802                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 803                 extra = {
 804                     'n_entries': n_entries,
 805                     'playlist': playlist,
 806                     'playlist_id': ie_result.get('id'),
 807                     'playlist_title': ie_result.get('title'),
 808                     'playlist_index': i + playliststart,
 809                     'extractor': ie_result['extractor'],
 810                     'webpage_url': ie_result['webpage_url'],
 811                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 812                     'extractor_key': ie_result['extractor_key'],
 813                 }
 814
 815                 reason = self._match_entry(entry, incomplete=True)
 816                 if reason is not None:
 817                     self.to_screen('[download] ' + reason)
 818                     continue
 819
 820                 entry_result = self.process_ie_result(entry,
 821                                                       download=download,
 822                                                       extra_info=extra)
 823                 playlist_results.append(entry_result)
 824             ie_result['entries'] = playlist_results
 825             return ie_result
 826         elif result_type == 'compat_list':
 827             self.report_warning(
 828                 'Extractor %s returned a compat_list result. '
 829                 'It needs to be updated.' % ie_result.get('extractor'))
 830
 831             def _fixup(r):
 832                 self.add_extra_info(
 833                     r,
 834                     {
 835                         'extractor': ie_result['extractor'],
 836                         'webpage_url': ie_result['webpage_url'],
 837                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 838                         'extractor_key': ie_result['extractor_key'],
 839                     }
 840                 )
 841                 return r
 842             ie_result['entries'] = [
 843                 self.process_ie_result(_fixup(r), download, extra_info)
 844                 for r in ie_result['entries']
 845             ]
 846             return ie_result
 847         else:
 848             raise Exception('Invalid result type: %s' % result_type)
 849
 850     def _apply_format_filter(self, format_spec, available_formats):
 851         " Returns a tuple of the remaining format_spec and filtered formats "
 852
 853         OPERATORS = {
 854             '<': operator.lt,
 855             '<=': operator.le,
 856             '>': operator.gt,
 857             '>=': operator.ge,
 858             '=': operator.eq,
 859             '!=': operator.ne,
 860         }
 861         operator_rex = re.compile(r'''(?x)\s*\[
 862             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 863             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 864             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 865             \]$
 866             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 867         m = operator_rex.search(format_spec)
 868         if m:
 869             try:
 870                 comparison_value = int(m.group('value'))
 871             except ValueError:
 872                 comparison_value = parse_filesize(m.group('value'))
 873                 if comparison_value is None:
 874                     comparison_value = parse_filesize(m.group('value') + 'B')
 875                 if comparison_value is None:
 876                     raise ValueError(
 877                         'Invalid value %r in format specification %r' % (
 878                             m.group('value'), format_spec))
 879             op = OPERATORS[m.group('op')]
 880
 881         if not m:
 882             STR_OPERATORS = {
 883                 '=': operator.eq,
 884                 '!=': operator.ne,
 885             }
 886             str_operator_rex = re.compile(r'''(?x)\s*\[
 887                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
 888                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 889                 \s*(?P<value>[a-zA-Z0-9_-]+)
 890                 \s*\]$
 891                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 892             m = str_operator_rex.search(format_spec)
 893             if m:
 894                 comparison_value = m.group('value')
 895                 op = STR_OPERATORS[m.group('op')]
 896
 897         if not m:
 898             raise ValueError('Invalid format specification %r' % format_spec)
 899
 900         def _filter(f):
 901             actual_value = f.get(m.group('key'))
 902             if actual_value is None:
 903                 return m.group('none_inclusive')
 904             return op(actual_value, comparison_value)
 905         new_formats = [f for f in available_formats if _filter(f)]
 906
 907         new_format_spec = format_spec[:-len(m.group(0))]
 908         if not new_format_spec:
 909             new_format_spec = 'best'
 910
 911         return (new_format_spec, new_formats)
 912
 913     def select_format(self, format_spec, available_formats):
 914         while format_spec.endswith(']'):
 915             format_spec, available_formats = self._apply_format_filter(
 916                 format_spec, available_formats)
 917         if not available_formats:
 918             return None
 919
 920         if format_spec in ['best', 'worst', None]:
 921             format_idx = 0 if format_spec == 'worst' else -1
 922             audiovideo_formats = [
 923                 f for f in available_formats
 924                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
 925             if audiovideo_formats:
 926                 return audiovideo_formats[format_idx]
 927             # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
 928             elif (all(f.get('acodec') != 'none' for f in available_formats) or
 929                   all(f.get('vcodec') != 'none' for f in available_formats)):
 930                 return available_formats[format_idx]
 931         elif format_spec == 'bestaudio':
 932             audio_formats = [
 933                 f for f in available_formats
 934                 if f.get('vcodec') == 'none']
 935             if audio_formats:
 936                 return audio_formats[-1]
 937         elif format_spec == 'worstaudio':
 938             audio_formats = [
 939                 f for f in available_formats
 940                 if f.get('vcodec') == 'none']
 941             if audio_formats:
 942                 return audio_formats[0]
 943         elif format_spec == 'bestvideo':
 944             video_formats = [
 945                 f for f in available_formats
 946                 if f.get('acodec') == 'none']
 947             if video_formats:
 948                 return video_formats[-1]
 949         elif format_spec == 'worstvideo':
 950             video_formats = [
 951                 f for f in available_formats
 952                 if f.get('acodec') == 'none']
 953             if video_formats:
 954                 return video_formats[0]
 955         else:
 956             extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
 957             if format_spec in extensions:
 958                 filter_f = lambda f: f['ext'] == format_spec
 959             else:
 960                 filter_f = lambda f: f['format_id'] == format_spec
 961             matches = list(filter(filter_f, available_formats))
 962             if matches:
 963                 return matches[-1]
 964         return None
 965
 966     def _calc_headers(self, info_dict):
 967         res = std_headers.copy()
 968
 969         add_headers = info_dict.get('http_headers')
 970         if add_headers:
 971             res.update(add_headers)
 972
 973         cookies = self._calc_cookies(info_dict)
 974         if cookies:
 975             res['Cookie'] = cookies
 976
 977         return res
 978
 979     def _calc_cookies(self, info_dict):
 980         pr = compat_urllib_request.Request(info_dict['url'])
 981         self.cookiejar.add_cookie_header(pr)
 982         return pr.get_header('Cookie')
 983
 984     def process_video_result(self, info_dict, download=True):
 985         assert info_dict.get('_type', 'video') == 'video'
 986
 987         if 'id' not in info_dict:
 988             raise ExtractorError('Missing "id" field in extractor result')
 989         if 'title' not in info_dict:
 990             raise ExtractorError('Missing "title" field in extractor result')
 991
 992         if 'playlist' not in info_dict:
 993             # It isn't part of a playlist
 994             info_dict['playlist'] = None
 995             info_dict['playlist_index'] = None
 996
 997         thumbnails = info_dict.get('thumbnails')
 998         if thumbnails is None:
 999             thumbnail = info_dict.get('thumbnail')
1000             if thumbnail:
1001                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1002         if thumbnails:
1003             thumbnails.sort(key=lambda t: (
1004                 t.get('preference'), t.get('width'), t.get('height'),
1005                 t.get('id'), t.get('url')))
1006             for i, t in enumerate(thumbnails):
1007                 if 'width' in t and 'height' in t:
1008                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1009                 if t.get('id') is None:
1010                     t['id'] = '%d' % i
1011
1012         if thumbnails and 'thumbnail' not in info_dict:
1013             info_dict['thumbnail'] = thumbnails[-1]['url']
1014
1015         if 'display_id' not in info_dict and 'id' in info_dict:
1016             info_dict['display_id'] = info_dict['id']
1017
1018         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1019             # Working around negative timestamps in Windows
1020             # (see http://bugs.python.org/issue1646728)
1021             if info_dict['timestamp'] < 0 and os.name == 'nt':
1022                 info_dict['timestamp'] = 0
1023             upload_date = datetime.datetime.utcfromtimestamp(
1024                 info_dict['timestamp'])
1025             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1026
1027         if self.params.get('listsubtitles', False):
1028             if 'automatic_captions' in info_dict:
1029                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1030             self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1031             return
1032         info_dict['requested_subtitles'] = self.process_subtitles(
1033             info_dict['id'], info_dict.get('subtitles'),
1034             info_dict.get('automatic_captions'))
1035
1036         # This extractors handle format selection themselves
1037         if info_dict['extractor'] in ['Youku']:
1038             if download:
1039                 self.process_info(info_dict)
1040             return info_dict
1041
1042         # We now pick which formats have to be downloaded
1043         if info_dict.get('formats') is None:
1044             # There's only one format available
1045             formats = [info_dict]
1046         else:
1047             formats = info_dict['formats']
1048
1049         if not formats:
1050             raise ExtractorError('No video formats found!')
1051
1052         formats_dict = {}
1053
1054         # We check that all the formats have the format and format_id fields
1055         for i, format in enumerate(formats):
1056             if 'url' not in format:
1057                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1058
1059             if format.get('format_id') is None:
1060                 format['format_id'] = compat_str(i)
1061             format_id = format['format_id']
1062             if format_id not in formats_dict:
1063                 formats_dict[format_id] = []
1064             formats_dict[format_id].append(format)
1065
1066         # Make sure all formats have unique format_id
1067         for format_id, ambiguous_formats in formats_dict.items():
1068             if len(ambiguous_formats) > 1:
1069                 for i, format in enumerate(ambiguous_formats):
1070                     format['format_id'] = '%s-%d' % (format_id, i)
1071
1072         for i, format in enumerate(formats):
1073             if format.get('format') is None:
1074                 format['format'] = '{id} - {res}{note}'.format(
1075                     id=format['format_id'],
1076                     res=self.format_resolution(format),
1077                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1078                 )
1079             # Automatically determine file extension if missing
1080             if 'ext' not in format:
1081                 format['ext'] = determine_ext(format['url']).lower()
1082             # Add HTTP headers, so that external programs can use them from the
1083             # json output
1084             full_format_info = info_dict.copy()
1085             full_format_info.update(format)
1086             format['http_headers'] = self._calc_headers(full_format_info)
1087
1088         # TODO Central sorting goes here
1089
1090         if formats[0] is not info_dict:
1091             # only set the 'formats' fields if the original info_dict list them
1092             # otherwise we end up with a circular reference, the first (and unique)
1093             # element in the 'formats' field in info_dict is info_dict itself,
1094             # wich can't be exported to json
1095             info_dict['formats'] = formats
1096         if self.params.get('listformats'):
1097             self.list_formats(info_dict)
1098             return
1099         if self.params.get('list_thumbnails'):
1100             self.list_thumbnails(info_dict)
1101             return
1102
1103         req_format = self.params.get('format')
1104         if req_format is None:
1105             req_format_list = []
1106             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1107                     info_dict['extractor'] in ['youtube', 'ted']):
1108                 merger = FFmpegMergerPP(self)
1109                 if merger.available and merger.can_merge():
1110                     req_format_list.append('bestvideo+bestaudio')
1111             req_format_list.append('best')
1112             req_format = '/'.join(req_format_list)
1113         formats_to_download = []
1114         if req_format == 'all':
1115             formats_to_download = formats
1116         else:
1117             for rfstr in req_format.split(','):
1118                 # We can accept formats requested in the format: 34/5/best, we pick
1119                 # the first that is available, starting from left
1120                 req_formats = rfstr.split('/')
1121                 for rf in req_formats:
1122                     if re.match(r'.+?\+.+?', rf) is not None:
1123                         # Two formats have been requested like '137+139'
1124                         format_1, format_2 = rf.split('+')
1125                         formats_info = (self.select_format(format_1, formats),
1126                                         self.select_format(format_2, formats))
1127                         if all(formats_info):
1128                             # The first format must contain the video and the
1129                             # second the audio
1130                             if formats_info[0].get('vcodec') == 'none':
1131                                 self.report_error('The first format must '
1132                                                   'contain the video, try using '
1133                                                   '"-f %s+%s"' % (format_2, format_1))
1134                                 return
1135                             output_ext = (
1136                                 formats_info[0]['ext']
1137                                 if self.params.get('merge_output_format') is None
1138                                 else self.params['merge_output_format'])
1139                             selected_format = {
1140                                 'requested_formats': formats_info,
1141                                 'format': '%s+%s' % (formats_info[0].get('format'),
1142                                                      formats_info[1].get('format')),
1143                                 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1144                                                         formats_info[1].get('format_id')),
1145                                 'width': formats_info[0].get('width'),
1146                                 'height': formats_info[0].get('height'),
1147                                 'resolution': formats_info[0].get('resolution'),
1148                                 'fps': formats_info[0].get('fps'),
1149                                 'vcodec': formats_info[0].get('vcodec'),
1150                                 'vbr': formats_info[0].get('vbr'),
1151                                 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1152                                 'acodec': formats_info[1].get('acodec'),
1153                                 'abr': formats_info[1].get('abr'),
1154                                 'ext': output_ext,
1155                             }
1156                         else:
1157                             selected_format = None
1158                     else:
1159                         selected_format = self.select_format(rf, formats)
1160                     if selected_format is not None:
1161                         formats_to_download.append(selected_format)
1162                         break
1163         if not formats_to_download:
1164             raise ExtractorError('requested format not available',
1165                                  expected=True)
1166
1167         if download:
1168             if len(formats_to_download) > 1:
1169                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1170             for format in formats_to_download:
1171                 new_info = dict(info_dict)
1172                 new_info.update(format)
1173                 self.process_info(new_info)
1174         # We update the info dict with the best quality format (backwards compatibility)
1175         info_dict.update(formats_to_download[-1])
1176         return info_dict
1177
1178     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1179         """Select the requested subtitles and their format"""
1180         available_subs = {}
1181         if normal_subtitles and self.params.get('writesubtitles'):
1182             available_subs.update(normal_subtitles)
1183         if automatic_captions and self.params.get('writeautomaticsub'):
1184             for lang, cap_info in automatic_captions.items():
1185                 if lang not in available_subs:
1186                     available_subs[lang] = cap_info
1187
1188         if (not self.params.get('writesubtitles') and not
1189                 self.params.get('writeautomaticsub') or not
1190                 available_subs):
1191             return None
1192
1193         if self.params.get('allsubtitles', False):
1194             requested_langs = available_subs.keys()
1195         else:
1196             if self.params.get('subtitleslangs', False):
1197                 requested_langs = self.params.get('subtitleslangs')
1198             elif 'en' in available_subs:
1199                 requested_langs = ['en']
1200             else:
1201                 requested_langs = [list(available_subs.keys())[0]]
1202
1203         formats_query = self.params.get('subtitlesformat', 'best')
1204         formats_preference = formats_query.split('/') if formats_query else []
1205         subs = {}
1206         for lang in requested_langs:
1207             formats = available_subs.get(lang)
1208             if formats is None:
1209                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1210                 continue
1211             for ext in formats_preference:
1212                 if ext == 'best':
1213                     f = formats[-1]
1214                     break
1215                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1216                 if matches:
1217                     f = matches[-1]
1218                     break
1219             else:
1220                 f = formats[-1]
1221                 self.report_warning(
1222                     'No subtitle format found matching "%s" for language %s, '
1223                     'using %s' % (formats_query, lang, f['ext']))
1224             subs[lang] = f
1225         return subs
1226
1227     def process_info(self, info_dict):
1228         """Process a single resolved IE result."""
1229
1230         assert info_dict.get('_type', 'video') == 'video'
1231
1232         max_downloads = self.params.get('max_downloads')
1233         if max_downloads is not None:
1234             if self._num_downloads >= int(max_downloads):
1235                 raise MaxDownloadsReached()
1236
1237         info_dict['fulltitle'] = info_dict['title']
1238         if len(info_dict['title']) > 200:
1239             info_dict['title'] = info_dict['title'][:197] + '...'
1240
1241         if 'format' not in info_dict:
1242             info_dict['format'] = info_dict['ext']
1243
1244         reason = self._match_entry(info_dict, incomplete=False)
1245         if reason is not None:
1246             self.to_screen('[download] ' + reason)
1247             return
1248
1249         self._num_downloads += 1
1250
1251         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1252
1253         # Forced printings
1254         if self.params.get('forcetitle', False):
1255             self.to_stdout(info_dict['fulltitle'])
1256         if self.params.get('forceid', False):
1257             self.to_stdout(info_dict['id'])
1258         if self.params.get('forceurl', False):
1259             if info_dict.get('requested_formats') is not None:
1260                 for f in info_dict['requested_formats']:
1261                     self.to_stdout(f['url'] + f.get('play_path', ''))
1262             else:
1263                 # For RTMP URLs, also include the playpath
1264                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1265         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1266             self.to_stdout(info_dict['thumbnail'])
1267         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1268             self.to_stdout(info_dict['description'])
1269         if self.params.get('forcefilename', False) and filename is not None:
1270             self.to_stdout(filename)
1271         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1272             self.to_stdout(formatSeconds(info_dict['duration']))
1273         if self.params.get('forceformat', False):
1274             self.to_stdout(info_dict['format'])
1275         if self.params.get('forcejson', False):
1276             self.to_stdout(json.dumps(info_dict))
1277
1278         # Do nothing else if in simulate mode
1279         if self.params.get('simulate', False):
1280             return
1281
1282         if filename is None:
1283             return
1284
1285         try:
1286             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1287             if dn and not os.path.exists(dn):
1288                 os.makedirs(dn)
1289         except (OSError, IOError) as err:
1290             self.report_error('unable to create directory ' + compat_str(err))
1291             return
1292
1293         if self.params.get('writedescription', False):
1294             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1295             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1296                 self.to_screen('[info] Video description is already present')
1297             elif info_dict.get('description') is None:
1298                 self.report_warning('There\'s no description to write.')
1299             else:
1300                 try:
1301                     self.to_screen('[info] Writing video description to: ' + descfn)
1302                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1303                         descfile.write(info_dict['description'])
1304                 except (OSError, IOError):
1305                     self.report_error('Cannot write description file ' + descfn)
1306                     return
1307
1308         if self.params.get('writeannotations', False):
1309             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1310             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1311                 self.to_screen('[info] Video annotations are already present')
1312             else:
1313                 try:
1314                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1315                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1316                         annofile.write(info_dict['annotations'])
1317                 except (KeyError, TypeError):
1318                     self.report_warning('There are no annotations to write.')
1319                 except (OSError, IOError):
1320                     self.report_error('Cannot write annotations file: ' + annofn)
1321                     return
1322
1323         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1324                                        self.params.get('writeautomaticsub')])
1325
1326         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1327             # subtitles download errors are already managed as troubles in relevant IE
1328             # that way it will silently go on when used with unsupporting IE
1329             subtitles = info_dict['requested_subtitles']
1330             ie = self.get_info_extractor(info_dict['extractor_key'])
1331             for sub_lang, sub_info in subtitles.items():
1332                 sub_format = sub_info['ext']
1333                 if sub_info.get('data') is not None:
1334                     sub_data = sub_info['data']
1335                 else:
1336                     try:
1337                         sub_data = ie._download_webpage(
1338                             sub_info['url'], info_dict['id'], note=False)
1339                     except ExtractorError as err:
1340                         self.report_warning('Unable to download subtitle for "%s": %s' %
1341                                             (sub_lang, compat_str(err.cause)))
1342                         continue
1343                 try:
1344                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1345                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1346                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1347                     else:
1348                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1349                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1350                             subfile.write(sub_data)
1351                 except (OSError, IOError):
1352                     self.report_error('Cannot write subtitles file ' + sub_filename)
1353                     return
1354
1355         if self.params.get('writeinfojson', False):
1356             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1357             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1358                 self.to_screen('[info] Video description metadata is already present')
1359             else:
1360                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1361                 try:
1362                     write_json_file(self.filter_requested_info(info_dict), infofn)
1363                 except (OSError, IOError):
1364                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1365                     return
1366
1367         self._write_thumbnails(info_dict, filename)
1368
1369         if not self.params.get('skip_download', False):
1370             try:
1371                 def dl(name, info):
1372                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1373                     for ph in self._progress_hooks:
1374                         fd.add_progress_hook(ph)
1375                     if self.params.get('verbose'):
1376                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1377                     return fd.download(name, info)
1378
1379                 if info_dict.get('requested_formats') is not None:
1380                     downloaded = []
1381                     success = True
1382                     merger = FFmpegMergerPP(self)
1383                     if not merger.available:
1384                         postprocessors = []
1385                         self.report_warning('You have requested multiple '
1386                                             'formats but ffmpeg or avconv are not installed.'
1387                                             ' The formats won\'t be merged.')
1388                     else:
1389                         postprocessors = [merger]
1390
1391                     def compatible_formats(formats):
1392                         video, audio = formats
1393                         # Check extension
1394                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1395                         if video_ext and audio_ext:
1396                             COMPATIBLE_EXTS = (
1397                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1398                                 ('webm')
1399                             )
1400                             for exts in COMPATIBLE_EXTS:
1401                                 if video_ext in exts and audio_ext in exts:
1402                                     return True
1403                         # TODO: Check acodec/vcodec
1404                         return False
1405
1406                     filename_real_ext = os.path.splitext(filename)[1][1:]
1407                     filename_wo_ext = (
1408                         os.path.splitext(filename)[0]
1409                         if filename_real_ext == info_dict['ext']
1410                         else filename)
1411                     requested_formats = info_dict['requested_formats']
1412                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1413                         info_dict['ext'] = 'mkv'
1414                         self.report_warning(
1415                             'Requested formats are incompatible for merge and will be merged into mkv.')
1416                     # Ensure filename always has a correct extension for successful merge
1417                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1418                     if os.path.exists(encodeFilename(filename)):
1419                         self.to_screen(
1420                             '[download] %s has already been downloaded and '
1421                             'merged' % filename)
1422                     else:
1423                         for f in requested_formats:
1424                             new_info = dict(info_dict)
1425                             new_info.update(f)
1426                             fname = self.prepare_filename(new_info)
1427                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1428                             downloaded.append(fname)
1429                             partial_success = dl(fname, new_info)
1430                             success = success and partial_success
1431                         info_dict['__postprocessors'] = postprocessors
1432                         info_dict['__files_to_merge'] = downloaded
1433                 else:
1434                     # Just a single file
1435                     success = dl(filename, info_dict)
1436             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1437                 self.report_error('unable to download video data: %s' % str(err))
1438                 return
1439             except (OSError, IOError) as err:
1440                 raise UnavailableVideoError(err)
1441             except (ContentTooShortError, ) as err:
1442                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1443                 return
1444
1445             if success:
1446                 # Fixup content
1447                 fixup_policy = self.params.get('fixup')
1448                 if fixup_policy is None:
1449                     fixup_policy = 'detect_or_warn'
1450
1451                 stretched_ratio = info_dict.get('stretched_ratio')
1452                 if stretched_ratio is not None and stretched_ratio != 1:
1453                     if fixup_policy == 'warn':
1454                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1455                             info_dict['id'], stretched_ratio))
1456                     elif fixup_policy == 'detect_or_warn':
1457                         stretched_pp = FFmpegFixupStretchedPP(self)
1458                         if stretched_pp.available:
1459                             info_dict.setdefault('__postprocessors', [])
1460                             info_dict['__postprocessors'].append(stretched_pp)
1461                         else:
1462                             self.report_warning(
1463                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1464                                     info_dict['id'], stretched_ratio))
1465                     else:
1466                         assert fixup_policy in ('ignore', 'never')
1467
1468                 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1469                     if fixup_policy == 'warn':
1470                         self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1471                             info_dict['id']))
1472                     elif fixup_policy == 'detect_or_warn':
1473                         fixup_pp = FFmpegFixupM4aPP(self)
1474                         if fixup_pp.available:
1475                             info_dict.setdefault('__postprocessors', [])
1476                             info_dict['__postprocessors'].append(fixup_pp)
1477                         else:
1478                             self.report_warning(
1479                                 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1480                                     info_dict['id']))
1481                     else:
1482                         assert fixup_policy in ('ignore', 'never')
1483
1484                 try:
1485                     self.post_process(filename, info_dict)
1486                 except (PostProcessingError) as err:
1487                     self.report_error('postprocessing: %s' % str(err))
1488                     return
1489                 self.record_download_archive(info_dict)
1490
1491     def download(self, url_list):
1492         """Download a given list of URLs."""
1493         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1494         if (len(url_list) > 1 and
1495                 '%' not in outtmpl and
1496                 self.params.get('max_downloads') != 1):
1497             raise SameFileError(outtmpl)
1498
1499         for url in url_list:
1500             try:
1501                 # It also downloads the videos
1502                 res = self.extract_info(url)
1503             except UnavailableVideoError:
1504                 self.report_error('unable to download video')
1505             except MaxDownloadsReached:
1506                 self.to_screen('[info] Maximum number of downloaded files reached.')
1507                 raise
1508             else:
1509                 if self.params.get('dump_single_json', False):
1510                     self.to_stdout(json.dumps(res))
1511
1512         return self._download_retcode
1513
1514     def download_with_info_file(self, info_filename):
1515         with contextlib.closing(fileinput.FileInput(
1516                 [info_filename], mode='r',
1517                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1518             # FileInput doesn't have a read method, we can't call json.load
1519             info = self.filter_requested_info(json.loads('\n'.join(f)))
1520         try:
1521             self.process_ie_result(info, download=True)
1522         except DownloadError:
1523             webpage_url = info.get('webpage_url')
1524             if webpage_url is not None:
1525                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1526                 return self.download([webpage_url])
1527             else:
1528                 raise
1529         return self._download_retcode
1530
1531     @staticmethod
1532     def filter_requested_info(info_dict):
1533         return dict(
1534             (k, v) for k, v in info_dict.items()
1535             if k not in ['requested_formats', 'requested_subtitles'])
1536
1537     def post_process(self, filename, ie_info):
1538         """Run all the postprocessors on the given file."""
1539         info = dict(ie_info)
1540         info['filepath'] = filename
1541         pps_chain = []
1542         if ie_info.get('__postprocessors') is not None:
1543             pps_chain.extend(ie_info['__postprocessors'])
1544         pps_chain.extend(self._pps)
1545         for pp in pps_chain:
1546             files_to_delete = []
1547             try:
1548                 files_to_delete, info = pp.run(info)
1549             except PostProcessingError as e:
1550                 self.report_error(e.msg)
1551             if files_to_delete and not self.params.get('keepvideo', False):
1552                 for old_filename in files_to_delete:
1553                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1554                     try:
1555                         os.remove(encodeFilename(old_filename))
1556                     except (IOError, OSError):
1557                         self.report_warning('Unable to remove downloaded original file')
1558
1559     def _make_archive_id(self, info_dict):
1560         # Future-proof against any change in case
1561         # and backwards compatibility with prior versions
1562         extractor = info_dict.get('extractor_key')
1563         if extractor is None:
1564             if 'id' in info_dict:
1565                 extractor = info_dict.get('ie_key')  # key in a playlist
1566         if extractor is None:
1567             return None  # Incomplete video information
1568         return extractor.lower() + ' ' + info_dict['id']
1569
1570     def in_download_archive(self, info_dict):
1571         fn = self.params.get('download_archive')
1572         if fn is None:
1573             return False
1574
1575         vid_id = self._make_archive_id(info_dict)
1576         if vid_id is None:
1577             return False  # Incomplete video information
1578
1579         try:
1580             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1581                 for line in archive_file:
1582                     if line.strip() == vid_id:
1583                         return True
1584         except IOError as ioe:
1585             if ioe.errno != errno.ENOENT:
1586                 raise
1587         return False
1588
1589     def record_download_archive(self, info_dict):
1590         fn = self.params.get('download_archive')
1591         if fn is None:
1592             return
1593         vid_id = self._make_archive_id(info_dict)
1594         assert vid_id
1595         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1596             archive_file.write(vid_id + '\n')
1597
1598     @staticmethod
1599     def format_resolution(format, default='unknown'):
1600         if format.get('vcodec') == 'none':
1601             return 'audio only'
1602         if format.get('resolution') is not None:
1603             return format['resolution']
1604         if format.get('height') is not None:
1605             if format.get('width') is not None:
1606                 res = '%sx%s' % (format['width'], format['height'])
1607             else:
1608                 res = '%sp' % format['height']
1609         elif format.get('width') is not None:
1610             res = '?x%d' % format['width']
1611         else:
1612             res = default
1613         return res
1614
1615     def _format_note(self, fdict):
1616         res = ''
1617         if fdict.get('ext') in ['f4f', 'f4m']:
1618             res += '(unsupported) '
1619         if fdict.get('format_note') is not None:
1620             res += fdict['format_note'] + ' '
1621         if fdict.get('tbr') is not None:
1622             res += '%4dk ' % fdict['tbr']
1623         if fdict.get('container') is not None:
1624             if res:
1625                 res += ', '
1626             res += '%s container' % fdict['container']
1627         if (fdict.get('vcodec') is not None and
1628                 fdict.get('vcodec') != 'none'):
1629             if res:
1630                 res += ', '
1631             res += fdict['vcodec']
1632             if fdict.get('vbr') is not None:
1633                 res += '@'
1634         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1635             res += 'video@'
1636         if fdict.get('vbr') is not None:
1637             res += '%4dk' % fdict['vbr']
1638         if fdict.get('fps') is not None:
1639             res += ', %sfps' % fdict['fps']
1640         if fdict.get('acodec') is not None:
1641             if res:
1642                 res += ', '
1643             if fdict['acodec'] == 'none':
1644                 res += 'video only'
1645             else:
1646                 res += '%-5s' % fdict['acodec']
1647         elif fdict.get('abr') is not None:
1648             if res:
1649                 res += ', '
1650             res += 'audio'
1651         if fdict.get('abr') is not None:
1652             res += '@%3dk' % fdict['abr']
1653         if fdict.get('asr') is not None:
1654             res += ' (%5dHz)' % fdict['asr']
1655         if fdict.get('filesize') is not None:
1656             if res:
1657                 res += ', '
1658             res += format_bytes(fdict['filesize'])
1659         elif fdict.get('filesize_approx') is not None:
1660             if res:
1661                 res += ', '
1662             res += '~' + format_bytes(fdict['filesize_approx'])
1663         return res
1664
1665     def list_formats(self, info_dict):
1666         formats = info_dict.get('formats', [info_dict])
1667         table = [
1668             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1669             for f in formats
1670             if f.get('preference') is None or f['preference'] >= -1000]
1671         if len(formats) > 1:
1672             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1673
1674         header_line = ['format code', 'extension', 'resolution', 'note']
1675         self.to_screen(
1676             '[info] Available formats for %s:\n%s' %
1677             (info_dict['id'], render_table(header_line, table)))
1678
1679     def list_thumbnails(self, info_dict):
1680         thumbnails = info_dict.get('thumbnails')
1681         if not thumbnails:
1682             tn_url = info_dict.get('thumbnail')
1683             if tn_url:
1684                 thumbnails = [{'id': '0', 'url': tn_url}]
1685             else:
1686                 self.to_screen(
1687                     '[info] No thumbnails present for %s' % info_dict['id'])
1688                 return
1689
1690         self.to_screen(
1691             '[info] Thumbnails for %s:' % info_dict['id'])
1692         self.to_screen(render_table(
1693             ['ID', 'width', 'height', 'URL'],
1694             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1695
1696     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1697         if not subtitles:
1698             self.to_screen('%s has no %s' % (video_id, name))
1699             return
1700         self.to_screen(
1701             'Available %s for %s:' % (name, video_id))
1702         self.to_screen(render_table(
1703             ['Language', 'formats'],
1704             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1705                 for lang, formats in subtitles.items()]))
1706
1707     def urlopen(self, req):
1708         """ Start an HTTP download """
1709
1710         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1711         # always respected by websites, some tend to give out URLs with non percent-encoded
1712         # non-ASCII characters (see telemb.py, ard.py [#3412])
1713         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1714         # To work around aforementioned issue we will replace request's original URL with
1715         # percent-encoded one
1716         req_is_string = isinstance(req, compat_basestring)
1717         url = req if req_is_string else req.get_full_url()
1718         url_escaped = escape_url(url)
1719
1720         # Substitute URL if any change after escaping
1721         if url != url_escaped:
1722             if req_is_string:
1723                 req = url_escaped
1724             else:
1725                 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
1726                 req = req_type(
1727                     url_escaped, data=req.data, headers=req.headers,
1728                     origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1729
1730         return self._opener.open(req, timeout=self._socket_timeout)
1731
1732     def print_debug_header(self):
1733         if not self.params.get('verbose'):
1734             return
1735
1736         if type('') is not compat_str:
1737             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1738             self.report_warning(
1739                 'Your Python is broken! Update to a newer and supported version')
1740
1741         stdout_encoding = getattr(
1742             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1743         encoding_str = (
1744             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1745                 locale.getpreferredencoding(),
1746                 sys.getfilesystemencoding(),
1747                 stdout_encoding,
1748                 self.get_encoding()))
1749         write_string(encoding_str, encoding=None)
1750
1751         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1752         try:
1753             sp = subprocess.Popen(
1754                 ['git', 'rev-parse', '--short', 'HEAD'],
1755                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1756                 cwd=os.path.dirname(os.path.abspath(__file__)))
1757             out, err = sp.communicate()
1758             out = out.decode().strip()
1759             if re.match('[0-9a-f]+', out):
1760                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1761         except Exception:
1762             try:
1763                 sys.exc_clear()
1764             except Exception:
1765                 pass
1766         self._write_string('[debug] Python version %s - %s\n' % (
1767             platform.python_version(), platform_name()))
1768
1769         exe_versions = FFmpegPostProcessor.get_versions(self)
1770         exe_versions['rtmpdump'] = rtmpdump_version()
1771         exe_str = ', '.join(
1772             '%s %s' % (exe, v)
1773             for exe, v in sorted(exe_versions.items())
1774             if v
1775         )
1776         if not exe_str:
1777             exe_str = 'none'
1778         self._write_string('[debug] exe versions: %s\n' % exe_str)
1779
1780         proxy_map = {}
1781         for handler in self._opener.handlers:
1782             if hasattr(handler, 'proxies'):
1783                 proxy_map.update(handler.proxies)
1784         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1785
1786         if self.params.get('call_home', False):
1787             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1788             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1789             latest_version = self.urlopen(
1790                 'https://yt-dl.org/latest/version').read().decode('utf-8')
1791             if version_tuple(latest_version) > version_tuple(__version__):
1792                 self.report_warning(
1793                     'You are using an outdated version (newest version: %s)! '
1794                     'See https://yt-dl.org/update if you need help updating.' %
1795                     latest_version)
1796
1797     def _setup_opener(self):
1798         timeout_val = self.params.get('socket_timeout')
1799         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1800
1801         opts_cookiefile = self.params.get('cookiefile')
1802         opts_proxy = self.params.get('proxy')
1803
1804         if opts_cookiefile is None:
1805             self.cookiejar = compat_cookiejar.CookieJar()
1806         else:
1807             self.cookiejar = compat_cookiejar.MozillaCookieJar(
1808                 opts_cookiefile)
1809             if os.access(opts_cookiefile, os.R_OK):
1810                 self.cookiejar.load()
1811
1812         cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1813             self.cookiejar)
1814         if opts_proxy is not None:
1815             if opts_proxy == '':
1816                 proxies = {}
1817             else:
1818                 proxies = {'http': opts_proxy, 'https': opts_proxy}
1819         else:
1820             proxies = compat_urllib_request.getproxies()
1821             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1822             if 'http' in proxies and 'https' not in proxies:
1823                 proxies['https'] = proxies['http']
1824         proxy_handler = PerRequestProxyHandler(proxies)
1825
1826         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1827         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1828         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1829         opener = compat_urllib_request.build_opener(
1830             proxy_handler, https_handler, cookie_processor, ydlh)
1831
1832         # Delete the default user-agent header, which would otherwise apply in
1833         # cases where our custom HTTP handler doesn't come into play
1834         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1835         opener.addheaders = []
1836         self._opener = opener
1837
1838     def encode(self, s):
1839         if isinstance(s, bytes):
1840             return s  # Already encoded
1841
1842         try:
1843             return s.encode(self.get_encoding())
1844         except UnicodeEncodeError as err:
1845             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1846             raise
1847
1848     def get_encoding(self):
1849         encoding = self.params.get('encoding')
1850         if encoding is None:
1851             encoding = preferredencoding()
1852         return encoding
1853
1854     def _write_thumbnails(self, info_dict, filename):
1855         if self.params.get('writethumbnail', False):
1856             thumbnails = info_dict.get('thumbnails')
1857             if thumbnails:
1858                 thumbnails = [thumbnails[-1]]
1859         elif self.params.get('write_all_thumbnails', False):
1860             thumbnails = info_dict.get('thumbnails')
1861         else:
1862             return
1863
1864         if not thumbnails:
1865             # No thumbnails present, so return immediately
1866             return
1867
1868         for t in thumbnails:
1869             thumb_ext = determine_ext(t['url'], 'jpg')
1870             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1871             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1872             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1873
1874             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1875                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1876                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
1877             else:
1878                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1879                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
1880                 try:
1881                     uf = self.urlopen(t['url'])
1882                     with open(thumb_filename, 'wb') as thumbf:
1883                         shutil.copyfileobj(uf, thumbf)
1884                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1885                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1886                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1887                     self.report_warning('Unable to download thumbnail "%s": %s' %
1888                                         (t['url'], compat_str(err)))