Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67 )
  68
  69 __license__ = 'Public Domain'
  70
  71 import codecs
  72 import io
  73 import locale
  74 import optparse
  75 import os
  76 import random
  77 import re
  78 import shlex
  79 import sys
  80
  81
  82 from .utils import (
  83     compat_getpass,
  84     compat_print,
  85     DateRange,
  86     DEFAULT_OUTTMPL,
  87     decodeOption,
  88     get_term_width,
  89     DownloadError,
  90     get_cachedir,
  91     MaxDownloadsReached,
  92     preferredencoding,
  93     read_batch_urls,
  94     SameFileError,
  95     setproctitle,
  96     std_headers,
  97     write_string,
  98 )
  99 from .update import update_self
 100 from .FileDownloader import (
 101     FileDownloader,
 102 )
 103 from .extractor import gen_extractors
 104 from .version import __version__
 105 from .YoutubeDL import YoutubeDL
 106 from .postprocessor import (
 107     AtomicParsleyPP,
 108     FFmpegAudioFixPP,
 109     FFmpegMetadataPP,
 110     FFmpegVideoConvertor,
 111     FFmpegExtractAudioPP,
 112     FFmpegEmbedSubtitlePP,
 113     XAttrMetadataPP,
 114 )
 115
 116
 117 def parseOpts(overrideArguments=None):
 118     def _readOptions(filename_bytes, default=[]):
 119         try:
 120             optionf = open(filename_bytes)
 121         except IOError:
 122             return default  # silently skip if file is not present
 123         try:
 124             res = []
 125             for l in optionf:
 126                 res += shlex.split(l, comments=True)
 127         finally:
 128             optionf.close()
 129         return res
 130
 131     def _readUserConf():
 132         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 133         if xdg_config_home:
 134             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 135             if not os.path.isfile(userConfFile):
 136                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 137         else:
 138             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 139             if not os.path.isfile(userConfFile):
 140                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 141         userConf = _readOptions(userConfFile, None)
 142
 143         if userConf is None:
 144             appdata_dir = os.environ.get('appdata')
 145             if appdata_dir:
 146                 userConf = _readOptions(
 147                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 148                     default=None)
 149                 if userConf is None:
 150                     userConf = _readOptions(
 151                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 152                         default=None)
 153
 154         if userConf is None:
 155             userConf = _readOptions(
 156                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 157                 default=None)
 158         if userConf is None:
 159             userConf = _readOptions(
 160                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 161                 default=None)
 162
 163         if userConf is None:
 164             userConf = []
 165
 166         return userConf
 167
 168     def _format_option_string(option):
 169         ''' ('-o', '--option') -> -o, --format METAVAR'''
 170
 171         opts = []
 172
 173         if option._short_opts:
 174             opts.append(option._short_opts[0])
 175         if option._long_opts:
 176             opts.append(option._long_opts[0])
 177         if len(opts) > 1:
 178             opts.insert(1, ', ')
 179
 180         if option.takes_value(): opts.append(' %s' % option.metavar)
 181
 182         return "".join(opts)
 183
 184     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 185         setattr(parser.values, option.dest, value.split(','))
 186
 187     def _hide_login_info(opts):
 188         opts = list(opts)
 189         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 190             try:
 191                 i = opts.index(private_opt)
 192                 opts[i+1] = '<PRIVATE>'
 193             except ValueError:
 194                 pass
 195         return opts
 196
 197     max_width = 80
 198     max_help_position = 80
 199
 200     # No need to wrap help messages if we're on a wide console
 201     columns = get_term_width()
 202     if columns: max_width = columns
 203
 204     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 205     fmt.format_option_strings = _format_option_string
 206
 207     kw = {
 208         'version'   : __version__,
 209         'formatter' : fmt,
 210         'usage' : '%prog [options] url [url...]',
 211         'conflict_handler' : 'resolve',
 212     }
 213
 214     parser = optparse.OptionParser(**kw)
 215
 216     # option groups
 217     general        = optparse.OptionGroup(parser, 'General Options')
 218     selection      = optparse.OptionGroup(parser, 'Video Selection')
 219     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 220     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 221     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 222     downloader     = optparse.OptionGroup(parser, 'Download Options')
 223     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 224     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 225     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 226
 227     general.add_option('-h', '--help',
 228             action='help', help='print this help text and exit')
 229     general.add_option('-v', '--version',
 230             action='version', help='print program version and exit')
 231     general.add_option('-U', '--update',
 232             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 233     general.add_option('-i', '--ignore-errors',
 234             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 235     general.add_option('--abort-on-error',
 236             action='store_false', dest='ignoreerrors',
 237             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 238     general.add_option('--dump-user-agent',
 239             action='store_true', dest='dump_user_agent',
 240             help='display the current browser identification', default=False)
 241     general.add_option('--user-agent',
 242             dest='user_agent', help='specify a custom user agent', metavar='UA')
 243     general.add_option('--referer',
 244             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 245             metavar='REF', default=None)
 246     general.add_option('--add-header',
 247             dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
 248             metavar='FIELD:VALUE')
 249     general.add_option('--list-extractors',
 250             action='store_true', dest='list_extractors',
 251             help='List all supported extractors and the URLs they would handle', default=False)
 252     general.add_option('--extractor-descriptions',
 253             action='store_true', dest='list_extractor_descriptions',
 254             help='Output descriptions of all supported extractors', default=False)
 255     general.add_option(
 256         '--proxy', dest='proxy', default=None, metavar='URL',
 257         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 258     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 259     general.add_option(
 260         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 261         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 262     general.add_option(
 263         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 264         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 265     general.add_option(
 266         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 267         help='Disable filesystem caching')
 268     general.add_option(
 269         '--socket-timeout', dest='socket_timeout',
 270         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 271     general.add_option(
 272         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 273         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 274     general.add_option(
 275         '--default-search',
 276         dest='default_search', metavar='PREFIX',
 277         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess. The default value "error" just throws an error.')
 278     general.add_option(
 279         '--ignore-config',
 280         action='store_true',
 281         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 282     general.add_option(
 283         '--encoding', dest='encoding', metavar='ENCODING',
 284         help='Force the specified encoding (experimental)')
 285
 286     selection.add_option(
 287         '--playlist-start',
 288         dest='playliststart', metavar='NUMBER', default=1, type=int,
 289         help='playlist video to start at (default is %default)')
 290     selection.add_option(
 291         '--playlist-end',
 292         dest='playlistend', metavar='NUMBER', default=None, type=int,
 293         help='playlist video to end at (default is last)')
 294     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 295     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 296     selection.add_option('--max-downloads', metavar='NUMBER',
 297                          dest='max_downloads', type=int, default=None,
 298                          help='Abort after downloading NUMBER files')
 299     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 300     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 301     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 302     selection.add_option(
 303         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 304         help='download only videos uploaded on or before this date (i.e. inclusive)')
 305     selection.add_option(
 306         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 307         help='download only videos uploaded on or after this date (i.e. inclusive)')
 308     selection.add_option(
 309         '--min-views', metavar='COUNT', dest='min_views',
 310         default=None, type=int,
 311         help="Do not download any videos with less than COUNT views",)
 312     selection.add_option(
 313         '--max-views', metavar='COUNT', dest='max_views',
 314         default=None, type=int,
 315         help="Do not download any videos with more than COUNT views",)
 316     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 317     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 318                          help='download only videos suitable for the given age',
 319                          default=None, type=int)
 320     selection.add_option('--download-archive', metavar='FILE',
 321                          dest='download_archive',
 322                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 323     selection.add_option(
 324         '--include-ads', dest='include_ads',
 325         action='store_true',
 326         help='Download advertisements as well (experimental)')
 327     selection.add_option(
 328         '--youtube-include-dash-manifest', action='store_true',
 329         dest='youtube_include_dash_manifest', default=False,
 330         help='Try to download the DASH manifest on YouTube videos (experimental)')
 331
 332     authentication.add_option('-u', '--username',
 333             dest='username', metavar='USERNAME', help='account username')
 334     authentication.add_option('-p', '--password',
 335             dest='password', metavar='PASSWORD', help='account password')
 336     authentication.add_option('-n', '--netrc',
 337             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 338     authentication.add_option('--video-password',
 339             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 340
 341
 342     video_format.add_option('-f', '--format',
 343             action='store', dest='format', metavar='FORMAT', default=None,
 344             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 345     video_format.add_option('--all-formats',
 346             action='store_const', dest='format', help='download all available video formats', const='all')
 347     video_format.add_option('--prefer-free-formats',
 348             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 349     video_format.add_option('--max-quality',
 350             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 351     video_format.add_option('-F', '--list-formats',
 352             action='store_true', dest='listformats', help='list all available formats')
 353
 354     subtitles.add_option('--write-sub', '--write-srt',
 355             action='store_true', dest='writesubtitles',
 356             help='write subtitle file', default=False)
 357     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 358             action='store_true', dest='writeautomaticsub',
 359             help='write automatic subtitle file (youtube only)', default=False)
 360     subtitles.add_option('--all-subs',
 361             action='store_true', dest='allsubtitles',
 362             help='downloads all the available subtitles of the video', default=False)
 363     subtitles.add_option('--list-subs',
 364             action='store_true', dest='listsubtitles',
 365             help='lists all available subtitles for the video', default=False)
 366     subtitles.add_option('--sub-format',
 367             action='store', dest='subtitlesformat', metavar='FORMAT',
 368             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 369     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 370             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 371             default=[], callback=_comma_separated_values_options_callback,
 372             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 373
 374     downloader.add_option('-r', '--rate-limit',
 375             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 376     downloader.add_option('-R', '--retries',
 377             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 378     downloader.add_option('--buffer-size',
 379             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 380     downloader.add_option('--no-resize-buffer',
 381             action='store_true', dest='noresizebuffer',
 382             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 383     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 384
 385     verbosity.add_option('-q', '--quiet',
 386             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 387     verbosity.add_option(
 388         '--no-warnings',
 389         dest='no_warnings', action='store_true', default=False,
 390         help='Ignore warnings')
 391     verbosity.add_option('-s', '--simulate',
 392             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 393     verbosity.add_option('--skip-download',
 394             action='store_true', dest='skip_download', help='do not download the video', default=False)
 395     verbosity.add_option('-g', '--get-url',
 396             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 397     verbosity.add_option('-e', '--get-title',
 398             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 399     verbosity.add_option('--get-id',
 400             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 401     verbosity.add_option('--get-thumbnail',
 402             action='store_true', dest='getthumbnail',
 403             help='simulate, quiet but print thumbnail URL', default=False)
 404     verbosity.add_option('--get-description',
 405             action='store_true', dest='getdescription',
 406             help='simulate, quiet but print video description', default=False)
 407     verbosity.add_option('--get-duration',
 408             action='store_true', dest='getduration',
 409             help='simulate, quiet but print video length', default=False)
 410     verbosity.add_option('--get-filename',
 411             action='store_true', dest='getfilename',
 412             help='simulate, quiet but print output filename', default=False)
 413     verbosity.add_option('--get-format',
 414             action='store_true', dest='getformat',
 415             help='simulate, quiet but print output format', default=False)
 416     verbosity.add_option('-j', '--dump-json',
 417             action='store_true', dest='dumpjson',
 418             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 419     verbosity.add_option('--newline',
 420             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 421     verbosity.add_option('--no-progress',
 422             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 423     verbosity.add_option('--console-title',
 424             action='store_true', dest='consoletitle',
 425             help='display progress in console titlebar', default=False)
 426     verbosity.add_option('-v', '--verbose',
 427             action='store_true', dest='verbose', help='print various debugging information', default=False)
 428     verbosity.add_option('--dump-intermediate-pages',
 429             action='store_true', dest='dump_intermediate_pages', default=False,
 430             help='print downloaded pages to debug problems (very verbose)')
 431     verbosity.add_option('--write-pages',
 432             action='store_true', dest='write_pages', default=False,
 433             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 434     verbosity.add_option('--youtube-print-sig-code',
 435             action='store_true', dest='youtube_print_sig_code', default=False,
 436             help=optparse.SUPPRESS_HELP)
 437     verbosity.add_option('--print-traffic',
 438             dest='debug_printtraffic', action='store_true', default=False,
 439             help='Display sent and read HTTP traffic')
 440
 441
 442     filesystem.add_option('-t', '--title',
 443             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 444     filesystem.add_option('--id',
 445             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 446     filesystem.add_option('-l', '--literal',
 447             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 448     filesystem.add_option('-A', '--auto-number',
 449             action='store_true', dest='autonumber',
 450             help='number downloaded files starting from 00000', default=False)
 451     filesystem.add_option('-o', '--output',
 452             dest='outtmpl', metavar='TEMPLATE',
 453             help=('output filename template. Use %(title)s to get the title, '
 454                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 455                   '%(autonumber)s to get an automatically incremented number, '
 456                   '%(ext)s for the filename extension, '
 457                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 458                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 459                   '%(upload_date)s for the upload date (YYYYMMDD), '
 460                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 461                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 462                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 463                   '%(height)s and %(width)s for the width and height of the video format. '
 464                   '%(resolution)s for a textual description of the resolution of the video format. '
 465                   'Use - to output to stdout. Can also be used to download to a different directory, '
 466                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 467     filesystem.add_option('--autonumber-size',
 468             dest='autonumber_size', metavar='NUMBER',
 469             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 470     filesystem.add_option('--restrict-filenames',
 471             action='store_true', dest='restrictfilenames',
 472             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 473     filesystem.add_option('-a', '--batch-file',
 474             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 475     filesystem.add_option('--load-info',
 476             dest='load_info_filename', metavar='FILE',
 477             help='json file containing the video information (created with the "--write-json" option)')
 478     filesystem.add_option('-w', '--no-overwrites',
 479             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 480     filesystem.add_option('-c', '--continue',
 481             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 482     filesystem.add_option('--no-continue',
 483             action='store_false', dest='continue_dl',
 484             help='do not resume partially downloaded files (restart from beginning)')
 485     filesystem.add_option('--cookies',
 486             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 487     filesystem.add_option('--no-part',
 488             action='store_true', dest='nopart', help='do not use .part files', default=False)
 489     filesystem.add_option('--no-mtime',
 490             action='store_false', dest='updatetime',
 491             help='do not use the Last-modified header to set the file modification time', default=True)
 492     filesystem.add_option('--write-description',
 493             action='store_true', dest='writedescription',
 494             help='write video description to a .description file', default=False)
 495     filesystem.add_option('--write-info-json',
 496             action='store_true', dest='writeinfojson',
 497             help='write video metadata to a .info.json file', default=False)
 498     filesystem.add_option('--write-annotations',
 499             action='store_true', dest='writeannotations',
 500             help='write video annotations to a .annotation file', default=False)
 501     filesystem.add_option('--write-thumbnail',
 502             action='store_true', dest='writethumbnail',
 503             help='write thumbnail image to disk', default=False)
 504
 505
 506     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 507             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 508     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 509             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 510     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 511             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 512     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 513             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 514     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 515             help='keeps the video file on disk after the post-processing; the video is erased by default')
 516     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 517             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 518     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 519             help='embed subtitles in the video (only for mp4 videos)')
 520     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 521             help='embed thumbnail in the audio as cover art')
 522     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 523             help='write metadata to the video file')
 524     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 525             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 526     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 527         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 528     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 529         help='Prefer ffmpeg over avconv for running the postprocessors')
 530
 531
 532     parser.add_option_group(general)
 533     parser.add_option_group(selection)
 534     parser.add_option_group(downloader)
 535     parser.add_option_group(filesystem)
 536     parser.add_option_group(verbosity)
 537     parser.add_option_group(video_format)
 538     parser.add_option_group(subtitles)
 539     parser.add_option_group(authentication)
 540     parser.add_option_group(postproc)
 541
 542     if overrideArguments is not None:
 543         opts, args = parser.parse_args(overrideArguments)
 544         if opts.verbose:
 545             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 546     else:
 547         commandLineConf = sys.argv[1:]
 548         if '--ignore-config' in commandLineConf:
 549             systemConf = []
 550             userConf = []
 551         else:
 552             systemConf = _readOptions('/etc/youtube-dl.conf')
 553             if '--ignore-config' in systemConf:
 554                 userConf = []
 555             else:
 556                 userConf = _readUserConf()
 557         argv = systemConf + userConf + commandLineConf
 558
 559         opts, args = parser.parse_args(argv)
 560         if opts.verbose:
 561             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 562             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 563             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 564
 565     return parser, opts, args
 566
 567
 568 def _real_main(argv=None):
 569     # Compatibility fixes for Windows
 570     if sys.platform == 'win32':
 571         # https://github.com/rg3/youtube-dl/issues/820
 572         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 573
 574     setproctitle(u'youtube-dl')
 575
 576     parser, opts, args = parseOpts(argv)
 577
 578     # Set user agent
 579     if opts.user_agent is not None:
 580         std_headers['User-Agent'] = opts.user_agent
 581
 582     # Set referer
 583     if opts.referer is not None:
 584         std_headers['Referer'] = opts.referer
 585
 586     # Custom HTTP headers
 587     if opts.headers is not None:
 588         for h in opts.headers:
 589             if h.find(':', 1) < 0:
 590                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 591             key, value = h.split(':', 2)
 592             if opts.verbose:
 593                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 594             std_headers[key] = value
 595
 596     # Dump user agent
 597     if opts.dump_user_agent:
 598         compat_print(std_headers['User-Agent'])
 599         sys.exit(0)
 600
 601     # Batch file verification
 602     batch_urls = []
 603     if opts.batchfile is not None:
 604         try:
 605             if opts.batchfile == '-':
 606                 batchfd = sys.stdin
 607             else:
 608                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 609             batch_urls = read_batch_urls(batchfd)
 610             if opts.verbose:
 611                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 612         except IOError:
 613             sys.exit(u'ERROR: batch file could not be read')
 614     all_urls = batch_urls + args
 615     all_urls = [url.strip() for url in all_urls]
 616     _enc = preferredencoding()
 617     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 618
 619     extractors = gen_extractors()
 620
 621     if opts.list_extractors:
 622         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 623             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 624             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 625             for mu in matchedUrls:
 626                 compat_print(u'  ' + mu)
 627         sys.exit(0)
 628     if opts.list_extractor_descriptions:
 629         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 630             if not ie._WORKING:
 631                 continue
 632             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 633             if desc is False:
 634                 continue
 635             if hasattr(ie, 'SEARCH_KEY'):
 636                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 637                 _COUNTS = (u'', u'5', u'10', u'all')
 638                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 639             compat_print(desc)
 640         sys.exit(0)
 641
 642
 643     # Conflicting, missing and erroneous options
 644     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 645         parser.error(u'using .netrc conflicts with giving username/password')
 646     if opts.password is not None and opts.username is None:
 647         parser.error(u'account username missing\n')
 648     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 649         parser.error(u'using output template conflicts with using title, video ID or auto number')
 650     if opts.usetitle and opts.useid:
 651         parser.error(u'using title conflicts with using video ID')
 652     if opts.username is not None and opts.password is None:
 653         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 654     if opts.ratelimit is not None:
 655         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 656         if numeric_limit is None:
 657             parser.error(u'invalid rate limit specified')
 658         opts.ratelimit = numeric_limit
 659     if opts.min_filesize is not None:
 660         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 661         if numeric_limit is None:
 662             parser.error(u'invalid min_filesize specified')
 663         opts.min_filesize = numeric_limit
 664     if opts.max_filesize is not None:
 665         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 666         if numeric_limit is None:
 667             parser.error(u'invalid max_filesize specified')
 668         opts.max_filesize = numeric_limit
 669     if opts.retries is not None:
 670         try:
 671             opts.retries = int(opts.retries)
 672         except (TypeError, ValueError):
 673             parser.error(u'invalid retry count specified')
 674     if opts.buffersize is not None:
 675         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 676         if numeric_buffersize is None:
 677             parser.error(u'invalid buffer size specified')
 678         opts.buffersize = numeric_buffersize
 679     if opts.playliststart <= 0:
 680         raise ValueError(u'Playlist start must be positive')
 681     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 682         raise ValueError(u'Playlist end must be greater than playlist start')
 683     if opts.extractaudio:
 684         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 685             parser.error(u'invalid audio format specified')
 686     if opts.audioquality:
 687         opts.audioquality = opts.audioquality.strip('k').strip('K')
 688         if not opts.audioquality.isdigit():
 689             parser.error(u'invalid audio quality specified')
 690     if opts.recodevideo is not None:
 691         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 692             parser.error(u'invalid video recode format specified')
 693     if opts.date is not None:
 694         date = DateRange.day(opts.date)
 695     else:
 696         date = DateRange(opts.dateafter, opts.datebefore)
 697     if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
 698         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 699
 700     # Do not download videos when there are audio-only formats
 701     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 702         opts.format = 'bestaudio/best'
 703
 704     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 705     # this was the old behaviour if only --all-sub was given.
 706     if opts.allsubtitles and (opts.writeautomaticsub == False):
 707         opts.writesubtitles = True
 708
 709     if sys.version_info < (3,):
 710         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 711         if opts.outtmpl is not None:
 712             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 713     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 714             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 715             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 716             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 717             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 718             or (opts.useid and u'%(id)s.%(ext)s')
 719             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 720             or DEFAULT_OUTTMPL)
 721     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 722         parser.error(u'Cannot download a video and extract audio into the same'
 723                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 724                      u' template'.format(outtmpl))
 725
 726     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 727     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 728
 729     ydl_opts = {
 730         'usenetrc': opts.usenetrc,
 731         'username': opts.username,
 732         'password': opts.password,
 733         'videopassword': opts.videopassword,
 734         'quiet': (opts.quiet or any_printing),
 735         'no_warnings': opts.no_warnings,
 736         'forceurl': opts.geturl,
 737         'forcetitle': opts.gettitle,
 738         'forceid': opts.getid,
 739         'forcethumbnail': opts.getthumbnail,
 740         'forcedescription': opts.getdescription,
 741         'forceduration': opts.getduration,
 742         'forcefilename': opts.getfilename,
 743         'forceformat': opts.getformat,
 744         'forcejson': opts.dumpjson,
 745         'simulate': opts.simulate,
 746         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 747         'format': opts.format,
 748         'format_limit': opts.format_limit,
 749         'listformats': opts.listformats,
 750         'outtmpl': outtmpl,
 751         'autonumber_size': opts.autonumber_size,
 752         'restrictfilenames': opts.restrictfilenames,
 753         'ignoreerrors': opts.ignoreerrors,
 754         'ratelimit': opts.ratelimit,
 755         'nooverwrites': opts.nooverwrites,
 756         'retries': opts.retries,
 757         'buffersize': opts.buffersize,
 758         'noresizebuffer': opts.noresizebuffer,
 759         'continuedl': opts.continue_dl,
 760         'noprogress': opts.noprogress,
 761         'progress_with_newline': opts.progress_with_newline,
 762         'playliststart': opts.playliststart,
 763         'playlistend': opts.playlistend,
 764         'noplaylist': opts.noplaylist,
 765         'logtostderr': opts.outtmpl == '-',
 766         'consoletitle': opts.consoletitle,
 767         'nopart': opts.nopart,
 768         'updatetime': opts.updatetime,
 769         'writedescription': opts.writedescription,
 770         'writeannotations': opts.writeannotations,
 771         'writeinfojson': opts.writeinfojson,
 772         'writethumbnail': opts.writethumbnail,
 773         'writesubtitles': opts.writesubtitles,
 774         'writeautomaticsub': opts.writeautomaticsub,
 775         'allsubtitles': opts.allsubtitles,
 776         'listsubtitles': opts.listsubtitles,
 777         'subtitlesformat': opts.subtitlesformat,
 778         'subtitleslangs': opts.subtitleslangs,
 779         'matchtitle': decodeOption(opts.matchtitle),
 780         'rejecttitle': decodeOption(opts.rejecttitle),
 781         'max_downloads': opts.max_downloads,
 782         'prefer_free_formats': opts.prefer_free_formats,
 783         'verbose': opts.verbose,
 784         'dump_intermediate_pages': opts.dump_intermediate_pages,
 785         'write_pages': opts.write_pages,
 786         'test': opts.test,
 787         'keepvideo': opts.keepvideo,
 788         'min_filesize': opts.min_filesize,
 789         'max_filesize': opts.max_filesize,
 790         'min_views': opts.min_views,
 791         'max_views': opts.max_views,
 792         'daterange': date,
 793         'cachedir': opts.cachedir,
 794         'youtube_print_sig_code': opts.youtube_print_sig_code,
 795         'age_limit': opts.age_limit,
 796         'download_archive': download_archive_fn,
 797         'cookiefile': opts.cookiefile,
 798         'nocheckcertificate': opts.no_check_certificate,
 799         'prefer_insecure': opts.prefer_insecure,
 800         'proxy': opts.proxy,
 801         'socket_timeout': opts.socket_timeout,
 802         'bidi_workaround': opts.bidi_workaround,
 803         'debug_printtraffic': opts.debug_printtraffic,
 804         'prefer_ffmpeg': opts.prefer_ffmpeg,
 805         'include_ads': opts.include_ads,
 806         'default_search': opts.default_search,
 807         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 808         'encoding': opts.encoding,
 809     }
 810
 811     with YoutubeDL(ydl_opts) as ydl:
 812         ydl.print_debug_header()
 813         ydl.add_default_info_extractors()
 814
 815         # PostProcessors
 816         # Add the metadata pp first, the other pps will copy it
 817         if opts.addmetadata:
 818             ydl.add_post_processor(FFmpegMetadataPP())
 819         if opts.extractaudio:
 820             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 821         if opts.recodevideo:
 822             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 823         if opts.embedsubtitles:
 824             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 825         if opts.xattrs:
 826             ydl.add_post_processor(XAttrMetadataPP())
 827         if opts.embedthumbnail:
 828             if not opts.addmetadata:
 829                 ydl.add_post_processor(FFmpegAudioFixPP())
 830             ydl.add_post_processor(AtomicParsleyPP())
 831
 832         # Update version
 833         if opts.update_self:
 834             update_self(ydl.to_screen, opts.verbose)
 835
 836         # Maybe do nothing
 837         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 838             if not opts.update_self:
 839                 parser.error(u'you must provide at least one URL')
 840             else:
 841                 sys.exit()
 842
 843         try:
 844             if opts.load_info_filename is not None:
 845                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 846             else:
 847                 retcode = ydl.download(all_urls)
 848         except MaxDownloadsReached:
 849             ydl.to_screen(u'--max-download limit reached, aborting.')
 850             retcode = 101
 851
 852     sys.exit(retcode)
 853
 854
 855 def main(argv=None):
 856     try:
 857         _real_main(argv)
 858     except DownloadError:
 859         sys.exit(1)
 860     except SameFileError:
 861         sys.exit(u'ERROR: fixed output name but more than one file to download')
 862     except KeyboardInterrupt:
 863         sys.exit(u'\nERROR: Interrupted by user')