Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63     'Ariset Llerena',
  64     'Adam Malcontenti-Wilson',
  65     'Tobias Bell',
  66     'Naglis Jonaitis',
  67     'Charles Chen',
  68     'Hassaan Ali',
  69     'Dobrosław Żybort',
  70 )
  71
  72 __license__ = 'Public Domain'
  73
  74 import codecs
  75 import io
  76 import optparse
  77 import os
  78 import random
  79 import shlex
  80 import shutil
  81 import sys
  82
  83
  84 from .utils import (
  85     compat_getpass,
  86     compat_print,
  87     DateRange,
  88     DEFAULT_OUTTMPL,
  89     decodeOption,
  90     get_term_width,
  91     DownloadError,
  92     get_cachedir,
  93     MaxDownloadsReached,
  94     preferredencoding,
  95     read_batch_urls,
  96     SameFileError,
  97     setproctitle,
  98     std_headers,
  99     write_string,
 100 )
 101 from .update import update_self
 102 from .downloader import (
 103     FileDownloader,
 104 )
 105 from .extractor import gen_extractors
 106 from .version import __version__
 107 from .YoutubeDL import YoutubeDL
 108 from .postprocessor import (
 109     AtomicParsleyPP,
 110     FFmpegAudioFixPP,
 111     FFmpegMetadataPP,
 112     FFmpegVideoConvertor,
 113     FFmpegExtractAudioPP,
 114     FFmpegEmbedSubtitlePP,
 115     XAttrMetadataPP,
 116 )
 117
 118
 119 def parseOpts(overrideArguments=None):
 120     def _readOptions(filename_bytes, default=[]):
 121         try:
 122             optionf = open(filename_bytes)
 123         except IOError:
 124             return default  # silently skip if file is not present
 125         try:
 126             res = []
 127             for l in optionf:
 128                 res += shlex.split(l, comments=True)
 129         finally:
 130             optionf.close()
 131         return res
 132
 133     def _readUserConf():
 134         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 135         if xdg_config_home:
 136             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 137             if not os.path.isfile(userConfFile):
 138                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 139         else:
 140             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 141             if not os.path.isfile(userConfFile):
 142                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 143         userConf = _readOptions(userConfFile, None)
 144
 145         if userConf is None:
 146             appdata_dir = os.environ.get('appdata')
 147             if appdata_dir:
 148                 userConf = _readOptions(
 149                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 150                     default=None)
 151                 if userConf is None:
 152                     userConf = _readOptions(
 153                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 154                         default=None)
 155
 156         if userConf is None:
 157             userConf = _readOptions(
 158                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 159                 default=None)
 160         if userConf is None:
 161             userConf = _readOptions(
 162                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 163                 default=None)
 164
 165         if userConf is None:
 166             userConf = []
 167
 168         return userConf
 169
 170     def _format_option_string(option):
 171         ''' ('-o', '--option') -> -o, --format METAVAR'''
 172
 173         opts = []
 174
 175         if option._short_opts:
 176             opts.append(option._short_opts[0])
 177         if option._long_opts:
 178             opts.append(option._long_opts[0])
 179         if len(opts) > 1:
 180             opts.insert(1, ', ')
 181
 182         if option.takes_value(): opts.append(' %s' % option.metavar)
 183
 184         return "".join(opts)
 185
 186     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 187         setattr(parser.values, option.dest, value.split(','))
 188
 189     def _hide_login_info(opts):
 190         opts = list(opts)
 191         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 192             try:
 193                 i = opts.index(private_opt)
 194                 opts[i+1] = '<PRIVATE>'
 195             except ValueError:
 196                 pass
 197         return opts
 198
 199     max_width = 80
 200     max_help_position = 80
 201
 202     # No need to wrap help messages if we're on a wide console
 203     columns = get_term_width()
 204     if columns: max_width = columns
 205
 206     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 207     fmt.format_option_strings = _format_option_string
 208
 209     kw = {
 210         'version'   : __version__,
 211         'formatter' : fmt,
 212         'usage' : '%prog [options] url [url...]',
 213         'conflict_handler' : 'resolve',
 214     }
 215
 216     parser = optparse.OptionParser(**kw)
 217
 218     # option groups
 219     general        = optparse.OptionGroup(parser, 'General Options')
 220     selection      = optparse.OptionGroup(parser, 'Video Selection')
 221     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 222     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 223     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 224     downloader     = optparse.OptionGroup(parser, 'Download Options')
 225     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 226     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 227     workarounds    = optparse.OptionGroup(parser, 'Workarounds')
 228     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 229
 230     general.add_option('-h', '--help',
 231             action='help', help='print this help text and exit')
 232     general.add_option('-v', '--version',
 233             action='version', help='print program version and exit')
 234     general.add_option('-U', '--update',
 235             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 236     general.add_option('-i', '--ignore-errors',
 237             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 238     general.add_option('--abort-on-error',
 239             action='store_false', dest='ignoreerrors',
 240             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 241     general.add_option('--dump-user-agent',
 242             action='store_true', dest='dump_user_agent',
 243             help='display the current browser identification', default=False)
 244     general.add_option('--list-extractors',
 245             action='store_true', dest='list_extractors',
 246             help='List all supported extractors and the URLs they would handle', default=False)
 247     general.add_option('--extractor-descriptions',
 248             action='store_true', dest='list_extractor_descriptions',
 249             help='Output descriptions of all supported extractors', default=False)
 250     general.add_option(
 251         '--proxy', dest='proxy', default=None, metavar='URL',
 252         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 253     general.add_option(
 254         '--socket-timeout', dest='socket_timeout',
 255         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 256     general.add_option(
 257         '--default-search',
 258         dest='default_search', metavar='PREFIX',
 259         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
 260     general.add_option(
 261         '--ignore-config',
 262         action='store_true',
 263         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 264
 265     selection.add_option(
 266         '--playlist-start',
 267         dest='playliststart', metavar='NUMBER', default=1, type=int,
 268         help='playlist video to start at (default is %default)')
 269     selection.add_option(
 270         '--playlist-end',
 271         dest='playlistend', metavar='NUMBER', default=None, type=int,
 272         help='playlist video to end at (default is last)')
 273     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 274     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 275     selection.add_option('--max-downloads', metavar='NUMBER',
 276                          dest='max_downloads', type=int, default=None,
 277                          help='Abort after downloading NUMBER files')
 278     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 279     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 280     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 281     selection.add_option(
 282         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 283         help='download only videos uploaded on or before this date (i.e. inclusive)')
 284     selection.add_option(
 285         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 286         help='download only videos uploaded on or after this date (i.e. inclusive)')
 287     selection.add_option(
 288         '--min-views', metavar='COUNT', dest='min_views',
 289         default=None, type=int,
 290         help="Do not download any videos with less than COUNT views",)
 291     selection.add_option(
 292         '--max-views', metavar='COUNT', dest='max_views',
 293         default=None, type=int,
 294         help="Do not download any videos with more than COUNT views",)
 295     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 296     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 297                          help='download only videos suitable for the given age',
 298                          default=None, type=int)
 299     selection.add_option('--download-archive', metavar='FILE',
 300                          dest='download_archive',
 301                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 302     selection.add_option(
 303         '--include-ads', dest='include_ads',
 304         action='store_true',
 305         help='Download advertisements as well (experimental)')
 306     selection.add_option(
 307         '--youtube-include-dash-manifest', action='store_true',
 308         dest='youtube_include_dash_manifest', default=False,
 309         help='Try to download the DASH manifest on YouTube videos (experimental)')
 310
 311     authentication.add_option('-u', '--username',
 312             dest='username', metavar='USERNAME', help='account username')
 313     authentication.add_option('-p', '--password',
 314             dest='password', metavar='PASSWORD', help='account password')
 315     authentication.add_option('-n', '--netrc',
 316             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 317     authentication.add_option('--video-password',
 318             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 319
 320
 321     video_format.add_option('-f', '--format',
 322             action='store', dest='format', metavar='FORMAT', default=None,
 323             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 324     video_format.add_option('--all-formats',
 325             action='store_const', dest='format', help='download all available video formats', const='all')
 326     video_format.add_option('--prefer-free-formats',
 327             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 328     video_format.add_option('--max-quality',
 329             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 330     video_format.add_option('-F', '--list-formats',
 331             action='store_true', dest='listformats', help='list all available formats')
 332
 333     subtitles.add_option('--write-sub', '--write-srt',
 334             action='store_true', dest='writesubtitles',
 335             help='write subtitle file', default=False)
 336     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 337             action='store_true', dest='writeautomaticsub',
 338             help='write automatic subtitle file (youtube only)', default=False)
 339     subtitles.add_option('--all-subs',
 340             action='store_true', dest='allsubtitles',
 341             help='downloads all the available subtitles of the video', default=False)
 342     subtitles.add_option('--list-subs',
 343             action='store_true', dest='listsubtitles',
 344             help='lists all available subtitles for the video', default=False)
 345     subtitles.add_option('--sub-format',
 346             action='store', dest='subtitlesformat', metavar='FORMAT',
 347             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 348     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 349             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 350             default=[], callback=_comma_separated_values_options_callback,
 351             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 352
 353     downloader.add_option('-r', '--rate-limit',
 354             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 355     downloader.add_option('-R', '--retries',
 356             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 357     downloader.add_option('--buffer-size',
 358             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 359     downloader.add_option('--no-resize-buffer',
 360             action='store_true', dest='noresizebuffer',
 361             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 362     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 363
 364     workarounds.add_option(
 365         '--encoding', dest='encoding', metavar='ENCODING',
 366         help='Force the specified encoding (experimental)')
 367     workarounds.add_option(
 368         '--no-check-certificate', action='store_true',
 369         dest='no_check_certificate', default=False,
 370         help='Suppress HTTPS certificate validation.')
 371     workarounds.add_option(
 372         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 373         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 374     workarounds.add_option(
 375         '--user-agent', metavar='UA',
 376         dest='user_agent', help='specify a custom user agent')
 377     workarounds.add_option(
 378         '--referer', metavar='REF',
 379         dest='referer', default=None,
 380         help='specify a custom referer, use if the video access is restricted to one domain',
 381     )
 382     workarounds.add_option(
 383         '--add-header', metavar='FIELD:VALUE',
 384         dest='headers', action='append',
 385         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
 386     )
 387     workarounds.add_option(
 388         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 389         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 390
 391     verbosity.add_option('-q', '--quiet',
 392             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 393     verbosity.add_option(
 394         '--no-warnings',
 395         dest='no_warnings', action='store_true', default=False,
 396         help='Ignore warnings')
 397     verbosity.add_option('-s', '--simulate',
 398             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 399     verbosity.add_option('--skip-download',
 400             action='store_true', dest='skip_download', help='do not download the video', default=False)
 401     verbosity.add_option('-g', '--get-url',
 402             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 403     verbosity.add_option('-e', '--get-title',
 404             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 405     verbosity.add_option('--get-id',
 406             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 407     verbosity.add_option('--get-thumbnail',
 408             action='store_true', dest='getthumbnail',
 409             help='simulate, quiet but print thumbnail URL', default=False)
 410     verbosity.add_option('--get-description',
 411             action='store_true', dest='getdescription',
 412             help='simulate, quiet but print video description', default=False)
 413     verbosity.add_option('--get-duration',
 414             action='store_true', dest='getduration',
 415             help='simulate, quiet but print video length', default=False)
 416     verbosity.add_option('--get-filename',
 417             action='store_true', dest='getfilename',
 418             help='simulate, quiet but print output filename', default=False)
 419     verbosity.add_option('--get-format',
 420             action='store_true', dest='getformat',
 421             help='simulate, quiet but print output format', default=False)
 422     verbosity.add_option('-j', '--dump-json',
 423             action='store_true', dest='dumpjson',
 424             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 425     verbosity.add_option('--newline',
 426             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 427     verbosity.add_option('--no-progress',
 428             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 429     verbosity.add_option('--console-title',
 430             action='store_true', dest='consoletitle',
 431             help='display progress in console titlebar', default=False)
 432     verbosity.add_option('-v', '--verbose',
 433             action='store_true', dest='verbose', help='print various debugging information', default=False)
 434     verbosity.add_option('--dump-intermediate-pages',
 435             action='store_true', dest='dump_intermediate_pages', default=False,
 436             help='print downloaded pages to debug problems (very verbose)')
 437     verbosity.add_option('--write-pages',
 438             action='store_true', dest='write_pages', default=False,
 439             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 440     verbosity.add_option('--youtube-print-sig-code',
 441             action='store_true', dest='youtube_print_sig_code', default=False,
 442             help=optparse.SUPPRESS_HELP)
 443     verbosity.add_option('--print-traffic',
 444             dest='debug_printtraffic', action='store_true', default=False,
 445             help='Display sent and read HTTP traffic')
 446
 447
 448     filesystem.add_option('-a', '--batch-file',
 449             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 450     filesystem.add_option('--id',
 451             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 452     filesystem.add_option('-A', '--auto-number',
 453             action='store_true', dest='autonumber',
 454             help='number downloaded files starting from 00000', default=False)
 455     filesystem.add_option('-o', '--output',
 456             dest='outtmpl', metavar='TEMPLATE',
 457             help=('output filename template. Use %(title)s to get the title, '
 458                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 459                   '%(autonumber)s to get an automatically incremented number, '
 460                   '%(ext)s for the filename extension, '
 461                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 462                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 463                   '%(upload_date)s for the upload date (YYYYMMDD), '
 464                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 465                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 466                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 467                   '%(height)s and %(width)s for the width and height of the video format. '
 468                   '%(resolution)s for a textual description of the resolution of the video format. '
 469                   'Use - to output to stdout. Can also be used to download to a different directory, '
 470                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 471     filesystem.add_option('--autonumber-size',
 472             dest='autonumber_size', metavar='NUMBER',
 473             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 474     filesystem.add_option('--restrict-filenames',
 475             action='store_true', dest='restrictfilenames',
 476             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 477     filesystem.add_option('-t', '--title',
 478             action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
 479     filesystem.add_option('-l', '--literal',
 480             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 481     filesystem.add_option('-w', '--no-overwrites',
 482             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 483     filesystem.add_option('-c', '--continue',
 484             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 485     filesystem.add_option('--no-continue',
 486             action='store_false', dest='continue_dl',
 487             help='do not resume partially downloaded files (restart from beginning)')
 488     filesystem.add_option('--no-part',
 489             action='store_true', dest='nopart', help='do not use .part files', default=False)
 490     filesystem.add_option('--no-mtime',
 491             action='store_false', dest='updatetime',
 492             help='do not use the Last-modified header to set the file modification time', default=True)
 493     filesystem.add_option('--write-description',
 494             action='store_true', dest='writedescription',
 495             help='write video description to a .description file', default=False)
 496     filesystem.add_option('--write-info-json',
 497             action='store_true', dest='writeinfojson',
 498             help='write video metadata to a .info.json file', default=False)
 499     filesystem.add_option('--write-annotations',
 500             action='store_true', dest='writeannotations',
 501             help='write video annotations to a .annotation file', default=False)
 502     filesystem.add_option('--write-thumbnail',
 503             action='store_true', dest='writethumbnail',
 504             help='write thumbnail image to disk', default=False)
 505     filesystem.add_option('--load-info',
 506             dest='load_info_filename', metavar='FILE',
 507             help='json file containing the video information (created with the "--write-json" option)')
 508     filesystem.add_option('--cookies',
 509             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 510     filesystem.add_option(
 511         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 512         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 513     filesystem.add_option(
 514         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 515         help='Disable filesystem caching')
 516     filesystem.add_option(
 517         '--rm-cache-dir', action='store_true', dest='rm_cachedir',
 518         help='Delete all filesystem cache files')
 519
 520
 521     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 522             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 523     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 524             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 525     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 526             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 527     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 528             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
 529     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 530             help='keeps the video file on disk after the post-processing; the video is erased by default')
 531     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 532             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 533     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 534             help='embed subtitles in the video (only for mp4 videos)')
 535     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 536             help='embed thumbnail in the audio as cover art')
 537     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 538             help='write metadata to the video file')
 539     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 540             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 541     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 542         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 543     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 544         help='Prefer ffmpeg over avconv for running the postprocessors')
 545
 546
 547     parser.add_option_group(general)
 548     parser.add_option_group(selection)
 549     parser.add_option_group(downloader)
 550     parser.add_option_group(filesystem)
 551     parser.add_option_group(verbosity)
 552     parser.add_option_group(workarounds)
 553     parser.add_option_group(video_format)
 554     parser.add_option_group(subtitles)
 555     parser.add_option_group(authentication)
 556     parser.add_option_group(postproc)
 557
 558     if overrideArguments is not None:
 559         opts, args = parser.parse_args(overrideArguments)
 560         if opts.verbose:
 561             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 562     else:
 563         commandLineConf = sys.argv[1:]
 564         if '--ignore-config' in commandLineConf:
 565             systemConf = []
 566             userConf = []
 567         else:
 568             systemConf = _readOptions('/etc/youtube-dl.conf')
 569             if '--ignore-config' in systemConf:
 570                 userConf = []
 571             else:
 572                 userConf = _readUserConf()
 573         argv = systemConf + userConf + commandLineConf
 574
 575         opts, args = parser.parse_args(argv)
 576         if opts.verbose:
 577             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 578             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 579             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 580
 581     return parser, opts, args
 582
 583
 584 def _real_main(argv=None):
 585     # Compatibility fixes for Windows
 586     if sys.platform == 'win32':
 587         # https://github.com/rg3/youtube-dl/issues/820
 588         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 589
 590     setproctitle(u'youtube-dl')
 591
 592     parser, opts, args = parseOpts(argv)
 593
 594     # Set user agent
 595     if opts.user_agent is not None:
 596         std_headers['User-Agent'] = opts.user_agent
 597
 598     # Set referer
 599     if opts.referer is not None:
 600         std_headers['Referer'] = opts.referer
 601
 602     # Custom HTTP headers
 603     if opts.headers is not None:
 604         for h in opts.headers:
 605             if h.find(':', 1) < 0:
 606                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 607             key, value = h.split(':', 2)
 608             if opts.verbose:
 609                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 610             std_headers[key] = value
 611
 612     # Dump user agent
 613     if opts.dump_user_agent:
 614         compat_print(std_headers['User-Agent'])
 615         sys.exit(0)
 616
 617     # Batch file verification
 618     batch_urls = []
 619     if opts.batchfile is not None:
 620         try:
 621             if opts.batchfile == '-':
 622                 batchfd = sys.stdin
 623             else:
 624                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 625             batch_urls = read_batch_urls(batchfd)
 626             if opts.verbose:
 627                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 628         except IOError:
 629             sys.exit(u'ERROR: batch file could not be read')
 630     all_urls = batch_urls + args
 631     all_urls = [url.strip() for url in all_urls]
 632     _enc = preferredencoding()
 633     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 634
 635     extractors = gen_extractors()
 636
 637     if opts.list_extractors:
 638         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 639             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 640             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 641             for mu in matchedUrls:
 642                 compat_print(u'  ' + mu)
 643         sys.exit(0)
 644     if opts.list_extractor_descriptions:
 645         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 646             if not ie._WORKING:
 647                 continue
 648             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 649             if desc is False:
 650                 continue
 651             if hasattr(ie, 'SEARCH_KEY'):
 652                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
 653                 _COUNTS = (u'', u'5', u'10', u'all')
 654                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 655             compat_print(desc)
 656         sys.exit(0)
 657
 658
 659     # Conflicting, missing and erroneous options
 660     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 661         parser.error(u'using .netrc conflicts with giving username/password')
 662     if opts.password is not None and opts.username is None:
 663         parser.error(u'account username missing\n')
 664     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 665         parser.error(u'using output template conflicts with using title, video ID or auto number')
 666     if opts.usetitle and opts.useid:
 667         parser.error(u'using title conflicts with using video ID')
 668     if opts.username is not None and opts.password is None:
 669         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 670     if opts.ratelimit is not None:
 671         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 672         if numeric_limit is None:
 673             parser.error(u'invalid rate limit specified')
 674         opts.ratelimit = numeric_limit
 675     if opts.min_filesize is not None:
 676         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 677         if numeric_limit is None:
 678             parser.error(u'invalid min_filesize specified')
 679         opts.min_filesize = numeric_limit
 680     if opts.max_filesize is not None:
 681         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 682         if numeric_limit is None:
 683             parser.error(u'invalid max_filesize specified')
 684         opts.max_filesize = numeric_limit
 685     if opts.retries is not None:
 686         try:
 687             opts.retries = int(opts.retries)
 688         except (TypeError, ValueError):
 689             parser.error(u'invalid retry count specified')
 690     if opts.buffersize is not None:
 691         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 692         if numeric_buffersize is None:
 693             parser.error(u'invalid buffer size specified')
 694         opts.buffersize = numeric_buffersize
 695     if opts.playliststart <= 0:
 696         raise ValueError(u'Playlist start must be positive')
 697     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 698         raise ValueError(u'Playlist end must be greater than playlist start')
 699     if opts.extractaudio:
 700         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 701             parser.error(u'invalid audio format specified')
 702     if opts.audioquality:
 703         opts.audioquality = opts.audioquality.strip('k').strip('K')
 704         if not opts.audioquality.isdigit():
 705             parser.error(u'invalid audio quality specified')
 706     if opts.recodevideo is not None:
 707         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 708             parser.error(u'invalid video recode format specified')
 709     if opts.date is not None:
 710         date = DateRange.day(opts.date)
 711     else:
 712         date = DateRange(opts.dateafter, opts.datebefore)
 713     if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
 714         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 715
 716     # Do not download videos when there are audio-only formats
 717     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 718         opts.format = 'bestaudio/best'
 719
 720     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 721     # this was the old behaviour if only --all-sub was given.
 722     if opts.allsubtitles and (opts.writeautomaticsub == False):
 723         opts.writesubtitles = True
 724
 725     if sys.version_info < (3,):
 726         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 727         if opts.outtmpl is not None:
 728             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 729     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 730             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 731             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 732             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 733             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 734             or (opts.useid and u'%(id)s.%(ext)s')
 735             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 736             or DEFAULT_OUTTMPL)
 737     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 738         parser.error(u'Cannot download a video and extract audio into the same'
 739                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 740                      u' template'.format(outtmpl))
 741
 742     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 743     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 744
 745     ydl_opts = {
 746         'usenetrc': opts.usenetrc,
 747         'username': opts.username,
 748         'password': opts.password,
 749         'videopassword': opts.videopassword,
 750         'quiet': (opts.quiet or any_printing),
 751         'no_warnings': opts.no_warnings,
 752         'forceurl': opts.geturl,
 753         'forcetitle': opts.gettitle,
 754         'forceid': opts.getid,
 755         'forcethumbnail': opts.getthumbnail,
 756         'forcedescription': opts.getdescription,
 757         'forceduration': opts.getduration,
 758         'forcefilename': opts.getfilename,
 759         'forceformat': opts.getformat,
 760         'forcejson': opts.dumpjson,
 761         'simulate': opts.simulate,
 762         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 763         'format': opts.format,
 764         'format_limit': opts.format_limit,
 765         'listformats': opts.listformats,
 766         'outtmpl': outtmpl,
 767         'autonumber_size': opts.autonumber_size,
 768         'restrictfilenames': opts.restrictfilenames,
 769         'ignoreerrors': opts.ignoreerrors,
 770         'ratelimit': opts.ratelimit,
 771         'nooverwrites': opts.nooverwrites,
 772         'retries': opts.retries,
 773         'buffersize': opts.buffersize,
 774         'noresizebuffer': opts.noresizebuffer,
 775         'continuedl': opts.continue_dl,
 776         'noprogress': opts.noprogress,
 777         'progress_with_newline': opts.progress_with_newline,
 778         'playliststart': opts.playliststart,
 779         'playlistend': opts.playlistend,
 780         'noplaylist': opts.noplaylist,
 781         'logtostderr': opts.outtmpl == '-',
 782         'consoletitle': opts.consoletitle,
 783         'nopart': opts.nopart,
 784         'updatetime': opts.updatetime,
 785         'writedescription': opts.writedescription,
 786         'writeannotations': opts.writeannotations,
 787         'writeinfojson': opts.writeinfojson,
 788         'writethumbnail': opts.writethumbnail,
 789         'writesubtitles': opts.writesubtitles,
 790         'writeautomaticsub': opts.writeautomaticsub,
 791         'allsubtitles': opts.allsubtitles,
 792         'listsubtitles': opts.listsubtitles,
 793         'subtitlesformat': opts.subtitlesformat,
 794         'subtitleslangs': opts.subtitleslangs,
 795         'matchtitle': decodeOption(opts.matchtitle),
 796         'rejecttitle': decodeOption(opts.rejecttitle),
 797         'max_downloads': opts.max_downloads,
 798         'prefer_free_formats': opts.prefer_free_formats,
 799         'verbose': opts.verbose,
 800         'dump_intermediate_pages': opts.dump_intermediate_pages,
 801         'write_pages': opts.write_pages,
 802         'test': opts.test,
 803         'keepvideo': opts.keepvideo,
 804         'min_filesize': opts.min_filesize,
 805         'max_filesize': opts.max_filesize,
 806         'min_views': opts.min_views,
 807         'max_views': opts.max_views,
 808         'daterange': date,
 809         'cachedir': opts.cachedir,
 810         'youtube_print_sig_code': opts.youtube_print_sig_code,
 811         'age_limit': opts.age_limit,
 812         'download_archive': download_archive_fn,
 813         'cookiefile': opts.cookiefile,
 814         'nocheckcertificate': opts.no_check_certificate,
 815         'prefer_insecure': opts.prefer_insecure,
 816         'proxy': opts.proxy,
 817         'socket_timeout': opts.socket_timeout,
 818         'bidi_workaround': opts.bidi_workaround,
 819         'debug_printtraffic': opts.debug_printtraffic,
 820         'prefer_ffmpeg': opts.prefer_ffmpeg,
 821         'include_ads': opts.include_ads,
 822         'default_search': opts.default_search,
 823         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 824         'encoding': opts.encoding,
 825     }
 826
 827     with YoutubeDL(ydl_opts) as ydl:
 828         ydl.print_debug_header()
 829         ydl.add_default_info_extractors()
 830
 831         # PostProcessors
 832         # Add the metadata pp first, the other pps will copy it
 833         if opts.addmetadata:
 834             ydl.add_post_processor(FFmpegMetadataPP())
 835         if opts.extractaudio:
 836             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 837         if opts.recodevideo:
 838             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 839         if opts.embedsubtitles:
 840             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 841         if opts.xattrs:
 842             ydl.add_post_processor(XAttrMetadataPP())
 843         if opts.embedthumbnail:
 844             if not opts.addmetadata:
 845                 ydl.add_post_processor(FFmpegAudioFixPP())
 846             ydl.add_post_processor(AtomicParsleyPP())
 847
 848         # Update version
 849         if opts.update_self:
 850             update_self(ydl.to_screen, opts.verbose)
 851
 852         # Remove cache dir
 853         if opts.rm_cachedir:
 854             if opts.cachedir is None:
 855                 ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)')
 856             else:
 857                 if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir):
 858                     ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir')
 859                     retcode = 141
 860                 else:
 861                     ydl.to_screen(
 862                         u'Removing cache dir %s .' % opts.cachedir,
 863                         skip_eol=True)
 864                     if os.path.exists(opts.cachedir):
 865                         ydl.to_screen(u'.', skip_eol=True)
 866                         shutil.rmtree(opts.cachedir)
 867                     ydl.to_screen(u'.')
 868
 869         # Maybe do nothing
 870         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 871             if not (opts.update_self or opts.rm_cachedir):
 872                 parser.error(u'you must provide at least one URL')
 873             else:
 874                 sys.exit()
 875
 876         try:
 877             if opts.load_info_filename is not None:
 878                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 879             else:
 880                 retcode = ydl.download(all_urls)
 881         except MaxDownloadsReached:
 882             ydl.to_screen(u'--max-download limit reached, aborting.')
 883             retcode = 101
 884
 885     sys.exit(retcode)
 886
 887
 888 def main(argv=None):
 889     try:
 890         _real_main(argv)
 891     except DownloadError:
 892         sys.exit(1)
 893     except SameFileError:
 894         sys.exit(u'ERROR: fixed output name but more than one file to download')
 895     except KeyboardInterrupt:
 896         sys.exit(u'\nERROR: Interrupted by user')