Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62 )
  63
  64 __license__ = 'Public Domain'
  65
  66 import codecs
  67 import io
  68 import locale
  69 import optparse
  70 import os
  71 import random
  72 import re
  73 import shlex
  74 import sys
  75
  76
  77 from .utils import (
  78     compat_getpass,
  79     compat_print,
  80     DateRange,
  81     DEFAULT_OUTTMPL,
  82     decodeOption,
  83     get_term_width,
  84     DownloadError,
  85     get_cachedir,
  86     MaxDownloadsReached,
  87     preferredencoding,
  88     read_batch_urls,
  89     SameFileError,
  90     setproctitle,
  91     std_headers,
  92     write_string,
  93 )
  94 from .update import update_self
  95 from .FileDownloader import (
  96     FileDownloader,
  97 )
  98 from .extractor import gen_extractors
  99 from .version import __version__
 100 from .YoutubeDL import YoutubeDL
 101 from .postprocessor import (
 102     AtomicParsleyPP,
 103     FFmpegAudioFixPP,
 104     FFmpegMetadataPP,
 105     FFmpegVideoConvertor,
 106     FFmpegExtractAudioPP,
 107     FFmpegEmbedSubtitlePP,
 108     XAttrMetadataPP,
 109 )
 110
 111
 112 def parseOpts(overrideArguments=None):
 113     def _readOptions(filename_bytes, default=[]):
 114         try:
 115             optionf = open(filename_bytes)
 116         except IOError:
 117             return default  # silently skip if file is not present
 118         try:
 119             res = []
 120             for l in optionf:
 121                 res += shlex.split(l, comments=True)
 122         finally:
 123             optionf.close()
 124         return res
 125
 126     def _readUserConf():
 127         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 128         if xdg_config_home:
 129             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 130             if not os.path.isfile(userConfFile):
 131                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 132         else:
 133             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 134             if not os.path.isfile(userConfFile):
 135                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 136         userConf = _readOptions(userConfFile, None)
 137
 138         if userConf is None:
 139             appdata_dir = os.environ.get('appdata')
 140             if appdata_dir:
 141                 userConf = _readOptions(
 142                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 143                     default=None)
 144                 if userConf is None:
 145                     userConf = _readOptions(
 146                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 147                         default=None)
 148
 149         if userConf is None:
 150             userConf = _readOptions(
 151                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 152                 default=None)
 153         if userConf is None:
 154             userConf = _readOptions(
 155                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 156                 default=None)
 157
 158         if userConf is None:
 159             userConf = []
 160
 161         return userConf
 162
 163     def _format_option_string(option):
 164         ''' ('-o', '--option') -> -o, --format METAVAR'''
 165
 166         opts = []
 167
 168         if option._short_opts:
 169             opts.append(option._short_opts[0])
 170         if option._long_opts:
 171             opts.append(option._long_opts[0])
 172         if len(opts) > 1:
 173             opts.insert(1, ', ')
 174
 175         if option.takes_value(): opts.append(' %s' % option.metavar)
 176
 177         return "".join(opts)
 178
 179     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 180         setattr(parser.values, option.dest, value.split(','))
 181
 182     def _hide_login_info(opts):
 183         opts = list(opts)
 184         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 185             try:
 186                 i = opts.index(private_opt)
 187                 opts[i+1] = '<PRIVATE>'
 188             except ValueError:
 189                 pass
 190         return opts
 191
 192     max_width = 80
 193     max_help_position = 80
 194
 195     # No need to wrap help messages if we're on a wide console
 196     columns = get_term_width()
 197     if columns: max_width = columns
 198
 199     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 200     fmt.format_option_strings = _format_option_string
 201
 202     kw = {
 203         'version'   : __version__,
 204         'formatter' : fmt,
 205         'usage' : '%prog [options] url [url...]',
 206         'conflict_handler' : 'resolve',
 207     }
 208
 209     parser = optparse.OptionParser(**kw)
 210
 211     # option groups
 212     general        = optparse.OptionGroup(parser, 'General Options')
 213     selection      = optparse.OptionGroup(parser, 'Video Selection')
 214     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 215     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 216     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 217     downloader     = optparse.OptionGroup(parser, 'Download Options')
 218     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 219     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 220     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 221
 222     general.add_option('-h', '--help',
 223             action='help', help='print this help text and exit')
 224     general.add_option('-v', '--version',
 225             action='version', help='print program version and exit')
 226     general.add_option('-U', '--update',
 227             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 228     general.add_option('-i', '--ignore-errors',
 229             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 230     general.add_option('--abort-on-error',
 231             action='store_false', dest='ignoreerrors',
 232             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 233     general.add_option('--dump-user-agent',
 234             action='store_true', dest='dump_user_agent',
 235             help='display the current browser identification', default=False)
 236     general.add_option('--user-agent',
 237             dest='user_agent', help='specify a custom user agent', metavar='UA')
 238     general.add_option('--referer',
 239             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 240             metavar='REF', default=None)
 241     general.add_option('--add-header',
 242             dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
 243             metavar='FIELD:VALUE')
 244     general.add_option('--list-extractors',
 245             action='store_true', dest='list_extractors',
 246             help='List all supported extractors and the URLs they would handle', default=False)
 247     general.add_option('--extractor-descriptions',
 248             action='store_true', dest='list_extractor_descriptions',
 249             help='Output descriptions of all supported extractors', default=False)
 250     general.add_option(
 251         '--proxy', dest='proxy', default=None, metavar='URL',
 252         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 253     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 254     general.add_option(
 255         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 256         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 257     general.add_option(
 258         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 259         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 260     general.add_option(
 261         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 262         help='Disable filesystem caching')
 263     general.add_option(
 264         '--socket-timeout', dest='socket_timeout',
 265         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 266     general.add_option(
 267         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 268         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 269     general.add_option(
 270         '--default-search',
 271         dest='default_search', metavar='PREFIX',
 272         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
 273     general.add_option(
 274         '--ignore-config',
 275         action='store_true',
 276         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 277     general.add_option(
 278         '--encoding', dest='encoding', metavar='ENCODING',
 279         help='Force the specified encoding (experimental)')
 280
 281     selection.add_option(
 282         '--playlist-start',
 283         dest='playliststart', metavar='NUMBER', default=1, type=int,
 284         help='playlist video to start at (default is %default)')
 285     selection.add_option(
 286         '--playlist-end',
 287         dest='playlistend', metavar='NUMBER', default=None, type=int,
 288         help='playlist video to end at (default is last)')
 289     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 290     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 291     selection.add_option('--max-downloads', metavar='NUMBER',
 292                          dest='max_downloads', type=int, default=None,
 293                          help='Abort after downloading NUMBER files')
 294     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 295     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 296     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 297     selection.add_option(
 298         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 299         help='download only videos uploaded on or before this date (i.e. inclusive)')
 300     selection.add_option(
 301         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 302         help='download only videos uploaded on or after this date (i.e. inclusive)')
 303     selection.add_option(
 304         '--min-views', metavar='COUNT', dest='min_views',
 305         default=None, type=int,
 306         help="Do not download any videos with less than COUNT views",)
 307     selection.add_option(
 308         '--max-views', metavar='COUNT', dest='max_views',
 309         default=None, type=int,
 310         help="Do not download any videos with more than COUNT views",)
 311     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 312     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 313                          help='download only videos suitable for the given age',
 314                          default=None, type=int)
 315     selection.add_option('--download-archive', metavar='FILE',
 316                          dest='download_archive',
 317                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 318     selection.add_option(
 319         '--include-ads', dest='include_ads',
 320         action='store_true',
 321         help='Download advertisements as well (experimental)')
 322     selection.add_option(
 323         '--youtube-include-dash-manifest', action='store_true',
 324         dest='youtube_include_dash_manifest', default=False,
 325         help='Try to download the DASH manifest on YouTube videos (experimental)')
 326
 327     authentication.add_option('-u', '--username',
 328             dest='username', metavar='USERNAME', help='account username')
 329     authentication.add_option('-p', '--password',
 330             dest='password', metavar='PASSWORD', help='account password')
 331     authentication.add_option('-n', '--netrc',
 332             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 333     authentication.add_option('--video-password',
 334             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 335
 336
 337     video_format.add_option('-f', '--format',
 338             action='store', dest='format', metavar='FORMAT', default=None,
 339             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 340     video_format.add_option('--all-formats',
 341             action='store_const', dest='format', help='download all available video formats', const='all')
 342     video_format.add_option('--prefer-free-formats',
 343             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 344     video_format.add_option('--max-quality',
 345             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 346     video_format.add_option('-F', '--list-formats',
 347             action='store_true', dest='listformats', help='list all available formats')
 348
 349     subtitles.add_option('--write-sub', '--write-srt',
 350             action='store_true', dest='writesubtitles',
 351             help='write subtitle file', default=False)
 352     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 353             action='store_true', dest='writeautomaticsub',
 354             help='write automatic subtitle file (youtube only)', default=False)
 355     subtitles.add_option('--all-subs',
 356             action='store_true', dest='allsubtitles',
 357             help='downloads all the available subtitles of the video', default=False)
 358     subtitles.add_option('--list-subs',
 359             action='store_true', dest='listsubtitles',
 360             help='lists all available subtitles for the video', default=False)
 361     subtitles.add_option('--sub-format',
 362             action='store', dest='subtitlesformat', metavar='FORMAT',
 363             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 364     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 365             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 366             default=[], callback=_comma_separated_values_options_callback,
 367             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 368
 369     downloader.add_option('-r', '--rate-limit',
 370             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 371     downloader.add_option('-R', '--retries',
 372             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 373     downloader.add_option('--buffer-size',
 374             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 375     downloader.add_option('--no-resize-buffer',
 376             action='store_true', dest='noresizebuffer',
 377             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 378     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 379
 380     verbosity.add_option('-q', '--quiet',
 381             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 382     verbosity.add_option(
 383         '--no-warnings',
 384         dest='no_warnings', action='store_true', default=False,
 385         help='Ignore warnings')
 386     verbosity.add_option('-s', '--simulate',
 387             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 388     verbosity.add_option('--skip-download',
 389             action='store_true', dest='skip_download', help='do not download the video', default=False)
 390     verbosity.add_option('-g', '--get-url',
 391             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 392     verbosity.add_option('-e', '--get-title',
 393             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 394     verbosity.add_option('--get-id',
 395             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 396     verbosity.add_option('--get-thumbnail',
 397             action='store_true', dest='getthumbnail',
 398             help='simulate, quiet but print thumbnail URL', default=False)
 399     verbosity.add_option('--get-description',
 400             action='store_true', dest='getdescription',
 401             help='simulate, quiet but print video description', default=False)
 402     verbosity.add_option('--get-duration',
 403             action='store_true', dest='getduration',
 404             help='simulate, quiet but print video length', default=False)
 405     verbosity.add_option('--get-filename',
 406             action='store_true', dest='getfilename',
 407             help='simulate, quiet but print output filename', default=False)
 408     verbosity.add_option('--get-format',
 409             action='store_true', dest='getformat',
 410             help='simulate, quiet but print output format', default=False)
 411     verbosity.add_option('-j', '--dump-json',
 412             action='store_true', dest='dumpjson',
 413             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 414     verbosity.add_option('--newline',
 415             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 416     verbosity.add_option('--no-progress',
 417             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 418     verbosity.add_option('--console-title',
 419             action='store_true', dest='consoletitle',
 420             help='display progress in console titlebar', default=False)
 421     verbosity.add_option('-v', '--verbose',
 422             action='store_true', dest='verbose', help='print various debugging information', default=False)
 423     verbosity.add_option('--dump-intermediate-pages',
 424             action='store_true', dest='dump_intermediate_pages', default=False,
 425             help='print downloaded pages to debug problems (very verbose)')
 426     verbosity.add_option('--write-pages',
 427             action='store_true', dest='write_pages', default=False,
 428             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 429     verbosity.add_option('--youtube-print-sig-code',
 430             action='store_true', dest='youtube_print_sig_code', default=False,
 431             help=optparse.SUPPRESS_HELP)
 432     verbosity.add_option('--print-traffic',
 433             dest='debug_printtraffic', action='store_true', default=False,
 434             help='Display sent and read HTTP traffic')
 435
 436
 437     filesystem.add_option('-t', '--title',
 438             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 439     filesystem.add_option('--id',
 440             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 441     filesystem.add_option('-l', '--literal',
 442             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 443     filesystem.add_option('-A', '--auto-number',
 444             action='store_true', dest='autonumber',
 445             help='number downloaded files starting from 00000', default=False)
 446     filesystem.add_option('-o', '--output',
 447             dest='outtmpl', metavar='TEMPLATE',
 448             help=('output filename template. Use %(title)s to get the title, '
 449                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 450                   '%(autonumber)s to get an automatically incremented number, '
 451                   '%(ext)s for the filename extension, '
 452                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 453                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 454                   '%(upload_date)s for the upload date (YYYYMMDD), '
 455                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 456                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 457                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 458                   '%(height)s and %(width)s for the width and height of the video format. '
 459                   '%(resolution)s for a textual description of the resolution of the video format. '
 460                   'Use - to output to stdout. Can also be used to download to a different directory, '
 461                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 462     filesystem.add_option('--autonumber-size',
 463             dest='autonumber_size', metavar='NUMBER',
 464             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 465     filesystem.add_option('--restrict-filenames',
 466             action='store_true', dest='restrictfilenames',
 467             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 468     filesystem.add_option('-a', '--batch-file',
 469             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 470     filesystem.add_option('--load-info',
 471             dest='load_info_filename', metavar='FILE',
 472             help='json file containing the video information (created with the "--write-json" option)')
 473     filesystem.add_option('-w', '--no-overwrites',
 474             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 475     filesystem.add_option('-c', '--continue',
 476             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 477     filesystem.add_option('--no-continue',
 478             action='store_false', dest='continue_dl',
 479             help='do not resume partially downloaded files (restart from beginning)')
 480     filesystem.add_option('--cookies',
 481             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 482     filesystem.add_option('--no-part',
 483             action='store_true', dest='nopart', help='do not use .part files', default=False)
 484     filesystem.add_option('--no-mtime',
 485             action='store_false', dest='updatetime',
 486             help='do not use the Last-modified header to set the file modification time', default=True)
 487     filesystem.add_option('--write-description',
 488             action='store_true', dest='writedescription',
 489             help='write video description to a .description file', default=False)
 490     filesystem.add_option('--write-info-json',
 491             action='store_true', dest='writeinfojson',
 492             help='write video metadata to a .info.json file', default=False)
 493     filesystem.add_option('--write-annotations',
 494             action='store_true', dest='writeannotations',
 495             help='write video annotations to a .annotation file', default=False)
 496     filesystem.add_option('--write-thumbnail',
 497             action='store_true', dest='writethumbnail',
 498             help='write thumbnail image to disk', default=False)
 499
 500
 501     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 502             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 503     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 504             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 505     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 506             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 507     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 508             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 509     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 510             help='keeps the video file on disk after the post-processing; the video is erased by default')
 511     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 512             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 513     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 514             help='embed subtitles in the video (only for mp4 videos)')
 515     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 516             help='embed thumbnail in the audio as cover art')
 517     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 518             help='write metadata to the video file')
 519     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 520             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 521     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 522         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 523     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 524         help='Prefer ffmpeg over avconv for running the postprocessors')
 525
 526
 527     parser.add_option_group(general)
 528     parser.add_option_group(selection)
 529     parser.add_option_group(downloader)
 530     parser.add_option_group(filesystem)
 531     parser.add_option_group(verbosity)
 532     parser.add_option_group(video_format)
 533     parser.add_option_group(subtitles)
 534     parser.add_option_group(authentication)
 535     parser.add_option_group(postproc)
 536
 537     if overrideArguments is not None:
 538         opts, args = parser.parse_args(overrideArguments)
 539         if opts.verbose:
 540             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 541     else:
 542         commandLineConf = sys.argv[1:]
 543         if '--ignore-config' in commandLineConf:
 544             systemConf = []
 545             userConf = []
 546         else:
 547             systemConf = _readOptions('/etc/youtube-dl.conf')
 548             if '--ignore-config' in systemConf:
 549                 userConf = []
 550             else:
 551                 userConf = _readUserConf()
 552         argv = systemConf + userConf + commandLineConf
 553
 554         opts, args = parser.parse_args(argv)
 555         if opts.verbose:
 556             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 557             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 558             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 559
 560     return parser, opts, args
 561
 562
 563 def _real_main(argv=None):
 564     # Compatibility fixes for Windows
 565     if sys.platform == 'win32':
 566         # https://github.com/rg3/youtube-dl/issues/820
 567         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 568
 569     setproctitle(u'youtube-dl')
 570
 571     parser, opts, args = parseOpts(argv)
 572
 573     # Set user agent
 574     if opts.user_agent is not None:
 575         std_headers['User-Agent'] = opts.user_agent
 576
 577     # Set referer
 578     if opts.referer is not None:
 579         std_headers['Referer'] = opts.referer
 580
 581     # Custom HTTP headers
 582     if opts.headers is not None:
 583         for h in opts.headers:
 584             if h.find(':', 1) < 0:
 585                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 586             key, value = h.split(':', 2)
 587             if opts.verbose:
 588                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 589             std_headers[key] = value
 590
 591     # Dump user agent
 592     if opts.dump_user_agent:
 593         compat_print(std_headers['User-Agent'])
 594         sys.exit(0)
 595
 596     # Batch file verification
 597     batch_urls = []
 598     if opts.batchfile is not None:
 599         try:
 600             if opts.batchfile == '-':
 601                 batchfd = sys.stdin
 602             else:
 603                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 604             batch_urls = read_batch_urls(batchfd)
 605             if opts.verbose:
 606                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 607         except IOError:
 608             sys.exit(u'ERROR: batch file could not be read')
 609     all_urls = batch_urls + args
 610     all_urls = [url.strip() for url in all_urls]
 611     _enc = preferredencoding()
 612     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 613
 614     extractors = gen_extractors()
 615
 616     if opts.list_extractors:
 617         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 618             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 619             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 620             for mu in matchedUrls:
 621                 compat_print(u'  ' + mu)
 622         sys.exit(0)
 623     if opts.list_extractor_descriptions:
 624         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 625             if not ie._WORKING:
 626                 continue
 627             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 628             if desc is False:
 629                 continue
 630             if hasattr(ie, 'SEARCH_KEY'):
 631                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 632                 _COUNTS = (u'', u'5', u'10', u'all')
 633                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 634             compat_print(desc)
 635         sys.exit(0)
 636
 637
 638     # Conflicting, missing and erroneous options
 639     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 640         parser.error(u'using .netrc conflicts with giving username/password')
 641     if opts.password is not None and opts.username is None:
 642         parser.error(u'account username missing\n')
 643     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 644         parser.error(u'using output template conflicts with using title, video ID or auto number')
 645     if opts.usetitle and opts.useid:
 646         parser.error(u'using title conflicts with using video ID')
 647     if opts.username is not None and opts.password is None:
 648         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 649     if opts.ratelimit is not None:
 650         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 651         if numeric_limit is None:
 652             parser.error(u'invalid rate limit specified')
 653         opts.ratelimit = numeric_limit
 654     if opts.min_filesize is not None:
 655         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 656         if numeric_limit is None:
 657             parser.error(u'invalid min_filesize specified')
 658         opts.min_filesize = numeric_limit
 659     if opts.max_filesize is not None:
 660         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 661         if numeric_limit is None:
 662             parser.error(u'invalid max_filesize specified')
 663         opts.max_filesize = numeric_limit
 664     if opts.retries is not None:
 665         try:
 666             opts.retries = int(opts.retries)
 667         except (TypeError, ValueError):
 668             parser.error(u'invalid retry count specified')
 669     if opts.buffersize is not None:
 670         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 671         if numeric_buffersize is None:
 672             parser.error(u'invalid buffer size specified')
 673         opts.buffersize = numeric_buffersize
 674     if opts.playliststart <= 0:
 675         raise ValueError(u'Playlist start must be positive')
 676     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 677         raise ValueError(u'Playlist end must be greater than playlist start')
 678     if opts.extractaudio:
 679         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 680             parser.error(u'invalid audio format specified')
 681     if opts.audioquality:
 682         opts.audioquality = opts.audioquality.strip('k').strip('K')
 683         if not opts.audioquality.isdigit():
 684             parser.error(u'invalid audio quality specified')
 685     if opts.recodevideo is not None:
 686         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 687             parser.error(u'invalid video recode format specified')
 688     if opts.date is not None:
 689         date = DateRange.day(opts.date)
 690     else:
 691         date = DateRange(opts.dateafter, opts.datebefore)
 692     if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
 693         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 694
 695     # Do not download videos when there are audio-only formats
 696     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 697         opts.format = 'bestaudio/best'
 698
 699     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 700     # this was the old behaviour if only --all-sub was given.
 701     if opts.allsubtitles and (opts.writeautomaticsub == False):
 702         opts.writesubtitles = True
 703
 704     if sys.version_info < (3,):
 705         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 706         if opts.outtmpl is not None:
 707             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 708     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 709             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 710             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 711             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 712             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 713             or (opts.useid and u'%(id)s.%(ext)s')
 714             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 715             or DEFAULT_OUTTMPL)
 716     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 717         parser.error(u'Cannot download a video and extract audio into the same'
 718                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 719                      u' template'.format(outtmpl))
 720
 721     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 722     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 723
 724     ydl_opts = {
 725         'usenetrc': opts.usenetrc,
 726         'username': opts.username,
 727         'password': opts.password,
 728         'videopassword': opts.videopassword,
 729         'quiet': (opts.quiet or any_printing),
 730         'no_warnings': opts.no_warnings,
 731         'forceurl': opts.geturl,
 732         'forcetitle': opts.gettitle,
 733         'forceid': opts.getid,
 734         'forcethumbnail': opts.getthumbnail,
 735         'forcedescription': opts.getdescription,
 736         'forceduration': opts.getduration,
 737         'forcefilename': opts.getfilename,
 738         'forceformat': opts.getformat,
 739         'forcejson': opts.dumpjson,
 740         'simulate': opts.simulate,
 741         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 742         'format': opts.format,
 743         'format_limit': opts.format_limit,
 744         'listformats': opts.listformats,
 745         'outtmpl': outtmpl,
 746         'autonumber_size': opts.autonumber_size,
 747         'restrictfilenames': opts.restrictfilenames,
 748         'ignoreerrors': opts.ignoreerrors,
 749         'ratelimit': opts.ratelimit,
 750         'nooverwrites': opts.nooverwrites,
 751         'retries': opts.retries,
 752         'buffersize': opts.buffersize,
 753         'noresizebuffer': opts.noresizebuffer,
 754         'continuedl': opts.continue_dl,
 755         'noprogress': opts.noprogress,
 756         'progress_with_newline': opts.progress_with_newline,
 757         'playliststart': opts.playliststart,
 758         'playlistend': opts.playlistend,
 759         'noplaylist': opts.noplaylist,
 760         'logtostderr': opts.outtmpl == '-',
 761         'consoletitle': opts.consoletitle,
 762         'nopart': opts.nopart,
 763         'updatetime': opts.updatetime,
 764         'writedescription': opts.writedescription,
 765         'writeannotations': opts.writeannotations,
 766         'writeinfojson': opts.writeinfojson,
 767         'writethumbnail': opts.writethumbnail,
 768         'writesubtitles': opts.writesubtitles,
 769         'writeautomaticsub': opts.writeautomaticsub,
 770         'allsubtitles': opts.allsubtitles,
 771         'listsubtitles': opts.listsubtitles,
 772         'subtitlesformat': opts.subtitlesformat,
 773         'subtitleslangs': opts.subtitleslangs,
 774         'matchtitle': decodeOption(opts.matchtitle),
 775         'rejecttitle': decodeOption(opts.rejecttitle),
 776         'max_downloads': opts.max_downloads,
 777         'prefer_free_formats': opts.prefer_free_formats,
 778         'verbose': opts.verbose,
 779         'dump_intermediate_pages': opts.dump_intermediate_pages,
 780         'write_pages': opts.write_pages,
 781         'test': opts.test,
 782         'keepvideo': opts.keepvideo,
 783         'min_filesize': opts.min_filesize,
 784         'max_filesize': opts.max_filesize,
 785         'min_views': opts.min_views,
 786         'max_views': opts.max_views,
 787         'daterange': date,
 788         'cachedir': opts.cachedir,
 789         'youtube_print_sig_code': opts.youtube_print_sig_code,
 790         'age_limit': opts.age_limit,
 791         'download_archive': download_archive_fn,
 792         'cookiefile': opts.cookiefile,
 793         'nocheckcertificate': opts.no_check_certificate,
 794         'prefer_insecure': opts.prefer_insecure,
 795         'proxy': opts.proxy,
 796         'socket_timeout': opts.socket_timeout,
 797         'bidi_workaround': opts.bidi_workaround,
 798         'debug_printtraffic': opts.debug_printtraffic,
 799         'prefer_ffmpeg': opts.prefer_ffmpeg,
 800         'include_ads': opts.include_ads,
 801         'default_search': opts.default_search,
 802         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 803         'encoding': opts.encoding,
 804     }
 805
 806     with YoutubeDL(ydl_opts) as ydl:
 807         ydl.print_debug_header()
 808         ydl.add_default_info_extractors()
 809
 810         # PostProcessors
 811         # Add the metadata pp first, the other pps will copy it
 812         if opts.addmetadata:
 813             ydl.add_post_processor(FFmpegMetadataPP())
 814         if opts.extractaudio:
 815             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 816         if opts.recodevideo:
 817             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 818         if opts.embedsubtitles:
 819             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 820         if opts.xattrs:
 821             ydl.add_post_processor(XAttrMetadataPP())
 822         if opts.embedthumbnail:
 823             if not opts.addmetadata:
 824                 ydl.add_post_processor(FFmpegAudioFixPP())
 825             ydl.add_post_processor(AtomicParsleyPP())
 826
 827         # Update version
 828         if opts.update_self:
 829             update_self(ydl.to_screen, opts.verbose)
 830
 831         # Maybe do nothing
 832         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 833             if not opts.update_self:
 834                 parser.error(u'you must provide at least one URL')
 835             else:
 836                 sys.exit()
 837
 838         try:
 839             if opts.load_info_filename is not None:
 840                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 841             else:
 842                 retcode = ydl.download(all_urls)
 843         except MaxDownloadsReached:
 844             ydl.to_screen(u'--max-download limit reached, aborting.')
 845             retcode = 101
 846
 847     sys.exit(retcode)
 848
 849
 850 def main(argv=None):
 851     try:
 852         _real_main(argv)
 853     except DownloadError:
 854         sys.exit(1)
 855     except SameFileError:
 856         sys.exit(u'ERROR: fixed output name but more than one file to download')
 857     except KeyboardInterrupt:
 858         sys.exit(u'\nERROR: Interrupted by user')