Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49     'David Triendl',
  50     'Anthony Weems',
  51     'David Wagner',
  52     'Juan C. Olivares',
  53     'Mattias Harrysson',
  54     'phaer',
  55     'Sainyam Kapoor',
  56     'Nicolas Évrard',
  57     'Jason Normore',
  58     'Hoje Lee',
  59     'Adam Thalhammer',
  60     'Georg Jähnig',
  61     'Ralf Haring',
  62     'Koki Takahashi',
  63 )
  64
  65 __license__ = 'Public Domain'
  66
  67 import codecs
  68 import io
  69 import locale
  70 import optparse
  71 import os
  72 import random
  73 import re
  74 import shlex
  75 import sys
  76
  77
  78 from .utils import (
  79     compat_getpass,
  80     compat_print,
  81     DateRange,
  82     DEFAULT_OUTTMPL,
  83     decodeOption,
  84     get_term_width,
  85     DownloadError,
  86     get_cachedir,
  87     MaxDownloadsReached,
  88     preferredencoding,
  89     read_batch_urls,
  90     SameFileError,
  91     setproctitle,
  92     std_headers,
  93     write_string,
  94 )
  95 from .update import update_self
  96 from .FileDownloader import (
  97     FileDownloader,
  98 )
  99 from .extractor import gen_extractors
 100 from .version import __version__
 101 from .YoutubeDL import YoutubeDL
 102 from .postprocessor import (
 103     AtomicParsleyPP,
 104     FFmpegAudioFixPP,
 105     FFmpegMetadataPP,
 106     FFmpegVideoConvertor,
 107     FFmpegExtractAudioPP,
 108     FFmpegEmbedSubtitlePP,
 109     XAttrMetadataPP,
 110 )
 111
 112
 113 def parseOpts(overrideArguments=None):
 114     def _readOptions(filename_bytes, default=[]):
 115         try:
 116             optionf = open(filename_bytes)
 117         except IOError:
 118             return default  # silently skip if file is not present
 119         try:
 120             res = []
 121             for l in optionf:
 122                 res += shlex.split(l, comments=True)
 123         finally:
 124             optionf.close()
 125         return res
 126
 127     def _readUserConf():
 128         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 129         if xdg_config_home:
 130             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 131             if not os.path.isfile(userConfFile):
 132                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 133         else:
 134             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 135             if not os.path.isfile(userConfFile):
 136                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 137         userConf = _readOptions(userConfFile, None)
 138
 139         if userConf is None:
 140             appdata_dir = os.environ.get('appdata')
 141             if appdata_dir:
 142                 userConf = _readOptions(
 143                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 144                     default=None)
 145                 if userConf is None:
 146                     userConf = _readOptions(
 147                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 148                         default=None)
 149
 150         if userConf is None:
 151             userConf = _readOptions(
 152                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 153                 default=None)
 154         if userConf is None:
 155             userConf = _readOptions(
 156                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 157                 default=None)
 158
 159         if userConf is None:
 160             userConf = []
 161
 162         return userConf
 163
 164     def _format_option_string(option):
 165         ''' ('-o', '--option') -> -o, --format METAVAR'''
 166
 167         opts = []
 168
 169         if option._short_opts:
 170             opts.append(option._short_opts[0])
 171         if option._long_opts:
 172             opts.append(option._long_opts[0])
 173         if len(opts) > 1:
 174             opts.insert(1, ', ')
 175
 176         if option.takes_value(): opts.append(' %s' % option.metavar)
 177
 178         return "".join(opts)
 179
 180     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 181         setattr(parser.values, option.dest, value.split(','))
 182
 183     def _hide_login_info(opts):
 184         opts = list(opts)
 185         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 186             try:
 187                 i = opts.index(private_opt)
 188                 opts[i+1] = '<PRIVATE>'
 189             except ValueError:
 190                 pass
 191         return opts
 192
 193     max_width = 80
 194     max_help_position = 80
 195
 196     # No need to wrap help messages if we're on a wide console
 197     columns = get_term_width()
 198     if columns: max_width = columns
 199
 200     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 201     fmt.format_option_strings = _format_option_string
 202
 203     kw = {
 204         'version'   : __version__,
 205         'formatter' : fmt,
 206         'usage' : '%prog [options] url [url...]',
 207         'conflict_handler' : 'resolve',
 208     }
 209
 210     parser = optparse.OptionParser(**kw)
 211
 212     # option groups
 213     general        = optparse.OptionGroup(parser, 'General Options')
 214     selection      = optparse.OptionGroup(parser, 'Video Selection')
 215     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 216     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 217     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 218     downloader     = optparse.OptionGroup(parser, 'Download Options')
 219     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 220     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 221     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 222
 223     general.add_option('-h', '--help',
 224             action='help', help='print this help text and exit')
 225     general.add_option('-v', '--version',
 226             action='version', help='print program version and exit')
 227     general.add_option('-U', '--update',
 228             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 229     general.add_option('-i', '--ignore-errors',
 230             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
 231     general.add_option('--abort-on-error',
 232             action='store_false', dest='ignoreerrors',
 233             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 234     general.add_option('--dump-user-agent',
 235             action='store_true', dest='dump_user_agent',
 236             help='display the current browser identification', default=False)
 237     general.add_option('--user-agent',
 238             dest='user_agent', help='specify a custom user agent', metavar='UA')
 239     general.add_option('--referer',
 240             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 241             metavar='REF', default=None)
 242     general.add_option('--add-header',
 243             dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
 244             metavar='FIELD:VALUE')
 245     general.add_option('--list-extractors',
 246             action='store_true', dest='list_extractors',
 247             help='List all supported extractors and the URLs they would handle', default=False)
 248     general.add_option('--extractor-descriptions',
 249             action='store_true', dest='list_extractor_descriptions',
 250             help='Output descriptions of all supported extractors', default=False)
 251     general.add_option(
 252         '--proxy', dest='proxy', default=None, metavar='URL',
 253         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 254     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 255     general.add_option(
 256         '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
 257         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
 258     general.add_option(
 259         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 260         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 261     general.add_option(
 262         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 263         help='Disable filesystem caching')
 264     general.add_option(
 265         '--socket-timeout', dest='socket_timeout',
 266         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 267     general.add_option(
 268         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 269         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 270     general.add_option(
 271         '--default-search',
 272         dest='default_search', metavar='PREFIX',
 273         help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". Use the value "auto" to let youtube-dl guess. The default value "error" just throws an error.')
 274     general.add_option(
 275         '--ignore-config',
 276         action='store_true',
 277         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 278     general.add_option(
 279         '--encoding', dest='encoding', metavar='ENCODING',
 280         help='Force the specified encoding (experimental)')
 281
 282     selection.add_option(
 283         '--playlist-start',
 284         dest='playliststart', metavar='NUMBER', default=1, type=int,
 285         help='playlist video to start at (default is %default)')
 286     selection.add_option(
 287         '--playlist-end',
 288         dest='playlistend', metavar='NUMBER', default=None, type=int,
 289         help='playlist video to end at (default is last)')
 290     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 291     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 292     selection.add_option('--max-downloads', metavar='NUMBER',
 293                          dest='max_downloads', type=int, default=None,
 294                          help='Abort after downloading NUMBER files')
 295     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 296     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 297     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 298     selection.add_option(
 299         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 300         help='download only videos uploaded on or before this date (i.e. inclusive)')
 301     selection.add_option(
 302         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 303         help='download only videos uploaded on or after this date (i.e. inclusive)')
 304     selection.add_option(
 305         '--min-views', metavar='COUNT', dest='min_views',
 306         default=None, type=int,
 307         help="Do not download any videos with less than COUNT views",)
 308     selection.add_option(
 309         '--max-views', metavar='COUNT', dest='max_views',
 310         default=None, type=int,
 311         help="Do not download any videos with more than COUNT views",)
 312     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 313     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 314                          help='download only videos suitable for the given age',
 315                          default=None, type=int)
 316     selection.add_option('--download-archive', metavar='FILE',
 317                          dest='download_archive',
 318                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 319     selection.add_option(
 320         '--include-ads', dest='include_ads',
 321         action='store_true',
 322         help='Download advertisements as well (experimental)')
 323     selection.add_option(
 324         '--youtube-include-dash-manifest', action='store_true',
 325         dest='youtube_include_dash_manifest', default=False,
 326         help='Try to download the DASH manifest on YouTube videos (experimental)')
 327
 328     authentication.add_option('-u', '--username',
 329             dest='username', metavar='USERNAME', help='account username')
 330     authentication.add_option('-p', '--password',
 331             dest='password', metavar='PASSWORD', help='account password')
 332     authentication.add_option('-n', '--netrc',
 333             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 334     authentication.add_option('--video-password',
 335             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 336
 337
 338     video_format.add_option('-f', '--format',
 339             action='store', dest='format', metavar='FORMAT', default=None,
 340             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
 341     video_format.add_option('--all-formats',
 342             action='store_const', dest='format', help='download all available video formats', const='all')
 343     video_format.add_option('--prefer-free-formats',
 344             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 345     video_format.add_option('--max-quality',
 346             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 347     video_format.add_option('-F', '--list-formats',
 348             action='store_true', dest='listformats', help='list all available formats')
 349
 350     subtitles.add_option('--write-sub', '--write-srt',
 351             action='store_true', dest='writesubtitles',
 352             help='write subtitle file', default=False)
 353     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 354             action='store_true', dest='writeautomaticsub',
 355             help='write automatic subtitle file (youtube only)', default=False)
 356     subtitles.add_option('--all-subs',
 357             action='store_true', dest='allsubtitles',
 358             help='downloads all the available subtitles of the video', default=False)
 359     subtitles.add_option('--list-subs',
 360             action='store_true', dest='listsubtitles',
 361             help='lists all available subtitles for the video', default=False)
 362     subtitles.add_option('--sub-format',
 363             action='store', dest='subtitlesformat', metavar='FORMAT',
 364             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 365     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 366             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 367             default=[], callback=_comma_separated_values_options_callback,
 368             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 369
 370     downloader.add_option('-r', '--rate-limit',
 371             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 372     downloader.add_option('-R', '--retries',
 373             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 374     downloader.add_option('--buffer-size',
 375             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 376     downloader.add_option('--no-resize-buffer',
 377             action='store_true', dest='noresizebuffer',
 378             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 379     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 380
 381     verbosity.add_option('-q', '--quiet',
 382             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 383     verbosity.add_option(
 384         '--no-warnings',
 385         dest='no_warnings', action='store_true', default=False,
 386         help='Ignore warnings')
 387     verbosity.add_option('-s', '--simulate',
 388             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 389     verbosity.add_option('--skip-download',
 390             action='store_true', dest='skip_download', help='do not download the video', default=False)
 391     verbosity.add_option('-g', '--get-url',
 392             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 393     verbosity.add_option('-e', '--get-title',
 394             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 395     verbosity.add_option('--get-id',
 396             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 397     verbosity.add_option('--get-thumbnail',
 398             action='store_true', dest='getthumbnail',
 399             help='simulate, quiet but print thumbnail URL', default=False)
 400     verbosity.add_option('--get-description',
 401             action='store_true', dest='getdescription',
 402             help='simulate, quiet but print video description', default=False)
 403     verbosity.add_option('--get-duration',
 404             action='store_true', dest='getduration',
 405             help='simulate, quiet but print video length', default=False)
 406     verbosity.add_option('--get-filename',
 407             action='store_true', dest='getfilename',
 408             help='simulate, quiet but print output filename', default=False)
 409     verbosity.add_option('--get-format',
 410             action='store_true', dest='getformat',
 411             help='simulate, quiet but print output format', default=False)
 412     verbosity.add_option('-j', '--dump-json',
 413             action='store_true', dest='dumpjson',
 414             help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
 415     verbosity.add_option('--newline',
 416             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 417     verbosity.add_option('--no-progress',
 418             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 419     verbosity.add_option('--console-title',
 420             action='store_true', dest='consoletitle',
 421             help='display progress in console titlebar', default=False)
 422     verbosity.add_option('-v', '--verbose',
 423             action='store_true', dest='verbose', help='print various debugging information', default=False)
 424     verbosity.add_option('--dump-intermediate-pages',
 425             action='store_true', dest='dump_intermediate_pages', default=False,
 426             help='print downloaded pages to debug problems (very verbose)')
 427     verbosity.add_option('--write-pages',
 428             action='store_true', dest='write_pages', default=False,
 429             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 430     verbosity.add_option('--youtube-print-sig-code',
 431             action='store_true', dest='youtube_print_sig_code', default=False,
 432             help=optparse.SUPPRESS_HELP)
 433     verbosity.add_option('--print-traffic',
 434             dest='debug_printtraffic', action='store_true', default=False,
 435             help='Display sent and read HTTP traffic')
 436
 437
 438     filesystem.add_option('-t', '--title',
 439             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 440     filesystem.add_option('--id',
 441             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 442     filesystem.add_option('-l', '--literal',
 443             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 444     filesystem.add_option('-A', '--auto-number',
 445             action='store_true', dest='autonumber',
 446             help='number downloaded files starting from 00000', default=False)
 447     filesystem.add_option('-o', '--output',
 448             dest='outtmpl', metavar='TEMPLATE',
 449             help=('output filename template. Use %(title)s to get the title, '
 450                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 451                   '%(autonumber)s to get an automatically incremented number, '
 452                   '%(ext)s for the filename extension, '
 453                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 454                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 455                   '%(upload_date)s for the upload date (YYYYMMDD), '
 456                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 457                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 458                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 459                   '%(height)s and %(width)s for the width and height of the video format. '
 460                   '%(resolution)s for a textual description of the resolution of the video format. '
 461                   'Use - to output to stdout. Can also be used to download to a different directory, '
 462                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 463     filesystem.add_option('--autonumber-size',
 464             dest='autonumber_size', metavar='NUMBER',
 465             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 466     filesystem.add_option('--restrict-filenames',
 467             action='store_true', dest='restrictfilenames',
 468             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 469     filesystem.add_option('-a', '--batch-file',
 470             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 471     filesystem.add_option('--load-info',
 472             dest='load_info_filename', metavar='FILE',
 473             help='json file containing the video information (created with the "--write-json" option)')
 474     filesystem.add_option('-w', '--no-overwrites',
 475             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 476     filesystem.add_option('-c', '--continue',
 477             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 478     filesystem.add_option('--no-continue',
 479             action='store_false', dest='continue_dl',
 480             help='do not resume partially downloaded files (restart from beginning)')
 481     filesystem.add_option('--cookies',
 482             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 483     filesystem.add_option('--no-part',
 484             action='store_true', dest='nopart', help='do not use .part files', default=False)
 485     filesystem.add_option('--no-mtime',
 486             action='store_false', dest='updatetime',
 487             help='do not use the Last-modified header to set the file modification time', default=True)
 488     filesystem.add_option('--write-description',
 489             action='store_true', dest='writedescription',
 490             help='write video description to a .description file', default=False)
 491     filesystem.add_option('--write-info-json',
 492             action='store_true', dest='writeinfojson',
 493             help='write video metadata to a .info.json file', default=False)
 494     filesystem.add_option('--write-annotations',
 495             action='store_true', dest='writeannotations',
 496             help='write video annotations to a .annotation file', default=False)
 497     filesystem.add_option('--write-thumbnail',
 498             action='store_true', dest='writethumbnail',
 499             help='write thumbnail image to disk', default=False)
 500
 501
 502     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 503             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 504     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 505             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 506     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 507             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 508     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 509             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 510     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 511             help='keeps the video file on disk after the post-processing; the video is erased by default')
 512     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 513             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 514     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 515             help='embed subtitles in the video (only for mp4 videos)')
 516     postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
 517             help='embed thumbnail in the audio as cover art')
 518     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 519             help='write metadata to the video file')
 520     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 521             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 522     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 523         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 524     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 525         help='Prefer ffmpeg over avconv for running the postprocessors')
 526
 527
 528     parser.add_option_group(general)
 529     parser.add_option_group(selection)
 530     parser.add_option_group(downloader)
 531     parser.add_option_group(filesystem)
 532     parser.add_option_group(verbosity)
 533     parser.add_option_group(video_format)
 534     parser.add_option_group(subtitles)
 535     parser.add_option_group(authentication)
 536     parser.add_option_group(postproc)
 537
 538     if overrideArguments is not None:
 539         opts, args = parser.parse_args(overrideArguments)
 540         if opts.verbose:
 541             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 542     else:
 543         commandLineConf = sys.argv[1:]
 544         if '--ignore-config' in commandLineConf:
 545             systemConf = []
 546             userConf = []
 547         else:
 548             systemConf = _readOptions('/etc/youtube-dl.conf')
 549             if '--ignore-config' in systemConf:
 550                 userConf = []
 551             else:
 552                 userConf = _readUserConf()
 553         argv = systemConf + userConf + commandLineConf
 554
 555         opts, args = parser.parse_args(argv)
 556         if opts.verbose:
 557             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 558             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 559             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 560
 561     return parser, opts, args
 562
 563
 564 def _real_main(argv=None):
 565     # Compatibility fixes for Windows
 566     if sys.platform == 'win32':
 567         # https://github.com/rg3/youtube-dl/issues/820
 568         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 569
 570     setproctitle(u'youtube-dl')
 571
 572     parser, opts, args = parseOpts(argv)
 573
 574     # Set user agent
 575     if opts.user_agent is not None:
 576         std_headers['User-Agent'] = opts.user_agent
 577
 578     # Set referer
 579     if opts.referer is not None:
 580         std_headers['Referer'] = opts.referer
 581
 582     # Custom HTTP headers
 583     if opts.headers is not None:
 584         for h in opts.headers:
 585             if h.find(':', 1) < 0:
 586                 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
 587             key, value = h.split(':', 2)
 588             if opts.verbose:
 589                 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
 590             std_headers[key] = value
 591
 592     # Dump user agent
 593     if opts.dump_user_agent:
 594         compat_print(std_headers['User-Agent'])
 595         sys.exit(0)
 596
 597     # Batch file verification
 598     batch_urls = []
 599     if opts.batchfile is not None:
 600         try:
 601             if opts.batchfile == '-':
 602                 batchfd = sys.stdin
 603             else:
 604                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
 605             batch_urls = read_batch_urls(batchfd)
 606             if opts.verbose:
 607                 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
 608         except IOError:
 609             sys.exit(u'ERROR: batch file could not be read')
 610     all_urls = batch_urls + args
 611     all_urls = [url.strip() for url in all_urls]
 612     _enc = preferredencoding()
 613     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 614
 615     extractors = gen_extractors()
 616
 617     if opts.list_extractors:
 618         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 619             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 620             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 621             for mu in matchedUrls:
 622                 compat_print(u'  ' + mu)
 623         sys.exit(0)
 624     if opts.list_extractor_descriptions:
 625         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 626             if not ie._WORKING:
 627                 continue
 628             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 629             if desc is False:
 630                 continue
 631             if hasattr(ie, 'SEARCH_KEY'):
 632                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 633                 _COUNTS = (u'', u'5', u'10', u'all')
 634                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 635             compat_print(desc)
 636         sys.exit(0)
 637
 638
 639     # Conflicting, missing and erroneous options
 640     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 641         parser.error(u'using .netrc conflicts with giving username/password')
 642     if opts.password is not None and opts.username is None:
 643         parser.error(u'account username missing\n')
 644     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 645         parser.error(u'using output template conflicts with using title, video ID or auto number')
 646     if opts.usetitle and opts.useid:
 647         parser.error(u'using title conflicts with using video ID')
 648     if opts.username is not None and opts.password is None:
 649         opts.password = compat_getpass(u'Type account password and press [Return]: ')
 650     if opts.ratelimit is not None:
 651         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 652         if numeric_limit is None:
 653             parser.error(u'invalid rate limit specified')
 654         opts.ratelimit = numeric_limit
 655     if opts.min_filesize is not None:
 656         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 657         if numeric_limit is None:
 658             parser.error(u'invalid min_filesize specified')
 659         opts.min_filesize = numeric_limit
 660     if opts.max_filesize is not None:
 661         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 662         if numeric_limit is None:
 663             parser.error(u'invalid max_filesize specified')
 664         opts.max_filesize = numeric_limit
 665     if opts.retries is not None:
 666         try:
 667             opts.retries = int(opts.retries)
 668         except (TypeError, ValueError):
 669             parser.error(u'invalid retry count specified')
 670     if opts.buffersize is not None:
 671         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 672         if numeric_buffersize is None:
 673             parser.error(u'invalid buffer size specified')
 674         opts.buffersize = numeric_buffersize
 675     if opts.playliststart <= 0:
 676         raise ValueError(u'Playlist start must be positive')
 677     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 678         raise ValueError(u'Playlist end must be greater than playlist start')
 679     if opts.extractaudio:
 680         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 681             parser.error(u'invalid audio format specified')
 682     if opts.audioquality:
 683         opts.audioquality = opts.audioquality.strip('k').strip('K')
 684         if not opts.audioquality.isdigit():
 685             parser.error(u'invalid audio quality specified')
 686     if opts.recodevideo is not None:
 687         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
 688             parser.error(u'invalid video recode format specified')
 689     if opts.date is not None:
 690         date = DateRange.day(opts.date)
 691     else:
 692         date = DateRange(opts.dateafter, opts.datebefore)
 693     if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
 694         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 695
 696     # Do not download videos when there are audio-only formats
 697     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 698         opts.format = 'bestaudio/best'
 699
 700     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 701     # this was the old behaviour if only --all-sub was given.
 702     if opts.allsubtitles and (opts.writeautomaticsub == False):
 703         opts.writesubtitles = True
 704
 705     if sys.version_info < (3,):
 706         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 707         if opts.outtmpl is not None:
 708             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 709     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 710             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 711             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 712             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 713             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 714             or (opts.useid and u'%(id)s.%(ext)s')
 715             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 716             or DEFAULT_OUTTMPL)
 717     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 718         parser.error(u'Cannot download a video and extract audio into the same'
 719                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 720                      u' template'.format(outtmpl))
 721
 722     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 723     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 724
 725     ydl_opts = {
 726         'usenetrc': opts.usenetrc,
 727         'username': opts.username,
 728         'password': opts.password,
 729         'videopassword': opts.videopassword,
 730         'quiet': (opts.quiet or any_printing),
 731         'no_warnings': opts.no_warnings,
 732         'forceurl': opts.geturl,
 733         'forcetitle': opts.gettitle,
 734         'forceid': opts.getid,
 735         'forcethumbnail': opts.getthumbnail,
 736         'forcedescription': opts.getdescription,
 737         'forceduration': opts.getduration,
 738         'forcefilename': opts.getfilename,
 739         'forceformat': opts.getformat,
 740         'forcejson': opts.dumpjson,
 741         'simulate': opts.simulate,
 742         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 743         'format': opts.format,
 744         'format_limit': opts.format_limit,
 745         'listformats': opts.listformats,
 746         'outtmpl': outtmpl,
 747         'autonumber_size': opts.autonumber_size,
 748         'restrictfilenames': opts.restrictfilenames,
 749         'ignoreerrors': opts.ignoreerrors,
 750         'ratelimit': opts.ratelimit,
 751         'nooverwrites': opts.nooverwrites,
 752         'retries': opts.retries,
 753         'buffersize': opts.buffersize,
 754         'noresizebuffer': opts.noresizebuffer,
 755         'continuedl': opts.continue_dl,
 756         'noprogress': opts.noprogress,
 757         'progress_with_newline': opts.progress_with_newline,
 758         'playliststart': opts.playliststart,
 759         'playlistend': opts.playlistend,
 760         'noplaylist': opts.noplaylist,
 761         'logtostderr': opts.outtmpl == '-',
 762         'consoletitle': opts.consoletitle,
 763         'nopart': opts.nopart,
 764         'updatetime': opts.updatetime,
 765         'writedescription': opts.writedescription,
 766         'writeannotations': opts.writeannotations,
 767         'writeinfojson': opts.writeinfojson,
 768         'writethumbnail': opts.writethumbnail,
 769         'writesubtitles': opts.writesubtitles,
 770         'writeautomaticsub': opts.writeautomaticsub,
 771         'allsubtitles': opts.allsubtitles,
 772         'listsubtitles': opts.listsubtitles,
 773         'subtitlesformat': opts.subtitlesformat,
 774         'subtitleslangs': opts.subtitleslangs,
 775         'matchtitle': decodeOption(opts.matchtitle),
 776         'rejecttitle': decodeOption(opts.rejecttitle),
 777         'max_downloads': opts.max_downloads,
 778         'prefer_free_formats': opts.prefer_free_formats,
 779         'verbose': opts.verbose,
 780         'dump_intermediate_pages': opts.dump_intermediate_pages,
 781         'write_pages': opts.write_pages,
 782         'test': opts.test,
 783         'keepvideo': opts.keepvideo,
 784         'min_filesize': opts.min_filesize,
 785         'max_filesize': opts.max_filesize,
 786         'min_views': opts.min_views,
 787         'max_views': opts.max_views,
 788         'daterange': date,
 789         'cachedir': opts.cachedir,
 790         'youtube_print_sig_code': opts.youtube_print_sig_code,
 791         'age_limit': opts.age_limit,
 792         'download_archive': download_archive_fn,
 793         'cookiefile': opts.cookiefile,
 794         'nocheckcertificate': opts.no_check_certificate,
 795         'prefer_insecure': opts.prefer_insecure,
 796         'proxy': opts.proxy,
 797         'socket_timeout': opts.socket_timeout,
 798         'bidi_workaround': opts.bidi_workaround,
 799         'debug_printtraffic': opts.debug_printtraffic,
 800         'prefer_ffmpeg': opts.prefer_ffmpeg,
 801         'include_ads': opts.include_ads,
 802         'default_search': opts.default_search,
 803         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 804         'encoding': opts.encoding,
 805     }
 806
 807     with YoutubeDL(ydl_opts) as ydl:
 808         ydl.print_debug_header()
 809         ydl.add_default_info_extractors()
 810
 811         # PostProcessors
 812         # Add the metadata pp first, the other pps will copy it
 813         if opts.addmetadata:
 814             ydl.add_post_processor(FFmpegMetadataPP())
 815         if opts.extractaudio:
 816             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 817         if opts.recodevideo:
 818             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 819         if opts.embedsubtitles:
 820             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 821         if opts.xattrs:
 822             ydl.add_post_processor(XAttrMetadataPP())
 823         if opts.embedthumbnail:
 824             if not opts.addmetadata:
 825                 ydl.add_post_processor(FFmpegAudioFixPP())
 826             ydl.add_post_processor(AtomicParsleyPP())
 827
 828         # Update version
 829         if opts.update_self:
 830             update_self(ydl.to_screen, opts.verbose)
 831
 832         # Maybe do nothing
 833         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 834             if not opts.update_self:
 835                 parser.error(u'you must provide at least one URL')
 836             else:
 837                 sys.exit()
 838
 839         try:
 840             if opts.load_info_filename is not None:
 841                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 842             else:
 843                 retcode = ydl.download(all_urls)
 844         except MaxDownloadsReached:
 845             ydl.to_screen(u'--max-download limit reached, aborting.')
 846             retcode = 101
 847
 848     sys.exit(retcode)
 849
 850
 851 def main(argv=None):
 852     try:
 853         _real_main(argv)
 854     except DownloadError:
 855         sys.exit(1)
 856     except SameFileError:
 857         sys.exit(u'ERROR: fixed output name but more than one file to download')
 858     except KeyboardInterrupt:
 859         sys.exit(u'\nERROR: Interrupted by user')