Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41     'Chris Gahan',
  42     'Saimadhav Heblikar',
  43     'Mike Col',
  44     'Oleg Prutz',
  45     'pulpe',
  46     'Andreas Schmitz',
  47     'Michael Kaiser',
  48     'Niklas Laxström',
  49 )
  50
  51 __license__ = 'Public Domain'
  52
  53 import codecs
  54 import getpass
  55 import locale
  56 import optparse
  57 import os
  58 import random
  59 import re
  60 import shlex
  61 import sys
  62
  63
  64 from .utils import (
  65     compat_print,
  66     DateRange,
  67     decodeOption,
  68     get_term_width,
  69     DownloadError,
  70     get_cachedir,
  71     MaxDownloadsReached,
  72     preferredencoding,
  73     SameFileError,
  74     setproctitle,
  75     std_headers,
  76     write_string,
  77 )
  78 from .update import update_self
  79 from .FileDownloader import (
  80     FileDownloader,
  81 )
  82 from .extractor import gen_extractors
  83 from .version import __version__
  84 from .YoutubeDL import YoutubeDL
  85 from .postprocessor import (
  86     FFmpegMetadataPP,
  87     FFmpegVideoConvertor,
  88     FFmpegExtractAudioPP,
  89     FFmpegEmbedSubtitlePP,
  90     XAttrMetadataPP,
  91 )
  92
  93
  94 def parseOpts(overrideArguments=None):
  95     def _readOptions(filename_bytes, default=[]):
  96         try:
  97             optionf = open(filename_bytes)
  98         except IOError:
  99             return default  # silently skip if file is not present
 100         try:
 101             res = []
 102             for l in optionf:
 103                 res += shlex.split(l, comments=True)
 104         finally:
 105             optionf.close()
 106         return res
 107
 108     def _readUserConf():
 109         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 110         if xdg_config_home:
 111             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 112             if not os.path.isfile(userConfFile):
 113                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 114         else:
 115             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 116             if not os.path.isfile(userConfFile):
 117                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 118         userConf = _readOptions(userConfFile, None)
 119
 120         if userConf is None:
 121             appdata_dir = os.environ.get('appdata')
 122             if appdata_dir:
 123                 userConf = _readOptions(
 124                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 125                     default=None)
 126                 if userConf is None:
 127                     userConf = _readOptions(
 128                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 129                         default=None)
 130
 131         if userConf is None:
 132             userConf = _readOptions(
 133                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 134                 default=None)
 135         if userConf is None:
 136             userConf = _readOptions(
 137                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 138                 default=None)
 139
 140         if userConf is None:
 141             userConf = []
 142
 143         return userConf
 144
 145     def _format_option_string(option):
 146         ''' ('-o', '--option') -> -o, --format METAVAR'''
 147
 148         opts = []
 149
 150         if option._short_opts:
 151             opts.append(option._short_opts[0])
 152         if option._long_opts:
 153             opts.append(option._long_opts[0])
 154         if len(opts) > 1:
 155             opts.insert(1, ', ')
 156
 157         if option.takes_value(): opts.append(' %s' % option.metavar)
 158
 159         return "".join(opts)
 160
 161     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 162         setattr(parser.values, option.dest, value.split(','))
 163
 164     def _hide_login_info(opts):
 165         opts = list(opts)
 166         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 167             try:
 168                 i = opts.index(private_opt)
 169                 opts[i+1] = '<PRIVATE>'
 170             except ValueError:
 171                 pass
 172         return opts
 173
 174     max_width = 80
 175     max_help_position = 80
 176
 177     # No need to wrap help messages if we're on a wide console
 178     columns = get_term_width()
 179     if columns: max_width = columns
 180
 181     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 182     fmt.format_option_strings = _format_option_string
 183
 184     kw = {
 185         'version'   : __version__,
 186         'formatter' : fmt,
 187         'usage' : '%prog [options] url [url...]',
 188         'conflict_handler' : 'resolve',
 189     }
 190
 191     parser = optparse.OptionParser(**kw)
 192
 193     # option groups
 194     general        = optparse.OptionGroup(parser, 'General Options')
 195     selection      = optparse.OptionGroup(parser, 'Video Selection')
 196     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 197     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 198     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 199     downloader     = optparse.OptionGroup(parser, 'Download Options')
 200     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 201     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 202     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 203
 204     general.add_option('-h', '--help',
 205             action='help', help='print this help text and exit')
 206     general.add_option('-v', '--version',
 207             action='version', help='print program version and exit')
 208     general.add_option('-U', '--update',
 209             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 210     general.add_option('-i', '--ignore-errors',
 211             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 212     general.add_option('--abort-on-error',
 213             action='store_false', dest='ignoreerrors',
 214             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 215     general.add_option('--dump-user-agent',
 216             action='store_true', dest='dump_user_agent',
 217             help='display the current browser identification', default=False)
 218     general.add_option('--user-agent',
 219             dest='user_agent', help='specify a custom user agent', metavar='UA')
 220     general.add_option('--referer',
 221             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 222             metavar='REF', default=None)
 223     general.add_option('--list-extractors',
 224             action='store_true', dest='list_extractors',
 225             help='List all supported extractors and the URLs they would handle', default=False)
 226     general.add_option('--extractor-descriptions',
 227             action='store_true', dest='list_extractor_descriptions',
 228             help='Output descriptions of all supported extractors', default=False)
 229     general.add_option(
 230         '--proxy', dest='proxy', default=None, metavar='URL',
 231         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 232     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 233     general.add_option(
 234         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 235         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
 236     general.add_option(
 237         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 238         help='Disable filesystem caching')
 239     general.add_option(
 240         '--socket-timeout', dest='socket_timeout',
 241         type=float, default=None, help=u'Time to wait before giving up, in seconds')
 242     general.add_option(
 243         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 244         help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 245     general.add_option('--default-search',
 246             dest='default_search', metavar='PREFIX',
 247             help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
 248     general.add_option(
 249         '--ignore-config',
 250         action='store_true',
 251         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 252
 253
 254     selection.add_option(
 255         '--playlist-start',
 256         dest='playliststart', metavar='NUMBER', default=1, type=int,
 257         help='playlist video to start at (default is %default)')
 258     selection.add_option(
 259         '--playlist-end',
 260         dest='playlistend', metavar='NUMBER', default=None, type=int,
 261         help='playlist video to end at (default is last)')
 262     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 263     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 264     selection.add_option('--max-downloads', metavar='NUMBER',
 265                          dest='max_downloads', type=int, default=None,
 266                          help='Abort after downloading NUMBER files')
 267     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 268     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 269     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 270     selection.add_option(
 271         '--datebefore', metavar='DATE', dest='datebefore', default=None,
 272         help='download only videos uploaded on or before this date (i.e. inclusive)')
 273     selection.add_option(
 274         '--dateafter', metavar='DATE', dest='dateafter', default=None,
 275         help='download only videos uploaded on or after this date (i.e. inclusive)')
 276     selection.add_option(
 277         '--min-views', metavar='COUNT', dest='min_views',
 278         default=None, type=int,
 279         help="Do not download any videos with less than COUNT views",)
 280     selection.add_option(
 281         '--max-views', metavar='COUNT', dest='max_views',
 282         default=None, type=int,
 283         help="Do not download any videos with more than COUNT views",)
 284     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 285     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 286                          help='download only videos suitable for the given age',
 287                          default=None, type=int)
 288     selection.add_option('--download-archive', metavar='FILE',
 289                          dest='download_archive',
 290                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 291     selection.add_option(
 292         '--include-ads', dest='include_ads',
 293         action='store_true',
 294         help='Download advertisements as well (experimental)')
 295     selection.add_option(
 296         '--youtube-include-dash-manifest', action='store_true',
 297         dest='youtube_include_dash_manifest', default=False,
 298         help='Try to download the DASH manifest on YouTube videos (experimental)')
 299
 300     authentication.add_option('-u', '--username',
 301             dest='username', metavar='USERNAME', help='account username')
 302     authentication.add_option('-p', '--password',
 303             dest='password', metavar='PASSWORD', help='account password')
 304     authentication.add_option('-n', '--netrc',
 305             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 306     authentication.add_option('--video-password',
 307             dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
 308
 309
 310     video_format.add_option('-f', '--format',
 311             action='store', dest='format', metavar='FORMAT', default=None,
 312             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestaudio", "worst", and "worstaudio". By default, youtube-dl will pick the best quality.')
 313     video_format.add_option('--all-formats',
 314             action='store_const', dest='format', help='download all available video formats', const='all')
 315     video_format.add_option('--prefer-free-formats',
 316             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 317     video_format.add_option('--max-quality',
 318             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 319     video_format.add_option('-F', '--list-formats',
 320             action='store_true', dest='listformats', help='list all available formats')
 321
 322     subtitles.add_option('--write-sub', '--write-srt',
 323             action='store_true', dest='writesubtitles',
 324             help='write subtitle file', default=False)
 325     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 326             action='store_true', dest='writeautomaticsub',
 327             help='write automatic subtitle file (youtube only)', default=False)
 328     subtitles.add_option('--all-subs',
 329             action='store_true', dest='allsubtitles',
 330             help='downloads all the available subtitles of the video', default=False)
 331     subtitles.add_option('--list-subs',
 332             action='store_true', dest='listsubtitles',
 333             help='lists all available subtitles for the video', default=False)
 334     subtitles.add_option('--sub-format',
 335             action='store', dest='subtitlesformat', metavar='FORMAT',
 336             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 337     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 338             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 339             default=[], callback=_comma_separated_values_options_callback,
 340             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 341
 342     downloader.add_option('-r', '--rate-limit',
 343             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 344     downloader.add_option('-R', '--retries',
 345             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 346     downloader.add_option('--buffer-size',
 347             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 348     downloader.add_option('--no-resize-buffer',
 349             action='store_true', dest='noresizebuffer',
 350             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 351     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 352
 353     verbosity.add_option('-q', '--quiet',
 354             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 355     verbosity.add_option('-s', '--simulate',
 356             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 357     verbosity.add_option('--skip-download',
 358             action='store_true', dest='skip_download', help='do not download the video', default=False)
 359     verbosity.add_option('-g', '--get-url',
 360             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 361     verbosity.add_option('-e', '--get-title',
 362             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 363     verbosity.add_option('--get-id',
 364             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 365     verbosity.add_option('--get-thumbnail',
 366             action='store_true', dest='getthumbnail',
 367             help='simulate, quiet but print thumbnail URL', default=False)
 368     verbosity.add_option('--get-description',
 369             action='store_true', dest='getdescription',
 370             help='simulate, quiet but print video description', default=False)
 371     verbosity.add_option('--get-duration',
 372             action='store_true', dest='getduration',
 373             help='simulate, quiet but print video length', default=False)
 374     verbosity.add_option('--get-filename',
 375             action='store_true', dest='getfilename',
 376             help='simulate, quiet but print output filename', default=False)
 377     verbosity.add_option('--get-format',
 378             action='store_true', dest='getformat',
 379             help='simulate, quiet but print output format', default=False)
 380     verbosity.add_option('-j', '--dump-json',
 381             action='store_true', dest='dumpjson',
 382             help='simulate, quiet but print JSON information', default=False)
 383     verbosity.add_option('--newline',
 384             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 385     verbosity.add_option('--no-progress',
 386             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 387     verbosity.add_option('--console-title',
 388             action='store_true', dest='consoletitle',
 389             help='display progress in console titlebar', default=False)
 390     verbosity.add_option('-v', '--verbose',
 391             action='store_true', dest='verbose', help='print various debugging information', default=False)
 392     verbosity.add_option('--dump-intermediate-pages',
 393             action='store_true', dest='dump_intermediate_pages', default=False,
 394             help='print downloaded pages to debug problems (very verbose)')
 395     verbosity.add_option('--write-pages',
 396             action='store_true', dest='write_pages', default=False,
 397             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 398     verbosity.add_option('--youtube-print-sig-code',
 399             action='store_true', dest='youtube_print_sig_code', default=False,
 400             help=optparse.SUPPRESS_HELP)
 401     verbosity.add_option('--print-traffic',
 402             dest='debug_printtraffic', action='store_true', default=False,
 403             help='Display sent and read HTTP traffic')
 404
 405
 406     filesystem.add_option('-t', '--title',
 407             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 408     filesystem.add_option('--id',
 409             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 410     filesystem.add_option('-l', '--literal',
 411             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 412     filesystem.add_option('-A', '--auto-number',
 413             action='store_true', dest='autonumber',
 414             help='number downloaded files starting from 00000', default=False)
 415     filesystem.add_option('-o', '--output',
 416             dest='outtmpl', metavar='TEMPLATE',
 417             help=('output filename template. Use %(title)s to get the title, '
 418                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 419                   '%(autonumber)s to get an automatically incremented number, '
 420                   '%(ext)s for the filename extension, '
 421                   '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
 422                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
 423                   '%(upload_date)s for the upload date (YYYYMMDD), '
 424                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 425                   '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
 426                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 427                   'Use - to output to stdout. Can also be used to download to a different directory, '
 428                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 429     filesystem.add_option('--autonumber-size',
 430             dest='autonumber_size', metavar='NUMBER',
 431             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 432     filesystem.add_option('--restrict-filenames',
 433             action='store_true', dest='restrictfilenames',
 434             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 435     filesystem.add_option('-a', '--batch-file',
 436             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 437     filesystem.add_option('--load-info',
 438             dest='load_info_filename', metavar='FILE',
 439             help='json file containing the video information (created with the "--write-json" option)')
 440     filesystem.add_option('-w', '--no-overwrites',
 441             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 442     filesystem.add_option('-c', '--continue',
 443             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 444     filesystem.add_option('--no-continue',
 445             action='store_false', dest='continue_dl',
 446             help='do not resume partially downloaded files (restart from beginning)')
 447     filesystem.add_option('--cookies',
 448             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 449     filesystem.add_option('--no-part',
 450             action='store_true', dest='nopart', help='do not use .part files', default=False)
 451     filesystem.add_option('--no-mtime',
 452             action='store_false', dest='updatetime',
 453             help='do not use the Last-modified header to set the file modification time', default=True)
 454     filesystem.add_option('--write-description',
 455             action='store_true', dest='writedescription',
 456             help='write video description to a .description file', default=False)
 457     filesystem.add_option('--write-info-json',
 458             action='store_true', dest='writeinfojson',
 459             help='write video metadata to a .info.json file', default=False)
 460     filesystem.add_option('--write-annotations',
 461             action='store_true', dest='writeannotations',
 462             help='write video annotations to a .annotation file', default=False)
 463     filesystem.add_option('--write-thumbnail',
 464             action='store_true', dest='writethumbnail',
 465             help='write thumbnail image to disk', default=False)
 466
 467
 468     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 469             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 470     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 471             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 472     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 473             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 474     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 475             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 476     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 477             help='keeps the video file on disk after the post-processing; the video is erased by default')
 478     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 479             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 480     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 481             help='embed subtitles in the video (only for mp4 videos)')
 482     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 483             help='write metadata to the video file')
 484     postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
 485             help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
 486     postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
 487         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
 488     postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
 489         help='Prefer ffmpeg over avconv for running the postprocessors')
 490
 491
 492     parser.add_option_group(general)
 493     parser.add_option_group(selection)
 494     parser.add_option_group(downloader)
 495     parser.add_option_group(filesystem)
 496     parser.add_option_group(verbosity)
 497     parser.add_option_group(video_format)
 498     parser.add_option_group(subtitles)
 499     parser.add_option_group(authentication)
 500     parser.add_option_group(postproc)
 501
 502     if overrideArguments is not None:
 503         opts, args = parser.parse_args(overrideArguments)
 504         if opts.verbose:
 505             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 506     else:
 507         commandLineConf = sys.argv[1:]
 508         if '--ignore-config' in commandLineConf:
 509             systemConf = []
 510             userConf = []
 511         else:
 512             systemConf = _readOptions('/etc/youtube-dl.conf')
 513             if '--ignore-config' in systemConf:
 514                 userConf = []
 515             else:
 516                 userConf = _readUserConf()
 517         argv = systemConf + userConf + commandLineConf
 518
 519         opts, args = parser.parse_args(argv)
 520         if opts.verbose:
 521             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 522             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 523             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 524             write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
 525                          (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
 526
 527     return parser, opts, args
 528
 529
 530 def _real_main(argv=None):
 531     # Compatibility fixes for Windows
 532     if sys.platform == 'win32':
 533         # https://github.com/rg3/youtube-dl/issues/820
 534         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 535
 536     setproctitle(u'youtube-dl')
 537
 538     parser, opts, args = parseOpts(argv)
 539
 540     # Set user agent
 541     if opts.user_agent is not None:
 542         std_headers['User-Agent'] = opts.user_agent
 543
 544     # Set referer
 545     if opts.referer is not None:
 546         std_headers['Referer'] = opts.referer
 547
 548     # Dump user agent
 549     if opts.dump_user_agent:
 550         compat_print(std_headers['User-Agent'])
 551         sys.exit(0)
 552
 553     # Batch file verification
 554     batchurls = []
 555     if opts.batchfile is not None:
 556         try:
 557             if opts.batchfile == '-':
 558                 batchfd = sys.stdin
 559             else:
 560                 batchfd = open(opts.batchfile, 'r')
 561             batchurls = batchfd.readlines()
 562             batchurls = [x.strip() for x in batchurls]
 563             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 564             if opts.verbose:
 565                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 566         except IOError:
 567             sys.exit(u'ERROR: batch file could not be read')
 568     all_urls = batchurls + args
 569     all_urls = [url.strip() for url in all_urls]
 570     _enc = preferredencoding()
 571     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 572
 573     extractors = gen_extractors()
 574
 575     if opts.list_extractors:
 576         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 577             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 578             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 579             for mu in matchedUrls:
 580                 compat_print(u'  ' + mu)
 581         sys.exit(0)
 582     if opts.list_extractor_descriptions:
 583         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 584             if not ie._WORKING:
 585                 continue
 586             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 587             if desc is False:
 588                 continue
 589             if hasattr(ie, 'SEARCH_KEY'):
 590                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 591                 _COUNTS = (u'', u'5', u'10', u'all')
 592                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 593             compat_print(desc)
 594         sys.exit(0)
 595
 596
 597     # Conflicting, missing and erroneous options
 598     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 599         parser.error(u'using .netrc conflicts with giving username/password')
 600     if opts.password is not None and opts.username is None:
 601         parser.error(u'account username missing\n')
 602     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 603         parser.error(u'using output template conflicts with using title, video ID or auto number')
 604     if opts.usetitle and opts.useid:
 605         parser.error(u'using title conflicts with using video ID')
 606     if opts.username is not None and opts.password is None:
 607         opts.password = getpass.getpass(u'Type account password and press return:')
 608     if opts.ratelimit is not None:
 609         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 610         if numeric_limit is None:
 611             parser.error(u'invalid rate limit specified')
 612         opts.ratelimit = numeric_limit
 613     if opts.min_filesize is not None:
 614         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 615         if numeric_limit is None:
 616             parser.error(u'invalid min_filesize specified')
 617         opts.min_filesize = numeric_limit
 618     if opts.max_filesize is not None:
 619         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 620         if numeric_limit is None:
 621             parser.error(u'invalid max_filesize specified')
 622         opts.max_filesize = numeric_limit
 623     if opts.retries is not None:
 624         try:
 625             opts.retries = int(opts.retries)
 626         except (TypeError, ValueError):
 627             parser.error(u'invalid retry count specified')
 628     if opts.buffersize is not None:
 629         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 630         if numeric_buffersize is None:
 631             parser.error(u'invalid buffer size specified')
 632         opts.buffersize = numeric_buffersize
 633     if opts.playliststart <= 0:
 634         raise ValueError(u'Playlist start must be positive')
 635     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 636         raise ValueError(u'Playlist end must be greater than playlist start')
 637     if opts.extractaudio:
 638         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 639             parser.error(u'invalid audio format specified')
 640     if opts.audioquality:
 641         opts.audioquality = opts.audioquality.strip('k').strip('K')
 642         if not opts.audioquality.isdigit():
 643             parser.error(u'invalid audio quality specified')
 644     if opts.recodevideo is not None:
 645         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 646             parser.error(u'invalid video recode format specified')
 647     if opts.date is not None:
 648         date = DateRange.day(opts.date)
 649     else:
 650         date = DateRange(opts.dateafter, opts.datebefore)
 651     if opts.default_search not in ('auto', None) and ':' not in opts.default_search:
 652         parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 653
 654     # Do not download videos when there are audio-only formats
 655     if opts.extractaudio and not opts.keepvideo and opts.format is None:
 656         opts.format = 'bestaudio/best'
 657
 658     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 659     # this was the old behaviour if only --all-sub was given.
 660     if opts.allsubtitles and (opts.writeautomaticsub == False):
 661         opts.writesubtitles = True
 662
 663     if sys.version_info < (3,):
 664         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 665         if opts.outtmpl is not None:
 666             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 667     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 668             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 669             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 670             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 671             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 672             or (opts.useid and u'%(id)s.%(ext)s')
 673             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 674             or u'%(title)s-%(id)s.%(ext)s')
 675     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 676         parser.error(u'Cannot download a video and extract audio into the same'
 677                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 678                      u' template'.format(outtmpl))
 679
 680     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 681     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 682
 683     ydl_opts = {
 684         'usenetrc': opts.usenetrc,
 685         'username': opts.username,
 686         'password': opts.password,
 687         'videopassword': opts.videopassword,
 688         'quiet': (opts.quiet or any_printing),
 689         'forceurl': opts.geturl,
 690         'forcetitle': opts.gettitle,
 691         'forceid': opts.getid,
 692         'forcethumbnail': opts.getthumbnail,
 693         'forcedescription': opts.getdescription,
 694         'forceduration': opts.getduration,
 695         'forcefilename': opts.getfilename,
 696         'forceformat': opts.getformat,
 697         'forcejson': opts.dumpjson,
 698         'simulate': opts.simulate,
 699         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 700         'format': opts.format,
 701         'format_limit': opts.format_limit,
 702         'listformats': opts.listformats,
 703         'outtmpl': outtmpl,
 704         'autonumber_size': opts.autonumber_size,
 705         'restrictfilenames': opts.restrictfilenames,
 706         'ignoreerrors': opts.ignoreerrors,
 707         'ratelimit': opts.ratelimit,
 708         'nooverwrites': opts.nooverwrites,
 709         'retries': opts.retries,
 710         'buffersize': opts.buffersize,
 711         'noresizebuffer': opts.noresizebuffer,
 712         'continuedl': opts.continue_dl,
 713         'noprogress': opts.noprogress,
 714         'progress_with_newline': opts.progress_with_newline,
 715         'playliststart': opts.playliststart,
 716         'playlistend': opts.playlistend,
 717         'noplaylist': opts.noplaylist,
 718         'logtostderr': opts.outtmpl == '-',
 719         'consoletitle': opts.consoletitle,
 720         'nopart': opts.nopart,
 721         'updatetime': opts.updatetime,
 722         'writedescription': opts.writedescription,
 723         'writeannotations': opts.writeannotations,
 724         'writeinfojson': opts.writeinfojson,
 725         'writethumbnail': opts.writethumbnail,
 726         'writesubtitles': opts.writesubtitles,
 727         'writeautomaticsub': opts.writeautomaticsub,
 728         'allsubtitles': opts.allsubtitles,
 729         'listsubtitles': opts.listsubtitles,
 730         'subtitlesformat': opts.subtitlesformat,
 731         'subtitleslangs': opts.subtitleslangs,
 732         'matchtitle': decodeOption(opts.matchtitle),
 733         'rejecttitle': decodeOption(opts.rejecttitle),
 734         'max_downloads': opts.max_downloads,
 735         'prefer_free_formats': opts.prefer_free_formats,
 736         'verbose': opts.verbose,
 737         'dump_intermediate_pages': opts.dump_intermediate_pages,
 738         'write_pages': opts.write_pages,
 739         'test': opts.test,
 740         'keepvideo': opts.keepvideo,
 741         'min_filesize': opts.min_filesize,
 742         'max_filesize': opts.max_filesize,
 743         'min_views': opts.min_views,
 744         'max_views': opts.max_views,
 745         'daterange': date,
 746         'cachedir': opts.cachedir,
 747         'youtube_print_sig_code': opts.youtube_print_sig_code,
 748         'age_limit': opts.age_limit,
 749         'download_archive': download_archive_fn,
 750         'cookiefile': opts.cookiefile,
 751         'nocheckcertificate': opts.no_check_certificate,
 752         'proxy': opts.proxy,
 753         'socket_timeout': opts.socket_timeout,
 754         'bidi_workaround': opts.bidi_workaround,
 755         'debug_printtraffic': opts.debug_printtraffic,
 756         'prefer_ffmpeg': opts.prefer_ffmpeg,
 757         'include_ads': opts.include_ads,
 758         'default_search': opts.default_search,
 759         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
 760     }
 761
 762     with YoutubeDL(ydl_opts) as ydl:
 763         ydl.print_debug_header()
 764         ydl.add_default_info_extractors()
 765
 766         # PostProcessors
 767         # Add the metadata pp first, the other pps will copy it
 768         if opts.addmetadata:
 769             ydl.add_post_processor(FFmpegMetadataPP())
 770         if opts.extractaudio:
 771             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 772         if opts.recodevideo:
 773             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 774         if opts.embedsubtitles:
 775             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 776         if opts.xattrs:
 777             ydl.add_post_processor(XAttrMetadataPP())
 778
 779         # Update version
 780         if opts.update_self:
 781             update_self(ydl.to_screen, opts.verbose)
 782
 783         # Maybe do nothing
 784         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 785             if not opts.update_self:
 786                 parser.error(u'you must provide at least one URL')
 787             else:
 788                 sys.exit()
 789
 790         try:
 791             if opts.load_info_filename is not None:
 792                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 793             else:
 794                 retcode = ydl.download(all_urls)
 795         except MaxDownloadsReached:
 796             ydl.to_screen(u'--max-download limit reached, aborting.')
 797             retcode = 101
 798
 799     sys.exit(retcode)
 800
 801
 802 def main(argv=None):
 803     try:
 804         _real_main(argv)
 805     except DownloadError:
 806         sys.exit(1)
 807     except SameFileError:
 808         sys.exit(u'ERROR: fixed output name but more than one file to download')
 809     except KeyboardInterrupt:
 810         sys.exit(u'\nERROR: Interrupted by user')