Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35     'Jelle van der Waa',
  36     'Marcin Cieślak',
  37     'Anton Larionov',
  38     'Takuya Tsuchida',
  39     'Sergey M.',
  40     'Michael Orlitzky',
  41 )
  42
  43 __license__ = 'Public Domain'
  44
  45 import codecs
  46 import getpass
  47 import optparse
  48 import os
  49 import random
  50 import re
  51 import shlex
  52 import sys
  53
  54
  55 from .utils import (
  56     compat_print,
  57     DateRange,
  58     decodeOption,
  59     get_term_width,
  60     DownloadError,
  61     get_cachedir,
  62     MaxDownloadsReached,
  63     preferredencoding,
  64     SameFileError,
  65     setproctitle,
  66     std_headers,
  67     write_string,
  68 )
  69 from .update import update_self
  70 from .FileDownloader import (
  71     FileDownloader,
  72 )
  73 from .extractor import gen_extractors
  74 from .version import __version__
  75 from .YoutubeDL import YoutubeDL
  76 from .PostProcessor import (
  77     FFmpegMetadataPP,
  78     FFmpegVideoConvertor,
  79     FFmpegExtractAudioPP,
  80     FFmpegEmbedSubtitlePP,
  81 )
  82
  83
  84 def parseOpts(overrideArguments=None):
  85     def _readOptions(filename_bytes, default=[]):
  86         try:
  87             optionf = open(filename_bytes)
  88         except IOError:
  89             return default  # silently skip if file is not present
  90         try:
  91             res = []
  92             for l in optionf:
  93                 res += shlex.split(l, comments=True)
  94         finally:
  95             optionf.close()
  96         return res
  97
  98     def _format_option_string(option):
  99         ''' ('-o', '--option') -> -o, --format METAVAR'''
 100
 101         opts = []
 102
 103         if option._short_opts:
 104             opts.append(option._short_opts[0])
 105         if option._long_opts:
 106             opts.append(option._long_opts[0])
 107         if len(opts) > 1:
 108             opts.insert(1, ', ')
 109
 110         if option.takes_value(): opts.append(' %s' % option.metavar)
 111
 112         return "".join(opts)
 113
 114     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 115         setattr(parser.values, option.dest, value.split(','))
 116
 117     def _hide_login_info(opts):
 118         opts = list(opts)
 119         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
 120             try:
 121                 i = opts.index(private_opt)
 122                 opts[i+1] = '<PRIVATE>'
 123             except ValueError:
 124                 pass
 125         return opts
 126
 127     max_width = 80
 128     max_help_position = 80
 129
 130     # No need to wrap help messages if we're on a wide console
 131     columns = get_term_width()
 132     if columns: max_width = columns
 133
 134     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 135     fmt.format_option_strings = _format_option_string
 136
 137     kw = {
 138         'version'   : __version__,
 139         'formatter' : fmt,
 140         'usage' : '%prog [options] url [url...]',
 141         'conflict_handler' : 'resolve',
 142     }
 143
 144     parser = optparse.OptionParser(**kw)
 145
 146     # option groups
 147     general        = optparse.OptionGroup(parser, 'General Options')
 148     selection      = optparse.OptionGroup(parser, 'Video Selection')
 149     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 150     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 151     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 152     downloader     = optparse.OptionGroup(parser, 'Download Options')
 153     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 154     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 155     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 156
 157     general.add_option('-h', '--help',
 158             action='help', help='print this help text and exit')
 159     general.add_option('-v', '--version',
 160             action='version', help='print program version and exit')
 161     general.add_option('-U', '--update',
 162             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 163     general.add_option('-i', '--ignore-errors',
 164             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 165     general.add_option('--abort-on-error',
 166             action='store_false', dest='ignoreerrors',
 167             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 168     general.add_option('--dump-user-agent',
 169             action='store_true', dest='dump_user_agent',
 170             help='display the current browser identification', default=False)
 171     general.add_option('--user-agent',
 172             dest='user_agent', help='specify a custom user agent', metavar='UA')
 173     general.add_option('--referer',
 174             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 175             metavar='REF', default=None)
 176     general.add_option('--list-extractors',
 177             action='store_true', dest='list_extractors',
 178             help='List all supported extractors and the URLs they would handle', default=False)
 179     general.add_option('--extractor-descriptions',
 180             action='store_true', dest='list_extractor_descriptions',
 181             help='Output descriptions of all supported extractors', default=False)
 182     general.add_option(
 183         '--proxy', dest='proxy', default=None, metavar='URL',
 184         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
 185     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 186     general.add_option(
 187         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 188         help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
 189     general.add_option(
 190         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 191         help='Disable filesystem caching')
 192     general.add_option(
 193         '--socket-timeout', dest='socket_timeout',
 194         type=float, default=None, help=optparse.SUPPRESS_HELP)
 195     general.add_option(
 196         '--bidi-workaround', dest='bidi_workaround', action='store_true',
 197         help=u'Work around terminals that lack bidirectional text support. Requires fribidi executable in PATH')
 198
 199
 200     selection.add_option(
 201         '--playlist-start',
 202         dest='playliststart', metavar='NUMBER', default=1, type=int,
 203         help='playlist video to start at (default is %default)')
 204     selection.add_option(
 205         '--playlist-end',
 206         dest='playlistend', metavar='NUMBER', default=None, type=int,
 207         help='playlist video to end at (default is last)')
 208     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 209     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 210     selection.add_option('--max-downloads', metavar='NUMBER',
 211                          dest='max_downloads', type=int, default=None,
 212                          help='Abort after downloading NUMBER files')
 213     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 214     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 215     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 216     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 217     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 218     selection.add_option(
 219         '--min-views', metavar='COUNT', dest='min_views',
 220         default=None, type=int,
 221         help="Do not download any videos with less than COUNT views",)
 222     selection.add_option(
 223         '--max-views', metavar='COUNT', dest='max_views',
 224         default=None, type=int,
 225         help="Do not download any videos with more than COUNT views",)
 226     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 227     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 228                          help='download only videos suitable for the given age',
 229                          default=None, type=int)
 230     selection.add_option('--download-archive', metavar='FILE',
 231                          dest='download_archive',
 232                          help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
 233
 234
 235     authentication.add_option('-u', '--username',
 236             dest='username', metavar='USERNAME', help='account username')
 237     authentication.add_option('-p', '--password',
 238             dest='password', metavar='PASSWORD', help='account password')
 239     authentication.add_option('-n', '--netrc',
 240             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 241     authentication.add_option('--video-password',
 242             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 243
 244
 245     video_format.add_option('-f', '--format',
 246             action='store', dest='format', metavar='FORMAT', default='best',
 247             help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 248     video_format.add_option('--all-formats',
 249             action='store_const', dest='format', help='download all available video formats', const='all')
 250     video_format.add_option('--prefer-free-formats',
 251             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 252     video_format.add_option('--max-quality',
 253             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 254     video_format.add_option('-F', '--list-formats',
 255             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 256
 257     subtitles.add_option('--write-sub', '--write-srt',
 258             action='store_true', dest='writesubtitles',
 259             help='write subtitle file', default=False)
 260     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 261             action='store_true', dest='writeautomaticsub',
 262             help='write automatic subtitle file (youtube only)', default=False)
 263     subtitles.add_option('--all-subs',
 264             action='store_true', dest='allsubtitles',
 265             help='downloads all the available subtitles of the video', default=False)
 266     subtitles.add_option('--list-subs',
 267             action='store_true', dest='listsubtitles',
 268             help='lists all available subtitles for the video', default=False)
 269     subtitles.add_option('--sub-format',
 270             action='store', dest='subtitlesformat', metavar='FORMAT',
 271             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 272     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 273             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 274             default=[], callback=_comma_separated_values_options_callback,
 275             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 276
 277     downloader.add_option('-r', '--rate-limit',
 278             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 279     downloader.add_option('-R', '--retries',
 280             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 281     downloader.add_option('--buffer-size',
 282             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 283     downloader.add_option('--no-resize-buffer',
 284             action='store_true', dest='noresizebuffer',
 285             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 286     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 287
 288     verbosity.add_option('-q', '--quiet',
 289             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 290     verbosity.add_option('-s', '--simulate',
 291             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 292     verbosity.add_option('--skip-download',
 293             action='store_true', dest='skip_download', help='do not download the video', default=False)
 294     verbosity.add_option('-g', '--get-url',
 295             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 296     verbosity.add_option('-e', '--get-title',
 297             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 298     verbosity.add_option('--get-id',
 299             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 300     verbosity.add_option('--get-thumbnail',
 301             action='store_true', dest='getthumbnail',
 302             help='simulate, quiet but print thumbnail URL', default=False)
 303     verbosity.add_option('--get-description',
 304             action='store_true', dest='getdescription',
 305             help='simulate, quiet but print video description', default=False)
 306     verbosity.add_option('--get-duration',
 307             action='store_true', dest='getduration',
 308             help='simulate, quiet but print video length', default=False)
 309     verbosity.add_option('--get-filename',
 310             action='store_true', dest='getfilename',
 311             help='simulate, quiet but print output filename', default=False)
 312     verbosity.add_option('--get-format',
 313             action='store_true', dest='getformat',
 314             help='simulate, quiet but print output format', default=False)
 315     verbosity.add_option('-j', '--dump-json',
 316             action='store_true', dest='dumpjson',
 317             help='simulate, quiet but print JSON information', default=False)
 318     verbosity.add_option('--newline',
 319             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 320     verbosity.add_option('--no-progress',
 321             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 322     verbosity.add_option('--console-title',
 323             action='store_true', dest='consoletitle',
 324             help='display progress in console titlebar', default=False)
 325     verbosity.add_option('-v', '--verbose',
 326             action='store_true', dest='verbose', help='print various debugging information', default=False)
 327     verbosity.add_option('--dump-intermediate-pages',
 328             action='store_true', dest='dump_intermediate_pages', default=False,
 329             help='print downloaded pages to debug problems(very verbose)')
 330     verbosity.add_option('--write-pages',
 331             action='store_true', dest='write_pages', default=False,
 332             help='Write downloaded intermediary pages to files in the current directory to debug problems')
 333     verbosity.add_option('--youtube-print-sig-code',
 334             action='store_true', dest='youtube_print_sig_code', default=False,
 335             help=optparse.SUPPRESS_HELP)
 336
 337
 338     filesystem.add_option('-t', '--title',
 339             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 340     filesystem.add_option('--id',
 341             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 342     filesystem.add_option('-l', '--literal',
 343             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 344     filesystem.add_option('-A', '--auto-number',
 345             action='store_true', dest='autonumber',
 346             help='number downloaded files starting from 00000', default=False)
 347     filesystem.add_option('-o', '--output',
 348             dest='outtmpl', metavar='TEMPLATE',
 349             help=('output filename template. Use %(title)s to get the title, '
 350                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 351                   '%(autonumber)s to get an automatically incremented number, '
 352                   '%(ext)s for the filename extension, '
 353                   '%(format)s for the format description (like "22 - 1280x720" or "HD"),'
 354                   '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"),'
 355                   '%(upload_date)s for the upload date (YYYYMMDD), '
 356                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 357                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 358                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 359                   'Use - to output to stdout. Can also be used to download to a different directory, '
 360                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 361     filesystem.add_option('--autonumber-size',
 362             dest='autonumber_size', metavar='NUMBER',
 363             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
 364     filesystem.add_option('--restrict-filenames',
 365             action='store_true', dest='restrictfilenames',
 366             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 367     filesystem.add_option('-a', '--batch-file',
 368             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 369     filesystem.add_option('--load-info',
 370             dest='load_info_filename', metavar='FILE',
 371             help='json file containing the video information (created with the "--write-json" option')
 372     filesystem.add_option('-w', '--no-overwrites',
 373             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 374     filesystem.add_option('-c', '--continue',
 375             action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
 376     filesystem.add_option('--no-continue',
 377             action='store_false', dest='continue_dl',
 378             help='do not resume partially downloaded files (restart from beginning)')
 379     filesystem.add_option('--cookies',
 380             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 381     filesystem.add_option('--no-part',
 382             action='store_true', dest='nopart', help='do not use .part files', default=False)
 383     filesystem.add_option('--no-mtime',
 384             action='store_false', dest='updatetime',
 385             help='do not use the Last-modified header to set the file modification time', default=True)
 386     filesystem.add_option('--write-description',
 387             action='store_true', dest='writedescription',
 388             help='write video description to a .description file', default=False)
 389     filesystem.add_option('--write-info-json',
 390             action='store_true', dest='writeinfojson',
 391             help='write video metadata to a .info.json file', default=False)
 392     filesystem.add_option('--write-annotations',
 393             action='store_true', dest='writeannotations',
 394             help='write video annotations to a .annotation file', default=False)
 395     filesystem.add_option('--write-thumbnail',
 396             action='store_true', dest='writethumbnail',
 397             help='write thumbnail image to disk', default=False)
 398
 399
 400     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 401             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 402     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 403             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 404     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 405             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 406     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 407             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 408     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 409             help='keeps the video file on disk after the post-processing; the video is erased by default')
 410     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 411             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 412     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 413             help='embed subtitles in the video (only for mp4 videos)')
 414     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 415             help='add metadata to the files')
 416
 417
 418     parser.add_option_group(general)
 419     parser.add_option_group(selection)
 420     parser.add_option_group(downloader)
 421     parser.add_option_group(filesystem)
 422     parser.add_option_group(verbosity)
 423     parser.add_option_group(video_format)
 424     parser.add_option_group(subtitles)
 425     parser.add_option_group(authentication)
 426     parser.add_option_group(postproc)
 427
 428     if overrideArguments is not None:
 429         opts, args = parser.parse_args(overrideArguments)
 430         if opts.verbose:
 431             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 432     else:
 433         systemConf = _readOptions('/etc/youtube-dl.conf')
 434
 435         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 436         if xdg_config_home:
 437             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 438             if not os.path.isfile(userConfFile):
 439                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 440         else:
 441             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 442             if not os.path.isfile(userConfFile):
 443                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 444         userConf = _readOptions(userConfFile, None)
 445
 446         if userConf is None:
 447             appdata_dir = os.environ.get('appdata')
 448             if appdata_dir:
 449                 userConf = _readOptions(
 450                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
 451                     default=None)
 452                 if userConf is None:
 453                     userConf = _readOptions(
 454                         os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
 455                         default=None)
 456
 457         if userConf is None:
 458             userConf = _readOptions(
 459                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
 460                 default=None)
 461         if userConf is None:
 462             userConf = _readOptions(
 463                 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
 464                 default=None)
 465
 466         if userConf is None:
 467             userConf = []
 468
 469         commandLineConf = sys.argv[1:]
 470         argv = systemConf + userConf + commandLineConf
 471         opts, args = parser.parse_args(argv)
 472         if opts.verbose:
 473             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 474             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 475             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 476
 477     return parser, opts, args
 478
 479
 480 def _real_main(argv=None):
 481     # Compatibility fixes for Windows
 482     if sys.platform == 'win32':
 483         # https://github.com/rg3/youtube-dl/issues/820
 484         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 485
 486     setproctitle(u'youtube-dl')
 487
 488     parser, opts, args = parseOpts(argv)
 489
 490     # Set user agent
 491     if opts.user_agent is not None:
 492         std_headers['User-Agent'] = opts.user_agent
 493
 494     # Set referer
 495     if opts.referer is not None:
 496         std_headers['Referer'] = opts.referer
 497
 498     # Dump user agent
 499     if opts.dump_user_agent:
 500         compat_print(std_headers['User-Agent'])
 501         sys.exit(0)
 502
 503     # Batch file verification
 504     batchurls = []
 505     if opts.batchfile is not None:
 506         try:
 507             if opts.batchfile == '-':
 508                 batchfd = sys.stdin
 509             else:
 510                 batchfd = open(opts.batchfile, 'r')
 511             batchurls = batchfd.readlines()
 512             batchurls = [x.strip() for x in batchurls]
 513             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 514             if opts.verbose:
 515                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 516         except IOError:
 517             sys.exit(u'ERROR: batch file could not be read')
 518     all_urls = batchurls + args
 519     all_urls = [url.strip() for url in all_urls]
 520
 521     extractors = gen_extractors()
 522
 523     if opts.list_extractors:
 524         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 525             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 526             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 527             for mu in matchedUrls:
 528                 compat_print(u'  ' + mu)
 529         sys.exit(0)
 530     if opts.list_extractor_descriptions:
 531         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 532             if not ie._WORKING:
 533                 continue
 534             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 535             if desc is False:
 536                 continue
 537             if hasattr(ie, 'SEARCH_KEY'):
 538                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 539                 _COUNTS = (u'', u'5', u'10', u'all')
 540                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 541             compat_print(desc)
 542         sys.exit(0)
 543
 544
 545     # Conflicting, missing and erroneous options
 546     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 547         parser.error(u'using .netrc conflicts with giving username/password')
 548     if opts.password is not None and opts.username is None:
 549         parser.error(u' account username missing\n')
 550     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 551         parser.error(u'using output template conflicts with using title, video ID or auto number')
 552     if opts.usetitle and opts.useid:
 553         parser.error(u'using title conflicts with using video ID')
 554     if opts.username is not None and opts.password is None:
 555         opts.password = getpass.getpass(u'Type account password and press return:')
 556     if opts.ratelimit is not None:
 557         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 558         if numeric_limit is None:
 559             parser.error(u'invalid rate limit specified')
 560         opts.ratelimit = numeric_limit
 561     if opts.min_filesize is not None:
 562         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 563         if numeric_limit is None:
 564             parser.error(u'invalid min_filesize specified')
 565         opts.min_filesize = numeric_limit
 566     if opts.max_filesize is not None:
 567         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 568         if numeric_limit is None:
 569             parser.error(u'invalid max_filesize specified')
 570         opts.max_filesize = numeric_limit
 571     if opts.retries is not None:
 572         try:
 573             opts.retries = int(opts.retries)
 574         except (TypeError, ValueError):
 575             parser.error(u'invalid retry count specified')
 576     if opts.buffersize is not None:
 577         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 578         if numeric_buffersize is None:
 579             parser.error(u'invalid buffer size specified')
 580         opts.buffersize = numeric_buffersize
 581     if opts.playliststart <= 0:
 582         raise ValueError(u'Playlist start must be positive')
 583     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
 584         raise ValueError(u'Playlist end must be greater than playlist start')
 585     if opts.extractaudio:
 586         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 587             parser.error(u'invalid audio format specified')
 588     if opts.audioquality:
 589         opts.audioquality = opts.audioquality.strip('k').strip('K')
 590         if not opts.audioquality.isdigit():
 591             parser.error(u'invalid audio quality specified')
 592     if opts.recodevideo is not None:
 593         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 594             parser.error(u'invalid video recode format specified')
 595     if opts.date is not None:
 596         date = DateRange.day(opts.date)
 597     else:
 598         date = DateRange(opts.dateafter, opts.datebefore)
 599
 600     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 601     # this was the old behaviour if only --all-sub was given.
 602     if opts.allsubtitles and (opts.writeautomaticsub == False):
 603         opts.writesubtitles = True
 604
 605     if sys.version_info < (3,):
 606         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 607         if opts.outtmpl is not None:
 608             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 609     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 610             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 611             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 612             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 613             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 614             or (opts.useid and u'%(id)s.%(ext)s')
 615             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 616             or u'%(title)s-%(id)s.%(ext)s')
 617     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
 618         parser.error(u'Cannot download a video and extract audio into the same'
 619                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
 620                      u' template'.format(outtmpl))
 621
 622     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
 623
 624     ydl_opts = {
 625         'usenetrc': opts.usenetrc,
 626         'username': opts.username,
 627         'password': opts.password,
 628         'videopassword': opts.videopassword,
 629         'quiet': (opts.quiet or any_printing),
 630         'forceurl': opts.geturl,
 631         'forcetitle': opts.gettitle,
 632         'forceid': opts.getid,
 633         'forcethumbnail': opts.getthumbnail,
 634         'forcedescription': opts.getdescription,
 635         'forceduration': opts.getduration,
 636         'forcefilename': opts.getfilename,
 637         'forceformat': opts.getformat,
 638         'forcejson': opts.dumpjson,
 639         'simulate': opts.simulate,
 640         'skip_download': (opts.skip_download or opts.simulate or any_printing),
 641         'format': opts.format,
 642         'format_limit': opts.format_limit,
 643         'listformats': opts.listformats,
 644         'outtmpl': outtmpl,
 645         'autonumber_size': opts.autonumber_size,
 646         'restrictfilenames': opts.restrictfilenames,
 647         'ignoreerrors': opts.ignoreerrors,
 648         'ratelimit': opts.ratelimit,
 649         'nooverwrites': opts.nooverwrites,
 650         'retries': opts.retries,
 651         'buffersize': opts.buffersize,
 652         'noresizebuffer': opts.noresizebuffer,
 653         'continuedl': opts.continue_dl,
 654         'noprogress': opts.noprogress,
 655         'progress_with_newline': opts.progress_with_newline,
 656         'playliststart': opts.playliststart,
 657         'playlistend': opts.playlistend,
 658         'noplaylist': opts.noplaylist,
 659         'logtostderr': opts.outtmpl == '-',
 660         'consoletitle': opts.consoletitle,
 661         'nopart': opts.nopart,
 662         'updatetime': opts.updatetime,
 663         'writedescription': opts.writedescription,
 664         'writeannotations': opts.writeannotations,
 665         'writeinfojson': opts.writeinfojson,
 666         'writethumbnail': opts.writethumbnail,
 667         'writesubtitles': opts.writesubtitles,
 668         'writeautomaticsub': opts.writeautomaticsub,
 669         'allsubtitles': opts.allsubtitles,
 670         'listsubtitles': opts.listsubtitles,
 671         'subtitlesformat': opts.subtitlesformat,
 672         'subtitleslangs': opts.subtitleslangs,
 673         'matchtitle': decodeOption(opts.matchtitle),
 674         'rejecttitle': decodeOption(opts.rejecttitle),
 675         'max_downloads': opts.max_downloads,
 676         'prefer_free_formats': opts.prefer_free_formats,
 677         'verbose': opts.verbose,
 678         'dump_intermediate_pages': opts.dump_intermediate_pages,
 679         'write_pages': opts.write_pages,
 680         'test': opts.test,
 681         'keepvideo': opts.keepvideo,
 682         'min_filesize': opts.min_filesize,
 683         'max_filesize': opts.max_filesize,
 684         'min_views': opts.min_views,
 685         'max_views': opts.max_views,
 686         'daterange': date,
 687         'cachedir': opts.cachedir,
 688         'youtube_print_sig_code': opts.youtube_print_sig_code,
 689         'age_limit': opts.age_limit,
 690         'download_archive': opts.download_archive,
 691         'cookiefile': opts.cookiefile,
 692         'nocheckcertificate': opts.no_check_certificate,
 693         'proxy': opts.proxy,
 694         'socket_timeout': opts.socket_timeout,
 695         'bidi_workaround': opts.bidi_workaround,
 696     }
 697
 698     with YoutubeDL(ydl_opts) as ydl:
 699         ydl.print_debug_header()
 700         ydl.add_default_info_extractors()
 701
 702         # PostProcessors
 703         # Add the metadata pp first, the other pps will copy it
 704         if opts.addmetadata:
 705             ydl.add_post_processor(FFmpegMetadataPP())
 706         if opts.extractaudio:
 707             ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 708         if opts.recodevideo:
 709             ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 710         if opts.embedsubtitles:
 711             ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 712
 713         # Update version
 714         if opts.update_self:
 715             update_self(ydl.to_screen, opts.verbose)
 716
 717         # Maybe do nothing
 718         if (len(all_urls) < 1) and (opts.load_info_filename is None):
 719             if not opts.update_self:
 720                 parser.error(u'you must provide at least one URL')
 721             else:
 722                 sys.exit()
 723
 724         try:
 725             if opts.load_info_filename is not None:
 726                 retcode = ydl.download_with_info_file(opts.load_info_filename)
 727             else:
 728                 retcode = ydl.download(all_urls)
 729         except MaxDownloadsReached:
 730             ydl.to_screen(u'--max-download limit reached, aborting.')
 731             retcode = 101
 732
 733     sys.exit(retcode)
 734
 735
 736 def main(argv=None):
 737     try:
 738         _real_main(argv)
 739     except DownloadError:
 740         sys.exit(1)
 741     except SameFileError:
 742         sys.exit(u'ERROR: fixed output name but more than one file to download')
 743     except KeyboardInterrupt:
 744         sys.exit(u'\nERROR: Interrupted by user')