Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 __authors__  = (
   5     'Ricardo Garcia Gonzalez',
   6     'Danny Colligan',
   7     'Benjamin Johnson',
   8     'Vasyl\' Vavrychuk',
   9     'Witold Baryluk',
  10     'Paweł Paprota',
  11     'Gergely Imreh',
  12     'Rogério Brito',
  13     'Philipp Hagemeister',
  14     'Sören Schulze',
  15     'Kevin Ngo',
  16     'Ori Avtalion',
  17     'shizeeg',
  18     'Filippo Valsorda',
  19     'Christian Albrecht',
  20     'Dave Vasilevsky',
  21     'Jaime Marquínez Ferrándiz',
  22     'Jeff Crouse',
  23     'Osama Khalid',
  24     'Michael Walter',
  25     'M. Yasoob Ullah Khalid',
  26     'Julien Fraichard',
  27     'Johny Mo Swag',
  28     'Axel Noack',
  29     'Albert Kim',
  30     'Pierre Rudloff',
  31     'Huarong Huo',
  32     'Ismael Mejía',
  33     'Steffan \'Ruirize\' James',
  34     'Andras Elso',
  35 )
  36
  37 __license__ = 'Public Domain'
  38
  39 import codecs
  40 import collections
  41 import getpass
  42 import optparse
  43 import os
  44 import random
  45 import re
  46 import shlex
  47 import socket
  48 import subprocess
  49 import sys
  50 import traceback
  51 import platform
  52
  53
  54 from .utils import (
  55     compat_cookiejar,
  56     compat_print,
  57     compat_str,
  58     compat_urllib_request,
  59     DateRange,
  60     decodeOption,
  61     determine_ext,
  62     DownloadError,
  63     get_cachedir,
  64     make_HTTPS_handler,
  65     MaxDownloadsReached,
  66     platform_name,
  67     preferredencoding,
  68     SameFileError,
  69     std_headers,
  70     write_string,
  71     YoutubeDLHandler,
  72 )
  73 from .update import update_self
  74 from .version import __version__
  75 from .FileDownloader import (
  76     FileDownloader,
  77 )
  78 from .extractor import gen_extractors
  79 from .YoutubeDL import YoutubeDL
  80 from .PostProcessor import (
  81     FFmpegMetadataPP,
  82     FFmpegVideoConvertor,
  83     FFmpegExtractAudioPP,
  84     FFmpegEmbedSubtitlePP,
  85 )
  86
  87
  88 def parseOpts(overrideArguments=None):
  89     def _readOptions(filename_bytes):
  90         try:
  91             optionf = open(filename_bytes)
  92         except IOError:
  93             return [] # silently skip if file is not present
  94         try:
  95             res = []
  96             for l in optionf:
  97                 res += shlex.split(l, comments=True)
  98         finally:
  99             optionf.close()
 100         return res
 101
 102     def _format_option_string(option):
 103         ''' ('-o', '--option') -> -o, --format METAVAR'''
 104
 105         opts = []
 106
 107         if option._short_opts:
 108             opts.append(option._short_opts[0])
 109         if option._long_opts:
 110             opts.append(option._long_opts[0])
 111         if len(opts) > 1:
 112             opts.insert(1, ', ')
 113
 114         if option.takes_value(): opts.append(' %s' % option.metavar)
 115
 116         return "".join(opts)
 117
 118     def _comma_separated_values_options_callback(option, opt_str, value, parser):
 119         setattr(parser.values, option.dest, value.split(','))
 120
 121     def _find_term_columns():
 122         columns = os.environ.get('COLUMNS', None)
 123         if columns:
 124             return int(columns)
 125
 126         try:
 127             sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 128             out,err = sp.communicate()
 129             return int(out.split()[1])
 130         except:
 131             pass
 132         return None
 133
 134     def _hide_login_info(opts):
 135         opts = list(opts)
 136         for private_opt in ['-p', '--password', '-u', '--username']:
 137             try:
 138                 i = opts.index(private_opt)
 139                 opts[i+1] = '<PRIVATE>'
 140             except ValueError:
 141                 pass
 142         return opts
 143
 144     max_width = 80
 145     max_help_position = 80
 146
 147     # No need to wrap help messages if we're on a wide console
 148     columns = _find_term_columns()
 149     if columns: max_width = columns
 150
 151     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
 152     fmt.format_option_strings = _format_option_string
 153
 154     kw = {
 155         'version'   : __version__,
 156         'formatter' : fmt,
 157         'usage' : '%prog [options] url [url...]',
 158         'conflict_handler' : 'resolve',
 159     }
 160
 161     parser = optparse.OptionParser(**kw)
 162
 163     # option groups
 164     general        = optparse.OptionGroup(parser, 'General Options')
 165     selection      = optparse.OptionGroup(parser, 'Video Selection')
 166     authentication = optparse.OptionGroup(parser, 'Authentication Options')
 167     video_format   = optparse.OptionGroup(parser, 'Video Format Options')
 168     subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
 169     downloader     = optparse.OptionGroup(parser, 'Download Options')
 170     postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
 171     filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
 172     verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
 173
 174     general.add_option('-h', '--help',
 175             action='help', help='print this help text and exit')
 176     general.add_option('-v', '--version',
 177             action='version', help='print program version and exit')
 178     general.add_option('-U', '--update',
 179             action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
 180     general.add_option('-i', '--ignore-errors',
 181             action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
 182     general.add_option('--abort-on-error',
 183             action='store_false', dest='ignoreerrors',
 184             help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
 185     general.add_option('--dump-user-agent',
 186             action='store_true', dest='dump_user_agent',
 187             help='display the current browser identification', default=False)
 188     general.add_option('--user-agent',
 189             dest='user_agent', help='specify a custom user agent', metavar='UA')
 190     general.add_option('--referer',
 191             dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
 192             metavar='REF', default=None)
 193     general.add_option('--list-extractors',
 194             action='store_true', dest='list_extractors',
 195             help='List all supported extractors and the URLs they would handle', default=False)
 196     general.add_option('--extractor-descriptions',
 197             action='store_true', dest='list_extractor_descriptions',
 198             help='Output descriptions of all supported extractors', default=False)
 199     general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
 200     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
 201     general.add_option(
 202         '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
 203         help='Location in the filesystem where youtube-dl can store downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl .')
 204     general.add_option(
 205         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
 206         help='Disable filesystem caching')
 207
 208
 209     selection.add_option('--playlist-start',
 210             dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1)
 211     selection.add_option('--playlist-end',
 212             dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
 213     selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
 214     selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
 215     selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
 216     selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
 217     selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
 218     selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
 219     selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None)
 220     selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None)
 221     selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
 222     selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
 223                          help='download only videos suitable for the given age',
 224                          default=None, type=int)
 225     selection.add_option('--download-archive', metavar='FILE',
 226                          dest='download_archive',
 227                          help='Download only videos not present in the archive file. Record all downloaded videos in it.')
 228
 229
 230     authentication.add_option('-u', '--username',
 231             dest='username', metavar='USERNAME', help='account username')
 232     authentication.add_option('-p', '--password',
 233             dest='password', metavar='PASSWORD', help='account password')
 234     authentication.add_option('-n', '--netrc',
 235             action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
 236     authentication.add_option('--video-password',
 237             dest='videopassword', metavar='PASSWORD', help='video password (vimeo only)')
 238
 239
 240     video_format.add_option('-f', '--format',
 241             action='store', dest='format', metavar='FORMAT', default='best',
 242             help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
 243     video_format.add_option('--all-formats',
 244             action='store_const', dest='format', help='download all available video formats', const='all')
 245     video_format.add_option('--prefer-free-formats',
 246             action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
 247     video_format.add_option('--max-quality',
 248             action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
 249     video_format.add_option('-F', '--list-formats',
 250             action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
 251
 252     subtitles.add_option('--write-sub', '--write-srt',
 253             action='store_true', dest='writesubtitles',
 254             help='write subtitle file', default=False)
 255     subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
 256             action='store_true', dest='writeautomaticsub',
 257             help='write automatic subtitle file (youtube only)', default=False)
 258     subtitles.add_option('--all-subs',
 259             action='store_true', dest='allsubtitles',
 260             help='downloads all the available subtitles of the video', default=False)
 261     subtitles.add_option('--list-subs',
 262             action='store_true', dest='listsubtitles',
 263             help='lists all available subtitles for the video', default=False)
 264     subtitles.add_option('--sub-format',
 265             action='store', dest='subtitlesformat', metavar='FORMAT',
 266             help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
 267     subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
 268             action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
 269             default=[], callback=_comma_separated_values_options_callback,
 270             help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 271
 272     downloader.add_option('-r', '--rate-limit',
 273             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
 274     downloader.add_option('-R', '--retries',
 275             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
 276     downloader.add_option('--buffer-size',
 277             dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
 278     downloader.add_option('--no-resize-buffer',
 279             action='store_true', dest='noresizebuffer',
 280             help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 281     downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
 282
 283     verbosity.add_option('-q', '--quiet',
 284             action='store_true', dest='quiet', help='activates quiet mode', default=False)
 285     verbosity.add_option('-s', '--simulate',
 286             action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
 287     verbosity.add_option('--skip-download',
 288             action='store_true', dest='skip_download', help='do not download the video', default=False)
 289     verbosity.add_option('-g', '--get-url',
 290             action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
 291     verbosity.add_option('-e', '--get-title',
 292             action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
 293     verbosity.add_option('--get-id',
 294             action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
 295     verbosity.add_option('--get-thumbnail',
 296             action='store_true', dest='getthumbnail',
 297             help='simulate, quiet but print thumbnail URL', default=False)
 298     verbosity.add_option('--get-description',
 299             action='store_true', dest='getdescription',
 300             help='simulate, quiet but print video description', default=False)
 301     verbosity.add_option('--get-filename',
 302             action='store_true', dest='getfilename',
 303             help='simulate, quiet but print output filename', default=False)
 304     verbosity.add_option('--get-format',
 305             action='store_true', dest='getformat',
 306             help='simulate, quiet but print output format', default=False)
 307     verbosity.add_option('--newline',
 308             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
 309     verbosity.add_option('--no-progress',
 310             action='store_true', dest='noprogress', help='do not print progress bar', default=False)
 311     verbosity.add_option('--console-title',
 312             action='store_true', dest='consoletitle',
 313             help='display progress in console titlebar', default=False)
 314     verbosity.add_option('-v', '--verbose',
 315             action='store_true', dest='verbose', help='print various debugging information', default=False)
 316     verbosity.add_option('--dump-intermediate-pages',
 317             action='store_true', dest='dump_intermediate_pages', default=False,
 318             help='print downloaded pages to debug problems(very verbose)')
 319     verbosity.add_option('--youtube-print-sig-code',
 320             action='store_true', dest='youtube_print_sig_code', default=False,
 321             help=optparse.SUPPRESS_HELP)
 322
 323
 324     filesystem.add_option('-t', '--title',
 325             action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
 326     filesystem.add_option('--id',
 327             action='store_true', dest='useid', help='use only video ID in file name', default=False)
 328     filesystem.add_option('-l', '--literal',
 329             action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
 330     filesystem.add_option('-A', '--auto-number',
 331             action='store_true', dest='autonumber',
 332             help='number downloaded files starting from 00000', default=False)
 333     filesystem.add_option('-o', '--output',
 334             dest='outtmpl', metavar='TEMPLATE',
 335             help=('output filename template. Use %(title)s to get the title, '
 336                   '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
 337                   '%(autonumber)s to get an automatically incremented number, '
 338                   '%(ext)s for the filename extension, '
 339                   '%(format)s for the format description (like "22 - 1280x720" or "HD")'
 340                   '%(upload_date)s for the upload date (YYYYMMDD), '
 341                   '%(extractor)s for the provider (youtube, metacafe, etc), '
 342                   '%(id)s for the video id , %(playlist)s for the playlist the video is in, '
 343                   '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
 344                   'Use - to output to stdout. Can also be used to download to a different directory, '
 345                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
 346     filesystem.add_option('--autonumber-size',
 347             dest='autonumber_size', metavar='NUMBER',
 348             help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
 349     filesystem.add_option('--restrict-filenames',
 350             action='store_true', dest='restrictfilenames',
 351             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
 352     filesystem.add_option('-a', '--batch-file',
 353             dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
 354     filesystem.add_option('-w', '--no-overwrites',
 355             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
 356     filesystem.add_option('-c', '--continue',
 357             action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
 358     filesystem.add_option('--no-continue',
 359             action='store_false', dest='continue_dl',
 360             help='do not resume partially downloaded files (restart from beginning)')
 361     filesystem.add_option('--cookies',
 362             dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
 363     filesystem.add_option('--no-part',
 364             action='store_true', dest='nopart', help='do not use .part files', default=False)
 365     filesystem.add_option('--no-mtime',
 366             action='store_false', dest='updatetime',
 367             help='do not use the Last-modified header to set the file modification time', default=True)
 368     filesystem.add_option('--write-description',
 369             action='store_true', dest='writedescription',
 370             help='write video description to a .description file', default=False)
 371     filesystem.add_option('--write-info-json',
 372             action='store_true', dest='writeinfojson',
 373             help='write video metadata to a .info.json file', default=False)
 374     filesystem.add_option('--write-annotations',
 375             action='store_true', dest='writeannotations',
 376             help='write video annotations to a .annotation file', default=False)
 377     filesystem.add_option('--write-thumbnail',
 378             action='store_true', dest='writethumbnail',
 379             help='write thumbnail image to disk', default=False)
 380
 381
 382     postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
 383             help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
 384     postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
 385             help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
 386     postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
 387             help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
 388     postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
 389             help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
 390     postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
 391             help='keeps the video file on disk after the post-processing; the video is erased by default')
 392     postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
 393             help='do not overwrite post-processed files; the post-processed files are overwritten by default')
 394     postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
 395             help='embed subtitles in the video (only for mp4 videos)')
 396     postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
 397             help='add metadata to the files')
 398
 399
 400     parser.add_option_group(general)
 401     parser.add_option_group(selection)
 402     parser.add_option_group(downloader)
 403     parser.add_option_group(filesystem)
 404     parser.add_option_group(verbosity)
 405     parser.add_option_group(video_format)
 406     parser.add_option_group(subtitles)
 407     parser.add_option_group(authentication)
 408     parser.add_option_group(postproc)
 409
 410     if overrideArguments is not None:
 411         opts, args = parser.parse_args(overrideArguments)
 412         if opts.verbose:
 413             write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
 414     else:
 415         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
 416         if xdg_config_home:
 417             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
 418             if not os.path.isfile(userConfFile):
 419                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
 420         else:
 421             userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
 422             if not os.path.isfile(userConfFile):
 423                 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
 424         systemConf = _readOptions('/etc/youtube-dl.conf')
 425         userConf = _readOptions(userConfFile)
 426         commandLineConf = sys.argv[1:]
 427         argv = systemConf + userConf + commandLineConf
 428         opts, args = parser.parse_args(argv)
 429         if opts.verbose:
 430             write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
 431             write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
 432             write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 433
 434     return parser, opts, args
 435
 436 def _real_main(argv=None):
 437     # Compatibility fixes for Windows
 438     if sys.platform == 'win32':
 439         # https://github.com/rg3/youtube-dl/issues/820
 440         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 441
 442     parser, opts, args = parseOpts(argv)
 443
 444     # Open appropriate CookieJar
 445     if opts.cookiefile is None:
 446         jar = compat_cookiejar.CookieJar()
 447     else:
 448         try:
 449             jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile)
 450             if os.access(opts.cookiefile, os.R_OK):
 451                 jar.load()
 452         except (IOError, OSError) as err:
 453             if opts.verbose:
 454                 traceback.print_exc()
 455             write_string(u'ERROR: unable to open cookie file\n')
 456             sys.exit(101)
 457     # Set user agent
 458     if opts.user_agent is not None:
 459         std_headers['User-Agent'] = opts.user_agent
 460
 461     # Set referer
 462     if opts.referer is not None:
 463         std_headers['Referer'] = opts.referer
 464
 465     # Dump user agent
 466     if opts.dump_user_agent:
 467         compat_print(std_headers['User-Agent'])
 468         sys.exit(0)
 469
 470     # Batch file verification
 471     batchurls = []
 472     if opts.batchfile is not None:
 473         try:
 474             if opts.batchfile == '-':
 475                 batchfd = sys.stdin
 476             else:
 477                 batchfd = open(opts.batchfile, 'r')
 478             batchurls = batchfd.readlines()
 479             batchurls = [x.strip() for x in batchurls]
 480             batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
 481             if opts.verbose:
 482                 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
 483         except IOError:
 484             sys.exit(u'ERROR: batch file could not be read')
 485     all_urls = batchurls + args
 486     all_urls = [url.strip() for url in all_urls]
 487
 488     opener = _setup_opener(jar=jar, opts=opts)
 489
 490     extractors = gen_extractors()
 491
 492     if opts.list_extractors:
 493         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 494             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
 495             matchedUrls = [url for url in all_urls if ie.suitable(url)]
 496             all_urls = [url for url in all_urls if url not in matchedUrls]
 497             for mu in matchedUrls:
 498                 compat_print(u'  ' + mu)
 499         sys.exit(0)
 500     if opts.list_extractor_descriptions:
 501         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
 502             if not ie._WORKING:
 503                 continue
 504             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
 505             if desc is False:
 506                 continue
 507             if hasattr(ie, 'SEARCH_KEY'):
 508                 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
 509                 _COUNTS = (u'', u'5', u'10', u'all')
 510                 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
 511             compat_print(desc)
 512         sys.exit(0)
 513
 514
 515     # Conflicting, missing and erroneous options
 516     if opts.usenetrc and (opts.username is not None or opts.password is not None):
 517         parser.error(u'using .netrc conflicts with giving username/password')
 518     if opts.password is not None and opts.username is None:
 519         parser.error(u' account username missing\n')
 520     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
 521         parser.error(u'using output template conflicts with using title, video ID or auto number')
 522     if opts.usetitle and opts.useid:
 523         parser.error(u'using title conflicts with using video ID')
 524     if opts.username is not None and opts.password is None:
 525         opts.password = getpass.getpass(u'Type account password and press return:')
 526     if opts.ratelimit is not None:
 527         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
 528         if numeric_limit is None:
 529             parser.error(u'invalid rate limit specified')
 530         opts.ratelimit = numeric_limit
 531     if opts.min_filesize is not None:
 532         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
 533         if numeric_limit is None:
 534             parser.error(u'invalid min_filesize specified')
 535         opts.min_filesize = numeric_limit
 536     if opts.max_filesize is not None:
 537         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
 538         if numeric_limit is None:
 539             parser.error(u'invalid max_filesize specified')
 540         opts.max_filesize = numeric_limit
 541     if opts.retries is not None:
 542         try:
 543             opts.retries = int(opts.retries)
 544         except (TypeError, ValueError) as err:
 545             parser.error(u'invalid retry count specified')
 546     if opts.buffersize is not None:
 547         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
 548         if numeric_buffersize is None:
 549             parser.error(u'invalid buffer size specified')
 550         opts.buffersize = numeric_buffersize
 551     try:
 552         opts.playliststart = int(opts.playliststart)
 553         if opts.playliststart <= 0:
 554             raise ValueError(u'Playlist start must be positive')
 555     except (TypeError, ValueError) as err:
 556         parser.error(u'invalid playlist start number specified')
 557     try:
 558         opts.playlistend = int(opts.playlistend)
 559         if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
 560             raise ValueError(u'Playlist end must be greater than playlist start')
 561     except (TypeError, ValueError) as err:
 562         parser.error(u'invalid playlist end number specified')
 563     if opts.extractaudio:
 564         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
 565             parser.error(u'invalid audio format specified')
 566     if opts.audioquality:
 567         opts.audioquality = opts.audioquality.strip('k').strip('K')
 568         if not opts.audioquality.isdigit():
 569             parser.error(u'invalid audio quality specified')
 570     if opts.recodevideo is not None:
 571         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
 572             parser.error(u'invalid video recode format specified')
 573     if opts.date is not None:
 574         date = DateRange.day(opts.date)
 575     else:
 576         date = DateRange(opts.dateafter, opts.datebefore)
 577
 578     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
 579     # this was the old behaviour if only --all-sub was given.
 580     if opts.allsubtitles and (opts.writeautomaticsub == False):
 581         opts.writesubtitles = True
 582
 583     if sys.version_info < (3,):
 584         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
 585         if opts.outtmpl is not None:
 586             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
 587     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
 588             or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
 589             or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
 590             or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
 591             or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
 592             or (opts.useid and u'%(id)s.%(ext)s')
 593             or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
 594             or u'%(title)s-%(id)s.%(ext)s')
 595     if '%(ext)s' not in outtmpl and opts.extractaudio:
 596         parser.error(u'Cannot download a video and extract audio into the same'
 597                      u' file! Use "%%(ext)s" instead of %r' %
 598                      determine_ext(outtmpl, u''))
 599
 600     # YoutubeDL
 601     ydl = YoutubeDL({
 602         'usenetrc': opts.usenetrc,
 603         'username': opts.username,
 604         'password': opts.password,
 605         'videopassword': opts.videopassword,
 606         'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 607         'forceurl': opts.geturl,
 608         'forcetitle': opts.gettitle,
 609         'forceid': opts.getid,
 610         'forcethumbnail': opts.getthumbnail,
 611         'forcedescription': opts.getdescription,
 612         'forcefilename': opts.getfilename,
 613         'forceformat': opts.getformat,
 614         'simulate': opts.simulate,
 615         'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
 616         'format': opts.format,
 617         'format_limit': opts.format_limit,
 618         'listformats': opts.listformats,
 619         'outtmpl': outtmpl,
 620         'autonumber_size': opts.autonumber_size,
 621         'restrictfilenames': opts.restrictfilenames,
 622         'ignoreerrors': opts.ignoreerrors,
 623         'ratelimit': opts.ratelimit,
 624         'nooverwrites': opts.nooverwrites,
 625         'retries': opts.retries,
 626         'buffersize': opts.buffersize,
 627         'noresizebuffer': opts.noresizebuffer,
 628         'continuedl': opts.continue_dl,
 629         'noprogress': opts.noprogress,
 630         'progress_with_newline': opts.progress_with_newline,
 631         'playliststart': opts.playliststart,
 632         'playlistend': opts.playlistend,
 633         'noplaylist': opts.noplaylist,
 634         'logtostderr': opts.outtmpl == '-',
 635         'consoletitle': opts.consoletitle,
 636         'nopart': opts.nopart,
 637         'updatetime': opts.updatetime,
 638         'writedescription': opts.writedescription,
 639         'writeannotations': opts.writeannotations,
 640         'writeinfojson': opts.writeinfojson,
 641         'writethumbnail': opts.writethumbnail,
 642         'writesubtitles': opts.writesubtitles,
 643         'writeautomaticsub': opts.writeautomaticsub,
 644         'allsubtitles': opts.allsubtitles,
 645         'listsubtitles': opts.listsubtitles,
 646         'subtitlesformat': opts.subtitlesformat,
 647         'subtitleslangs': opts.subtitleslangs,
 648         'matchtitle': decodeOption(opts.matchtitle),
 649         'rejecttitle': decodeOption(opts.rejecttitle),
 650         'max_downloads': opts.max_downloads,
 651         'prefer_free_formats': opts.prefer_free_formats,
 652         'verbose': opts.verbose,
 653         'dump_intermediate_pages': opts.dump_intermediate_pages,
 654         'test': opts.test,
 655         'keepvideo': opts.keepvideo,
 656         'min_filesize': opts.min_filesize,
 657         'max_filesize': opts.max_filesize,
 658         'daterange': date,
 659         'cachedir': opts.cachedir,
 660         'youtube_print_sig_code': opts.youtube_print_sig_code,
 661         'age_limit': opts.age_limit,
 662         'download_archive': opts.download_archive,
 663         })
 664
 665     if opts.verbose:
 666         write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
 667         try:
 668             sp = subprocess.Popen(
 669                 ['git', 'rev-parse', '--short', 'HEAD'],
 670                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 671                 cwd=os.path.dirname(os.path.abspath(__file__)))
 672             out, err = sp.communicate()
 673             out = out.decode().strip()
 674             if re.match('[0-9a-f]+', out):
 675                 write_string(u'[debug] Git HEAD: ' + out + u'\n')
 676         except:
 677             try:
 678                 sys.exc_clear()
 679             except:
 680                 pass
 681         write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
 682
 683         proxy_map = {}
 684         for handler in opener.handlers:
 685             if hasattr(handler, 'proxies'):
 686                 proxy_map.update(handler.proxies)
 687         write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
 688
 689     ydl.add_default_info_extractors()
 690
 691     # PostProcessors
 692     # Add the metadata pp first, the other pps will copy it
 693     if opts.addmetadata:
 694         ydl.add_post_processor(FFmpegMetadataPP())
 695     if opts.extractaudio:
 696         ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
 697     if opts.recodevideo:
 698         ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
 699     if opts.embedsubtitles:
 700         ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 701
 702     # Update version
 703     if opts.update_self:
 704         update_self(ydl.to_screen, opts.verbose)
 705
 706     # Maybe do nothing
 707     if len(all_urls) < 1:
 708         if not opts.update_self:
 709             parser.error(u'you must provide at least one URL')
 710         else:
 711             sys.exit()
 712
 713     try:
 714         retcode = ydl.download(all_urls)
 715     except MaxDownloadsReached:
 716         ydl.to_screen(u'--max-download limit reached, aborting.')
 717         retcode = 101
 718
 719     # Dump cookie jar if requested
 720     if opts.cookiefile is not None:
 721         try:
 722             jar.save()
 723         except (IOError, OSError):
 724             sys.exit(u'ERROR: unable to save cookie jar')
 725
 726     sys.exit(retcode)
 727
 728
 729 def _setup_opener(jar=None, opts=None, timeout=300):
 730     if opts is None:
 731         FakeOptions = collections.namedtuple(
 732             'FakeOptions', ['proxy', 'no_check_certificate'])
 733         opts = FakeOptions(proxy=None, no_check_certificate=False)
 734
 735     cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
 736     if opts.proxy is not None:
 737         if opts.proxy == '':
 738             proxies = {}
 739         else:
 740             proxies = {'http': opts.proxy, 'https': opts.proxy}
 741     else:
 742         proxies = compat_urllib_request.getproxies()
 743         # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
 744         if 'http' in proxies and 'https' not in proxies:
 745             proxies['https'] = proxies['http']
 746     proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 747     https_handler = make_HTTPS_handler(opts)
 748     opener = compat_urllib_request.build_opener(
 749         https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
 750     # Delete the default user-agent header, which would otherwise apply in
 751     # cases where our custom HTTP handler doesn't come into play
 752     # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
 753     opener.addheaders = []
 754     compat_urllib_request.install_opener(opener)
 755     socket.setdefaulttimeout(timeout)
 756     return opener
 757
 758
 759 def main(argv=None):
 760     try:
 761         _real_main(argv)
 762     except DownloadError:
 763         sys.exit(1)
 764     except SameFileError:
 765         sys.exit(u'ERROR: fixed output name but more than one file to download')
 766     except KeyboardInterrupt:
 767         sys.exit(u'\nERROR: Interrupted by user')