]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py
82b1ff4f45e5ad032493093101f0c9439c4590a4
[youtubedl] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __authors__ = (
5 'Ricardo Garcia Gonzalez',
6 'Danny Colligan',
7 'Benjamin Johnson',
8 'Vasyl\' Vavrychuk',
9 'Witold Baryluk',
10 'Paweł Paprota',
11 'Gergely Imreh',
12 'Rogério Brito',
13 'Philipp Hagemeister',
14 'Sören Schulze',
15 'Kevin Ngo',
16 'Ori Avtalion',
17 'shizeeg',
18 'Filippo Valsorda',
19 'Christian Albrecht',
20 'Dave Vasilevsky',
21 'Jaime Marquínez Ferrándiz',
22 'Jeff Crouse',
23 'Osama Khalid',
24 'Michael Walter',
25 'M. Yasoob Ullah Khalid',
26 'Julien Fraichard',
27 'Johny Mo Swag',
28 'Axel Noack',
29 'Albert Kim',
30 'Pierre Rudloff',
31 'Huarong Huo',
32 'Ismael Mejía',
33 'Steffan \'Ruirize\' James',
34 'Andras Elso',
35 'Jelle van der Waa',
36 'Marcin Cieślak',
37 'Anton Larionov',
38 'Takuya Tsuchida',
39 'Sergey M.',
40 'Michael Orlitzky',
41 'Chris Gahan',
42 'Saimadhav Heblikar',
43 )
44
45 __license__ = 'Public Domain'
46
47 import codecs
48 import getpass
49 import locale
50 import optparse
51 import os
52 import random
53 import re
54 import shlex
55 import sys
56
57
58 from .utils import (
59 compat_print,
60 DateRange,
61 decodeOption,
62 get_term_width,
63 DownloadError,
64 get_cachedir,
65 MaxDownloadsReached,
66 preferredencoding,
67 SameFileError,
68 setproctitle,
69 std_headers,
70 write_string,
71 )
72 from .update import update_self
73 from .FileDownloader import (
74 FileDownloader,
75 )
76 from .extractor import gen_extractors
77 from .version import __version__
78 from .YoutubeDL import YoutubeDL
79 from .postprocessor import (
80 FFmpegMetadataPP,
81 FFmpegVideoConvertor,
82 FFmpegExtractAudioPP,
83 FFmpegEmbedSubtitlePP,
84 XAttrMetadataPP,
85 )
86
87
88 def parseOpts(overrideArguments=None):
89 def _readOptions(filename_bytes, default=[]):
90 try:
91 optionf = open(filename_bytes)
92 except IOError:
93 return default # silently skip if file is not present
94 try:
95 res = []
96 for l in optionf:
97 res += shlex.split(l, comments=True)
98 finally:
99 optionf.close()
100 return res
101
102 def _format_option_string(option):
103 ''' ('-o', '--option') -> -o, --format METAVAR'''
104
105 opts = []
106
107 if option._short_opts:
108 opts.append(option._short_opts[0])
109 if option._long_opts:
110 opts.append(option._long_opts[0])
111 if len(opts) > 1:
112 opts.insert(1, ', ')
113
114 if option.takes_value(): opts.append(' %s' % option.metavar)
115
116 return "".join(opts)
117
118 def _comma_separated_values_options_callback(option, opt_str, value, parser):
119 setattr(parser.values, option.dest, value.split(','))
120
121 def _hide_login_info(opts):
122 opts = list(opts)
123 for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
124 try:
125 i = opts.index(private_opt)
126 opts[i+1] = '<PRIVATE>'
127 except ValueError:
128 pass
129 return opts
130
131 max_width = 80
132 max_help_position = 80
133
134 # No need to wrap help messages if we're on a wide console
135 columns = get_term_width()
136 if columns: max_width = columns
137
138 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
139 fmt.format_option_strings = _format_option_string
140
141 kw = {
142 'version' : __version__,
143 'formatter' : fmt,
144 'usage' : '%prog [options] url [url...]',
145 'conflict_handler' : 'resolve',
146 }
147
148 parser = optparse.OptionParser(**kw)
149
150 # option groups
151 general = optparse.OptionGroup(parser, 'General Options')
152 selection = optparse.OptionGroup(parser, 'Video Selection')
153 authentication = optparse.OptionGroup(parser, 'Authentication Options')
154 video_format = optparse.OptionGroup(parser, 'Video Format Options')
155 subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
156 downloader = optparse.OptionGroup(parser, 'Download Options')
157 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
158 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
159 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
160
161 general.add_option('-h', '--help',
162 action='help', help='print this help text and exit')
163 general.add_option('-v', '--version',
164 action='version', help='print program version and exit')
165 general.add_option('-U', '--update',
166 action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
167 general.add_option('-i', '--ignore-errors',
168 action='store_true', dest='ignoreerrors', help='continue on download errors, for example to to skip unavailable videos in a playlist', default=False)
169 general.add_option('--abort-on-error',
170 action='store_false', dest='ignoreerrors',
171 help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
172 general.add_option('--dump-user-agent',
173 action='store_true', dest='dump_user_agent',
174 help='display the current browser identification', default=False)
175 general.add_option('--user-agent',
176 dest='user_agent', help='specify a custom user agent', metavar='UA')
177 general.add_option('--referer',
178 dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
179 metavar='REF', default=None)
180 general.add_option('--list-extractors',
181 action='store_true', dest='list_extractors',
182 help='List all supported extractors and the URLs they would handle', default=False)
183 general.add_option('--extractor-descriptions',
184 action='store_true', dest='list_extractor_descriptions',
185 help='Output descriptions of all supported extractors', default=False)
186 general.add_option(
187 '--proxy', dest='proxy', default=None, metavar='URL',
188 help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
189 general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
190 general.add_option(
191 '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
192 help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
193 general.add_option(
194 '--no-cache-dir', action='store_const', const=None, dest='cachedir',
195 help='Disable filesystem caching')
196 general.add_option(
197 '--socket-timeout', dest='socket_timeout',
198 type=float, default=None, help=u'Time to wait before giving up, in seconds')
199 general.add_option(
200 '--bidi-workaround', dest='bidi_workaround', action='store_true',
201 help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
202
203
204 selection.add_option(
205 '--playlist-start',
206 dest='playliststart', metavar='NUMBER', default=1, type=int,
207 help='playlist video to start at (default is %default)')
208 selection.add_option(
209 '--playlist-end',
210 dest='playlistend', metavar='NUMBER', default=None, type=int,
211 help='playlist video to end at (default is last)')
212 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
213 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
214 selection.add_option('--max-downloads', metavar='NUMBER',
215 dest='max_downloads', type=int, default=None,
216 help='Abort after downloading NUMBER files')
217 selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
218 selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
219 selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
220 selection.add_option(
221 '--datebefore', metavar='DATE', dest='datebefore', default=None,
222 help='download only videos uploaded on or before this date (i.e. inclusive)')
223 selection.add_option(
224 '--dateafter', metavar='DATE', dest='dateafter', default=None,
225 help='download only videos uploaded on or after this date (i.e. inclusive)')
226 selection.add_option(
227 '--min-views', metavar='COUNT', dest='min_views',
228 default=None, type=int,
229 help="Do not download any videos with less than COUNT views",)
230 selection.add_option(
231 '--max-views', metavar='COUNT', dest='max_views',
232 default=None, type=int,
233 help="Do not download any videos with more than COUNT views",)
234 selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
235 selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
236 help='download only videos suitable for the given age',
237 default=None, type=int)
238 selection.add_option('--download-archive', metavar='FILE',
239 dest='download_archive',
240 help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
241
242
243 authentication.add_option('-u', '--username',
244 dest='username', metavar='USERNAME', help='account username')
245 authentication.add_option('-p', '--password',
246 dest='password', metavar='PASSWORD', help='account password')
247 authentication.add_option('-n', '--netrc',
248 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
249 authentication.add_option('--video-password',
250 dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
251
252
253 video_format.add_option('-f', '--format',
254 action='store', dest='format', metavar='FORMAT', default='best',
255 help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported')
256 video_format.add_option('--all-formats',
257 action='store_const', dest='format', help='download all available video formats', const='all')
258 video_format.add_option('--prefer-free-formats',
259 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
260 video_format.add_option('--max-quality',
261 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
262 video_format.add_option('-F', '--list-formats',
263 action='store_true', dest='listformats', help='list all available formats')
264
265 subtitles.add_option('--write-sub', '--write-srt',
266 action='store_true', dest='writesubtitles',
267 help='write subtitle file', default=False)
268 subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
269 action='store_true', dest='writeautomaticsub',
270 help='write automatic subtitle file (youtube only)', default=False)
271 subtitles.add_option('--all-subs',
272 action='store_true', dest='allsubtitles',
273 help='downloads all the available subtitles of the video', default=False)
274 subtitles.add_option('--list-subs',
275 action='store_true', dest='listsubtitles',
276 help='lists all available subtitles for the video', default=False)
277 subtitles.add_option('--sub-format',
278 action='store', dest='subtitlesformat', metavar='FORMAT',
279 help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
280 subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
281 action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
282 default=[], callback=_comma_separated_values_options_callback,
283 help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
284
285 downloader.add_option('-r', '--rate-limit',
286 dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
287 downloader.add_option('-R', '--retries',
288 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
289 downloader.add_option('--buffer-size',
290 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
291 downloader.add_option('--no-resize-buffer',
292 action='store_true', dest='noresizebuffer',
293 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
294 downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
295
296 verbosity.add_option('-q', '--quiet',
297 action='store_true', dest='quiet', help='activates quiet mode', default=False)
298 verbosity.add_option('-s', '--simulate',
299 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
300 verbosity.add_option('--skip-download',
301 action='store_true', dest='skip_download', help='do not download the video', default=False)
302 verbosity.add_option('-g', '--get-url',
303 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
304 verbosity.add_option('-e', '--get-title',
305 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
306 verbosity.add_option('--get-id',
307 action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
308 verbosity.add_option('--get-thumbnail',
309 action='store_true', dest='getthumbnail',
310 help='simulate, quiet but print thumbnail URL', default=False)
311 verbosity.add_option('--get-description',
312 action='store_true', dest='getdescription',
313 help='simulate, quiet but print video description', default=False)
314 verbosity.add_option('--get-duration',
315 action='store_true', dest='getduration',
316 help='simulate, quiet but print video length', default=False)
317 verbosity.add_option('--get-filename',
318 action='store_true', dest='getfilename',
319 help='simulate, quiet but print output filename', default=False)
320 verbosity.add_option('--get-format',
321 action='store_true', dest='getformat',
322 help='simulate, quiet but print output format', default=False)
323 verbosity.add_option('-j', '--dump-json',
324 action='store_true', dest='dumpjson',
325 help='simulate, quiet but print JSON information', default=False)
326 verbosity.add_option('--newline',
327 action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
328 verbosity.add_option('--no-progress',
329 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
330 verbosity.add_option('--console-title',
331 action='store_true', dest='consoletitle',
332 help='display progress in console titlebar', default=False)
333 verbosity.add_option('-v', '--verbose',
334 action='store_true', dest='verbose', help='print various debugging information', default=False)
335 verbosity.add_option('--dump-intermediate-pages',
336 action='store_true', dest='dump_intermediate_pages', default=False,
337 help='print downloaded pages to debug problems (very verbose)')
338 verbosity.add_option('--write-pages',
339 action='store_true', dest='write_pages', default=False,
340 help='Write downloaded intermediary pages to files in the current directory to debug problems')
341 verbosity.add_option('--youtube-print-sig-code',
342 action='store_true', dest='youtube_print_sig_code', default=False,
343 help=optparse.SUPPRESS_HELP)
344 verbosity.add_option('--print-traffic',
345 dest='debug_printtraffic', action='store_true', default=False,
346 help=optparse.SUPPRESS_HELP)
347
348 filesystem.add_option('-t', '--title',
349 action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
350 filesystem.add_option('--id',
351 action='store_true', dest='useid', help='use only video ID in file name', default=False)
352 filesystem.add_option('-l', '--literal',
353 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
354 filesystem.add_option('-A', '--auto-number',
355 action='store_true', dest='autonumber',
356 help='number downloaded files starting from 00000', default=False)
357 filesystem.add_option('-o', '--output',
358 dest='outtmpl', metavar='TEMPLATE',
359 help=('output filename template. Use %(title)s to get the title, '
360 '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
361 '%(autonumber)s to get an automatically incremented number, '
362 '%(ext)s for the filename extension, '
363 '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
364 '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
365 '%(upload_date)s for the upload date (YYYYMMDD), '
366 '%(extractor)s for the provider (youtube, metacafe, etc), '
367 '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
368 '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
369 'Use - to output to stdout. Can also be used to download to a different directory, '
370 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
371 filesystem.add_option('--autonumber-size',
372 dest='autonumber_size', metavar='NUMBER',
373 help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
374 filesystem.add_option('--restrict-filenames',
375 action='store_true', dest='restrictfilenames',
376 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
377 filesystem.add_option('-a', '--batch-file',
378 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
379 filesystem.add_option('--load-info',
380 dest='load_info_filename', metavar='FILE',
381 help='json file containing the video information (created with the "--write-json" option)')
382 filesystem.add_option('-w', '--no-overwrites',
383 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
384 filesystem.add_option('-c', '--continue',
385 action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
386 filesystem.add_option('--no-continue',
387 action='store_false', dest='continue_dl',
388 help='do not resume partially downloaded files (restart from beginning)')
389 filesystem.add_option('--cookies',
390 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
391 filesystem.add_option('--no-part',
392 action='store_true', dest='nopart', help='do not use .part files', default=False)
393 filesystem.add_option('--no-mtime',
394 action='store_false', dest='updatetime',
395 help='do not use the Last-modified header to set the file modification time', default=True)
396 filesystem.add_option('--write-description',
397 action='store_true', dest='writedescription',
398 help='write video description to a .description file', default=False)
399 filesystem.add_option('--write-info-json',
400 action='store_true', dest='writeinfojson',
401 help='write video metadata to a .info.json file', default=False)
402 filesystem.add_option('--write-annotations',
403 action='store_true', dest='writeannotations',
404 help='write video annotations to a .annotation file', default=False)
405 filesystem.add_option('--write-thumbnail',
406 action='store_true', dest='writethumbnail',
407 help='write thumbnail image to disk', default=False)
408
409
410 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
411 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
412 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
413 help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
414 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
415 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
416 postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
417 help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
418 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
419 help='keeps the video file on disk after the post-processing; the video is erased by default')
420 postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
421 help='do not overwrite post-processed files; the post-processed files are overwritten by default')
422 postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
423 help='embed subtitles in the video (only for mp4 videos)')
424 postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
425 help='write metadata to the video file')
426 postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
427 help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
428 postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
429 help='Prefer avconv over ffmpeg for running the postprocessors (default)')
430 postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
431 help='Prefer ffmpeg over avconv for running the postprocessors')
432
433
434 parser.add_option_group(general)
435 parser.add_option_group(selection)
436 parser.add_option_group(downloader)
437 parser.add_option_group(filesystem)
438 parser.add_option_group(verbosity)
439 parser.add_option_group(video_format)
440 parser.add_option_group(subtitles)
441 parser.add_option_group(authentication)
442 parser.add_option_group(postproc)
443
444 if overrideArguments is not None:
445 opts, args = parser.parse_args(overrideArguments)
446 if opts.verbose:
447 write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
448 else:
449 systemConf = _readOptions('/etc/youtube-dl.conf')
450
451 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
452 if xdg_config_home:
453 userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
454 if not os.path.isfile(userConfFile):
455 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
456 else:
457 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
458 if not os.path.isfile(userConfFile):
459 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
460 userConf = _readOptions(userConfFile, None)
461
462 if userConf is None:
463 appdata_dir = os.environ.get('appdata')
464 if appdata_dir:
465 userConf = _readOptions(
466 os.path.join(appdata_dir, 'youtube-dl', 'config'),
467 default=None)
468 if userConf is None:
469 userConf = _readOptions(
470 os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
471 default=None)
472
473 if userConf is None:
474 userConf = _readOptions(
475 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
476 default=None)
477 if userConf is None:
478 userConf = _readOptions(
479 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
480 default=None)
481
482 if userConf is None:
483 userConf = []
484
485 commandLineConf = sys.argv[1:]
486 argv = systemConf + userConf + commandLineConf
487 opts, args = parser.parse_args(argv)
488 if opts.verbose:
489 write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
490 write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
491 write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
492 write_string(u'[debug] Encodings: locale %r, fs %r, out %r, pref: %r\n' %
493 (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, preferredencoding()))
494
495 return parser, opts, args
496
497
498 def _real_main(argv=None):
499 # Compatibility fixes for Windows
500 if sys.platform == 'win32':
501 # https://github.com/rg3/youtube-dl/issues/820
502 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
503
504 setproctitle(u'youtube-dl')
505
506 parser, opts, args = parseOpts(argv)
507
508 # Set user agent
509 if opts.user_agent is not None:
510 std_headers['User-Agent'] = opts.user_agent
511
512 # Set referer
513 if opts.referer is not None:
514 std_headers['Referer'] = opts.referer
515
516 # Dump user agent
517 if opts.dump_user_agent:
518 compat_print(std_headers['User-Agent'])
519 sys.exit(0)
520
521 # Batch file verification
522 batchurls = []
523 if opts.batchfile is not None:
524 try:
525 if opts.batchfile == '-':
526 batchfd = sys.stdin
527 else:
528 batchfd = open(opts.batchfile, 'r')
529 batchurls = batchfd.readlines()
530 batchurls = [x.strip() for x in batchurls]
531 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
532 if opts.verbose:
533 write_string(u'[debug] Batch file urls: ' + repr(batchurls) + u'\n')
534 except IOError:
535 sys.exit(u'ERROR: batch file could not be read')
536 all_urls = batchurls + args
537 all_urls = [url.strip() for url in all_urls]
538 _enc = preferredencoding()
539 all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
540
541 extractors = gen_extractors()
542
543 if opts.list_extractors:
544 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
545 compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
546 matchedUrls = [url for url in all_urls if ie.suitable(url)]
547 for mu in matchedUrls:
548 compat_print(u' ' + mu)
549 sys.exit(0)
550 if opts.list_extractor_descriptions:
551 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
552 if not ie._WORKING:
553 continue
554 desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
555 if desc is False:
556 continue
557 if hasattr(ie, 'SEARCH_KEY'):
558 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
559 _COUNTS = (u'', u'5', u'10', u'all')
560 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
561 compat_print(desc)
562 sys.exit(0)
563
564
565 # Conflicting, missing and erroneous options
566 if opts.usenetrc and (opts.username is not None or opts.password is not None):
567 parser.error(u'using .netrc conflicts with giving username/password')
568 if opts.password is not None and opts.username is None:
569 parser.error(u'account username missing\n')
570 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
571 parser.error(u'using output template conflicts with using title, video ID or auto number')
572 if opts.usetitle and opts.useid:
573 parser.error(u'using title conflicts with using video ID')
574 if opts.username is not None and opts.password is None:
575 opts.password = getpass.getpass(u'Type account password and press return:')
576 if opts.ratelimit is not None:
577 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
578 if numeric_limit is None:
579 parser.error(u'invalid rate limit specified')
580 opts.ratelimit = numeric_limit
581 if opts.min_filesize is not None:
582 numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
583 if numeric_limit is None:
584 parser.error(u'invalid min_filesize specified')
585 opts.min_filesize = numeric_limit
586 if opts.max_filesize is not None:
587 numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
588 if numeric_limit is None:
589 parser.error(u'invalid max_filesize specified')
590 opts.max_filesize = numeric_limit
591 if opts.retries is not None:
592 try:
593 opts.retries = int(opts.retries)
594 except (TypeError, ValueError):
595 parser.error(u'invalid retry count specified')
596 if opts.buffersize is not None:
597 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
598 if numeric_buffersize is None:
599 parser.error(u'invalid buffer size specified')
600 opts.buffersize = numeric_buffersize
601 if opts.playliststart <= 0:
602 raise ValueError(u'Playlist start must be positive')
603 if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
604 raise ValueError(u'Playlist end must be greater than playlist start')
605 if opts.extractaudio:
606 if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
607 parser.error(u'invalid audio format specified')
608 if opts.audioquality:
609 opts.audioquality = opts.audioquality.strip('k').strip('K')
610 if not opts.audioquality.isdigit():
611 parser.error(u'invalid audio quality specified')
612 if opts.recodevideo is not None:
613 if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg']:
614 parser.error(u'invalid video recode format specified')
615 if opts.date is not None:
616 date = DateRange.day(opts.date)
617 else:
618 date = DateRange(opts.dateafter, opts.datebefore)
619
620 # --all-sub automatically sets --write-sub if --write-auto-sub is not given
621 # this was the old behaviour if only --all-sub was given.
622 if opts.allsubtitles and (opts.writeautomaticsub == False):
623 opts.writesubtitles = True
624
625 if sys.version_info < (3,):
626 # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
627 if opts.outtmpl is not None:
628 opts.outtmpl = opts.outtmpl.decode(preferredencoding())
629 outtmpl =((opts.outtmpl is not None and opts.outtmpl)
630 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
631 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
632 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
633 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
634 or (opts.useid and u'%(id)s.%(ext)s')
635 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
636 or u'%(title)s-%(id)s.%(ext)s')
637 if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
638 parser.error(u'Cannot download a video and extract audio into the same'
639 u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
640 u' template'.format(outtmpl))
641
642 any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
643 download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
644
645 ydl_opts = {
646 'usenetrc': opts.usenetrc,
647 'username': opts.username,
648 'password': opts.password,
649 'videopassword': opts.videopassword,
650 'quiet': (opts.quiet or any_printing),
651 'forceurl': opts.geturl,
652 'forcetitle': opts.gettitle,
653 'forceid': opts.getid,
654 'forcethumbnail': opts.getthumbnail,
655 'forcedescription': opts.getdescription,
656 'forceduration': opts.getduration,
657 'forcefilename': opts.getfilename,
658 'forceformat': opts.getformat,
659 'forcejson': opts.dumpjson,
660 'simulate': opts.simulate,
661 'skip_download': (opts.skip_download or opts.simulate or any_printing),
662 'format': opts.format,
663 'format_limit': opts.format_limit,
664 'listformats': opts.listformats,
665 'outtmpl': outtmpl,
666 'autonumber_size': opts.autonumber_size,
667 'restrictfilenames': opts.restrictfilenames,
668 'ignoreerrors': opts.ignoreerrors,
669 'ratelimit': opts.ratelimit,
670 'nooverwrites': opts.nooverwrites,
671 'retries': opts.retries,
672 'buffersize': opts.buffersize,
673 'noresizebuffer': opts.noresizebuffer,
674 'continuedl': opts.continue_dl,
675 'noprogress': opts.noprogress,
676 'progress_with_newline': opts.progress_with_newline,
677 'playliststart': opts.playliststart,
678 'playlistend': opts.playlistend,
679 'noplaylist': opts.noplaylist,
680 'logtostderr': opts.outtmpl == '-',
681 'consoletitle': opts.consoletitle,
682 'nopart': opts.nopart,
683 'updatetime': opts.updatetime,
684 'writedescription': opts.writedescription,
685 'writeannotations': opts.writeannotations,
686 'writeinfojson': opts.writeinfojson,
687 'writethumbnail': opts.writethumbnail,
688 'writesubtitles': opts.writesubtitles,
689 'writeautomaticsub': opts.writeautomaticsub,
690 'allsubtitles': opts.allsubtitles,
691 'listsubtitles': opts.listsubtitles,
692 'subtitlesformat': opts.subtitlesformat,
693 'subtitleslangs': opts.subtitleslangs,
694 'matchtitle': decodeOption(opts.matchtitle),
695 'rejecttitle': decodeOption(opts.rejecttitle),
696 'max_downloads': opts.max_downloads,
697 'prefer_free_formats': opts.prefer_free_formats,
698 'verbose': opts.verbose,
699 'dump_intermediate_pages': opts.dump_intermediate_pages,
700 'write_pages': opts.write_pages,
701 'test': opts.test,
702 'keepvideo': opts.keepvideo,
703 'min_filesize': opts.min_filesize,
704 'max_filesize': opts.max_filesize,
705 'min_views': opts.min_views,
706 'max_views': opts.max_views,
707 'daterange': date,
708 'cachedir': opts.cachedir,
709 'youtube_print_sig_code': opts.youtube_print_sig_code,
710 'age_limit': opts.age_limit,
711 'download_archive': download_archive_fn,
712 'cookiefile': opts.cookiefile,
713 'nocheckcertificate': opts.no_check_certificate,
714 'proxy': opts.proxy,
715 'socket_timeout': opts.socket_timeout,
716 'bidi_workaround': opts.bidi_workaround,
717 'debug_printtraffic': opts.debug_printtraffic,
718 'prefer_ffmpeg': opts.prefer_ffmpeg,
719 }
720
721 with YoutubeDL(ydl_opts) as ydl:
722 ydl.print_debug_header()
723 ydl.add_default_info_extractors()
724
725 # PostProcessors
726 # Add the metadata pp first, the other pps will copy it
727 if opts.addmetadata:
728 ydl.add_post_processor(FFmpegMetadataPP())
729 if opts.extractaudio:
730 ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
731 if opts.recodevideo:
732 ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
733 if opts.embedsubtitles:
734 ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
735 if opts.xattrs:
736 ydl.add_post_processor(XAttrMetadataPP())
737
738 # Update version
739 if opts.update_self:
740 update_self(ydl.to_screen, opts.verbose)
741
742 # Maybe do nothing
743 if (len(all_urls) < 1) and (opts.load_info_filename is None):
744 if not opts.update_self:
745 parser.error(u'you must provide at least one URL')
746 else:
747 sys.exit()
748
749 try:
750 if opts.load_info_filename is not None:
751 retcode = ydl.download_with_info_file(opts.load_info_filename)
752 else:
753 retcode = ydl.download(all_urls)
754 except MaxDownloadsReached:
755 ydl.to_screen(u'--max-download limit reached, aborting.')
756 retcode = 101
757
758 sys.exit(retcode)
759
760
761 def main(argv=None):
762 try:
763 _real_main(argv)
764 except DownloadError:
765 sys.exit(1)
766 except SameFileError:
767 sys.exit(u'ERROR: fixed output name but more than one file to download')
768 except KeyboardInterrupt:
769 sys.exit(u'\nERROR: Interrupted by user')