]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py
f9c16e426d1daa207ecec420a47a8d4692de00f6
[youtubedl] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __authors__ = (
5 'Ricardo Garcia Gonzalez',
6 'Danny Colligan',
7 'Benjamin Johnson',
8 'Vasyl\' Vavrychuk',
9 'Witold Baryluk',
10 'Paweł Paprota',
11 'Gergely Imreh',
12 'Rogério Brito',
13 'Philipp Hagemeister',
14 'Sören Schulze',
15 'Kevin Ngo',
16 'Ori Avtalion',
17 'shizeeg',
18 'Filippo Valsorda',
19 'Christian Albrecht',
20 'Dave Vasilevsky',
21 'Jaime Marquínez Ferrándiz',
22 'Jeff Crouse',
23 'Osama Khalid',
24 'Michael Walter',
25 'M. Yasoob Ullah Khalid',
26 'Julien Fraichard',
27 'Johny Mo Swag',
28 'Axel Noack',
29 'Albert Kim',
30 'Pierre Rudloff',
31 'Huarong Huo',
32 'Ismael Mejía',
33 'Steffan \'Ruirize\' James',
34 'Andras Elso',
35 'Jelle van der Waa',
36 'Marcin Cieślak',
37 'Anton Larionov',
38 'Takuya Tsuchida',
39 'Sergey M.',
40 'Michael Orlitzky',
41 'Chris Gahan',
42 'Saimadhav Heblikar',
43 'Mike Col',
44 'Oleg Prutz',
45 'pulpe',
46 'Andreas Schmitz',
47 'Michael Kaiser',
48 'Niklas Laxström',
49 'David Triendl',
50 'Anthony Weems',
51 'David Wagner',
52 'Juan C. Olivares',
53 'Mattias Harrysson',
54 'phaer',
55 'Sainyam Kapoor',
56 'Nicolas Évrard',
57 'Jason Normore',
58 'Hoje Lee',
59 'Adam Thalhammer',
60 'Georg Jähnig',
61 'Ralf Haring',
62 'Koki Takahashi',
63 'Ariset Llerena',
64 'Adam Malcontenti-Wilson',
65 'Tobias Bell',
66 'Naglis Jonaitis',
67 'Charles Chen',
68 'Hassaan Ali',
69 'Dobrosław Żybort',
70 )
71
72 __license__ = 'Public Domain'
73
74 import codecs
75 import io
76 import optparse
77 import os
78 import random
79 import shlex
80 import shutil
81 import sys
82
83
84 from .utils import (
85 compat_getpass,
86 compat_print,
87 DateRange,
88 DEFAULT_OUTTMPL,
89 decodeOption,
90 get_term_width,
91 DownloadError,
92 get_cachedir,
93 MaxDownloadsReached,
94 preferredencoding,
95 read_batch_urls,
96 SameFileError,
97 setproctitle,
98 std_headers,
99 write_string,
100 )
101 from .update import update_self
102 from .downloader import (
103 FileDownloader,
104 )
105 from .extractor import gen_extractors
106 from .version import __version__
107 from .YoutubeDL import YoutubeDL
108 from .postprocessor import (
109 AtomicParsleyPP,
110 FFmpegAudioFixPP,
111 FFmpegMetadataPP,
112 FFmpegVideoConvertor,
113 FFmpegExtractAudioPP,
114 FFmpegEmbedSubtitlePP,
115 XAttrMetadataPP,
116 )
117
118
119 def parseOpts(overrideArguments=None):
120 def _readOptions(filename_bytes, default=[]):
121 try:
122 optionf = open(filename_bytes)
123 except IOError:
124 return default # silently skip if file is not present
125 try:
126 res = []
127 for l in optionf:
128 res += shlex.split(l, comments=True)
129 finally:
130 optionf.close()
131 return res
132
133 def _readUserConf():
134 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
135 if xdg_config_home:
136 userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
137 if not os.path.isfile(userConfFile):
138 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
139 else:
140 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
141 if not os.path.isfile(userConfFile):
142 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
143 userConf = _readOptions(userConfFile, None)
144
145 if userConf is None:
146 appdata_dir = os.environ.get('appdata')
147 if appdata_dir:
148 userConf = _readOptions(
149 os.path.join(appdata_dir, 'youtube-dl', 'config'),
150 default=None)
151 if userConf is None:
152 userConf = _readOptions(
153 os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
154 default=None)
155
156 if userConf is None:
157 userConf = _readOptions(
158 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
159 default=None)
160 if userConf is None:
161 userConf = _readOptions(
162 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
163 default=None)
164
165 if userConf is None:
166 userConf = []
167
168 return userConf
169
170 def _format_option_string(option):
171 ''' ('-o', '--option') -> -o, --format METAVAR'''
172
173 opts = []
174
175 if option._short_opts:
176 opts.append(option._short_opts[0])
177 if option._long_opts:
178 opts.append(option._long_opts[0])
179 if len(opts) > 1:
180 opts.insert(1, ', ')
181
182 if option.takes_value(): opts.append(' %s' % option.metavar)
183
184 return "".join(opts)
185
186 def _comma_separated_values_options_callback(option, opt_str, value, parser):
187 setattr(parser.values, option.dest, value.split(','))
188
189 def _hide_login_info(opts):
190 opts = list(opts)
191 for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
192 try:
193 i = opts.index(private_opt)
194 opts[i+1] = '<PRIVATE>'
195 except ValueError:
196 pass
197 return opts
198
199 max_width = 80
200 max_help_position = 80
201
202 # No need to wrap help messages if we're on a wide console
203 columns = get_term_width()
204 if columns: max_width = columns
205
206 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
207 fmt.format_option_strings = _format_option_string
208
209 kw = {
210 'version' : __version__,
211 'formatter' : fmt,
212 'usage' : '%prog [options] url [url...]',
213 'conflict_handler' : 'resolve',
214 }
215
216 parser = optparse.OptionParser(**kw)
217
218 # option groups
219 general = optparse.OptionGroup(parser, 'General Options')
220 selection = optparse.OptionGroup(parser, 'Video Selection')
221 authentication = optparse.OptionGroup(parser, 'Authentication Options')
222 video_format = optparse.OptionGroup(parser, 'Video Format Options')
223 subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
224 downloader = optparse.OptionGroup(parser, 'Download Options')
225 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
226 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
227 workarounds = optparse.OptionGroup(parser, 'Workarounds')
228 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
229
230 general.add_option('-h', '--help',
231 action='help', help='print this help text and exit')
232 general.add_option('-v', '--version',
233 action='version', help='print program version and exit')
234 general.add_option('-U', '--update',
235 action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
236 general.add_option('-i', '--ignore-errors',
237 action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
238 general.add_option('--abort-on-error',
239 action='store_false', dest='ignoreerrors',
240 help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
241 general.add_option('--dump-user-agent',
242 action='store_true', dest='dump_user_agent',
243 help='display the current browser identification', default=False)
244 general.add_option('--list-extractors',
245 action='store_true', dest='list_extractors',
246 help='List all supported extractors and the URLs they would handle', default=False)
247 general.add_option('--extractor-descriptions',
248 action='store_true', dest='list_extractor_descriptions',
249 help='Output descriptions of all supported extractors', default=False)
250 general.add_option(
251 '--proxy', dest='proxy', default=None, metavar='URL',
252 help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
253 general.add_option(
254 '--socket-timeout', dest='socket_timeout',
255 type=float, default=None, help=u'Time to wait before giving up, in seconds')
256 general.add_option(
257 '--default-search',
258 dest='default_search', metavar='PREFIX',
259 help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.')
260 general.add_option(
261 '--ignore-config',
262 action='store_true',
263 help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
264
265 selection.add_option(
266 '--playlist-start',
267 dest='playliststart', metavar='NUMBER', default=1, type=int,
268 help='playlist video to start at (default is %default)')
269 selection.add_option(
270 '--playlist-end',
271 dest='playlistend', metavar='NUMBER', default=None, type=int,
272 help='playlist video to end at (default is last)')
273 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
274 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
275 selection.add_option('--max-downloads', metavar='NUMBER',
276 dest='max_downloads', type=int, default=None,
277 help='Abort after downloading NUMBER files')
278 selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
279 selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
280 selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
281 selection.add_option(
282 '--datebefore', metavar='DATE', dest='datebefore', default=None,
283 help='download only videos uploaded on or before this date (i.e. inclusive)')
284 selection.add_option(
285 '--dateafter', metavar='DATE', dest='dateafter', default=None,
286 help='download only videos uploaded on or after this date (i.e. inclusive)')
287 selection.add_option(
288 '--min-views', metavar='COUNT', dest='min_views',
289 default=None, type=int,
290 help="Do not download any videos with less than COUNT views",)
291 selection.add_option(
292 '--max-views', metavar='COUNT', dest='max_views',
293 default=None, type=int,
294 help="Do not download any videos with more than COUNT views",)
295 selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
296 selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
297 help='download only videos suitable for the given age',
298 default=None, type=int)
299 selection.add_option('--download-archive', metavar='FILE',
300 dest='download_archive',
301 help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
302 selection.add_option(
303 '--include-ads', dest='include_ads',
304 action='store_true',
305 help='Download advertisements as well (experimental)')
306 selection.add_option(
307 '--youtube-include-dash-manifest', action='store_true',
308 dest='youtube_include_dash_manifest', default=False,
309 help='Try to download the DASH manifest on YouTube videos (experimental)')
310
311 authentication.add_option('-u', '--username',
312 dest='username', metavar='USERNAME', help='account username')
313 authentication.add_option('-p', '--password',
314 dest='password', metavar='PASSWORD', help='account password')
315 authentication.add_option('-n', '--netrc',
316 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
317 authentication.add_option('--video-password',
318 dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
319
320
321 video_format.add_option('-f', '--format',
322 action='store', dest='format', metavar='FORMAT', default=None,
323 help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
324 video_format.add_option('--all-formats',
325 action='store_const', dest='format', help='download all available video formats', const='all')
326 video_format.add_option('--prefer-free-formats',
327 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
328 video_format.add_option('--max-quality',
329 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
330 video_format.add_option('-F', '--list-formats',
331 action='store_true', dest='listformats', help='list all available formats')
332
333 subtitles.add_option('--write-sub', '--write-srt',
334 action='store_true', dest='writesubtitles',
335 help='write subtitle file', default=False)
336 subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
337 action='store_true', dest='writeautomaticsub',
338 help='write automatic subtitle file (youtube only)', default=False)
339 subtitles.add_option('--all-subs',
340 action='store_true', dest='allsubtitles',
341 help='downloads all the available subtitles of the video', default=False)
342 subtitles.add_option('--list-subs',
343 action='store_true', dest='listsubtitles',
344 help='lists all available subtitles for the video', default=False)
345 subtitles.add_option('--sub-format',
346 action='store', dest='subtitlesformat', metavar='FORMAT',
347 help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
348 subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
349 action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
350 default=[], callback=_comma_separated_values_options_callback,
351 help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
352
353 downloader.add_option('-r', '--rate-limit',
354 dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
355 downloader.add_option('-R', '--retries',
356 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
357 downloader.add_option('--buffer-size',
358 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
359 downloader.add_option('--no-resize-buffer',
360 action='store_true', dest='noresizebuffer',
361 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
362 downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
363
364 workarounds.add_option(
365 '--encoding', dest='encoding', metavar='ENCODING',
366 help='Force the specified encoding (experimental)')
367 workarounds.add_option(
368 '--no-check-certificate', action='store_true',
369 dest='no_check_certificate', default=False,
370 help='Suppress HTTPS certificate validation.')
371 workarounds.add_option(
372 '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
373 help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
374 workarounds.add_option(
375 '--user-agent', metavar='UA',
376 dest='user_agent', help='specify a custom user agent')
377 workarounds.add_option(
378 '--referer', metavar='REF',
379 dest='referer', default=None,
380 help='specify a custom referer, use if the video access is restricted to one domain',
381 )
382 workarounds.add_option(
383 '--add-header', metavar='FIELD:VALUE',
384 dest='headers', action='append',
385 help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
386 )
387 workarounds.add_option(
388 '--bidi-workaround', dest='bidi_workaround', action='store_true',
389 help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
390
391 verbosity.add_option('-q', '--quiet',
392 action='store_true', dest='quiet', help='activates quiet mode', default=False)
393 verbosity.add_option(
394 '--no-warnings',
395 dest='no_warnings', action='store_true', default=False,
396 help='Ignore warnings')
397 verbosity.add_option('-s', '--simulate',
398 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
399 verbosity.add_option('--skip-download',
400 action='store_true', dest='skip_download', help='do not download the video', default=False)
401 verbosity.add_option('-g', '--get-url',
402 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
403 verbosity.add_option('-e', '--get-title',
404 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
405 verbosity.add_option('--get-id',
406 action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
407 verbosity.add_option('--get-thumbnail',
408 action='store_true', dest='getthumbnail',
409 help='simulate, quiet but print thumbnail URL', default=False)
410 verbosity.add_option('--get-description',
411 action='store_true', dest='getdescription',
412 help='simulate, quiet but print video description', default=False)
413 verbosity.add_option('--get-duration',
414 action='store_true', dest='getduration',
415 help='simulate, quiet but print video length', default=False)
416 verbosity.add_option('--get-filename',
417 action='store_true', dest='getfilename',
418 help='simulate, quiet but print output filename', default=False)
419 verbosity.add_option('--get-format',
420 action='store_true', dest='getformat',
421 help='simulate, quiet but print output format', default=False)
422 verbosity.add_option('-j', '--dump-json',
423 action='store_true', dest='dumpjson',
424 help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
425 verbosity.add_option('--newline',
426 action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
427 verbosity.add_option('--no-progress',
428 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
429 verbosity.add_option('--console-title',
430 action='store_true', dest='consoletitle',
431 help='display progress in console titlebar', default=False)
432 verbosity.add_option('-v', '--verbose',
433 action='store_true', dest='verbose', help='print various debugging information', default=False)
434 verbosity.add_option('--dump-intermediate-pages',
435 action='store_true', dest='dump_intermediate_pages', default=False,
436 help='print downloaded pages to debug problems (very verbose)')
437 verbosity.add_option('--write-pages',
438 action='store_true', dest='write_pages', default=False,
439 help='Write downloaded intermediary pages to files in the current directory to debug problems')
440 verbosity.add_option('--youtube-print-sig-code',
441 action='store_true', dest='youtube_print_sig_code', default=False,
442 help=optparse.SUPPRESS_HELP)
443 verbosity.add_option('--print-traffic',
444 dest='debug_printtraffic', action='store_true', default=False,
445 help='Display sent and read HTTP traffic')
446
447
448 filesystem.add_option('-a', '--batch-file',
449 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
450 filesystem.add_option('--id',
451 action='store_true', dest='useid', help='use only video ID in file name', default=False)
452 filesystem.add_option('-A', '--auto-number',
453 action='store_true', dest='autonumber',
454 help='number downloaded files starting from 00000', default=False)
455 filesystem.add_option('-o', '--output',
456 dest='outtmpl', metavar='TEMPLATE',
457 help=('output filename template. Use %(title)s to get the title, '
458 '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
459 '%(autonumber)s to get an automatically incremented number, '
460 '%(ext)s for the filename extension, '
461 '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
462 '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
463 '%(upload_date)s for the upload date (YYYYMMDD), '
464 '%(extractor)s for the provider (youtube, metacafe, etc), '
465 '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
466 '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
467 '%(height)s and %(width)s for the width and height of the video format. '
468 '%(resolution)s for a textual description of the resolution of the video format. '
469 'Use - to output to stdout. Can also be used to download to a different directory, '
470 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
471 filesystem.add_option('--autonumber-size',
472 dest='autonumber_size', metavar='NUMBER',
473 help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
474 filesystem.add_option('--restrict-filenames',
475 action='store_true', dest='restrictfilenames',
476 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
477 filesystem.add_option('-t', '--title',
478 action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
479 filesystem.add_option('-l', '--literal',
480 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
481 filesystem.add_option('-w', '--no-overwrites',
482 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
483 filesystem.add_option('-c', '--continue',
484 action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
485 filesystem.add_option('--no-continue',
486 action='store_false', dest='continue_dl',
487 help='do not resume partially downloaded files (restart from beginning)')
488 filesystem.add_option('--no-part',
489 action='store_true', dest='nopart', help='do not use .part files', default=False)
490 filesystem.add_option('--no-mtime',
491 action='store_false', dest='updatetime',
492 help='do not use the Last-modified header to set the file modification time', default=True)
493 filesystem.add_option('--write-description',
494 action='store_true', dest='writedescription',
495 help='write video description to a .description file', default=False)
496 filesystem.add_option('--write-info-json',
497 action='store_true', dest='writeinfojson',
498 help='write video metadata to a .info.json file', default=False)
499 filesystem.add_option('--write-annotations',
500 action='store_true', dest='writeannotations',
501 help='write video annotations to a .annotation file', default=False)
502 filesystem.add_option('--write-thumbnail',
503 action='store_true', dest='writethumbnail',
504 help='write thumbnail image to disk', default=False)
505 filesystem.add_option('--load-info',
506 dest='load_info_filename', metavar='FILE',
507 help='json file containing the video information (created with the "--write-json" option)')
508 filesystem.add_option('--cookies',
509 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
510 filesystem.add_option(
511 '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
512 help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
513 filesystem.add_option(
514 '--no-cache-dir', action='store_const', const=None, dest='cachedir',
515 help='Disable filesystem caching')
516 filesystem.add_option(
517 '--rm-cache-dir', action='store_true', dest='rm_cachedir',
518 help='Delete all filesystem cache files')
519
520
521 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
522 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
523 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
524 help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
525 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
526 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
527 postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
528 help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
529 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
530 help='keeps the video file on disk after the post-processing; the video is erased by default')
531 postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
532 help='do not overwrite post-processed files; the post-processed files are overwritten by default')
533 postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
534 help='embed subtitles in the video (only for mp4 videos)')
535 postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
536 help='embed thumbnail in the audio as cover art')
537 postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
538 help='write metadata to the video file')
539 postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
540 help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
541 postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
542 help='Prefer avconv over ffmpeg for running the postprocessors (default)')
543 postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
544 help='Prefer ffmpeg over avconv for running the postprocessors')
545
546
547 parser.add_option_group(general)
548 parser.add_option_group(selection)
549 parser.add_option_group(downloader)
550 parser.add_option_group(filesystem)
551 parser.add_option_group(verbosity)
552 parser.add_option_group(workarounds)
553 parser.add_option_group(video_format)
554 parser.add_option_group(subtitles)
555 parser.add_option_group(authentication)
556 parser.add_option_group(postproc)
557
558 if overrideArguments is not None:
559 opts, args = parser.parse_args(overrideArguments)
560 if opts.verbose:
561 write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
562 else:
563 commandLineConf = sys.argv[1:]
564 if '--ignore-config' in commandLineConf:
565 systemConf = []
566 userConf = []
567 else:
568 systemConf = _readOptions('/etc/youtube-dl.conf')
569 if '--ignore-config' in systemConf:
570 userConf = []
571 else:
572 userConf = _readUserConf()
573 argv = systemConf + userConf + commandLineConf
574
575 opts, args = parser.parse_args(argv)
576 if opts.verbose:
577 write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
578 write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
579 write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
580
581 return parser, opts, args
582
583
584 def _real_main(argv=None):
585 # Compatibility fixes for Windows
586 if sys.platform == 'win32':
587 # https://github.com/rg3/youtube-dl/issues/820
588 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
589
590 setproctitle(u'youtube-dl')
591
592 parser, opts, args = parseOpts(argv)
593
594 # Set user agent
595 if opts.user_agent is not None:
596 std_headers['User-Agent'] = opts.user_agent
597
598 # Set referer
599 if opts.referer is not None:
600 std_headers['Referer'] = opts.referer
601
602 # Custom HTTP headers
603 if opts.headers is not None:
604 for h in opts.headers:
605 if h.find(':', 1) < 0:
606 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
607 key, value = h.split(':', 2)
608 if opts.verbose:
609 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
610 std_headers[key] = value
611
612 # Dump user agent
613 if opts.dump_user_agent:
614 compat_print(std_headers['User-Agent'])
615 sys.exit(0)
616
617 # Batch file verification
618 batch_urls = []
619 if opts.batchfile is not None:
620 try:
621 if opts.batchfile == '-':
622 batchfd = sys.stdin
623 else:
624 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
625 batch_urls = read_batch_urls(batchfd)
626 if opts.verbose:
627 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
628 except IOError:
629 sys.exit(u'ERROR: batch file could not be read')
630 all_urls = batch_urls + args
631 all_urls = [url.strip() for url in all_urls]
632 _enc = preferredencoding()
633 all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
634
635 extractors = gen_extractors()
636
637 if opts.list_extractors:
638 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
639 compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
640 matchedUrls = [url for url in all_urls if ie.suitable(url)]
641 for mu in matchedUrls:
642 compat_print(u' ' + mu)
643 sys.exit(0)
644 if opts.list_extractor_descriptions:
645 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
646 if not ie._WORKING:
647 continue
648 desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
649 if desc is False:
650 continue
651 if hasattr(ie, 'SEARCH_KEY'):
652 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
653 _COUNTS = (u'', u'5', u'10', u'all')
654 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
655 compat_print(desc)
656 sys.exit(0)
657
658
659 # Conflicting, missing and erroneous options
660 if opts.usenetrc and (opts.username is not None or opts.password is not None):
661 parser.error(u'using .netrc conflicts with giving username/password')
662 if opts.password is not None and opts.username is None:
663 parser.error(u'account username missing\n')
664 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
665 parser.error(u'using output template conflicts with using title, video ID or auto number')
666 if opts.usetitle and opts.useid:
667 parser.error(u'using title conflicts with using video ID')
668 if opts.username is not None and opts.password is None:
669 opts.password = compat_getpass(u'Type account password and press [Return]: ')
670 if opts.ratelimit is not None:
671 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
672 if numeric_limit is None:
673 parser.error(u'invalid rate limit specified')
674 opts.ratelimit = numeric_limit
675 if opts.min_filesize is not None:
676 numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
677 if numeric_limit is None:
678 parser.error(u'invalid min_filesize specified')
679 opts.min_filesize = numeric_limit
680 if opts.max_filesize is not None:
681 numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
682 if numeric_limit is None:
683 parser.error(u'invalid max_filesize specified')
684 opts.max_filesize = numeric_limit
685 if opts.retries is not None:
686 try:
687 opts.retries = int(opts.retries)
688 except (TypeError, ValueError):
689 parser.error(u'invalid retry count specified')
690 if opts.buffersize is not None:
691 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
692 if numeric_buffersize is None:
693 parser.error(u'invalid buffer size specified')
694 opts.buffersize = numeric_buffersize
695 if opts.playliststart <= 0:
696 raise ValueError(u'Playlist start must be positive')
697 if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
698 raise ValueError(u'Playlist end must be greater than playlist start')
699 if opts.extractaudio:
700 if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
701 parser.error(u'invalid audio format specified')
702 if opts.audioquality:
703 opts.audioquality = opts.audioquality.strip('k').strip('K')
704 if not opts.audioquality.isdigit():
705 parser.error(u'invalid audio quality specified')
706 if opts.recodevideo is not None:
707 if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
708 parser.error(u'invalid video recode format specified')
709 if opts.date is not None:
710 date = DateRange.day(opts.date)
711 else:
712 date = DateRange(opts.dateafter, opts.datebefore)
713 if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
714 parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
715
716 # Do not download videos when there are audio-only formats
717 if opts.extractaudio and not opts.keepvideo and opts.format is None:
718 opts.format = 'bestaudio/best'
719
720 # --all-sub automatically sets --write-sub if --write-auto-sub is not given
721 # this was the old behaviour if only --all-sub was given.
722 if opts.allsubtitles and (opts.writeautomaticsub == False):
723 opts.writesubtitles = True
724
725 if sys.version_info < (3,):
726 # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
727 if opts.outtmpl is not None:
728 opts.outtmpl = opts.outtmpl.decode(preferredencoding())
729 outtmpl =((opts.outtmpl is not None and opts.outtmpl)
730 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
731 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
732 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
733 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
734 or (opts.useid and u'%(id)s.%(ext)s')
735 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
736 or DEFAULT_OUTTMPL)
737 if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
738 parser.error(u'Cannot download a video and extract audio into the same'
739 u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
740 u' template'.format(outtmpl))
741
742 any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
743 download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
744
745 ydl_opts = {
746 'usenetrc': opts.usenetrc,
747 'username': opts.username,
748 'password': opts.password,
749 'videopassword': opts.videopassword,
750 'quiet': (opts.quiet or any_printing),
751 'no_warnings': opts.no_warnings,
752 'forceurl': opts.geturl,
753 'forcetitle': opts.gettitle,
754 'forceid': opts.getid,
755 'forcethumbnail': opts.getthumbnail,
756 'forcedescription': opts.getdescription,
757 'forceduration': opts.getduration,
758 'forcefilename': opts.getfilename,
759 'forceformat': opts.getformat,
760 'forcejson': opts.dumpjson,
761 'simulate': opts.simulate,
762 'skip_download': (opts.skip_download or opts.simulate or any_printing),
763 'format': opts.format,
764 'format_limit': opts.format_limit,
765 'listformats': opts.listformats,
766 'outtmpl': outtmpl,
767 'autonumber_size': opts.autonumber_size,
768 'restrictfilenames': opts.restrictfilenames,
769 'ignoreerrors': opts.ignoreerrors,
770 'ratelimit': opts.ratelimit,
771 'nooverwrites': opts.nooverwrites,
772 'retries': opts.retries,
773 'buffersize': opts.buffersize,
774 'noresizebuffer': opts.noresizebuffer,
775 'continuedl': opts.continue_dl,
776 'noprogress': opts.noprogress,
777 'progress_with_newline': opts.progress_with_newline,
778 'playliststart': opts.playliststart,
779 'playlistend': opts.playlistend,
780 'noplaylist': opts.noplaylist,
781 'logtostderr': opts.outtmpl == '-',
782 'consoletitle': opts.consoletitle,
783 'nopart': opts.nopart,
784 'updatetime': opts.updatetime,
785 'writedescription': opts.writedescription,
786 'writeannotations': opts.writeannotations,
787 'writeinfojson': opts.writeinfojson,
788 'writethumbnail': opts.writethumbnail,
789 'writesubtitles': opts.writesubtitles,
790 'writeautomaticsub': opts.writeautomaticsub,
791 'allsubtitles': opts.allsubtitles,
792 'listsubtitles': opts.listsubtitles,
793 'subtitlesformat': opts.subtitlesformat,
794 'subtitleslangs': opts.subtitleslangs,
795 'matchtitle': decodeOption(opts.matchtitle),
796 'rejecttitle': decodeOption(opts.rejecttitle),
797 'max_downloads': opts.max_downloads,
798 'prefer_free_formats': opts.prefer_free_formats,
799 'verbose': opts.verbose,
800 'dump_intermediate_pages': opts.dump_intermediate_pages,
801 'write_pages': opts.write_pages,
802 'test': opts.test,
803 'keepvideo': opts.keepvideo,
804 'min_filesize': opts.min_filesize,
805 'max_filesize': opts.max_filesize,
806 'min_views': opts.min_views,
807 'max_views': opts.max_views,
808 'daterange': date,
809 'cachedir': opts.cachedir,
810 'youtube_print_sig_code': opts.youtube_print_sig_code,
811 'age_limit': opts.age_limit,
812 'download_archive': download_archive_fn,
813 'cookiefile': opts.cookiefile,
814 'nocheckcertificate': opts.no_check_certificate,
815 'prefer_insecure': opts.prefer_insecure,
816 'proxy': opts.proxy,
817 'socket_timeout': opts.socket_timeout,
818 'bidi_workaround': opts.bidi_workaround,
819 'debug_printtraffic': opts.debug_printtraffic,
820 'prefer_ffmpeg': opts.prefer_ffmpeg,
821 'include_ads': opts.include_ads,
822 'default_search': opts.default_search,
823 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
824 'encoding': opts.encoding,
825 }
826
827 with YoutubeDL(ydl_opts) as ydl:
828 ydl.print_debug_header()
829 ydl.add_default_info_extractors()
830
831 # PostProcessors
832 # Add the metadata pp first, the other pps will copy it
833 if opts.addmetadata:
834 ydl.add_post_processor(FFmpegMetadataPP())
835 if opts.extractaudio:
836 ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
837 if opts.recodevideo:
838 ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
839 if opts.embedsubtitles:
840 ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
841 if opts.xattrs:
842 ydl.add_post_processor(XAttrMetadataPP())
843 if opts.embedthumbnail:
844 if not opts.addmetadata:
845 ydl.add_post_processor(FFmpegAudioFixPP())
846 ydl.add_post_processor(AtomicParsleyPP())
847
848 # Update version
849 if opts.update_self:
850 update_self(ydl.to_screen, opts.verbose)
851
852 # Remove cache dir
853 if opts.rm_cachedir:
854 if opts.cachedir is None:
855 ydl.to_screen(u'No cache dir specified (Did you combine --no-cache-dir and --rm-cache-dir?)')
856 else:
857 if ('.cache' not in opts.cachedir) or ('youtube-dl' not in opts.cachedir):
858 ydl.to_screen(u'Not removing directory %s - this does not look like a cache dir')
859 retcode = 141
860 else:
861 ydl.to_screen(
862 u'Removing cache dir %s .' % opts.cachedir,
863 skip_eol=True)
864 if os.path.exists(opts.cachedir):
865 ydl.to_screen(u'.', skip_eol=True)
866 shutil.rmtree(opts.cachedir)
867 ydl.to_screen(u'.')
868
869 # Maybe do nothing
870 if (len(all_urls) < 1) and (opts.load_info_filename is None):
871 if not (opts.update_self or opts.rm_cachedir):
872 parser.error(u'you must provide at least one URL')
873 else:
874 sys.exit()
875
876 try:
877 if opts.load_info_filename is not None:
878 retcode = ydl.download_with_info_file(opts.load_info_filename)
879 else:
880 retcode = ydl.download(all_urls)
881 except MaxDownloadsReached:
882 ydl.to_screen(u'--max-download limit reached, aborting.')
883 retcode = 101
884
885 sys.exit(retcode)
886
887
888 def main(argv=None):
889 try:
890 _real_main(argv)
891 except DownloadError:
892 sys.exit(1)
893 except SameFileError:
894 sys.exit(u'ERROR: fixed output name but more than one file to download')
895 except KeyboardInterrupt:
896 sys.exit(u'\nERROR: Interrupted by user')