]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py
debian/changelog: Start new release.
[youtubedl] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __authors__ = (
5 'Ricardo Garcia Gonzalez',
6 'Danny Colligan',
7 'Benjamin Johnson',
8 'Vasyl\' Vavrychuk',
9 'Witold Baryluk',
10 'Paweł Paprota',
11 'Gergely Imreh',
12 'Rogério Brito',
13 'Philipp Hagemeister',
14 'Sören Schulze',
15 'Kevin Ngo',
16 'Ori Avtalion',
17 'shizeeg',
18 'Filippo Valsorda',
19 'Christian Albrecht',
20 'Dave Vasilevsky',
21 'Jaime Marquínez Ferrándiz',
22 'Jeff Crouse',
23 'Osama Khalid',
24 'Michael Walter',
25 'M. Yasoob Ullah Khalid',
26 'Julien Fraichard',
27 'Johny Mo Swag',
28 'Axel Noack',
29 'Albert Kim',
30 'Pierre Rudloff',
31 'Huarong Huo',
32 'Ismael Mejía',
33 'Steffan \'Ruirize\' James',
34 'Andras Elso',
35 'Jelle van der Waa',
36 'Marcin Cieślak',
37 'Anton Larionov',
38 'Takuya Tsuchida',
39 'Sergey M.',
40 'Michael Orlitzky',
41 'Chris Gahan',
42 'Saimadhav Heblikar',
43 'Mike Col',
44 'Oleg Prutz',
45 'pulpe',
46 'Andreas Schmitz',
47 'Michael Kaiser',
48 'Niklas Laxström',
49 'David Triendl',
50 'Anthony Weems',
51 'David Wagner',
52 'Juan C. Olivares',
53 'Mattias Harrysson',
54 'phaer',
55 'Sainyam Kapoor',
56 'Nicolas Évrard',
57 'Jason Normore',
58 'Hoje Lee',
59 'Adam Thalhammer',
60 'Georg Jähnig',
61 'Ralf Haring',
62 'Koki Takahashi',
63 )
64
65 __license__ = 'Public Domain'
66
67 import codecs
68 import io
69 import locale
70 import optparse
71 import os
72 import random
73 import re
74 import shlex
75 import sys
76
77
78 from .utils import (
79 compat_getpass,
80 compat_print,
81 DateRange,
82 DEFAULT_OUTTMPL,
83 decodeOption,
84 get_term_width,
85 DownloadError,
86 get_cachedir,
87 MaxDownloadsReached,
88 preferredencoding,
89 read_batch_urls,
90 SameFileError,
91 setproctitle,
92 std_headers,
93 write_string,
94 )
95 from .update import update_self
96 from .FileDownloader import (
97 FileDownloader,
98 )
99 from .extractor import gen_extractors
100 from .version import __version__
101 from .YoutubeDL import YoutubeDL
102 from .postprocessor import (
103 AtomicParsleyPP,
104 FFmpegAudioFixPP,
105 FFmpegMetadataPP,
106 FFmpegVideoConvertor,
107 FFmpegExtractAudioPP,
108 FFmpegEmbedSubtitlePP,
109 XAttrMetadataPP,
110 )
111
112
113 def parseOpts(overrideArguments=None):
114 def _readOptions(filename_bytes, default=[]):
115 try:
116 optionf = open(filename_bytes)
117 except IOError:
118 return default # silently skip if file is not present
119 try:
120 res = []
121 for l in optionf:
122 res += shlex.split(l, comments=True)
123 finally:
124 optionf.close()
125 return res
126
127 def _readUserConf():
128 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
129 if xdg_config_home:
130 userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
131 if not os.path.isfile(userConfFile):
132 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
133 else:
134 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
135 if not os.path.isfile(userConfFile):
136 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
137 userConf = _readOptions(userConfFile, None)
138
139 if userConf is None:
140 appdata_dir = os.environ.get('appdata')
141 if appdata_dir:
142 userConf = _readOptions(
143 os.path.join(appdata_dir, 'youtube-dl', 'config'),
144 default=None)
145 if userConf is None:
146 userConf = _readOptions(
147 os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
148 default=None)
149
150 if userConf is None:
151 userConf = _readOptions(
152 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
153 default=None)
154 if userConf is None:
155 userConf = _readOptions(
156 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
157 default=None)
158
159 if userConf is None:
160 userConf = []
161
162 return userConf
163
164 def _format_option_string(option):
165 ''' ('-o', '--option') -> -o, --format METAVAR'''
166
167 opts = []
168
169 if option._short_opts:
170 opts.append(option._short_opts[0])
171 if option._long_opts:
172 opts.append(option._long_opts[0])
173 if len(opts) > 1:
174 opts.insert(1, ', ')
175
176 if option.takes_value(): opts.append(' %s' % option.metavar)
177
178 return "".join(opts)
179
180 def _comma_separated_values_options_callback(option, opt_str, value, parser):
181 setattr(parser.values, option.dest, value.split(','))
182
183 def _hide_login_info(opts):
184 opts = list(opts)
185 for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
186 try:
187 i = opts.index(private_opt)
188 opts[i+1] = '<PRIVATE>'
189 except ValueError:
190 pass
191 return opts
192
193 max_width = 80
194 max_help_position = 80
195
196 # No need to wrap help messages if we're on a wide console
197 columns = get_term_width()
198 if columns: max_width = columns
199
200 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
201 fmt.format_option_strings = _format_option_string
202
203 kw = {
204 'version' : __version__,
205 'formatter' : fmt,
206 'usage' : '%prog [options] url [url...]',
207 'conflict_handler' : 'resolve',
208 }
209
210 parser = optparse.OptionParser(**kw)
211
212 # option groups
213 general = optparse.OptionGroup(parser, 'General Options')
214 selection = optparse.OptionGroup(parser, 'Video Selection')
215 authentication = optparse.OptionGroup(parser, 'Authentication Options')
216 video_format = optparse.OptionGroup(parser, 'Video Format Options')
217 subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
218 downloader = optparse.OptionGroup(parser, 'Download Options')
219 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
220 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
221 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
222
223 general.add_option('-h', '--help',
224 action='help', help='print this help text and exit')
225 general.add_option('-v', '--version',
226 action='version', help='print program version and exit')
227 general.add_option('-U', '--update',
228 action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
229 general.add_option('-i', '--ignore-errors',
230 action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
231 general.add_option('--abort-on-error',
232 action='store_false', dest='ignoreerrors',
233 help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
234 general.add_option('--dump-user-agent',
235 action='store_true', dest='dump_user_agent',
236 help='display the current browser identification', default=False)
237 general.add_option('--user-agent',
238 dest='user_agent', help='specify a custom user agent', metavar='UA')
239 general.add_option('--referer',
240 dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
241 metavar='REF', default=None)
242 general.add_option('--add-header',
243 dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
244 metavar='FIELD:VALUE')
245 general.add_option('--list-extractors',
246 action='store_true', dest='list_extractors',
247 help='List all supported extractors and the URLs they would handle', default=False)
248 general.add_option('--extractor-descriptions',
249 action='store_true', dest='list_extractor_descriptions',
250 help='Output descriptions of all supported extractors', default=False)
251 general.add_option(
252 '--proxy', dest='proxy', default=None, metavar='URL',
253 help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
254 general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
255 general.add_option(
256 '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
257 help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
258 general.add_option(
259 '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
260 help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
261 general.add_option(
262 '--no-cache-dir', action='store_const', const=None, dest='cachedir',
263 help='Disable filesystem caching')
264 general.add_option(
265 '--socket-timeout', dest='socket_timeout',
266 type=float, default=None, help=u'Time to wait before giving up, in seconds')
267 general.add_option(
268 '--bidi-workaround', dest='bidi_workaround', action='store_true',
269 help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
270 general.add_option(
271 '--default-search',
272 dest='default_search', metavar='PREFIX',
273 help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". Use the value "auto" to let youtube-dl guess. The default value "error" just throws an error.')
274 general.add_option(
275 '--ignore-config',
276 action='store_true',
277 help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
278 general.add_option(
279 '--encoding', dest='encoding', metavar='ENCODING',
280 help='Force the specified encoding (experimental)')
281
282 selection.add_option(
283 '--playlist-start',
284 dest='playliststart', metavar='NUMBER', default=1, type=int,
285 help='playlist video to start at (default is %default)')
286 selection.add_option(
287 '--playlist-end',
288 dest='playlistend', metavar='NUMBER', default=None, type=int,
289 help='playlist video to end at (default is last)')
290 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
291 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
292 selection.add_option('--max-downloads', metavar='NUMBER',
293 dest='max_downloads', type=int, default=None,
294 help='Abort after downloading NUMBER files')
295 selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
296 selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
297 selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
298 selection.add_option(
299 '--datebefore', metavar='DATE', dest='datebefore', default=None,
300 help='download only videos uploaded on or before this date (i.e. inclusive)')
301 selection.add_option(
302 '--dateafter', metavar='DATE', dest='dateafter', default=None,
303 help='download only videos uploaded on or after this date (i.e. inclusive)')
304 selection.add_option(
305 '--min-views', metavar='COUNT', dest='min_views',
306 default=None, type=int,
307 help="Do not download any videos with less than COUNT views",)
308 selection.add_option(
309 '--max-views', metavar='COUNT', dest='max_views',
310 default=None, type=int,
311 help="Do not download any videos with more than COUNT views",)
312 selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
313 selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
314 help='download only videos suitable for the given age',
315 default=None, type=int)
316 selection.add_option('--download-archive', metavar='FILE',
317 dest='download_archive',
318 help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
319 selection.add_option(
320 '--include-ads', dest='include_ads',
321 action='store_true',
322 help='Download advertisements as well (experimental)')
323 selection.add_option(
324 '--youtube-include-dash-manifest', action='store_true',
325 dest='youtube_include_dash_manifest', default=False,
326 help='Try to download the DASH manifest on YouTube videos (experimental)')
327
328 authentication.add_option('-u', '--username',
329 dest='username', metavar='USERNAME', help='account username')
330 authentication.add_option('-p', '--password',
331 dest='password', metavar='PASSWORD', help='account password')
332 authentication.add_option('-n', '--netrc',
333 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
334 authentication.add_option('--video-password',
335 dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
336
337
338 video_format.add_option('-f', '--format',
339 action='store', dest='format', metavar='FORMAT', default=None,
340 help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
341 video_format.add_option('--all-formats',
342 action='store_const', dest='format', help='download all available video formats', const='all')
343 video_format.add_option('--prefer-free-formats',
344 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
345 video_format.add_option('--max-quality',
346 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
347 video_format.add_option('-F', '--list-formats',
348 action='store_true', dest='listformats', help='list all available formats')
349
350 subtitles.add_option('--write-sub', '--write-srt',
351 action='store_true', dest='writesubtitles',
352 help='write subtitle file', default=False)
353 subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
354 action='store_true', dest='writeautomaticsub',
355 help='write automatic subtitle file (youtube only)', default=False)
356 subtitles.add_option('--all-subs',
357 action='store_true', dest='allsubtitles',
358 help='downloads all the available subtitles of the video', default=False)
359 subtitles.add_option('--list-subs',
360 action='store_true', dest='listsubtitles',
361 help='lists all available subtitles for the video', default=False)
362 subtitles.add_option('--sub-format',
363 action='store', dest='subtitlesformat', metavar='FORMAT',
364 help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
365 subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
366 action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
367 default=[], callback=_comma_separated_values_options_callback,
368 help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
369
370 downloader.add_option('-r', '--rate-limit',
371 dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
372 downloader.add_option('-R', '--retries',
373 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
374 downloader.add_option('--buffer-size',
375 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
376 downloader.add_option('--no-resize-buffer',
377 action='store_true', dest='noresizebuffer',
378 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
379 downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
380
381 verbosity.add_option('-q', '--quiet',
382 action='store_true', dest='quiet', help='activates quiet mode', default=False)
383 verbosity.add_option(
384 '--no-warnings',
385 dest='no_warnings', action='store_true', default=False,
386 help='Ignore warnings')
387 verbosity.add_option('-s', '--simulate',
388 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
389 verbosity.add_option('--skip-download',
390 action='store_true', dest='skip_download', help='do not download the video', default=False)
391 verbosity.add_option('-g', '--get-url',
392 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
393 verbosity.add_option('-e', '--get-title',
394 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
395 verbosity.add_option('--get-id',
396 action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
397 verbosity.add_option('--get-thumbnail',
398 action='store_true', dest='getthumbnail',
399 help='simulate, quiet but print thumbnail URL', default=False)
400 verbosity.add_option('--get-description',
401 action='store_true', dest='getdescription',
402 help='simulate, quiet but print video description', default=False)
403 verbosity.add_option('--get-duration',
404 action='store_true', dest='getduration',
405 help='simulate, quiet but print video length', default=False)
406 verbosity.add_option('--get-filename',
407 action='store_true', dest='getfilename',
408 help='simulate, quiet but print output filename', default=False)
409 verbosity.add_option('--get-format',
410 action='store_true', dest='getformat',
411 help='simulate, quiet but print output format', default=False)
412 verbosity.add_option('-j', '--dump-json',
413 action='store_true', dest='dumpjson',
414 help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
415 verbosity.add_option('--newline',
416 action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
417 verbosity.add_option('--no-progress',
418 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
419 verbosity.add_option('--console-title',
420 action='store_true', dest='consoletitle',
421 help='display progress in console titlebar', default=False)
422 verbosity.add_option('-v', '--verbose',
423 action='store_true', dest='verbose', help='print various debugging information', default=False)
424 verbosity.add_option('--dump-intermediate-pages',
425 action='store_true', dest='dump_intermediate_pages', default=False,
426 help='print downloaded pages to debug problems (very verbose)')
427 verbosity.add_option('--write-pages',
428 action='store_true', dest='write_pages', default=False,
429 help='Write downloaded intermediary pages to files in the current directory to debug problems')
430 verbosity.add_option('--youtube-print-sig-code',
431 action='store_true', dest='youtube_print_sig_code', default=False,
432 help=optparse.SUPPRESS_HELP)
433 verbosity.add_option('--print-traffic',
434 dest='debug_printtraffic', action='store_true', default=False,
435 help='Display sent and read HTTP traffic')
436
437
438 filesystem.add_option('-t', '--title',
439 action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
440 filesystem.add_option('--id',
441 action='store_true', dest='useid', help='use only video ID in file name', default=False)
442 filesystem.add_option('-l', '--literal',
443 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
444 filesystem.add_option('-A', '--auto-number',
445 action='store_true', dest='autonumber',
446 help='number downloaded files starting from 00000', default=False)
447 filesystem.add_option('-o', '--output',
448 dest='outtmpl', metavar='TEMPLATE',
449 help=('output filename template. Use %(title)s to get the title, '
450 '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
451 '%(autonumber)s to get an automatically incremented number, '
452 '%(ext)s for the filename extension, '
453 '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
454 '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
455 '%(upload_date)s for the upload date (YYYYMMDD), '
456 '%(extractor)s for the provider (youtube, metacafe, etc), '
457 '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
458 '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
459 '%(height)s and %(width)s for the width and height of the video format. '
460 '%(resolution)s for a textual description of the resolution of the video format. '
461 'Use - to output to stdout. Can also be used to download to a different directory, '
462 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
463 filesystem.add_option('--autonumber-size',
464 dest='autonumber_size', metavar='NUMBER',
465 help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
466 filesystem.add_option('--restrict-filenames',
467 action='store_true', dest='restrictfilenames',
468 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
469 filesystem.add_option('-a', '--batch-file',
470 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
471 filesystem.add_option('--load-info',
472 dest='load_info_filename', metavar='FILE',
473 help='json file containing the video information (created with the "--write-json" option)')
474 filesystem.add_option('-w', '--no-overwrites',
475 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
476 filesystem.add_option('-c', '--continue',
477 action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
478 filesystem.add_option('--no-continue',
479 action='store_false', dest='continue_dl',
480 help='do not resume partially downloaded files (restart from beginning)')
481 filesystem.add_option('--cookies',
482 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
483 filesystem.add_option('--no-part',
484 action='store_true', dest='nopart', help='do not use .part files', default=False)
485 filesystem.add_option('--no-mtime',
486 action='store_false', dest='updatetime',
487 help='do not use the Last-modified header to set the file modification time', default=True)
488 filesystem.add_option('--write-description',
489 action='store_true', dest='writedescription',
490 help='write video description to a .description file', default=False)
491 filesystem.add_option('--write-info-json',
492 action='store_true', dest='writeinfojson',
493 help='write video metadata to a .info.json file', default=False)
494 filesystem.add_option('--write-annotations',
495 action='store_true', dest='writeannotations',
496 help='write video annotations to a .annotation file', default=False)
497 filesystem.add_option('--write-thumbnail',
498 action='store_true', dest='writethumbnail',
499 help='write thumbnail image to disk', default=False)
500
501
502 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
503 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
504 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
505 help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
506 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
507 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
508 postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
509 help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
510 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
511 help='keeps the video file on disk after the post-processing; the video is erased by default')
512 postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
513 help='do not overwrite post-processed files; the post-processed files are overwritten by default')
514 postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
515 help='embed subtitles in the video (only for mp4 videos)')
516 postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
517 help='embed thumbnail in the audio as cover art')
518 postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
519 help='write metadata to the video file')
520 postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
521 help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
522 postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
523 help='Prefer avconv over ffmpeg for running the postprocessors (default)')
524 postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
525 help='Prefer ffmpeg over avconv for running the postprocessors')
526
527
528 parser.add_option_group(general)
529 parser.add_option_group(selection)
530 parser.add_option_group(downloader)
531 parser.add_option_group(filesystem)
532 parser.add_option_group(verbosity)
533 parser.add_option_group(video_format)
534 parser.add_option_group(subtitles)
535 parser.add_option_group(authentication)
536 parser.add_option_group(postproc)
537
538 if overrideArguments is not None:
539 opts, args = parser.parse_args(overrideArguments)
540 if opts.verbose:
541 write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
542 else:
543 commandLineConf = sys.argv[1:]
544 if '--ignore-config' in commandLineConf:
545 systemConf = []
546 userConf = []
547 else:
548 systemConf = _readOptions('/etc/youtube-dl.conf')
549 if '--ignore-config' in systemConf:
550 userConf = []
551 else:
552 userConf = _readUserConf()
553 argv = systemConf + userConf + commandLineConf
554
555 opts, args = parser.parse_args(argv)
556 if opts.verbose:
557 write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
558 write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
559 write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
560
561 return parser, opts, args
562
563
564 def _real_main(argv=None):
565 # Compatibility fixes for Windows
566 if sys.platform == 'win32':
567 # https://github.com/rg3/youtube-dl/issues/820
568 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
569
570 setproctitle(u'youtube-dl')
571
572 parser, opts, args = parseOpts(argv)
573
574 # Set user agent
575 if opts.user_agent is not None:
576 std_headers['User-Agent'] = opts.user_agent
577
578 # Set referer
579 if opts.referer is not None:
580 std_headers['Referer'] = opts.referer
581
582 # Custom HTTP headers
583 if opts.headers is not None:
584 for h in opts.headers:
585 if h.find(':', 1) < 0:
586 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
587 key, value = h.split(':', 2)
588 if opts.verbose:
589 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
590 std_headers[key] = value
591
592 # Dump user agent
593 if opts.dump_user_agent:
594 compat_print(std_headers['User-Agent'])
595 sys.exit(0)
596
597 # Batch file verification
598 batch_urls = []
599 if opts.batchfile is not None:
600 try:
601 if opts.batchfile == '-':
602 batchfd = sys.stdin
603 else:
604 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
605 batch_urls = read_batch_urls(batchfd)
606 if opts.verbose:
607 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
608 except IOError:
609 sys.exit(u'ERROR: batch file could not be read')
610 all_urls = batch_urls + args
611 all_urls = [url.strip() for url in all_urls]
612 _enc = preferredencoding()
613 all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
614
615 extractors = gen_extractors()
616
617 if opts.list_extractors:
618 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
619 compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
620 matchedUrls = [url for url in all_urls if ie.suitable(url)]
621 for mu in matchedUrls:
622 compat_print(u' ' + mu)
623 sys.exit(0)
624 if opts.list_extractor_descriptions:
625 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
626 if not ie._WORKING:
627 continue
628 desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
629 if desc is False:
630 continue
631 if hasattr(ie, 'SEARCH_KEY'):
632 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
633 _COUNTS = (u'', u'5', u'10', u'all')
634 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
635 compat_print(desc)
636 sys.exit(0)
637
638
639 # Conflicting, missing and erroneous options
640 if opts.usenetrc and (opts.username is not None or opts.password is not None):
641 parser.error(u'using .netrc conflicts with giving username/password')
642 if opts.password is not None and opts.username is None:
643 parser.error(u'account username missing\n')
644 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
645 parser.error(u'using output template conflicts with using title, video ID or auto number')
646 if opts.usetitle and opts.useid:
647 parser.error(u'using title conflicts with using video ID')
648 if opts.username is not None and opts.password is None:
649 opts.password = compat_getpass(u'Type account password and press [Return]: ')
650 if opts.ratelimit is not None:
651 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
652 if numeric_limit is None:
653 parser.error(u'invalid rate limit specified')
654 opts.ratelimit = numeric_limit
655 if opts.min_filesize is not None:
656 numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
657 if numeric_limit is None:
658 parser.error(u'invalid min_filesize specified')
659 opts.min_filesize = numeric_limit
660 if opts.max_filesize is not None:
661 numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
662 if numeric_limit is None:
663 parser.error(u'invalid max_filesize specified')
664 opts.max_filesize = numeric_limit
665 if opts.retries is not None:
666 try:
667 opts.retries = int(opts.retries)
668 except (TypeError, ValueError):
669 parser.error(u'invalid retry count specified')
670 if opts.buffersize is not None:
671 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
672 if numeric_buffersize is None:
673 parser.error(u'invalid buffer size specified')
674 opts.buffersize = numeric_buffersize
675 if opts.playliststart <= 0:
676 raise ValueError(u'Playlist start must be positive')
677 if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
678 raise ValueError(u'Playlist end must be greater than playlist start')
679 if opts.extractaudio:
680 if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
681 parser.error(u'invalid audio format specified')
682 if opts.audioquality:
683 opts.audioquality = opts.audioquality.strip('k').strip('K')
684 if not opts.audioquality.isdigit():
685 parser.error(u'invalid audio quality specified')
686 if opts.recodevideo is not None:
687 if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
688 parser.error(u'invalid video recode format specified')
689 if opts.date is not None:
690 date = DateRange.day(opts.date)
691 else:
692 date = DateRange(opts.dateafter, opts.datebefore)
693 if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
694 parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
695
696 # Do not download videos when there are audio-only formats
697 if opts.extractaudio and not opts.keepvideo and opts.format is None:
698 opts.format = 'bestaudio/best'
699
700 # --all-sub automatically sets --write-sub if --write-auto-sub is not given
701 # this was the old behaviour if only --all-sub was given.
702 if opts.allsubtitles and (opts.writeautomaticsub == False):
703 opts.writesubtitles = True
704
705 if sys.version_info < (3,):
706 # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
707 if opts.outtmpl is not None:
708 opts.outtmpl = opts.outtmpl.decode(preferredencoding())
709 outtmpl =((opts.outtmpl is not None and opts.outtmpl)
710 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
711 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
712 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
713 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
714 or (opts.useid and u'%(id)s.%(ext)s')
715 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
716 or DEFAULT_OUTTMPL)
717 if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
718 parser.error(u'Cannot download a video and extract audio into the same'
719 u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
720 u' template'.format(outtmpl))
721
722 any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
723 download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
724
725 ydl_opts = {
726 'usenetrc': opts.usenetrc,
727 'username': opts.username,
728 'password': opts.password,
729 'videopassword': opts.videopassword,
730 'quiet': (opts.quiet or any_printing),
731 'no_warnings': opts.no_warnings,
732 'forceurl': opts.geturl,
733 'forcetitle': opts.gettitle,
734 'forceid': opts.getid,
735 'forcethumbnail': opts.getthumbnail,
736 'forcedescription': opts.getdescription,
737 'forceduration': opts.getduration,
738 'forcefilename': opts.getfilename,
739 'forceformat': opts.getformat,
740 'forcejson': opts.dumpjson,
741 'simulate': opts.simulate,
742 'skip_download': (opts.skip_download or opts.simulate or any_printing),
743 'format': opts.format,
744 'format_limit': opts.format_limit,
745 'listformats': opts.listformats,
746 'outtmpl': outtmpl,
747 'autonumber_size': opts.autonumber_size,
748 'restrictfilenames': opts.restrictfilenames,
749 'ignoreerrors': opts.ignoreerrors,
750 'ratelimit': opts.ratelimit,
751 'nooverwrites': opts.nooverwrites,
752 'retries': opts.retries,
753 'buffersize': opts.buffersize,
754 'noresizebuffer': opts.noresizebuffer,
755 'continuedl': opts.continue_dl,
756 'noprogress': opts.noprogress,
757 'progress_with_newline': opts.progress_with_newline,
758 'playliststart': opts.playliststart,
759 'playlistend': opts.playlistend,
760 'noplaylist': opts.noplaylist,
761 'logtostderr': opts.outtmpl == '-',
762 'consoletitle': opts.consoletitle,
763 'nopart': opts.nopart,
764 'updatetime': opts.updatetime,
765 'writedescription': opts.writedescription,
766 'writeannotations': opts.writeannotations,
767 'writeinfojson': opts.writeinfojson,
768 'writethumbnail': opts.writethumbnail,
769 'writesubtitles': opts.writesubtitles,
770 'writeautomaticsub': opts.writeautomaticsub,
771 'allsubtitles': opts.allsubtitles,
772 'listsubtitles': opts.listsubtitles,
773 'subtitlesformat': opts.subtitlesformat,
774 'subtitleslangs': opts.subtitleslangs,
775 'matchtitle': decodeOption(opts.matchtitle),
776 'rejecttitle': decodeOption(opts.rejecttitle),
777 'max_downloads': opts.max_downloads,
778 'prefer_free_formats': opts.prefer_free_formats,
779 'verbose': opts.verbose,
780 'dump_intermediate_pages': opts.dump_intermediate_pages,
781 'write_pages': opts.write_pages,
782 'test': opts.test,
783 'keepvideo': opts.keepvideo,
784 'min_filesize': opts.min_filesize,
785 'max_filesize': opts.max_filesize,
786 'min_views': opts.min_views,
787 'max_views': opts.max_views,
788 'daterange': date,
789 'cachedir': opts.cachedir,
790 'youtube_print_sig_code': opts.youtube_print_sig_code,
791 'age_limit': opts.age_limit,
792 'download_archive': download_archive_fn,
793 'cookiefile': opts.cookiefile,
794 'nocheckcertificate': opts.no_check_certificate,
795 'prefer_insecure': opts.prefer_insecure,
796 'proxy': opts.proxy,
797 'socket_timeout': opts.socket_timeout,
798 'bidi_workaround': opts.bidi_workaround,
799 'debug_printtraffic': opts.debug_printtraffic,
800 'prefer_ffmpeg': opts.prefer_ffmpeg,
801 'include_ads': opts.include_ads,
802 'default_search': opts.default_search,
803 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
804 'encoding': opts.encoding,
805 }
806
807 with YoutubeDL(ydl_opts) as ydl:
808 ydl.print_debug_header()
809 ydl.add_default_info_extractors()
810
811 # PostProcessors
812 # Add the metadata pp first, the other pps will copy it
813 if opts.addmetadata:
814 ydl.add_post_processor(FFmpegMetadataPP())
815 if opts.extractaudio:
816 ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
817 if opts.recodevideo:
818 ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
819 if opts.embedsubtitles:
820 ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
821 if opts.xattrs:
822 ydl.add_post_processor(XAttrMetadataPP())
823 if opts.embedthumbnail:
824 if not opts.addmetadata:
825 ydl.add_post_processor(FFmpegAudioFixPP())
826 ydl.add_post_processor(AtomicParsleyPP())
827
828 # Update version
829 if opts.update_self:
830 update_self(ydl.to_screen, opts.verbose)
831
832 # Maybe do nothing
833 if (len(all_urls) < 1) and (opts.load_info_filename is None):
834 if not opts.update_self:
835 parser.error(u'you must provide at least one URL')
836 else:
837 sys.exit()
838
839 try:
840 if opts.load_info_filename is not None:
841 retcode = ydl.download_with_info_file(opts.load_info_filename)
842 else:
843 retcode = ydl.download(all_urls)
844 except MaxDownloadsReached:
845 ydl.to_screen(u'--max-download limit reached, aborting.')
846 retcode = 101
847
848 sys.exit(retcode)
849
850
851 def main(argv=None):
852 try:
853 _real_main(argv)
854 except DownloadError:
855 sys.exit(1)
856 except SameFileError:
857 sys.exit(u'ERROR: fixed output name but more than one file to download')
858 except KeyboardInterrupt:
859 sys.exit(u'\nERROR: Interrupted by user')