]> Raphaël G. Git Repositories - youtubedl/blob - youtube_dl/__init__.py
debian/watch: Make use of the signing key from upstream.
[youtubedl] / youtube_dl / __init__.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 __authors__ = (
5 'Ricardo Garcia Gonzalez',
6 'Danny Colligan',
7 'Benjamin Johnson',
8 'Vasyl\' Vavrychuk',
9 'Witold Baryluk',
10 'Paweł Paprota',
11 'Gergely Imreh',
12 'Rogério Brito',
13 'Philipp Hagemeister',
14 'Sören Schulze',
15 'Kevin Ngo',
16 'Ori Avtalion',
17 'shizeeg',
18 'Filippo Valsorda',
19 'Christian Albrecht',
20 'Dave Vasilevsky',
21 'Jaime Marquínez Ferrándiz',
22 'Jeff Crouse',
23 'Osama Khalid',
24 'Michael Walter',
25 'M. Yasoob Ullah Khalid',
26 'Julien Fraichard',
27 'Johny Mo Swag',
28 'Axel Noack',
29 'Albert Kim',
30 'Pierre Rudloff',
31 'Huarong Huo',
32 'Ismael Mejía',
33 'Steffan \'Ruirize\' James',
34 'Andras Elso',
35 'Jelle van der Waa',
36 'Marcin Cieślak',
37 'Anton Larionov',
38 'Takuya Tsuchida',
39 'Sergey M.',
40 'Michael Orlitzky',
41 'Chris Gahan',
42 'Saimadhav Heblikar',
43 'Mike Col',
44 'Oleg Prutz',
45 'pulpe',
46 'Andreas Schmitz',
47 'Michael Kaiser',
48 'Niklas Laxström',
49 'David Triendl',
50 'Anthony Weems',
51 'David Wagner',
52 'Juan C. Olivares',
53 'Mattias Harrysson',
54 'phaer',
55 'Sainyam Kapoor',
56 'Nicolas Évrard',
57 'Jason Normore',
58 'Hoje Lee',
59 'Adam Thalhammer',
60 'Georg Jähnig',
61 'Ralf Haring',
62 )
63
64 __license__ = 'Public Domain'
65
66 import codecs
67 import io
68 import locale
69 import optparse
70 import os
71 import random
72 import re
73 import shlex
74 import sys
75
76
77 from .utils import (
78 compat_getpass,
79 compat_print,
80 DateRange,
81 DEFAULT_OUTTMPL,
82 decodeOption,
83 get_term_width,
84 DownloadError,
85 get_cachedir,
86 MaxDownloadsReached,
87 preferredencoding,
88 read_batch_urls,
89 SameFileError,
90 setproctitle,
91 std_headers,
92 write_string,
93 )
94 from .update import update_self
95 from .FileDownloader import (
96 FileDownloader,
97 )
98 from .extractor import gen_extractors
99 from .version import __version__
100 from .YoutubeDL import YoutubeDL
101 from .postprocessor import (
102 AtomicParsleyPP,
103 FFmpegAudioFixPP,
104 FFmpegMetadataPP,
105 FFmpegVideoConvertor,
106 FFmpegExtractAudioPP,
107 FFmpegEmbedSubtitlePP,
108 XAttrMetadataPP,
109 )
110
111
112 def parseOpts(overrideArguments=None):
113 def _readOptions(filename_bytes, default=[]):
114 try:
115 optionf = open(filename_bytes)
116 except IOError:
117 return default # silently skip if file is not present
118 try:
119 res = []
120 for l in optionf:
121 res += shlex.split(l, comments=True)
122 finally:
123 optionf.close()
124 return res
125
126 def _readUserConf():
127 xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
128 if xdg_config_home:
129 userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
130 if not os.path.isfile(userConfFile):
131 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
132 else:
133 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
134 if not os.path.isfile(userConfFile):
135 userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
136 userConf = _readOptions(userConfFile, None)
137
138 if userConf is None:
139 appdata_dir = os.environ.get('appdata')
140 if appdata_dir:
141 userConf = _readOptions(
142 os.path.join(appdata_dir, 'youtube-dl', 'config'),
143 default=None)
144 if userConf is None:
145 userConf = _readOptions(
146 os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
147 default=None)
148
149 if userConf is None:
150 userConf = _readOptions(
151 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
152 default=None)
153 if userConf is None:
154 userConf = _readOptions(
155 os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
156 default=None)
157
158 if userConf is None:
159 userConf = []
160
161 return userConf
162
163 def _format_option_string(option):
164 ''' ('-o', '--option') -> -o, --format METAVAR'''
165
166 opts = []
167
168 if option._short_opts:
169 opts.append(option._short_opts[0])
170 if option._long_opts:
171 opts.append(option._long_opts[0])
172 if len(opts) > 1:
173 opts.insert(1, ', ')
174
175 if option.takes_value(): opts.append(' %s' % option.metavar)
176
177 return "".join(opts)
178
179 def _comma_separated_values_options_callback(option, opt_str, value, parser):
180 setattr(parser.values, option.dest, value.split(','))
181
182 def _hide_login_info(opts):
183 opts = list(opts)
184 for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
185 try:
186 i = opts.index(private_opt)
187 opts[i+1] = '<PRIVATE>'
188 except ValueError:
189 pass
190 return opts
191
192 max_width = 80
193 max_help_position = 80
194
195 # No need to wrap help messages if we're on a wide console
196 columns = get_term_width()
197 if columns: max_width = columns
198
199 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
200 fmt.format_option_strings = _format_option_string
201
202 kw = {
203 'version' : __version__,
204 'formatter' : fmt,
205 'usage' : '%prog [options] url [url...]',
206 'conflict_handler' : 'resolve',
207 }
208
209 parser = optparse.OptionParser(**kw)
210
211 # option groups
212 general = optparse.OptionGroup(parser, 'General Options')
213 selection = optparse.OptionGroup(parser, 'Video Selection')
214 authentication = optparse.OptionGroup(parser, 'Authentication Options')
215 video_format = optparse.OptionGroup(parser, 'Video Format Options')
216 subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
217 downloader = optparse.OptionGroup(parser, 'Download Options')
218 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
219 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
220 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
221
222 general.add_option('-h', '--help',
223 action='help', help='print this help text and exit')
224 general.add_option('-v', '--version',
225 action='version', help='print program version and exit')
226 general.add_option('-U', '--update',
227 action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
228 general.add_option('-i', '--ignore-errors',
229 action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
230 general.add_option('--abort-on-error',
231 action='store_false', dest='ignoreerrors',
232 help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
233 general.add_option('--dump-user-agent',
234 action='store_true', dest='dump_user_agent',
235 help='display the current browser identification', default=False)
236 general.add_option('--user-agent',
237 dest='user_agent', help='specify a custom user agent', metavar='UA')
238 general.add_option('--referer',
239 dest='referer', help='specify a custom referer, use if the video access is restricted to one domain',
240 metavar='REF', default=None)
241 general.add_option('--add-header',
242 dest='headers', help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times', action="append",
243 metavar='FIELD:VALUE')
244 general.add_option('--list-extractors',
245 action='store_true', dest='list_extractors',
246 help='List all supported extractors and the URLs they would handle', default=False)
247 general.add_option('--extractor-descriptions',
248 action='store_true', dest='list_extractor_descriptions',
249 help='Output descriptions of all supported extractors', default=False)
250 general.add_option(
251 '--proxy', dest='proxy', default=None, metavar='URL',
252 help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
253 general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
254 general.add_option(
255 '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
256 help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
257 general.add_option(
258 '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR',
259 help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
260 general.add_option(
261 '--no-cache-dir', action='store_const', const=None, dest='cachedir',
262 help='Disable filesystem caching')
263 general.add_option(
264 '--socket-timeout', dest='socket_timeout',
265 type=float, default=None, help=u'Time to wait before giving up, in seconds')
266 general.add_option(
267 '--bidi-workaround', dest='bidi_workaround', action='store_true',
268 help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
269 general.add_option(
270 '--default-search',
271 dest='default_search', metavar='PREFIX',
272 help='Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for youtube-dl "large apple". By default (with value "auto") youtube-dl guesses.')
273 general.add_option(
274 '--ignore-config',
275 action='store_true',
276 help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
277 general.add_option(
278 '--encoding', dest='encoding', metavar='ENCODING',
279 help='Force the specified encoding (experimental)')
280
281 selection.add_option(
282 '--playlist-start',
283 dest='playliststart', metavar='NUMBER', default=1, type=int,
284 help='playlist video to start at (default is %default)')
285 selection.add_option(
286 '--playlist-end',
287 dest='playlistend', metavar='NUMBER', default=None, type=int,
288 help='playlist video to end at (default is last)')
289 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
290 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
291 selection.add_option('--max-downloads', metavar='NUMBER',
292 dest='max_downloads', type=int, default=None,
293 help='Abort after downloading NUMBER files')
294 selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
295 selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
296 selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
297 selection.add_option(
298 '--datebefore', metavar='DATE', dest='datebefore', default=None,
299 help='download only videos uploaded on or before this date (i.e. inclusive)')
300 selection.add_option(
301 '--dateafter', metavar='DATE', dest='dateafter', default=None,
302 help='download only videos uploaded on or after this date (i.e. inclusive)')
303 selection.add_option(
304 '--min-views', metavar='COUNT', dest='min_views',
305 default=None, type=int,
306 help="Do not download any videos with less than COUNT views",)
307 selection.add_option(
308 '--max-views', metavar='COUNT', dest='max_views',
309 default=None, type=int,
310 help="Do not download any videos with more than COUNT views",)
311 selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
312 selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
313 help='download only videos suitable for the given age',
314 default=None, type=int)
315 selection.add_option('--download-archive', metavar='FILE',
316 dest='download_archive',
317 help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
318 selection.add_option(
319 '--include-ads', dest='include_ads',
320 action='store_true',
321 help='Download advertisements as well (experimental)')
322 selection.add_option(
323 '--youtube-include-dash-manifest', action='store_true',
324 dest='youtube_include_dash_manifest', default=False,
325 help='Try to download the DASH manifest on YouTube videos (experimental)')
326
327 authentication.add_option('-u', '--username',
328 dest='username', metavar='USERNAME', help='account username')
329 authentication.add_option('-p', '--password',
330 dest='password', metavar='PASSWORD', help='account password')
331 authentication.add_option('-n', '--netrc',
332 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
333 authentication.add_option('--video-password',
334 dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
335
336
337 video_format.add_option('-f', '--format',
338 action='store', dest='format', metavar='FORMAT', default=None,
339 help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
340 video_format.add_option('--all-formats',
341 action='store_const', dest='format', help='download all available video formats', const='all')
342 video_format.add_option('--prefer-free-formats',
343 action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
344 video_format.add_option('--max-quality',
345 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
346 video_format.add_option('-F', '--list-formats',
347 action='store_true', dest='listformats', help='list all available formats')
348
349 subtitles.add_option('--write-sub', '--write-srt',
350 action='store_true', dest='writesubtitles',
351 help='write subtitle file', default=False)
352 subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
353 action='store_true', dest='writeautomaticsub',
354 help='write automatic subtitle file (youtube only)', default=False)
355 subtitles.add_option('--all-subs',
356 action='store_true', dest='allsubtitles',
357 help='downloads all the available subtitles of the video', default=False)
358 subtitles.add_option('--list-subs',
359 action='store_true', dest='listsubtitles',
360 help='lists all available subtitles for the video', default=False)
361 subtitles.add_option('--sub-format',
362 action='store', dest='subtitlesformat', metavar='FORMAT',
363 help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
364 subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
365 action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
366 default=[], callback=_comma_separated_values_options_callback,
367 help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
368
369 downloader.add_option('-r', '--rate-limit',
370 dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
371 downloader.add_option('-R', '--retries',
372 dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
373 downloader.add_option('--buffer-size',
374 dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
375 downloader.add_option('--no-resize-buffer',
376 action='store_true', dest='noresizebuffer',
377 help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
378 downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
379
380 verbosity.add_option('-q', '--quiet',
381 action='store_true', dest='quiet', help='activates quiet mode', default=False)
382 verbosity.add_option(
383 '--no-warnings',
384 dest='no_warnings', action='store_true', default=False,
385 help='Ignore warnings')
386 verbosity.add_option('-s', '--simulate',
387 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
388 verbosity.add_option('--skip-download',
389 action='store_true', dest='skip_download', help='do not download the video', default=False)
390 verbosity.add_option('-g', '--get-url',
391 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
392 verbosity.add_option('-e', '--get-title',
393 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
394 verbosity.add_option('--get-id',
395 action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
396 verbosity.add_option('--get-thumbnail',
397 action='store_true', dest='getthumbnail',
398 help='simulate, quiet but print thumbnail URL', default=False)
399 verbosity.add_option('--get-description',
400 action='store_true', dest='getdescription',
401 help='simulate, quiet but print video description', default=False)
402 verbosity.add_option('--get-duration',
403 action='store_true', dest='getduration',
404 help='simulate, quiet but print video length', default=False)
405 verbosity.add_option('--get-filename',
406 action='store_true', dest='getfilename',
407 help='simulate, quiet but print output filename', default=False)
408 verbosity.add_option('--get-format',
409 action='store_true', dest='getformat',
410 help='simulate, quiet but print output format', default=False)
411 verbosity.add_option('-j', '--dump-json',
412 action='store_true', dest='dumpjson',
413 help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
414 verbosity.add_option('--newline',
415 action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
416 verbosity.add_option('--no-progress',
417 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
418 verbosity.add_option('--console-title',
419 action='store_true', dest='consoletitle',
420 help='display progress in console titlebar', default=False)
421 verbosity.add_option('-v', '--verbose',
422 action='store_true', dest='verbose', help='print various debugging information', default=False)
423 verbosity.add_option('--dump-intermediate-pages',
424 action='store_true', dest='dump_intermediate_pages', default=False,
425 help='print downloaded pages to debug problems (very verbose)')
426 verbosity.add_option('--write-pages',
427 action='store_true', dest='write_pages', default=False,
428 help='Write downloaded intermediary pages to files in the current directory to debug problems')
429 verbosity.add_option('--youtube-print-sig-code',
430 action='store_true', dest='youtube_print_sig_code', default=False,
431 help=optparse.SUPPRESS_HELP)
432 verbosity.add_option('--print-traffic',
433 dest='debug_printtraffic', action='store_true', default=False,
434 help='Display sent and read HTTP traffic')
435
436
437 filesystem.add_option('-t', '--title',
438 action='store_true', dest='usetitle', help='use title in file name (default)', default=False)
439 filesystem.add_option('--id',
440 action='store_true', dest='useid', help='use only video ID in file name', default=False)
441 filesystem.add_option('-l', '--literal',
442 action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
443 filesystem.add_option('-A', '--auto-number',
444 action='store_true', dest='autonumber',
445 help='number downloaded files starting from 00000', default=False)
446 filesystem.add_option('-o', '--output',
447 dest='outtmpl', metavar='TEMPLATE',
448 help=('output filename template. Use %(title)s to get the title, '
449 '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
450 '%(autonumber)s to get an automatically incremented number, '
451 '%(ext)s for the filename extension, '
452 '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
453 '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
454 '%(upload_date)s for the upload date (YYYYMMDD), '
455 '%(extractor)s for the provider (youtube, metacafe, etc), '
456 '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
457 '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
458 '%(height)s and %(width)s for the width and height of the video format. '
459 '%(resolution)s for a textual description of the resolution of the video format. '
460 'Use - to output to stdout. Can also be used to download to a different directory, '
461 'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
462 filesystem.add_option('--autonumber-size',
463 dest='autonumber_size', metavar='NUMBER',
464 help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
465 filesystem.add_option('--restrict-filenames',
466 action='store_true', dest='restrictfilenames',
467 help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
468 filesystem.add_option('-a', '--batch-file',
469 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
470 filesystem.add_option('--load-info',
471 dest='load_info_filename', metavar='FILE',
472 help='json file containing the video information (created with the "--write-json" option)')
473 filesystem.add_option('-w', '--no-overwrites',
474 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
475 filesystem.add_option('-c', '--continue',
476 action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
477 filesystem.add_option('--no-continue',
478 action='store_false', dest='continue_dl',
479 help='do not resume partially downloaded files (restart from beginning)')
480 filesystem.add_option('--cookies',
481 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
482 filesystem.add_option('--no-part',
483 action='store_true', dest='nopart', help='do not use .part files', default=False)
484 filesystem.add_option('--no-mtime',
485 action='store_false', dest='updatetime',
486 help='do not use the Last-modified header to set the file modification time', default=True)
487 filesystem.add_option('--write-description',
488 action='store_true', dest='writedescription',
489 help='write video description to a .description file', default=False)
490 filesystem.add_option('--write-info-json',
491 action='store_true', dest='writeinfojson',
492 help='write video metadata to a .info.json file', default=False)
493 filesystem.add_option('--write-annotations',
494 action='store_true', dest='writeannotations',
495 help='write video annotations to a .annotation file', default=False)
496 filesystem.add_option('--write-thumbnail',
497 action='store_true', dest='writethumbnail',
498 help='write thumbnail image to disk', default=False)
499
500
501 postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
502 help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
503 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
504 help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
505 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
506 help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
507 postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
508 help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm)')
509 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
510 help='keeps the video file on disk after the post-processing; the video is erased by default')
511 postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
512 help='do not overwrite post-processed files; the post-processed files are overwritten by default')
513 postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
514 help='embed subtitles in the video (only for mp4 videos)')
515 postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
516 help='embed thumbnail in the audio as cover art')
517 postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
518 help='write metadata to the video file')
519 postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
520 help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
521 postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
522 help='Prefer avconv over ffmpeg for running the postprocessors (default)')
523 postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
524 help='Prefer ffmpeg over avconv for running the postprocessors')
525
526
527 parser.add_option_group(general)
528 parser.add_option_group(selection)
529 parser.add_option_group(downloader)
530 parser.add_option_group(filesystem)
531 parser.add_option_group(verbosity)
532 parser.add_option_group(video_format)
533 parser.add_option_group(subtitles)
534 parser.add_option_group(authentication)
535 parser.add_option_group(postproc)
536
537 if overrideArguments is not None:
538 opts, args = parser.parse_args(overrideArguments)
539 if opts.verbose:
540 write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
541 else:
542 commandLineConf = sys.argv[1:]
543 if '--ignore-config' in commandLineConf:
544 systemConf = []
545 userConf = []
546 else:
547 systemConf = _readOptions('/etc/youtube-dl.conf')
548 if '--ignore-config' in systemConf:
549 userConf = []
550 else:
551 userConf = _readUserConf()
552 argv = systemConf + userConf + commandLineConf
553
554 opts, args = parser.parse_args(argv)
555 if opts.verbose:
556 write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
557 write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
558 write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
559
560 return parser, opts, args
561
562
563 def _real_main(argv=None):
564 # Compatibility fixes for Windows
565 if sys.platform == 'win32':
566 # https://github.com/rg3/youtube-dl/issues/820
567 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
568
569 setproctitle(u'youtube-dl')
570
571 parser, opts, args = parseOpts(argv)
572
573 # Set user agent
574 if opts.user_agent is not None:
575 std_headers['User-Agent'] = opts.user_agent
576
577 # Set referer
578 if opts.referer is not None:
579 std_headers['Referer'] = opts.referer
580
581 # Custom HTTP headers
582 if opts.headers is not None:
583 for h in opts.headers:
584 if h.find(':', 1) < 0:
585 parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
586 key, value = h.split(':', 2)
587 if opts.verbose:
588 write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
589 std_headers[key] = value
590
591 # Dump user agent
592 if opts.dump_user_agent:
593 compat_print(std_headers['User-Agent'])
594 sys.exit(0)
595
596 # Batch file verification
597 batch_urls = []
598 if opts.batchfile is not None:
599 try:
600 if opts.batchfile == '-':
601 batchfd = sys.stdin
602 else:
603 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
604 batch_urls = read_batch_urls(batchfd)
605 if opts.verbose:
606 write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
607 except IOError:
608 sys.exit(u'ERROR: batch file could not be read')
609 all_urls = batch_urls + args
610 all_urls = [url.strip() for url in all_urls]
611 _enc = preferredencoding()
612 all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
613
614 extractors = gen_extractors()
615
616 if opts.list_extractors:
617 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
618 compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
619 matchedUrls = [url for url in all_urls if ie.suitable(url)]
620 for mu in matchedUrls:
621 compat_print(u' ' + mu)
622 sys.exit(0)
623 if opts.list_extractor_descriptions:
624 for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
625 if not ie._WORKING:
626 continue
627 desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
628 if desc is False:
629 continue
630 if hasattr(ie, 'SEARCH_KEY'):
631 _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise')
632 _COUNTS = (u'', u'5', u'10', u'all')
633 desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
634 compat_print(desc)
635 sys.exit(0)
636
637
638 # Conflicting, missing and erroneous options
639 if opts.usenetrc and (opts.username is not None or opts.password is not None):
640 parser.error(u'using .netrc conflicts with giving username/password')
641 if opts.password is not None and opts.username is None:
642 parser.error(u'account username missing\n')
643 if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
644 parser.error(u'using output template conflicts with using title, video ID or auto number')
645 if opts.usetitle and opts.useid:
646 parser.error(u'using title conflicts with using video ID')
647 if opts.username is not None and opts.password is None:
648 opts.password = compat_getpass(u'Type account password and press [Return]: ')
649 if opts.ratelimit is not None:
650 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
651 if numeric_limit is None:
652 parser.error(u'invalid rate limit specified')
653 opts.ratelimit = numeric_limit
654 if opts.min_filesize is not None:
655 numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
656 if numeric_limit is None:
657 parser.error(u'invalid min_filesize specified')
658 opts.min_filesize = numeric_limit
659 if opts.max_filesize is not None:
660 numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
661 if numeric_limit is None:
662 parser.error(u'invalid max_filesize specified')
663 opts.max_filesize = numeric_limit
664 if opts.retries is not None:
665 try:
666 opts.retries = int(opts.retries)
667 except (TypeError, ValueError):
668 parser.error(u'invalid retry count specified')
669 if opts.buffersize is not None:
670 numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
671 if numeric_buffersize is None:
672 parser.error(u'invalid buffer size specified')
673 opts.buffersize = numeric_buffersize
674 if opts.playliststart <= 0:
675 raise ValueError(u'Playlist start must be positive')
676 if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
677 raise ValueError(u'Playlist end must be greater than playlist start')
678 if opts.extractaudio:
679 if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
680 parser.error(u'invalid audio format specified')
681 if opts.audioquality:
682 opts.audioquality = opts.audioquality.strip('k').strip('K')
683 if not opts.audioquality.isdigit():
684 parser.error(u'invalid audio quality specified')
685 if opts.recodevideo is not None:
686 if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
687 parser.error(u'invalid video recode format specified')
688 if opts.date is not None:
689 date = DateRange.day(opts.date)
690 else:
691 date = DateRange(opts.dateafter, opts.datebefore)
692 if opts.default_search not in ('auto', 'auto_warning', None) and ':' not in opts.default_search:
693 parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
694
695 # Do not download videos when there are audio-only formats
696 if opts.extractaudio and not opts.keepvideo and opts.format is None:
697 opts.format = 'bestaudio/best'
698
699 # --all-sub automatically sets --write-sub if --write-auto-sub is not given
700 # this was the old behaviour if only --all-sub was given.
701 if opts.allsubtitles and (opts.writeautomaticsub == False):
702 opts.writesubtitles = True
703
704 if sys.version_info < (3,):
705 # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
706 if opts.outtmpl is not None:
707 opts.outtmpl = opts.outtmpl.decode(preferredencoding())
708 outtmpl =((opts.outtmpl is not None and opts.outtmpl)
709 or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
710 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
711 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
712 or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
713 or (opts.useid and u'%(id)s.%(ext)s')
714 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
715 or DEFAULT_OUTTMPL)
716 if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
717 parser.error(u'Cannot download a video and extract audio into the same'
718 u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
719 u' template'.format(outtmpl))
720
721 any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
722 download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
723
724 ydl_opts = {
725 'usenetrc': opts.usenetrc,
726 'username': opts.username,
727 'password': opts.password,
728 'videopassword': opts.videopassword,
729 'quiet': (opts.quiet or any_printing),
730 'no_warnings': opts.no_warnings,
731 'forceurl': opts.geturl,
732 'forcetitle': opts.gettitle,
733 'forceid': opts.getid,
734 'forcethumbnail': opts.getthumbnail,
735 'forcedescription': opts.getdescription,
736 'forceduration': opts.getduration,
737 'forcefilename': opts.getfilename,
738 'forceformat': opts.getformat,
739 'forcejson': opts.dumpjson,
740 'simulate': opts.simulate,
741 'skip_download': (opts.skip_download or opts.simulate or any_printing),
742 'format': opts.format,
743 'format_limit': opts.format_limit,
744 'listformats': opts.listformats,
745 'outtmpl': outtmpl,
746 'autonumber_size': opts.autonumber_size,
747 'restrictfilenames': opts.restrictfilenames,
748 'ignoreerrors': opts.ignoreerrors,
749 'ratelimit': opts.ratelimit,
750 'nooverwrites': opts.nooverwrites,
751 'retries': opts.retries,
752 'buffersize': opts.buffersize,
753 'noresizebuffer': opts.noresizebuffer,
754 'continuedl': opts.continue_dl,
755 'noprogress': opts.noprogress,
756 'progress_with_newline': opts.progress_with_newline,
757 'playliststart': opts.playliststart,
758 'playlistend': opts.playlistend,
759 'noplaylist': opts.noplaylist,
760 'logtostderr': opts.outtmpl == '-',
761 'consoletitle': opts.consoletitle,
762 'nopart': opts.nopart,
763 'updatetime': opts.updatetime,
764 'writedescription': opts.writedescription,
765 'writeannotations': opts.writeannotations,
766 'writeinfojson': opts.writeinfojson,
767 'writethumbnail': opts.writethumbnail,
768 'writesubtitles': opts.writesubtitles,
769 'writeautomaticsub': opts.writeautomaticsub,
770 'allsubtitles': opts.allsubtitles,
771 'listsubtitles': opts.listsubtitles,
772 'subtitlesformat': opts.subtitlesformat,
773 'subtitleslangs': opts.subtitleslangs,
774 'matchtitle': decodeOption(opts.matchtitle),
775 'rejecttitle': decodeOption(opts.rejecttitle),
776 'max_downloads': opts.max_downloads,
777 'prefer_free_formats': opts.prefer_free_formats,
778 'verbose': opts.verbose,
779 'dump_intermediate_pages': opts.dump_intermediate_pages,
780 'write_pages': opts.write_pages,
781 'test': opts.test,
782 'keepvideo': opts.keepvideo,
783 'min_filesize': opts.min_filesize,
784 'max_filesize': opts.max_filesize,
785 'min_views': opts.min_views,
786 'max_views': opts.max_views,
787 'daterange': date,
788 'cachedir': opts.cachedir,
789 'youtube_print_sig_code': opts.youtube_print_sig_code,
790 'age_limit': opts.age_limit,
791 'download_archive': download_archive_fn,
792 'cookiefile': opts.cookiefile,
793 'nocheckcertificate': opts.no_check_certificate,
794 'prefer_insecure': opts.prefer_insecure,
795 'proxy': opts.proxy,
796 'socket_timeout': opts.socket_timeout,
797 'bidi_workaround': opts.bidi_workaround,
798 'debug_printtraffic': opts.debug_printtraffic,
799 'prefer_ffmpeg': opts.prefer_ffmpeg,
800 'include_ads': opts.include_ads,
801 'default_search': opts.default_search,
802 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
803 'encoding': opts.encoding,
804 }
805
806 with YoutubeDL(ydl_opts) as ydl:
807 ydl.print_debug_header()
808 ydl.add_default_info_extractors()
809
810 # PostProcessors
811 # Add the metadata pp first, the other pps will copy it
812 if opts.addmetadata:
813 ydl.add_post_processor(FFmpegMetadataPP())
814 if opts.extractaudio:
815 ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
816 if opts.recodevideo:
817 ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
818 if opts.embedsubtitles:
819 ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
820 if opts.xattrs:
821 ydl.add_post_processor(XAttrMetadataPP())
822 if opts.embedthumbnail:
823 if not opts.addmetadata:
824 ydl.add_post_processor(FFmpegAudioFixPP())
825 ydl.add_post_processor(AtomicParsleyPP())
826
827 # Update version
828 if opts.update_self:
829 update_self(ydl.to_screen, opts.verbose)
830
831 # Maybe do nothing
832 if (len(all_urls) < 1) and (opts.load_info_filename is None):
833 if not opts.update_self:
834 parser.error(u'you must provide at least one URL')
835 else:
836 sys.exit()
837
838 try:
839 if opts.load_info_filename is not None:
840 retcode = ydl.download_with_info_file(opts.load_info_filename)
841 else:
842 retcode = ydl.download(all_urls)
843 except MaxDownloadsReached:
844 ydl.to_screen(u'--max-download limit reached, aborting.')
845 retcode = 101
846
847 sys.exit(retcode)
848
849
850 def main(argv=None):
851 try:
852 _real_main(argv)
853 except DownloadError:
854 sys.exit(1)
855 except SameFileError:
856 sys.exit(u'ERROR: fixed output name but more than one file to download')
857 except KeyboardInterrupt:
858 sys.exit(u'\nERROR: Interrupted by user')